diff --git a/src/app/shared_dev/commands/configure/genesis.c b/src/app/shared_dev/commands/configure/genesis.c index 4895901f119..8cb8ca681dc 100644 --- a/src/app/shared_dev/commands/configure/genesis.c +++ b/src/app/shared_dev/commands/configure/genesis.c @@ -62,6 +62,8 @@ default_enable_features( fd_features_t * features ) { features->enable_big_mod_exp_syscall = 0UL; features->enable_alt_bn128_compression_syscall = 0UL; features->update_hashes_per_tick2 = 0UL; + features->account_data_direct_mapping = 0UL; + features->stricter_abi_and_runtime_constraints = 0UL; features->bpf_account_data_direct_mapping = 0UL; features->relax_authority_signer_check_for_lookup_table_creation = 0UL; features->update_hashes_per_tick6 = 0UL; diff --git a/src/flamenco/features/fd_features_generated.c b/src/flamenco/features/fd_features_generated.c index 16d93e761d2..4e4f60c674e 100644 --- a/src/flamenco/features/fd_features_generated.c +++ b/src/flamenco/features/fd_features_generated.c @@ -1026,7 +1026,8 @@ fd_feature_id_t const ids[] = { .id = {"\x90\x9d\x8a\x1a\x1b\xdb\xb4\x28\xec\x2a\x7c\xf2\xbc\x76\xaf\x8c\x72\x9e\xbb\xa0\x6f\xee\x98\xa4\x77\xdd\xe8\xc5\x08\x1b\x7f\x53"}, /* AjX3A4Nv2rzUuATEUWLP4rrBaBropyUnHxEvFDj1dKbx */ .name = "bpf_account_data_direct_mapping", - .cleaned_up = {UINT_MAX, UINT_MAX, UINT_MAX} }, + .cleaned_up = {UINT_MAX, UINT_MAX, UINT_MAX}, + .reverted = 1 }, { .index = offsetof(fd_features_t, add_set_tx_loaded_accounts_data_size_instruction)>>3, .id = {"\xe0\x63\xcf\x92\xc3\xa0\xd3\x55\x49\xd0\x52\xb1\x0c\xaf\xf1\x3f\x56\xfa\x06\x11\x1c\x63\x6f\x69\x75\x42\xd1\x31\x2a\x1e\xe2\x6d"}, @@ -1666,6 +1667,18 @@ fd_feature_id_t const ids[] = { .name = "raise_account_cu_limit", .cleaned_up = {UINT_MAX, UINT_MAX, UINT_MAX} }, + { .index = offsetof(fd_features_t, stricter_abi_and_runtime_constraints)>>3, + .id = {"\xb1\xb1\x8b\xfe\x0c\x8c\xa8\x90\xaf\x61\x7d\x45\x2d\x08\xd5\x33\x88\xea\x0b\x0b\x87\x1f\xb6\x1c\x38\xc8\xeb\x19\x0f\xd7\x9f\x0a"}, + /* CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM */ + .name = "stricter_abi_and_runtime_constraints", + .cleaned_up = {UINT_MAX, UINT_MAX, UINT_MAX} }, + + { .index = offsetof(fd_features_t, account_data_direct_mapping)>>3, + .id = {"\x83\xaf\x45\x7d\x2d\x4b\x60\xe8\xb6\x8b\xde\xea\x1f\x99\x51\x97\x42\x3e\x2d\x9a\xd0\xe0\x8c\xa8\x44\x7f\x6a\xd4\x68\x43\x4e\x19"}, + /* 9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ */ + .name = "account_data_direct_mapping", + .cleaned_up = {UINT_MAX, UINT_MAX, UINT_MAX} }, + { .index = ULONG_MAX } }; /* TODO replace this with fd_map_perfect */ @@ -1915,6 +1928,8 @@ fd_feature_id_query( ulong prefix ) { case 0x3711b30f40730240: return &ids[ 240 ]; case 0xc1309d1b0ae3e80c: return &ids[ 241 ]; case 0x5c64cc1a9be3790a: return &ids[ 242 ]; + case 0x90a88c0cfe8bb1b1: return &ids[ 243 ]; + case 0xe8604b2d7d45af83: return &ids[ 244 ]; default: break; } return NULL; @@ -2163,4 +2178,6 @@ FD_STATIC_ASSERT( offsetof( fd_features_t, require_static_nonce_account FD_STATIC_ASSERT( offsetof( fd_features_t, enable_vote_address_leader_schedule )>>3==240UL, layout ); FD_STATIC_ASSERT( offsetof( fd_features_t, enshrine_slashing_program )>>3==241UL, layout ); FD_STATIC_ASSERT( offsetof( fd_features_t, raise_account_cu_limit )>>3==242UL, layout ); +FD_STATIC_ASSERT( offsetof( fd_features_t, stricter_abi_and_runtime_constraints )>>3==243UL, layout ); +FD_STATIC_ASSERT( offsetof( fd_features_t, account_data_direct_mapping )>>3==244UL, layout ); FD_STATIC_ASSERT( sizeof( fd_features_t )>>3==FD_FEATURE_ID_CNT, layout ); diff --git a/src/flamenco/features/fd_features_generated.h b/src/flamenco/features/fd_features_generated.h index 94feba35c59..a0c3b74334f 100644 --- a/src/flamenco/features/fd_features_generated.h +++ b/src/flamenco/features/fd_features_generated.h @@ -8,7 +8,7 @@ #endif /* FEATURE_ID_CNT is the number of features in ids */ -#define FD_FEATURE_ID_CNT (243UL) +#define FD_FEATURE_ID_CNT (245UL) union fd_features { ulong f[ FD_FEATURE_ID_CNT ]; struct { @@ -255,5 +255,7 @@ union fd_features { /* 0x3711b30f40730240 */ ulong enable_vote_address_leader_schedule; /* 0xc1309d1b0ae3e80c */ ulong enshrine_slashing_program; /* 0x5c64cc1a9be3790a */ ulong raise_account_cu_limit; + /* 0x90a88c0cfe8bb1b1 */ ulong stricter_abi_and_runtime_constraints; + /* 0xe8604b2d7d45af83 */ ulong account_data_direct_mapping; }; }; diff --git a/src/flamenco/features/feature_map.json b/src/flamenco/features/feature_map.json index 585fe0abc10..c23bb1325e0 100644 --- a/src/flamenco/features/feature_map.json +++ b/src/flamenco/features/feature_map.json @@ -146,7 +146,7 @@ {"name":"delay_visibility_of_program_deployment","pubkey": "GmuBvtFb2aHfSfMXpuFeWZGHyDeCLPS79s48fmCWCfM5","cleaned_up":[1,18,0],"hardcode_for_fuzzing":1}, {"name":"apply_cost_tracker_during_replay","pubkey": "2ry7ygxiYURULZCrypHhveanvP5tzZ4toRwVp89oCNSj","cleaned_up":[2,3,0],"hardcode_for_fuzzing":1}, {"name":"deplete_cu_meter_on_vm_failure","pubkey": "B7H2caeia4ZFcpE3QcgMqbiWiBtWrdBRBSJ1DY6Ktxbq","comment":"do not set hardcode_for_fuzzing for this - it significantly degrades vm fuzzing discovery"}, - {"name":"bpf_account_data_direct_mapping","pubkey": "AjX3A4Nv2rzUuATEUWLP4rrBaBropyUnHxEvFDj1dKbx","old": "GJVDwRkUPNdk9QaK4VsU4g1N41QNxhy1hevjf8kz45Mq"}, + {"name":"bpf_account_data_direct_mapping","pubkey": "AjX3A4Nv2rzUuATEUWLP4rrBaBropyUnHxEvFDj1dKbx","old": "GJVDwRkUPNdk9QaK4VsU4g1N41QNxhy1hevjf8kz45Mq","reverted":1}, {"name":"add_set_tx_loaded_accounts_data_size_instruction","pubkey": "G6vbf1UBok8MWb8m25ex86aoQHeKTzDKzuZADHkShqm6","cleaned_up":[1,18,0],"hardcode_for_fuzzing":1}, {"name":"switch_to_new_elf_parser","pubkey": "Cdkc8PPTeTNUPoZEfCY5AyetUrEdkZtNPMgz58nqyaHD","cleaned_up":[2,1,0],"hardcode_for_fuzzing":1}, {"name":"round_up_heap_size","pubkey": "CE2et8pqgyQMP2mQRg3CgvX8nJBKUArMu3wfiQiQKY1y","cleaned_up":[1,18,0],"hardcode_for_fuzzing":1}, @@ -241,5 +241,7 @@ {"name":"require_static_nonce_account","pubkey":"7VVhpg5oAjAmnmz1zCcSHb2Z9ecZB2FQqpnEwReka9Zm"}, {"name":"enable_vote_address_leader_schedule","pubkey":"5JsG4NWH8Jbrqdd8uL6BNwnyZK3dQSoieRXG5vmofj9y"}, {"name":"enshrine_slashing_program","pubkey":"sProgVaNWkYdP2eTRAy1CPrgb3b9p8yXCASrPEqo6VJ"}, - {"name":"raise_account_cu_limit","pubkey":"htsptAwi2yRoZH83SKaUXykeZGtZHgxkS2QwW1pssR8"} + {"name":"raise_account_cu_limit","pubkey":"htsptAwi2yRoZH83SKaUXykeZGtZHgxkS2QwW1pssR8"}, + {"name":"stricter_abi_and_runtime_constraints","pubkey":"CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM"}, + {"name":"account_data_direct_mapping","pubkey":"9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ"} ] diff --git a/src/flamenco/progcache/fd_progcache_rec.c b/src/flamenco/progcache/fd_progcache_rec.c index 77ec6a96e24..56da035446e 100644 --- a/src/flamenco/progcache/fd_progcache_rec.c +++ b/src/flamenco/progcache/fd_progcache_rec.c @@ -88,6 +88,7 @@ fd_progcache_rec_new( void * mem, NULL, 0, FD_FEATURE_ACTIVE( load_slot, features, bpf_account_data_direct_mapping ), + FD_FEATURE_ACTIVE( load_slot, features, stricter_abi_and_runtime_constraints ), 0 ); if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_init failed" )); diff --git a/src/flamenco/runtime/fd_runtime.h b/src/flamenco/runtime/fd_runtime.h index 1d5b0c96a0d..2e1d2689dc8 100644 --- a/src/flamenco/runtime/fd_runtime.h +++ b/src/flamenco/runtime/fd_runtime.h @@ -80,7 +80,8 @@ FD_STATIC_ASSERT( (offsetof(fd_account_rec_t, data)%FD_ACCOUNT_REC_DATA_ALIGN )= #define MAX_PERMITTED_DATA_INCREASE (10240UL) // 10KB #define FD_BPF_ALIGN_OF_U128 (8UL ) FD_STATIC_ASSERT( FD_BPF_ALIGN_OF_U128==FD_ACCOUNT_REC_DATA_ALIGN, input_data_align ); -#define FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP (16UL) +/* https://github.com/anza-xyz/sbpf/blob/v0.12.2/src/ebpf.rs#L37-L38 */ +#define FD_RUNTIME_EBPF_HOST_ALIGN (16UL) /******** These macros bound out memory footprint ********/ @@ -123,7 +124,7 @@ FD_STATIC_ASSERT( FD_BPF_ALIGN_OF_U128==FD_ACCOUNT_REC_DATA_ALIGN, input_data_al sizeof(ulong) /* instr data len */ + \ /* No instr data */ \ sizeof(fd_pubkey_t)), /* program id */ \ - FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP ) + FD_BPF_ALIGN_OF_U128) + FD_RUNTIME_EBPF_HOST_ALIGN ) + FD_BPF_ALIGN_OF_U128) #define FD_RUNTIME_INPUT_REGION_TXN_FOOTPRINT(account_lock_limit, direct_mapping) \ ((FD_MAX_INSTRUCTION_STACK_DEPTH*FD_RUNTIME_INPUT_REGION_INSN_FOOTPRINT(account_lock_limit, direct_mapping)) + \ diff --git a/src/flamenco/runtime/program/fd_bpf_loader_program.c b/src/flamenco/runtime/program/fd_bpf_loader_program.c index 4a389c0fd37..3cd9543b7ba 100644 --- a/src/flamenco/runtime/program/fd_bpf_loader_program.c +++ b/src/flamenco/runtime/program/fd_bpf_loader_program.c @@ -120,8 +120,10 @@ fd_deploy_program( fd_exec_instr_ctx_t * instr_ctx, uchar const * programdata, ulong programdata_size, fd_spad_t * spad ) { - int deploy_mode = 1; - int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, bpf_account_data_direct_mapping ); + int deploy_mode = 1; + int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, account_data_direct_mapping ); + int stricter_abi_and_runtime_constraints = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, stricter_abi_and_runtime_constraints ); + fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_new( fd_spad_alloc( spad, fd_sbpf_syscalls_align(), fd_sbpf_syscalls_footprint() ) ); @@ -177,28 +179,29 @@ fd_deploy_program( fd_exec_instr_ctx_t * instr_ctx, fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) ); vm = fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ instr_ctx->txn_ctx->compute_budget_details.heap_size, - /* entry_cu */ instr_ctx->txn_ctx->compute_budget_details.compute_meter, - /* rodata */ prog->rodata, - /* rodata_sz */ prog->rodata_sz, - /* text */ prog->text, - /* text_cnt */ prog->info.text_cnt, - /* text_off */ prog->info.text_off, /* FIXME: What if text_off is not multiple of 8 */ - /* text_sz */ prog->info.text_sz, - /* entry_pc */ prog->entry_pc, - /* calldests */ prog->calldests, - /* sbpf_version */ elf_info->sbpf_version, - /* syscalls */ syscalls, - /* trace */ NULL, - /* sha */ NULL, - /* mem_regions */ NULL, - /* mem_regions_cnt */ 0, - /* mem_region_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ direct_mapping, - /* dump_syscall_to_pb */ 0 ); + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ instr_ctx->txn_ctx->compute_budget_details.heap_size, + /* entry_cu */ instr_ctx->txn_ctx->compute_budget_details.compute_meter, + /* rodata */ prog->rodata, + /* rodata_sz */ prog->rodata_sz, + /* text */ prog->text, + /* text_cnt */ prog->info.text_cnt, + /* text_off */ prog->info.text_off, /* FIXME: What if text_off is not multiple of 8 */ + /* text_sz */ prog->info.text_sz, + /* entry_pc */ prog->entry_pc, + /* calldests */ prog->calldests, + /* sbpf_version */ elf_info->sbpf_version, + /* syscalls */ syscalls, + /* trace */ NULL, + /* sha */ NULL, + /* mem_regions */ NULL, + /* mem_regions_cnt */ 0, + /* mem_region_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ direct_mapping, + /* stricter_abi_and_runtime_constraints */ stricter_abi_and_runtime_constraints, + /* dump_syscall_to_pb */ 0 ); if ( FD_UNLIKELY( vm == NULL ) ) { FD_LOG_WARNING(( "NULL vm" )); return FD_EXECUTOR_INSTR_ERR_PROGRAM_ENVIRONMENT_SETUP_FAILURE; @@ -388,18 +391,18 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx, 0 ); /* https://github.com/anza-xyz/agave/blob/574bae8fefc0ed256b55340b9d87b7689bcdf222/programs/bpf_loader/src/lib.rs#L1362-L1368 */ - ulong input_sz = 0UL; - ulong pre_lens[256] = {0}; - fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */ - fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */ - uint input_mem_regions_cnt = 0U; - int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, bpf_account_data_direct_mapping ); + ulong input_sz = 0UL; + ulong pre_lens[256] = {0}; + fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */ + fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */ + uint input_mem_regions_cnt = 0U; + int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, bpf_account_data_direct_mapping ); + int stricter_abi_and_runtime_constraints = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, stricter_abi_and_runtime_constraints ); uchar * input = NULL; err = fd_bpf_loader_input_serialize_parameters( instr_ctx, &input_sz, pre_lens, input_mem_regions, &input_mem_regions_cnt, - acc_region_metas, direct_mapping, - is_deprecated, &input ); + acc_region_metas, stricter_abi_and_runtime_constraints, direct_mapping, is_deprecated, &input ); if( FD_UNLIKELY( err ) ) { return err; } @@ -431,28 +434,29 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx, /* TODO: (topointon): correctly set check_size in vm setup */ vm = fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ heap_size, - /* entry_cu */ instr_ctx->txn_ctx->compute_budget_details.compute_meter, - /* rodata */ fd_progcache_rec_rodata( cache_entry ), - /* rodata_sz */ cache_entry->rodata_sz, - /* text */ (ulong *)((ulong)fd_progcache_rec_rodata( cache_entry ) + (ulong)cache_entry->text_off), /* Note: text_off is byte offset */ - /* text_cnt */ cache_entry->text_cnt, - /* text_off */ cache_entry->text_off, - /* text_sz */ cache_entry->text_sz, - /* entry_pc */ cache_entry->entry_pc, - /* calldests */ fd_progcache_rec_calldests( cache_entry ), - /* sbpf_version */ cache_entry->sbpf_version, - /* syscalls */ syscalls, - /* trace */ NULL, - /* sha */ sha, - /* input_mem_regions */ input_mem_regions, - /* input_mem_regions_cnt */ input_mem_regions_cnt, - /* acc_region_metas */ acc_region_metas, - /* is_deprecated */ is_deprecated, - /* direct_mapping */ direct_mapping, - /* dump_syscall_to_pb */ dump_syscall_to_pb ); + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ heap_size, + /* entry_cu */ instr_ctx->txn_ctx->compute_budget_details.compute_meter, + /* rodata */ fd_progcache_rec_rodata( cache_entry ), + /* rodata_sz */ cache_entry->rodata_sz, + /* text (note: text_off is byte offset) */ (ulong *)((ulong)fd_progcache_rec_rodata( cache_entry ) + (ulong)cache_entry->text_off), + /* text_cnt */ cache_entry->text_cnt, + /* text_off */ cache_entry->text_off, + /* text_sz */ cache_entry->text_sz, + /* entry_pc */ cache_entry->entry_pc, + /* calldests */ fd_progcache_rec_calldests( cache_entry ), + /* sbpf_version */ cache_entry->sbpf_version, + /* syscalls */ syscalls, + /* trace */ NULL, + /* sha */ sha, + /* input_mem_regions */ input_mem_regions, + /* input_mem_regions_cnt */ input_mem_regions_cnt, + /* acc_region_metas */ acc_region_metas, + /* is_deprecated */ is_deprecated, + /* direct_mapping */ direct_mapping, + /* stricter_abi_and_runtime_constraints */ stricter_abi_and_runtime_constraints, + /* dump_syscall_to_pb */ dump_syscall_to_pb ); if( FD_UNLIKELY( !vm ) ) { /* We throw an error here because it could be the case that the given heap_size > HEAP_MAX. In this case, Agave fails the transaction but does not error out. @@ -513,19 +517,16 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx, Edge case with error codes: if direct mapping is enabled, the EBPF error is an access violation, and the access type was a store, a different error code is returned to give developers more insight as to what caused the error. - https://github.com/anza-xyz/agave/blob/v2.0.9/programs/bpf_loader/src/lib.rs#L1436-L1470 */ - if( FD_UNLIKELY( direct_mapping && exec_err==FD_VM_ERR_EBPF_ACCESS_VIOLATION && - vm->segv_vaddr!=ULONG_MAX && - vm->segv_access_type==FD_VM_ACCESS_TYPE_ST ) ) { + https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1556-L1618 */ + if( FD_UNLIKELY( stricter_abi_and_runtime_constraints && + ( exec_err==FD_VM_ERR_EBPF_ACCESS_VIOLATION || instr_ctx->txn_ctx->exec_err==FD_VM_ERR_EBPF_ACCESS_VIOLATION ) && + vm->segv_vaddr!=ULONG_MAX ) ) { + /* vaddrs start at 0xFFFFFFFF + 1, so anything below it would not correspond to any account metadata. */ if( FD_UNLIKELY( vm->segv_vaddr>>32UL==0UL ) ) { return FD_EXECUTOR_INSTR_ERR_PROGRAM_FAILED_TO_COMPLETE; } - /* Find the account meta corresponding to the vaddr */ - ulong vaddr_offset = vm->segv_vaddr & FD_VM_OFFSET_MASK; - ulong acc_region_addl_off = is_deprecated ? 0UL : MAX_PERMITTED_DATA_INCREASE; - /* If the vaddr doesn't live in the input region, then we don't need to bother trying to iterate through all of the borrowed accounts. */ if( FD_VADDR_TO_REGION( vm->segv_vaddr )!=FD_VM_INPUT_REGION ) { @@ -537,23 +538,45 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx, vm error based on the account's accesss permissions. */ for( ushort i=0UL; iinstr->acct_cnt; i++ ) { /* https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/lib.rs#L1455 */ - fd_guarded_borrowed_account_t instr_acc = {0}; - FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( instr_ctx, i, &instr_acc ); + /* Find the input memory region that corresponds to the access + https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1566-L1617 */ ulong idx = acc_region_metas[i].region_idx; - if( input_mem_regions[idx].vaddr_offset<=vaddr_offset && vaddr_offsettxn_ctx->bank, remove_accounts_executable_flag_checks ) && - fd_borrowed_account_is_executable( &instr_acc ) ) { - err = FD_EXECUTOR_INSTR_ERR_EXECUTABLE_DATA_MODIFIED; - } else if( fd_borrowed_account_is_writable( &instr_acc ) ) { - err = FD_EXECUTOR_INSTR_ERR_EXTERNAL_DATA_MODIFIED; - } else { - err = FD_EXECUTOR_INSTR_ERR_READONLY_DATA_MODIFIED; + fd_vm_input_region_t const * input_mem_region = &input_mem_regions[idx]; + fd_vm_acc_region_meta_t const * acc_region_meta = &acc_region_metas[i]; + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1484-L1492 */ + ulong region_data_vaddr_start = FD_VM_MEM_MAP_INPUT_REGION_START + input_mem_region->vaddr_offset + input_mem_region->region_sz; + ulong region_data_vaddr_end = fd_ulong_sat_add( region_data_vaddr_start, acc_region_meta->original_data_len ); + if( FD_LIKELY( !is_deprecated ) ) { + region_data_vaddr_end = fd_ulong_sat_add( region_data_vaddr_end, MAX_PERMITTED_DATA_INCREASE ); + } + + if( vm->segv_vaddr >= region_data_vaddr_start && vm->segv_vaddr <= region_data_vaddr_end ) { + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1575-L1616 */ + fd_guarded_borrowed_account_t instr_acc = {0}; + FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( instr_ctx, i, &instr_acc ); + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1581-L1616 */ + if( fd_ulong_sat_add( vm->segv_vaddr, vm->segv_access_len ) <= region_data_vaddr_end ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/programs/bpf_loader/src/lib.rs#L1592-L1601 */ + if( vm->segv_access_type == FD_VM_ACCESS_TYPE_ST ) { + int borrow_err = FD_EXECUTOR_INSTR_SUCCESS; + if( !fd_borrowed_account_can_data_be_changed( &instr_acc, &borrow_err ) || borrow_err != FD_EXECUTOR_INSTR_SUCCESS ) { + return borrow_err; + } else { + return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; + } + } else if ( vm->segv_access_type == FD_VM_ACCESS_TYPE_LD ) { + int borrow_err = FD_EXECUTOR_INSTR_SUCCESS; + if( !fd_borrowed_account_can_data_be_changed( &instr_acc, &borrow_err ) || borrow_err != FD_EXECUTOR_INSTR_SUCCESS ) { + return FD_EXECUTOR_INSTR_ERR_ACC_DATA_TOO_SMALL; + } else { + return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; + } + } } - return err; } } } @@ -595,7 +618,8 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx, return FD_EXECUTOR_INSTR_ERR_PROGRAM_FAILED_TO_COMPLETE; } - err = fd_bpf_loader_input_deserialize_parameters( instr_ctx, pre_lens, input, input_sz, direct_mapping, is_deprecated ); + err = fd_bpf_loader_input_deserialize_parameters( + instr_ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping, is_deprecated ); if( FD_UNLIKELY( err ) ) { return err; } diff --git a/src/flamenco/runtime/program/fd_bpf_loader_serialization.c b/src/flamenco/runtime/program/fd_bpf_loader_serialization.c index bb7d939de65..60322e9b777 100644 --- a/src/flamenco/runtime/program/fd_bpf_loader_serialization.c +++ b/src/flamenco/runtime/program/fd_bpf_loader_serialization.c @@ -2,8 +2,7 @@ #include "../fd_borrowed_account.h" #include "../fd_runtime.h" -/* As a general note, copy_account_data implies that direct mapping is not being - used/is inactive. This file is responsible for serializing and deserializing +/* This file is responsible for serializing and deserializing the input region of the BPF virtual machine. The input region contains instruction information, account metadata, and account data. The high level format is as follows: @@ -33,7 +32,84 @@ also have different write permissions. This should solve the problem of having to memcpy/memcmp account data regions (which can be up to 10MiB each). There is some nuance to this, as the account data can be resized. This means - that memcpys for account data regions can't totally be avoided. */ + that memcpys for account data regions can't totally be avoided. + + SERIALIZATION BEHAVIOR + ========================================== + + This implementation supports three distinct serialization modes based on two + feature flags: stricter_abi_and_runtime_constraints and + account_data_direct_mapping. + + MODE 1 + -------------------------------------- + stricter_abi_and_runtime_constraints = false + account_data_direct_mapping = false + + Memory Layout: + - Single contiguous buffer in host memory + - Buffer contains: [metadata1, data1, realloc_buffer1, metadata2, data2, + realloc_buffer2, ..., metadataN, dataN, realloc_bufferN, instruction_info] + - Each account gets: original data + MAX_PERMITTED_DATA_INCREASE (10KiB) + - Padding added to maintain 16-byte alignment between accounts + - Entire buffer is writable + + Memory Regions: + - The entire input region buffer is mapped as one contiguous VM address + space region + + Serialization Process: + - Account data is memcpy'd into the buffer + - 10KiB realloc buffer is zeroed out and appended after each account's data + - Alignment padding is zeroed and added after realloc buffer + + Deserialization Process: + - Account data must be memcpy'd back from buffer to borrowed account + - For writable accounts: always copy data back + - For read-only accounts: memcmp to verify data unchanged, error if modified + - Account resizing allowed if account permissions permit it + + MODE 2 + ------------------------------------------- + stricter_abi_and_runtime_constraints = true + account_data_direct_mapping = false + + Memory Layout: + - Still uses a single contiguous buffer, but organized into fragmented + regions. + - Each account now has separate regions for metadata and data+realloc. + - Buffer contains: [metadata1, data1+realloc1, metadata2, data2+realloc2, ..., + metadataN, dataN+reallocN, instruction_info]. + - Each metadata region and data region tracked separately in + input_mem_regions. + + Memory Regions: + - For each account: + * Region 0: Account metadata (writable) + * Region 1: Account data + realloc space (writable if account is writable) + - If the account is owned by the deprecated loader, no realloc region is + created as the deprecated loader does not support resizing accounts. + + Serialization: + - Account metadata serialized first, added as a memory region. + - Account data memcpy'd into buffer - not directly mapped. + - 10KiB realloc buffer zeroed and appended (not direct mapped). + - Data region created pointing to copied data in buffer. + + MODE 3: Direct Mapping (requires stricter_abi_and_runtime_constraints) + ----------------------------------------------- + stricter_abi_and_runtime_constraints = true + account_data_direct_mapping = true + + This is very similar to stricter_abi_and_runtime_constraints, but account + data is NOT copied into the input region buffer. + + Instead, the data region points directly to the staging area for the + account in the transaction account's data. This staging area has enough + space to hold the account data and the realloc buffer. Changes to this + staging area will be written back to the account database in transaction + finalization. + */ /* Add a new memory region to represent the input region. All of the memory regions here have sorted virtual addresses. These regions may or may not @@ -44,22 +120,26 @@ new_input_mem_region( fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, const uchar * buffer, ulong region_sz, + ulong address_space_reserved, + ulong padding, uchar is_writable, - uchar is_acct_data ) { + ulong acc_region_meta_idx ) { /* The start vaddr of the new region should be equal to start of the previous - region added to its size. */ + region added to the address space reserved for the region. */ ulong vaddr_offset = *input_mem_regions_cnt==0UL ? 0UL : input_mem_regions[ *input_mem_regions_cnt-1U ].vaddr_offset + - input_mem_regions[ *input_mem_regions_cnt-1U ].region_sz; - input_mem_regions[ *input_mem_regions_cnt ].is_writable = is_writable; - input_mem_regions[ *input_mem_regions_cnt ].haddr = (ulong)buffer; - input_mem_regions[ *input_mem_regions_cnt ].region_sz = (uint)region_sz; - input_mem_regions[ *input_mem_regions_cnt ].vaddr_offset = vaddr_offset; - input_mem_regions[ *input_mem_regions_cnt ].is_acct_data = is_acct_data; + input_mem_regions[ *input_mem_regions_cnt-1U ].address_space_reserved; + input_mem_regions[ *input_mem_regions_cnt ].is_writable = is_writable; + input_mem_regions[ *input_mem_regions_cnt ].haddr = (ulong)buffer; + input_mem_regions[ *input_mem_regions_cnt ].region_sz = (uint)region_sz; + input_mem_regions[ *input_mem_regions_cnt ].address_space_reserved = address_space_reserved; + input_mem_regions[ *input_mem_regions_cnt ].vaddr_offset = vaddr_offset; + input_mem_regions[ *input_mem_regions_cnt ].padding = padding; + input_mem_regions[ *input_mem_regions_cnt ].acc_region_meta_idx = acc_region_meta_idx; (*input_mem_regions_cnt)++; } -/* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L93-L130 */ +/* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L127-L189 */ /* This function handles casing for direct mapping being enabled as well as if the alignment is being stored. In the case where direct mapping is not enabled, we copy in the account data and a 10KiB buffer into the input region. @@ -68,87 +148,115 @@ new_input_mem_region( fd_vm_input_region_t * input_mem_regions, different memory regions. In both cases, padding is used to maintain 8 byte alignment. If alignment is not required, then a resizing buffer is not used as the deprecated loader doesn't allow for resizing accounts. */ -static void +static ulong write_account( fd_borrowed_account_t * account, uchar instr_acc_idx, uchar * * serialized_params, uchar * * serialized_params_start, fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, + ulong padding, fd_vm_acc_region_meta_t * acc_region_metas, - int is_aligned, - int copy_account_data ) { + int is_loader_v1, + int stricter_abi_and_runtime_constraints, + int direct_mapping ) { uchar const * data = account ? fd_borrowed_account_get_data( account ) : NULL; ulong dlen = account ? fd_borrowed_account_get_data_len( account ) : 0UL; acc_region_metas[instr_acc_idx].original_data_len = dlen; - if( copy_account_data ) { + acc_region_metas[instr_acc_idx].acct = account->acct; + + /* Legacy behavior: no stricter_abi_and_runtime_constraints (also implies no direct mapping) + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L131-L140 */ + if( !stricter_abi_and_runtime_constraints ) { /* Copy the account data into input region buffer */ fd_memcpy( *serialized_params, data, dlen ); *serialized_params += dlen; - if( FD_LIKELY( is_aligned ) ) { + if( FD_LIKELY( !is_loader_v1 ) ) { /* Zero out padding bytes and max permitted data increase */ ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen; fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE + align_offset ); *serialized_params += MAX_PERMITTED_DATA_INCREASE + align_offset; } - /* In the non-DM case, we don't bother with setting up mem regions. - So has_data_region and has_resizing_region are set to 0. */ - acc_region_metas[instr_acc_idx].region_idx = UINT_MAX; - acc_region_metas[instr_acc_idx].has_data_region = 0U; - acc_region_metas[instr_acc_idx].has_resizing_region = 0U; - } else { /* direct_mapping == true */ + acc_region_metas[instr_acc_idx].region_idx = UINT_MAX; + } else { /* stricter_abi_and_runtime_constraints == true */ + + /* Set up account region metadata */ + acc_region_metas[instr_acc_idx].region_idx = *input_mem_regions_cnt; + /* First, push on the region for the metadata that has just been serialized. This function will push the metadata in the serialized_params from serialized_params_start to serialized_params as a region to the input - memory regions array. */ - /* TODO: This region always has length of 96 and this can be set as a constant. */ + memory regions array. + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L142 */ ulong region_sz = (ulong)(*serialized_params) - (ulong)(*serialized_params_start); - new_input_mem_region( input_mem_regions, input_mem_regions_cnt, *serialized_params_start, region_sz, 1U, 0U ); - - /* Next, push the region for the account data if there is account data. We - intentionally omit copy on write as a region type. */ - int err = 0; - uchar is_writable = !!(fd_borrowed_account_can_data_be_changed( account, &err ) && !err); - - /* Update the mapping from instruction account index to memory region index. - This is an optimization to avoid redundant lookups to find accounts. */ - acc_region_metas[instr_acc_idx].region_idx = *input_mem_regions_cnt; - acc_region_metas[instr_acc_idx].has_data_region = !!dlen; - acc_region_metas[instr_acc_idx].has_resizing_region = (uchar)is_aligned; + new_input_mem_region( input_mem_regions, input_mem_regions_cnt, *serialized_params_start, region_sz, region_sz, padding, 1U, ULONG_MAX ); + + /* If direct mapping isn't enabled, then copy the account data in directly + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L144-L150 */ + if( !direct_mapping ) { + fd_memcpy( *serialized_params, data, dlen ); + *serialized_params += dlen; + if( FD_LIKELY( !is_loader_v1 ) ) { + fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE ); + *serialized_params += MAX_PERMITTED_DATA_INCREASE; + } + } - if( dlen ) { - new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data, dlen, is_writable, 1U ); + /* Calculate address space reserved for account (data + realloc space) + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L151-L158 */ + ulong address_space_reserved = !is_loader_v1 ? + fd_ulong_sat_add( dlen, MAX_PERMITTED_DATA_INCREASE ) : dlen; + + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L159-L169 */ + if( address_space_reserved > 0 ) { + int err = 0; + uchar is_writable = !!(fd_borrowed_account_can_data_be_changed( account, &err ) && !err); + + if( !direct_mapping ) { + /* Create region pointing to the copied data in buffer + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L160-L164 */ + uchar * data_start = *serialized_params - address_space_reserved; + new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data_start, dlen, address_space_reserved, 0UL, is_writable, instr_acc_idx ); + } else { + /* Direct mapping: create region pointing directly to account data */ + new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data, dlen, address_space_reserved, 0UL, is_writable, instr_acc_idx ); + } } - if( FD_LIKELY( is_aligned ) ) { - /* Finally, push a third region for the max resizing data region. This is - done even if there is no account data. This must be aligned so padding - bytes must be inserted. This resizing region is also padded to result - in 8 byte alignment for the combination of the account data region with - the resizing region. + *serialized_params_start = *serialized_params; - We add the max permitted resizing limit along with 8 bytes of padding - to the serialization buffer. However, the padding bytes are used to - maintain alignment in the VM virtual address space. */ + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L170-L186 */ + if( FD_LIKELY( !is_loader_v1 ) ) { ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen; - - fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE + FD_BPF_ALIGN_OF_U128 ); - - /* Leave a gap for alignment */ - uchar * region_buffer = *serialized_params + (FD_BPF_ALIGN_OF_U128 - align_offset); - ulong region_sz = MAX_PERMITTED_DATA_INCREASE + align_offset; - new_input_mem_region( input_mem_regions, input_mem_regions_cnt, region_buffer, region_sz, is_writable, 1U ); - - *serialized_params += MAX_PERMITTED_DATA_INCREASE + FD_BPF_ALIGN_OF_U128; + if( !direct_mapping ) { + /* If direct mapping is not enabled, we do not align the start of each + region metadata to FD_BPF_ALIGN_OF_U128, but we do align the start + of the actual contents of the metadata region. + + This follows Agave's logic + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L173-L176 */ + fd_memset( *serialized_params, 0, align_offset ); + *serialized_params += align_offset; + } else { + /* If direct mapping is enabled, we align the start of each region + metadata to FD_BPF_ALIGN_OF_U128. */ + fd_memset( *serialized_params, 0, FD_BPF_ALIGN_OF_U128 ); + *serialized_params += FD_BPF_ALIGN_OF_U128; + *serialized_params_start += fd_ulong_sat_sub( FD_BPF_ALIGN_OF_U128, align_offset ); + } } - *serialized_params_start = *serialized_params; + + return (ulong)(*serialized_params - *serialized_params_start); } + + return 0UL; } +/* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L466 */ static uchar * fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, ulong * sz, @@ -156,13 +264,14 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, fd_vm_acc_region_meta_t * acc_region_metas, - int copy_account_data ) { + int stricter_abi_and_runtime_constraints, + int direct_mapping ) { fd_pubkey_t * txn_accs = ctx->txn_ctx->account_keys; uchar acc_idx_seen[256] = {0}; ushort dup_acc_idx[256] = {0}; - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L429-L459 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L482-L511 */ ulong serialized_size = 0UL; serialized_size += sizeof(ulong); // acct_cnt /* First pass is to calculate size of buffer to allocate */ @@ -177,7 +286,7 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, dup_acc_idx[acc_idx] = i; /* Borrow the account without checking the error, as it is guaranteed to exist - https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L225 */ + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L244-L257 */ fd_guarded_borrowed_account_t view_acc = {0}; fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc ); @@ -191,10 +300,10 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, + sizeof(fd_pubkey_t) // owner + sizeof(ulong) // lamports + sizeof(ulong) // data len - + MAX_PERMITTED_DATA_INCREASE + sizeof(ulong); // rent_epoch - if( copy_account_data ) { - serialized_size += fd_ulong_align_up( acc_data_len, FD_BPF_ALIGN_OF_U128 ); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L499-L505 */ + if( !(stricter_abi_and_runtime_constraints && direct_mapping) ) { + serialized_size += MAX_PERMITTED_DATA_INCREASE + fd_ulong_align_up( acc_data_len, FD_BPF_ALIGN_OF_U128 ); } else { serialized_size += FD_BPF_ALIGN_OF_U128; } @@ -206,74 +315,80 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, + sizeof(fd_pubkey_t); // program id /* 16-byte aligned buffer: - https://github.com/anza-xyz/agave/blob/v2.2.13/programs/bpf_loader/src/serialization.rs#L32 + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L60 */ - uchar * serialized_params = fd_spad_alloc( ctx->txn_ctx->spad, FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP, fd_ulong_align_up( serialized_size, FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP ) ); + uchar * serialized_params = fd_spad_alloc( + ctx->txn_ctx->spad, + FD_RUNTIME_EBPF_HOST_ALIGN, + fd_ulong_align_up( serialized_size, FD_RUNTIME_EBPF_HOST_ALIGN ) ); uchar * serialized_params_start = serialized_params; uchar * curr_serialized_params_start = serialized_params; + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L522 */ FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt ); serialized_params += sizeof(ulong); + ulong padding = sizeof(ulong); - /* Second pass over the account is to serialize into the buffer. */ + /* Second pass over the account is to serialize into the buffer. + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L523-L557 */ for( ushort i=0; iinstr->acct_cnt; i++ ) { uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction; fd_pubkey_t * acc = &txn_accs[acc_idx]; if( FD_UNLIKELY( acc_idx_seen[acc_idx] && dup_acc_idx[acc_idx] != i ) ) { /* Duplicate. Store 8 byte buffer to maintain alignment but store the - account index in the first byte.*/ + account index in the first byte. + + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L551-L555 */ FD_STORE( ulong, serialized_params, 0UL ); FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] ); serialized_params += sizeof(ulong); + padding += sizeof(ulong); } else { - /* Calculate and store the start of the actual metadata region for this account, - excluding any duplicate account markers at the beginning. - - We use this later for retrieving the serialized values later in the CPI security checks. */ - ulong metadata_region_offset_with_dups = *input_mem_regions_cnt==0UL ? 0UL : - input_mem_regions[ *input_mem_regions_cnt-1U ].vaddr_offset + - input_mem_regions[ *input_mem_regions_cnt-1U ].region_sz; - - acc_region_metas[i].metadata_region_offset = metadata_region_offset_with_dups + - (ulong)(serialized_params - curr_serialized_params_start); - + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L526 */ FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER ); serialized_params += sizeof(uchar); /* Borrow the account without checking the error, as it is guaranteed to exist - https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L225 */ + https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L244-L257 */ fd_guarded_borrowed_account_t view_acc = {0}; fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc ); - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L465 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L525 */ fd_account_meta_t const * metadata = fd_borrowed_account_get_acc_meta( &view_acc ); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L527 */ uchar is_signer = (uchar)fd_instr_acc_is_signer_idx( ctx->instr, (uchar)i, NULL ); FD_STORE( uchar, serialized_params, is_signer ); serialized_params += sizeof(uchar); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L528 */ uchar is_writable = (uchar)fd_instr_acc_is_writable_idx( ctx->instr, (uchar)i ); FD_STORE( uchar, serialized_params, is_writable ); serialized_params += sizeof(uchar); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L529-L530 */ uchar is_executable = (uchar)metadata->executable; FD_STORE( uchar, serialized_params, is_executable ); serialized_params += sizeof(uchar); /* The original data len field is intentionally NOT populated. */ + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L531 */ uint padding_0 = 0U; FD_STORE( uint, serialized_params, padding_0 ); serialized_params += sizeof(uint); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L532 */ fd_pubkey_t key = *acc; FD_STORE( fd_pubkey_t, serialized_params, key ); serialized_params += sizeof(fd_pubkey_t); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L533 */ fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner; FD_STORE( fd_pubkey_t, serialized_params, owner ); serialized_params += sizeof(fd_pubkey_t); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L534 */ ulong lamports = metadata->lamports; FD_STORE( ulong, serialized_params, lamports ); serialized_params += sizeof(ulong); @@ -281,73 +396,88 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx, ulong acc_data_len = metadata->dlen; pre_lens[i] = acc_data_len; + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L535 */ ulong data_len = acc_data_len; FD_STORE( ulong, serialized_params, data_len ); serialized_params += sizeof(ulong); - write_account( &view_acc, (uchar)i, &serialized_params, &curr_serialized_params_start, - input_mem_regions, input_mem_regions_cnt, acc_region_metas, 1, copy_account_data ); - + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L536 */ + padding = write_account( + &view_acc, + (uchar)i, + &serialized_params, + &curr_serialized_params_start, + input_mem_regions, + input_mem_regions_cnt, + padding, + acc_region_metas, + 0, + stricter_abi_and_runtime_constraints, + direct_mapping ); + + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L537-L541 */ FD_STORE( ulong, serialized_params, ULONG_MAX ); serialized_params += sizeof(ulong); + padding += sizeof(ulong); } } + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L558 */ ulong instr_data_len = ctx->instr->data_sz; FD_STORE( ulong, serialized_params, instr_data_len ); serialized_params += sizeof(ulong); + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L559 */ uchar * instr_data = ctx->instr->data; fd_memcpy( serialized_params, instr_data, instr_data_len ); serialized_params += instr_data_len; + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L560 */ FD_STORE( fd_pubkey_t, serialized_params, txn_accs[ctx->instr->program_id] ); serialized_params += sizeof(fd_pubkey_t); - if( FD_UNLIKELY( serialized_params!=serialized_params_start+serialized_size ) ) { - FD_LOG_ERR(( "Serializing error" )); /* TODO: we can likely get rid of this check altogether */ - } - /* Write out the final region. */ + ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start); new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start, - (ulong)(serialized_params - curr_serialized_params_start), 1U, 0U ); + region_sz, region_sz, padding, 1U, ULONG_MAX ); *sz = serialized_size; return serialized_params_start; } -/* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L500-L603 */ +/* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L566-L653 */ static int fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx, ulong const * pre_lens, uchar * buffer, ulong FD_FN_UNUSED buffer_sz, - int copy_account_data ) { + int stricter_abi_and_runtime_constraints, + int direct_mapping ) { /* TODO: An optimization would be to skip ahead through non-writable accounts */ - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L507 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L573 */ ulong start = 0UL; uchar acc_idx_seen[256] = {0}; start += sizeof(ulong); // number of accounts - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L508-L600 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L574-L650 */ for( ushort i=0; iinstr->acct_cnt; i++ ) { uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction; start++; // position /* get the borrowed account - https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L519 */ + https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L584-L585 */ fd_guarded_borrowed_account_t view_acc = {0}; FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( ctx, i, &view_acc ); if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) { - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L515-517 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L582 */ start += 7UL; } else { - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L518-524 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L586-L590 */ acc_idx_seen[acc_idx] = 1; start += sizeof(uchar) // is_signer + sizeof(uchar) // is_writable @@ -355,7 +485,7 @@ fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx, + sizeof(uint) // original_data_len + sizeof(fd_pubkey_t); // key - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L525-548 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L591-L593 */ fd_pubkey_t * owner = (fd_pubkey_t *)(buffer+start); start += sizeof(fd_pubkey_t); // owner @@ -383,8 +513,8 @@ fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx, return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; } - if( copy_account_data ) { - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L551-563 */ + if( !stricter_abi_and_runtime_constraints ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L617-L627 */ int err = 0; if( fd_borrowed_account_can_data_be_resized( &view_acc, post_len, &err ) && fd_borrowed_account_can_data_be_changed( &view_acc, &err ) ) { @@ -395,46 +525,34 @@ fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx, } } else if( FD_UNLIKELY( fd_borrowed_account_get_data_len( &view_acc )!=post_len || - memcmp( fd_borrowed_account_get_data( &view_acc ), post_data, post_len ) ) ) { + memcmp( fd_borrowed_account_get_data( &view_acc ), post_data, post_len ) ) ) { return err; } - start += pre_len; - } else { /* If direct mapping is enabled */ - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L564-587 */ - start += FD_BPF_ALIGN_OF_U128 - alignment_offset; + } else if( !direct_mapping ) { int err = 0; - if( fd_borrowed_account_can_data_be_resized( &view_acc, post_len, &err ) && - fd_borrowed_account_can_data_be_changed( &view_acc, &err ) ) { - - err = fd_borrowed_account_set_data_length( &view_acc, post_len ); + if( fd_borrowed_account_can_data_be_changed( &view_acc, &err ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L627-L633 */ + int err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len ); if( FD_UNLIKELY( err ) ) { return err; } - - ulong allocated_bytes = fd_ulong_sat_sub( post_len, pre_len ); - if( allocated_bytes ) { - uchar * acc_data = NULL; - ulong acc_dlen = 0UL; - err = fd_borrowed_account_get_data_mut( &view_acc, &acc_data, &acc_dlen ); - if( FD_UNLIKELY( err ) ) { - return err; - } - if( FD_UNLIKELY( pre_len+allocated_bytes>acc_dlen ) ) { - return FD_EXECUTOR_INSTR_ERR_INVALID_ARG; - } - /* We want to copy in the reallocated bytes from the input - buffer directly into the borrowed account data buffer - which has now been extended. */ - memcpy( acc_data+pre_len, buffer+start, allocated_bytes ); - } - } else if( FD_UNLIKELY( fd_borrowed_account_get_data_len( &view_acc )!=post_len ) ) { + } + } else if( fd_borrowed_account_get_data_len( &view_acc ) != post_len ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L633-L635 */ + int err = fd_borrowed_account_set_data_length( &view_acc, post_len ); + if( FD_UNLIKELY( err ) ) { return err; } } - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L593-598 */ - start += MAX_PERMITTED_DATA_INCREASE; - start += alignment_offset; + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L636-L644 */ + if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) { + start += fd_ulong_sat_add( MAX_PERMITTED_DATA_INCREASE, fd_ulong_sat_add( pre_len, alignment_offset ) ); + } else { + start += FD_BPF_ALIGN_OF_U128; + } + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L645 */ start += sizeof(ulong); // rent epoch if( memcmp( fd_borrowed_account_get_owner( &view_acc ), owner, sizeof(fd_pubkey_t) ) ) { int err = fd_borrowed_account_set_owner( &view_acc, owner ); @@ -455,7 +573,8 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, fd_vm_acc_region_meta_t * acc_region_metas, - int copy_account_data ) { + int stricter_abi_and_runtime_constraints, + int direct_mapping ) { ulong serialized_size = 0UL; fd_pubkey_t const * txn_accs = ctx->txn_ctx->account_keys; @@ -491,7 +610,7 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, + sizeof(fd_pubkey_t) // owner + sizeof(uchar) // executable + sizeof(ulong); // rent_epoch - if( copy_account_data ) { + if( !(stricter_abi_and_runtime_constraints && direct_mapping) ) { serialized_size += acc_data_len; } } @@ -503,12 +622,13 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, /* 16-byte aligned buffer: https://github.com/anza-xyz/agave/blob/v2.2.13/programs/bpf_loader/src/serialization.rs#L32 */ - uchar * serialized_params = fd_spad_alloc( ctx->txn_ctx->spad, FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP, serialized_size ); + uchar * serialized_params = fd_spad_alloc( ctx->txn_ctx->spad, FD_RUNTIME_EBPF_HOST_ALIGN, serialized_size ); uchar * serialized_params_start = serialized_params; uchar * curr_serialized_params_start = serialized_params; FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt ); serialized_params += sizeof(ulong); + ulong padding = sizeof(ulong); for( ushort i=0; iinstr->acct_cnt; i++ ) { uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction; @@ -518,18 +638,8 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, // Duplicate FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] ); serialized_params += sizeof(uchar); + padding += sizeof(uchar); } else { - /* Calculate and store the start of the actual metadata region for this account, - excluding any duplicate account markers at the beginning. - - We use this later for retrieving the serialized values later in the CPI security checks. */ - ulong metadata_region_offset_with_dups = *input_mem_regions_cnt==0UL ? 0UL : - input_mem_regions[ *input_mem_regions_cnt-1U ].vaddr_offset + - input_mem_regions[ *input_mem_regions_cnt-1U ].region_sz; - - acc_region_metas[i].metadata_region_offset = metadata_region_offset_with_dups + - (ulong)(serialized_params - curr_serialized_params_start); - FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER ); serialized_params += sizeof(uchar); @@ -560,8 +670,9 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, FD_STORE( ulong, serialized_params, acc_data_len ); serialized_params += sizeof(ulong); - write_account( &view_acc, (uchar)i, &serialized_params, &curr_serialized_params_start, - input_mem_regions, input_mem_regions_cnt, acc_region_metas, 0, copy_account_data ); + padding = write_account( &view_acc, (uchar)i, &serialized_params, &curr_serialized_params_start, + input_mem_regions, input_mem_regions_cnt, padding, acc_region_metas, 1, + stricter_abi_and_runtime_constraints, direct_mapping ); fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner; FD_STORE( fd_pubkey_t, serialized_params, owner ); @@ -590,18 +701,21 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx, FD_TEST( serialized_params == serialized_params_start + serialized_size ); *sz = serialized_size; + ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start); new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start, - (ulong)(serialized_params - curr_serialized_params_start), 1U, 0U ); + region_sz, region_sz, padding, 1U, ULONG_MAX ); return serialized_params_start; } +/* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L404 */ static int fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t * ctx, ulong const * pre_lens, uchar * input, ulong input_sz, - int copy_account_data ) { + int stricter_abi_and_runtime_constraints, + int direct_mapping ) { uchar * input_cursor = input; uchar acc_idx_seen[256] = {0}; @@ -632,24 +746,45 @@ fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t * ctx, } input_cursor += sizeof(ulong); /* lamports */ - input_cursor += sizeof(ulong); /* data length */ - if( copy_account_data ) { - ulong pre_len = pre_lens[i]; - uchar * post_data = input_cursor; - if( fd_borrowed_account_get_acc_meta( &view_acc ) ) { - int err = 0; + ulong post_len = FD_LOAD( ulong, input_cursor ); + input_cursor += sizeof(ulong); /* data length */ + + ulong pre_len = pre_lens[i]; + uchar * post_data = input_cursor; + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L436-L446 */ + if( !stricter_abi_and_runtime_constraints ) { + int err = 0; if( fd_borrowed_account_can_data_be_resized( &view_acc, pre_len, &err ) && fd_borrowed_account_can_data_be_changed( &view_acc, &err ) ) { - err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, pre_len ); - if( FD_UNLIKELY( err ) ) { - return err; - } - } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len || + err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, pre_len ); + if( FD_UNLIKELY( err ) ) { + return err; + } + } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len || memcmp( post_data, fd_borrowed_account_get_data( &view_acc ), pre_len ) ) { return err; } + } else if( !direct_mapping ) { + int err = 0; + if( fd_borrowed_account_can_data_be_changed( &view_acc, &err ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L446-L452 */ + err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len ); + if( FD_UNLIKELY( err ) ) { + return err; + } } + } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L452-L454 */ + int err = fd_borrowed_account_set_data_length( &view_acc, pre_len ); + if( FD_UNLIKELY( err ) ) { + return err; + } + } + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L455-L457 */ + if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) { input_cursor += pre_len; } input_cursor += sizeof(fd_pubkey_t) + /* owner */ @@ -665,7 +800,7 @@ fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t * ctx, return 0; } -/* https://github.com/anza-xyz/agave/blob/v2.1.11/programs/bpf_loader/src/serialization.rs#L191-L252 */ +/* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L221 */ int fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx, ulong * sz, @@ -673,11 +808,12 @@ fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx, fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, fd_vm_acc_region_meta_t * acc_region_metas, + int stricter_abi_and_runtime_constraints, int direct_mapping, uchar is_deprecated, uchar ** out /* output */ ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.11/programs/bpf_loader/src/serialization.rs#L203-L206 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L234-L237 */ ulong num_ix_accounts = instr_ctx->instr->acct_cnt; if( FD_UNLIKELY( num_ix_accounts>=FD_INSTR_ACCT_MAX ) ) { return FD_EXECUTOR_INSTR_ERR_MAX_ACCS_EXCEEDED; @@ -689,27 +825,32 @@ fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx, if( FD_UNLIKELY( is_deprecated ) ) { *out = fd_bpf_loader_input_serialize_unaligned( instr_ctx, sz, pre_lens, input_mem_regions, input_mem_regions_cnt, - acc_region_metas, !direct_mapping ); + acc_region_metas, stricter_abi_and_runtime_constraints, + direct_mapping ); } else { *out = fd_bpf_loader_input_serialize_aligned( instr_ctx, sz, pre_lens, input_mem_regions, input_mem_regions_cnt, - acc_region_metas, !direct_mapping ); + acc_region_metas, stricter_abi_and_runtime_constraints, + direct_mapping ); } return FD_EXECUTOR_INSTR_SUCCESS; } -/* https://github.com/anza-xyz/agave/blob/v2.1.11/programs/bpf_loader/src/serialization.rs#L254-L283 */ +/* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L284-L311 */ int fd_bpf_loader_input_deserialize_parameters( fd_exec_instr_ctx_t * ctx, ulong const * pre_lens, uchar * input, ulong input_sz, + int stricter_abi_and_runtime_constraints, int direct_mapping, uchar is_deprecated ) { if( FD_UNLIKELY( is_deprecated ) ) { - return fd_bpf_loader_input_deserialize_unaligned( ctx, pre_lens, input, input_sz, !direct_mapping ); + return fd_bpf_loader_input_deserialize_unaligned( + ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping ); } else { - return fd_bpf_loader_input_deserialize_aligned( ctx, pre_lens, input, input_sz, !direct_mapping ); + return fd_bpf_loader_input_deserialize_aligned( + ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping ); } } diff --git a/src/flamenco/runtime/program/fd_bpf_loader_serialization.h b/src/flamenco/runtime/program/fd_bpf_loader_serialization.h index f5945aa6f19..cf0c27347d6 100644 --- a/src/flamenco/runtime/program/fd_bpf_loader_serialization.h +++ b/src/flamenco/runtime/program/fd_bpf_loader_serialization.h @@ -15,6 +15,7 @@ fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx, fd_vm_input_region_t * input_mem_regions, uint * input_mem_regions_cnt, fd_vm_acc_region_meta_t * acc_region_metas, + int stricter_abi_and_runtime_constraints, int direct_mapping, uchar is_deprecated, uchar ** out /* output */ ); @@ -24,6 +25,7 @@ fd_bpf_loader_input_deserialize_parameters( fd_exec_instr_ctx_t * ctx, ulong const * pre_lens, uchar * input, ulong input_sz, + int stricter_abi_and_runtime_constraints, int direct_mapping, uchar is_deprecated ); diff --git a/src/flamenco/runtime/tests/fd_vm_harness.c b/src/flamenco/runtime/tests/fd_vm_harness.c index 5d751dc8c11..f15760b014c 100644 --- a/src/flamenco/runtime/tests/fd_vm_harness.c +++ b/src/flamenco/runtime/tests/fd_vm_harness.c @@ -145,8 +145,9 @@ do{ ulong pre_lens[256] = {0}; fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */ fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */ - uint input_mem_regions_cnt = 0U; - int direct_mapping = FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ); + uint input_mem_regions_cnt = 0UL; + int direct_mapping = FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ); + int stricter_abi_and_runtime_constraints = FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ); uchar * input_ptr = NULL; uchar program_id_idx = instr_ctx->instr->program_id; @@ -169,6 +170,7 @@ do{ input_mem_regions, &input_mem_regions_cnt, acc_region_metas, + stricter_abi_and_runtime_constraints, direct_mapping, is_deprecated, &input_ptr ); @@ -248,6 +250,7 @@ do{ acc_region_metas, /* vm_acc_region_meta*/ is_deprecated, /* is deprecated */ direct_mapping, /* direct mapping */ + stricter_abi_and_runtime_constraints, /* stricter_abi_and_runtime_constraints */ 0 /* dump_syscall_to_pb */ ); @@ -445,12 +448,13 @@ fd_solfuzz_syscall_run( fd_solfuzz_runner_t * runner, /* If the program ID account owner is the v1 BPF loader, then alignment is disabled (controlled by the `is_deprecated` flag) */ - ulong input_sz = 0UL; - ulong pre_lens[256] = {0}; - fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */ - fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */ - uint input_mem_regions_cnt = 0U; - int direct_mapping = FD_FEATURE_ACTIVE( ctx->txn_ctx->slot, &ctx->txn_ctx->features, bpf_account_data_direct_mapping ); + ulong input_sz = 0UL; + ulong pre_lens[256] = {0}; + fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */ + fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */ + uint input_mem_regions_cnt = 0U; + int direct_mapping = FD_FEATURE_ACTIVE( ctx->txn_ctx->slot, &ctx->txn_ctx->features, account_data_direct_mapping ); + int stricter_abi_and_runtime_constraints = FD_FEATURE_ACTIVE( ctx->txn_ctx->slot, &ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ); uchar * input_ptr = NULL; uchar program_id_idx = ctx->instr->program_id; @@ -472,6 +476,7 @@ fd_solfuzz_syscall_run( fd_solfuzz_runner_t * runner, input_mem_regions, &input_mem_regions_cnt, acc_region_metas, + stricter_abi_and_runtime_constraints, direct_mapping, is_deprecated, &input_ptr ); @@ -500,7 +505,8 @@ fd_solfuzz_syscall_run( fd_solfuzz_runner_t * runner, input_mem_regions_cnt, acc_region_metas, is_deprecated, - FD_FEATURE_ACTIVE( ctx->txn_ctx->slot, &ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + direct_mapping, + stricter_abi_and_runtime_constraints, 0 /* dump_syscall_to_pb */ ); // Override some execution state values from the syscall fuzzer input diff --git a/src/flamenco/runtime/tests/run_backtest_ci.sh b/src/flamenco/runtime/tests/run_backtest_ci.sh index cdad14315fc..11b74906e7c 100755 --- a/src/flamenco/runtime/tests/run_backtest_ci.sh +++ b/src/flamenco/runtime/tests/run_backtest_ci.sh @@ -25,3 +25,6 @@ src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-380592002-v2.3.0 -y src/flamenco/runtime/tests/run_ledger_backtest.sh -l local-multi-boundary -y 1 -m 1000 -e 2325 -c 2.3.0 src/flamenco/runtime/tests/run_ledger_backtest.sh -l genesis-v3.0 -y 1 -m 3000 -e 1280 -c 3.0.0 -g src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-stake-v3.0.0 -y 1 -m 3000 -e 541 -c 3.0.0 +src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-368528500-stricter-abi -y 5 -m 2000000 -e 368528527 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM +src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-368528500-direct-mapping -y 5 -m 2000000 -e 368528527 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM,9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ +src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-362107883-direct-mapping-2 -y 1 -m 2000000 -e 362219427 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM,9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ diff --git a/src/flamenco/runtime/tests/run_backtest_tests_all.sh b/src/flamenco/runtime/tests/run_backtest_tests_all.sh index e4e4755afaa..adc1959dd69 100755 --- a/src/flamenco/runtime/tests/run_backtest_tests_all.sh +++ b/src/flamenco/runtime/tests/run_backtest_tests_all.sh @@ -85,3 +85,6 @@ src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-bpf-loader-v2.3.0 -y src/flamenco/runtime/tests/run_ledger_backtest.sh -l local-multi-boundary -y 1 -m 1000 -e 2325 -c 2.3.0 src/flamenco/runtime/tests/run_ledger_backtest.sh -l genesis-v3.0 -y 1 -m 3000 -e 1280 -c 3.0.0 -g src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-stake-v3.0.0 -y 1 -m 3000 -e 541 -c 3.0.0 +src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-368528500-stricter-abi -y 5 -m 2000000 -e 368528527 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM +src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-368528500-direct-mapping -y 5 -m 2000000 -e 368528527 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM,9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ +src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-362107883-direct-mapping-2 -y 1 -m 2000000 -e 362219427 -c 3.0.3 -o CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM,9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ diff --git a/src/flamenco/vm/fd_vm.c b/src/flamenco/vm/fd_vm.c index e2ef98f7fb6..819ff8ff070 100644 --- a/src/flamenco/vm/fd_vm.c +++ b/src/flamenco/vm/fd_vm.c @@ -595,6 +595,7 @@ fd_vm_init( fd_vm_acc_region_meta_t * acc_region_metas, uchar is_deprecated, int direct_mapping, + int stricter_abi_and_runtime_constraints, int dump_syscall_to_pb ) { if ( FD_UNLIKELY( vm == NULL ) ) { @@ -621,30 +622,31 @@ fd_vm_init( } // Set the vm fields - vm->instr_ctx = instr_ctx; - vm->heap_max = heap_max; - vm->entry_cu = entry_cu; - vm->rodata = rodata; - vm->rodata_sz = rodata_sz; - vm->text = text; - vm->text_cnt = text_cnt; - vm->text_off = text_off; - vm->text_sz = text_sz; - vm->entry_pc = entry_pc; - vm->calldests = calldests; - vm->sbpf_version = sbpf_version; - vm->syscalls = syscalls; - vm->trace = trace; - vm->sha = sha; - vm->input_mem_regions = mem_regions; - vm->input_mem_regions_cnt = mem_regions_cnt; - vm->acc_region_metas = acc_region_metas; - vm->is_deprecated = is_deprecated; - vm->direct_mapping = direct_mapping; - vm->stack_frame_size = FD_VM_STACK_FRAME_SZ + ( direct_mapping ? 0UL : FD_VM_STACK_GUARD_SZ ); - vm->segv_vaddr = ULONG_MAX; - vm->segv_access_type = 0; - vm->dump_syscall_to_pb = dump_syscall_to_pb; + vm->instr_ctx = instr_ctx; + vm->heap_max = heap_max; + vm->entry_cu = entry_cu; + vm->rodata = rodata; + vm->rodata_sz = rodata_sz; + vm->text = text; + vm->text_cnt = text_cnt; + vm->text_off = text_off; + vm->text_sz = text_sz; + vm->entry_pc = entry_pc; + vm->calldests = calldests; + vm->sbpf_version = sbpf_version; + vm->syscalls = syscalls; + vm->trace = trace; + vm->sha = sha; + vm->input_mem_regions = mem_regions; + vm->input_mem_regions_cnt = mem_regions_cnt; + vm->acc_region_metas = acc_region_metas; + vm->is_deprecated = is_deprecated; + vm->direct_mapping = direct_mapping; + vm->stricter_abi_and_runtime_constraints = stricter_abi_and_runtime_constraints; + vm->segv_vaddr = ULONG_MAX; + vm->segv_access_len = 0UL; + vm->segv_access_type = 0; + vm->dump_syscall_to_pb = dump_syscall_to_pb; /* Unpack the configuration */ int err = fd_vm_setup_state_for_execution( vm ); diff --git a/src/flamenco/vm/fd_vm.h b/src/flamenco/vm/fd_vm.h index a5e01eb2775..353c3eff86b 100644 --- a/src/flamenco/vm/fd_vm.h +++ b/src/flamenco/vm/fd_vm.h @@ -23,11 +23,13 @@ typedef struct fd_vm_shadow fd_vm_shadow_t; within the larger input region. */ struct __attribute__((aligned(8UL))) fd_vm_input_region { - ulong vaddr_offset; /* Represents offset from the start of the input region. */ - ulong haddr; /* Host address corresponding to the start of the mem region. */ - uint region_sz; /* Size of the memory region. */ - uchar is_writable; /* If the region can be written to or is read-only */ - uchar is_acct_data; /* Set if this is an account data region (either orig data or resize buffer). */ + ulong vaddr_offset; /* Represents offset from the start of the input region. */ + ulong haddr; /* Host address corresponding to the start of the mem region. */ + uint region_sz; /* Size of the memory region. */ + ulong address_space_reserved; /* The amount of address space reserved for the region. */ + uchar is_writable; /* If the region can be written to or is read-only */ + ulong padding; /* Empty padding at the start of the region.*/ + ulong acc_region_meta_idx; /* Index of the acc_region_meta_t struct for the account corresponding to this region. */ }; typedef struct fd_vm_input_region fd_vm_input_region_t; @@ -36,17 +38,13 @@ typedef struct fd_vm_input_region fd_vm_input_region_t; region location. */ struct __attribute((aligned(8UL))) fd_vm_acc_region_meta { - uint region_idx; - uchar has_data_region; - uchar has_resizing_region; - /* offset of the accounts metadata region, relative to the start of the input region. - importantly, this excludes any duplicate account markers at the beginning of the "full" metadata region. */ - ulong metadata_region_offset; + uint region_idx; /* FIXME: We can get rid of this field once DM is activated. This is only a hack to make the non-DM code path happy. When DM is activated, we could query the input_mem_region array for the original data len. */ - ulong original_data_len; + ulong original_data_len; + fd_txn_account_t * acct; /* The transaction account corresponding to this account. */ }; typedef struct fd_vm_acc_region_meta fd_vm_acc_region_meta_t; @@ -208,14 +206,15 @@ struct __attribute__((aligned(FD_VM_HOST_REGION_ALIGN))) fd_vm { ulong magic; /* ==FD_VM_MAGIC */ - int direct_mapping; /* If direct mapping is enabled or not */ - ulong stack_frame_size; /* Size of a stack frame (varies depending on direct mapping being enabled or not) */ + int direct_mapping; /* If direct mapping feature flag is enabled */ + int stricter_abi_and_runtime_constraints; /* If stricter_abi_and_runtime_constraints feature flag is enabled */ /* Agave uses the segv vaddr in several different cases, including: - Determining whether or not to return a regular or stack access violation - (If direct mapping is enabled) determining the instruction error code to return on store operations. */ ulong segv_vaddr; + ulong segv_access_len; uchar segv_access_type; ulong sbpf_version; /* SBPF version, SIMD-0161 */ @@ -302,6 +301,7 @@ fd_vm_init( fd_vm_acc_region_meta_t * acc_region_metas, uchar is_deprecated, int direct_mapping, + int stricter_abi_and_runtime_constraints, int dump_syscall_to_pb ); /* fd_vm_leave leaves the caller's current local join to a vm. diff --git a/src/flamenco/vm/fd_vm_interp_core.c b/src/flamenco/vm/fd_vm_interp_core.c index 0a6cb34dbb4..5ab586384bd 100644 --- a/src/flamenco/vm/fd_vm_interp_core.c +++ b/src/flamenco/vm/fd_vm_interp_core.c @@ -264,15 +264,15 @@ /* FIXME: unvalidated code mucking with r10 */ -# define FD_VM_INTERP_STACK_PUSH \ +# define FD_VM_INTERP_STACK_PUSH \ shadow[ frame_cnt ].r6 = reg[6]; \ shadow[ frame_cnt ].r7 = reg[7]; \ shadow[ frame_cnt ].r8 = reg[8]; \ shadow[ frame_cnt ].r9 = reg[9]; \ shadow[ frame_cnt ].r10 = reg[10]; \ shadow[ frame_cnt ].pc = pc; \ - if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \ - if( !fd_sbpf_dynamic_stack_frames_enabled( sbpf_version ) ) reg[10] += vm->stack_frame_size; + if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \ + if( !fd_sbpf_dynamic_stack_frames_enabled( sbpf_version ) ) reg[10] += FD_VM_STACK_FRAME_SZ * 2UL; \ /* We subtract the heap cost in the BPF loader */ @@ -376,12 +376,12 @@ FD_VM_INTERP_BRANCH_END; FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL ); if( FD_UNLIKELY( !haddr ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; + vm->segv_access_len = 1UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ fd_vm_mem_st_1( haddr, (uchar)imm ); @@ -389,12 +389,12 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */ - uchar is_multi_region = 0; - ulong vaddr = reg_src + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region ); + ulong vaddr = reg_src + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL ); if( FD_UNLIKELY( !haddr ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; + vm->segv_access_len = 1UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ reg[ dst ] = fd_vm_mem_ld_1( haddr ); @@ -406,12 +406,12 @@ FD_VM_INTERP_BRANCH_END; FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL ); if( FD_UNLIKELY( !haddr ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; + vm->segv_access_len = 1UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */ fd_vm_mem_st_1( haddr, (uchar)reg_src ); @@ -447,41 +447,30 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* Only execute slow path partial store when direct mapping is enabled. - Note that Agave implements direct mapping as an UnalignedMemoryMapping. - When account memory regions are not aligned, there are edge cases that require - the slow path partial store. - https://github.com/anza-xyz/sbpf/blob/410a627313124252ab1abbd3a3b686c03301bb2a/src/memory_region.rs#L388-L419 */ - ushort val = (ushort)imm; - fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val ); - } - + vm->segv_access_len = 2UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)imm, is_multi_region ); + fd_vm_mem_st_2( haddr, (ushort)imm ); } FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */ - uchar is_multi_region = 0; - ulong vaddr = reg_src + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_src + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; + vm->segv_access_len = 2UL; goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ } - reg[ dst ] = fd_vm_mem_ld_2( vm, vaddr, haddr, is_multi_region ); + reg[ dst ] = fd_vm_mem_ld_2( haddr ); } FD_VM_INTERP_INSTR_END; @@ -490,23 +479,16 @@ FD_VM_INTERP_BRANCH_END; FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* See FD_SBPF_OP_STH for details */ - ushort val = (ushort)reg_src; - fd_vm_mem_st_try( vm, vaddr, sizeof(ushort), (uchar*)&val ); - } - + vm->segv_access_len = 2UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_2( vm, vaddr, haddr, (ushort)reg_src, is_multi_region ); + fd_vm_mem_st_2( haddr, (ushort)reg_src ); } FD_VM_INTERP_INSTR_END; @@ -748,23 +730,16 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* See FD_SBPF_OP_STH for details */ - uint val = (uint)imm; - fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val ); - } - + vm->segv_access_len = 4UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_4( vm, vaddr, haddr, imm, is_multi_region ); + fd_vm_mem_st_4( haddr, imm ); } FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */ @@ -772,16 +747,16 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */ - uchar is_multi_region = 0; - ulong vaddr = reg_src + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_src + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; + vm->segv_access_len = 4UL; goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ } - reg[ dst ] = fd_vm_mem_ld_4( vm, vaddr, haddr, is_multi_region ); + reg[ dst ] = fd_vm_mem_ld_4( haddr ); } FD_VM_INTERP_INSTR_END; @@ -819,23 +794,16 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* See FD_SBPF_OP_STH for details */ - uint val = (uint)reg_src; - fd_vm_mem_st_try( vm, vaddr, sizeof(uint), (uchar*)&val ); - } - + vm->segv_access_len = 4UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_4( vm, vaddr, haddr, (uint)reg_src, is_multi_region ); + fd_vm_mem_st_4( haddr, (uint)reg_src ); } FD_VM_INTERP_INSTR_END; @@ -859,37 +827,30 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* See FD_SBPF_OP_STH for details */ - ulong val = (ulong)(long)(int)imm; - fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)&val ); - } - + vm->segv_access_len = 8UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_8( vm, vaddr, haddr, (ulong)(long)(int)imm, is_multi_region ); + fd_vm_mem_st_8( haddr, (ulong)(long)(int)imm ); } FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */ - uchar is_multi_region = 0; - ulong vaddr = reg_src + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_src + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; + vm->segv_access_len = 8UL; goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ } - reg[ dst ] = fd_vm_mem_ld_8( vm, vaddr, haddr, is_multi_region ); + reg[ dst ] = fd_vm_mem_ld_8( haddr ); } FD_VM_INTERP_INSTR_END; @@ -914,22 +875,16 @@ FD_VM_INTERP_INSTR_END; FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */ - uchar is_multi_region = 0; - ulong vaddr = reg_dst + offset; - ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL, &is_multi_region ); - int sigsegv = !haddr; + ulong vaddr = reg_dst + offset; + ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL ); + int sigsegv = !haddr; if( FD_UNLIKELY( sigsegv ) ) { vm->segv_vaddr = vaddr; vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; - - if( vm->direct_mapping ) { - /* See FD_SBPF_OP_STH for details */ - fd_vm_mem_st_try( vm, vaddr, sizeof(ulong), (uchar*)®_src ); - } - + vm->segv_access_len = 8UL; goto sigsegv; } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */ - fd_vm_mem_st_8( vm, vaddr, haddr, reg_src, is_multi_region ); + fd_vm_mem_st_8( haddr, reg_src ); } FD_VM_INTERP_INSTR_END; diff --git a/src/flamenco/vm/fd_vm_private.h b/src/flamenco/vm/fd_vm_private.h index 6297b6af831..0f150f1f817 100644 --- a/src/flamenco/vm/fd_vm_private.h +++ b/src/flamenco/vm/fd_vm_private.h @@ -7,6 +7,7 @@ #include "../../ballet/sbpf/fd_sbpf_opcodes.h" #include "../../ballet/murmur3/fd_murmur3.h" #include "../runtime/context/fd_exec_txn_ctx.h" +#include "../runtime/fd_runtime_const.h" #include "../features/fd_features.h" #include "fd_vm_base.h" @@ -113,6 +114,9 @@ FD_STATIC_ASSERT( sizeof(fd_vm_vec_t)==FD_VM_VEC_SIZE, fd_vm_vec size mismatch ) #define FD_VM_OFFSET_MASK (0xffffffffUL) +/* https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L32 */ +#define FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION (FD_RUNTIME_ACC_SZ_MAX * 2UL) + FD_PROTOTYPES_BEGIN /* Error logging handholding assertions */ @@ -296,7 +300,7 @@ fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) { while( left=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].region_sz ) { + if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].address_space_reserved ) { left = mid + 1U; } else { right = mid; @@ -305,6 +309,69 @@ fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) { return left; } +/* If the region is an account, handle the resizing logic. This logic + corresponds to + solana_transaction_context::TransactionContext::access_violation_handler + + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */ +static inline void +fd_vm_handle_input_mem_region_oob( fd_vm_t const * vm, + ulong offset, + ulong sz, + ulong region_idx, + uchar write ) { + /* If stricter_abi_and_runtime_constraints is not enabled, we don't need to + do anything */ + if( FD_UNLIKELY( !vm->stricter_abi_and_runtime_constraints ) ) { + return; + } + + /* If the access is not a write, we don't need to do anything + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L523-L525 */ + if( FD_UNLIKELY( !write ) ) { + return; + } + + fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ]; + /* If the region is not writable, we don't need to do anything + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L526-L529 */ + if( FD_UNLIKELY( !region->is_writable ) ) { + return; + } + + /* Calculate the requested length + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L532-L535 */ + ulong requested_len = fd_ulong_sat_sub( fd_ulong_sat_add( offset, sz ), region->vaddr_offset ); + + /* Calculate the remaining allowed growth + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L549-L551 */ + ulong remaining_allowed_growth = fd_ulong_sat_sub( + FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION, + vm->instr_ctx->txn_ctx->accounts_resize_delta ); + + /* If the requested length is greater than the size of the region, + resize the region + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L553-L571 */ + if( FD_UNLIKELY( requested_len > region->region_sz ) ) { + /* Calculate the new region size + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L558-L560 */ + ulong new_region_sz = fd_ulong_min( + fd_ulong_min( region->address_space_reserved, FD_RUNTIME_ACC_SZ_MAX ), + fd_ulong_sat_add( region->region_sz, remaining_allowed_growth ) ); + + /* Resize the account and the region + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L569-L570 */ + if( FD_UNLIKELY( new_region_sz > region->region_sz ) ) { + vm->instr_ctx->txn_ctx->accounts_resize_delta = fd_ulong_sat_sub( + fd_ulong_sat_add( vm->instr_ctx->txn_ctx->accounts_resize_delta, new_region_sz ), + region->region_sz ); + + fd_txn_account_resize( vm->acc_region_metas[ region->acc_region_meta_idx ].acct, requested_len ); + region->region_sz = (uint)new_region_sz; + } + } +} + /* fd_vm_find_input_mem_region returns the translated haddr for a given offset into the input region. If an offset/sz is invalid or if an illegal write is performed, the sentinel value is returned. If the offset @@ -316,8 +383,7 @@ fd_vm_find_input_mem_region( fd_vm_t const * vm, ulong offset, ulong sz, uchar write, - ulong sentinel, - uchar * is_multi_region ) { + ulong sentinel ) { if( FD_UNLIKELY( vm->input_mem_regions_cnt==0 ) ) { return sentinel; /* Access is too large */ } @@ -325,10 +391,26 @@ fd_vm_find_input_mem_region( fd_vm_t const * vm, /* Binary search to find the correct memory region. If direct mapping is not enabled, then there is only 1 memory region which spans the input region. */ ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset ); + if( FD_UNLIKELY( region_idx>=vm->input_mem_regions_cnt ) ) { + return sentinel; /* Region not found */ + } + + ulong bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, + fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); - ulong bytes_left = sz; - ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, - fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); + /* If the access is out of bounds, invoke the callback to handle the out of bounds access. + This potentially resizes the region if necessary. */ + if( FD_UNLIKELY( sz>bytes_in_region ) ) { + fd_vm_handle_input_mem_region_oob( vm, offset, sz, region_idx, write ); + } + + /* After potentially resizing, re-check the bounds */ + bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, + fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); + /* If the access is still out of bounds, return the sentinel */ + if( FD_UNLIKELY( sz>bytes_in_region ) ) { + return sentinel; + } if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) { return sentinel; /* Illegal write */ @@ -336,48 +418,27 @@ fd_vm_find_input_mem_region( fd_vm_t const * vm, ulong start_region_idx = region_idx; - *is_multi_region = 0; - while( FD_UNLIKELY( bytes_left>bytes_in_cur_region ) ) { - *is_multi_region = 1; - FD_LOG_DEBUG(( "Size of access spans multiple memory regions" )); - bytes_left = fd_ulong_sat_sub( bytes_left, bytes_in_cur_region ); - - region_idx += 1U; - - if( FD_UNLIKELY( region_idx==vm->input_mem_regions_cnt ) ) { - return sentinel; /* Access is too large */ - } - bytes_in_cur_region = vm->input_mem_regions[ region_idx ].region_sz; - - if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) { - return sentinel; /* Illegal write */ - } - } - ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset; return adjusted_haddr; } static inline ulong -fd_vm_mem_haddr( fd_vm_t const * vm, - ulong vaddr, - ulong sz, - ulong const * vm_region_haddr, /* indexed [0,6) */ - uint const * vm_region_sz, /* indexed [0,6) */ - uchar write, /* 1 if the access is a write, 0 if it is a read */ - ulong sentinel, - uchar * is_multi_region ) { +fd_vm_mem_haddr( fd_vm_t const * vm, + ulong vaddr, + ulong sz, + ulong const * vm_region_haddr, /* indexed [0,6) */ + uint const * vm_region_sz, /* indexed [0,6) */ + uchar write, /* 1 if the access is a write, 0 if it is a read */ + ulong sentinel ) { ulong region = FD_VADDR_TO_REGION( vaddr ); ulong offset = vaddr & FD_VM_OFFSET_MASK; /* Stack memory regions have 4kB unmapped "gaps" in-between each frame, which only exist if... - - direct mapping is enabled (config.enable_stack_frame_gaps == !direct_mapping) - - dynamic stack frames are not enabled (!(SBPF version >= SBPF_V1)) + - dynamic stack frames are not enabled (!(SBPF version >= SBPF_V1)) https://github.com/anza-xyz/agave/blob/v2.2.12/programs/bpf_loader/src/lib.rs#L344-L351 */ if( FD_UNLIKELY( region==FD_VM_STACK_REGION && - !vm->direct_mapping && !fd_sbpf_dynamic_stack_frames_enabled( vm->sbpf_version ) ) ) { /* If an access starts in a gap region, that is an access violation */ if( FD_UNLIKELY( !!(vaddr & 0x1000) ) ) { @@ -396,8 +457,12 @@ fd_vm_mem_haddr( fd_vm_t const * vm, ulong region_sz = (ulong)vm_region_sz[ region ]; ulong sz_max = region_sz - fd_ulong_min( offset, region_sz ); + /* If the region is an account, handle the resizing logic. This logic corresponds to + solana_transaction_context::TransactionContext::access_violation_handler + + https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */ if( region==FD_VM_INPUT_REGION ) { - return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel, is_multi_region ); + return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel ); } # ifdef FD_VM_INTERP_MEM_TRACING_ENABLED @@ -412,162 +477,53 @@ static inline ulong fd_vm_mem_haddr_fast( fd_vm_t const * vm, ulong vaddr, ulong const * vm_region_haddr ) { /* indexed [0,6) */ - uchar is_multi = 0; ulong region = FD_VADDR_TO_REGION( vaddr ); ulong offset = vaddr & FD_VM_OFFSET_MASK; if( FD_UNLIKELY( region==FD_VM_INPUT_REGION ) ) { - return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL, &is_multi ); + return fd_vm_find_input_mem_region( vm, offset, 1UL, 0, 0UL ); } return vm_region_haddr[ region ] + offset; } -/* fd_vm_mem_ld_N loads N bytes from the host address location haddr, - zero extends it to a ulong and returns the ulong. haddr need not be - aligned. fd_vm_mem_ld_multi handles the case where the load spans - multiple input memory regions. */ - -static inline void fd_vm_mem_ld_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * dst ) { - - ulong offset = vaddr & FD_VM_OFFSET_MASK; - ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset ); - uint bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, - (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); - - while( sz-- ) { - if( !bytes_in_cur_region ) { - region_idx++; - bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, - (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); - haddr = vm->input_mem_regions[ region_idx ].haddr; - } - - *dst++ = *(uchar *)haddr++; - bytes_in_cur_region--; - } -} - FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) { return (ulong)*(uchar const *)haddr; } -FD_FN_PURE static inline ulong fd_vm_mem_ld_2( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) { +FD_FN_PURE static inline ulong fd_vm_mem_ld_2( ulong haddr ) { ushort t; - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( &t, (void const *)haddr, sizeof(ushort) ); - } else { - fd_vm_mem_ld_multi( vm, 2U, vaddr, haddr, (uchar *)&t ); - } + memcpy( &t, (void const *)haddr, sizeof(ushort) ); return (ulong)t; } -FD_FN_PURE static inline ulong fd_vm_mem_ld_4( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) { +FD_FN_PURE static inline ulong fd_vm_mem_ld_4( ulong haddr ) { uint t; - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( &t, (void const *)haddr, sizeof(uint) ); - } else { - fd_vm_mem_ld_multi( vm, 4U, vaddr, haddr, (uchar *)&t ); - } + memcpy( &t, (void const *)haddr, sizeof(uint) ); return (ulong)t; } -FD_FN_PURE static inline ulong fd_vm_mem_ld_8( fd_vm_t const * vm, ulong vaddr, ulong haddr, uint is_multi_region ) { +FD_FN_PURE static inline ulong fd_vm_mem_ld_8( ulong haddr ) { ulong t; - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( &t, (void const *)haddr, sizeof(ulong) ); - } else { - fd_vm_mem_ld_multi( vm, 8U, vaddr, haddr, (uchar *)&t ); - } + memcpy( &t, (void const *)haddr, sizeof(ulong) ); return t; } -/* fd_vm_mem_st_N stores val in little endian order to the host address - location haddr. haddr need not be aligned. fd_vm_mem_st_multi handles - the case where the store spans multiple input memory regions. */ - -static inline void fd_vm_mem_st_multi( fd_vm_t const * vm, uint sz, ulong vaddr, ulong haddr, uchar * src ) { - ulong offset = vaddr & FD_VM_OFFSET_MASK; - ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset ); - ulong bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, - (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); - uchar * dst = (uchar *)haddr; - - while( sz-- ) { - if( !bytes_in_cur_region ) { - region_idx++; - bytes_in_cur_region = fd_uint_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, - (uint)fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) ); - dst = (uchar *)vm->input_mem_regions[ region_idx ].haddr; - } - - *dst++ = *src++; - bytes_in_cur_region--; - } -} - static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) { *(uchar *)haddr = val; } -static inline void fd_vm_mem_st_2( fd_vm_t const * vm, - ulong vaddr, - ulong haddr, - ushort val, - uint is_multi_region ) { - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( (void *)haddr, &val, sizeof(ushort) ); - } else { - fd_vm_mem_st_multi( vm, 2U, vaddr, haddr, (uchar *)&val ); - } +static inline void fd_vm_mem_st_2( ulong haddr, + ushort val ) { + memcpy( (void *)haddr, &val, sizeof(ushort) ); } -static inline void fd_vm_mem_st_4( fd_vm_t const * vm, - ulong vaddr, - ulong haddr, - uint val, - uint is_multi_region ) { - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( (void *)haddr, &val, sizeof(uint) ); - } else { - fd_vm_mem_st_multi( vm, 4U, vaddr, haddr, (uchar *)&val ); - } +static inline void fd_vm_mem_st_4( ulong haddr, + uint val ) { + memcpy( (void *)haddr, &val, sizeof(uint) ); } -static inline void fd_vm_mem_st_8( fd_vm_t const * vm, - ulong vaddr, - ulong haddr, - ulong val, - uint is_multi_region ) { - if( FD_LIKELY( !is_multi_region ) ) { - memcpy( (void *)haddr, &val, sizeof(ulong) ); - } else { - fd_vm_mem_st_multi( vm, 8U, vaddr, haddr, (uchar *)&val ); - } -} - -/* fd_vm_mem_st_try is strictly not required for correctness and in - fact just slows down the performance of the firedancer vm. However, - this emulates the behavior of the agave client, where a store will - be attempted partially until it fails. This is useful for debugging - and fuzzing conformance. */ -static inline void fd_vm_mem_st_try( fd_vm_t const * vm, - ulong vaddr, - ulong sz, - uchar * val ) { - uchar is_multi_region = 0; - for( ulong i=0UL; iregion_haddr, - vm->region_st_sz, - 1, - 0UL, - &is_multi_region ); - if( !haddr ) { - return; - } - *(uchar *)haddr = *(val+i); - } +static inline void fd_vm_mem_st_8( ulong haddr, + ulong val ) { + memcpy( (void *)haddr, &val, sizeof(ulong) ); } FD_PROTOTYPES_END diff --git a/src/flamenco/vm/instr_test/v0/load.instr b/src/flamenco/vm/instr_test/v0/load.instr index e6228f0aa04..80a0d3d764d 100644 --- a/src/flamenco/vm/instr_test/v0/load.instr +++ b/src/flamenco/vm/instr_test/v0/load.instr @@ -5,7 +5,7 @@ $ op=71 dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=00 $ op=71 dst=1 src=2 off=0007 r2 =400000000 imm=01 : ok r1=07 $ op=71 dst=1 src=3 off=000f r3 =400000000 imm=02 : ok r1=15 $ op=71 dst=1 src=3 off=0010 r3 =400000000 imm=02 : err -$ op=71 dst=1 src=3 off=1000 r3 =400000000 imm=02 : err +$ op=71 dst=1 src=3 off=1000 r3 =400000000 imm=02 : err $ op=71 dst=2 src=4 off=7fff r4 =3ffff8002 imm=03 : ok r2=01 $ op=71 dst=2 src=5 off=8000 r5 =400008003 imm=04 : ok r2=03 $ op=71 dst=2 src=6 off=ffff r6 =400000008 imm=05 : ok r2=07 @@ -72,18 +72,18 @@ region_boundary=09 region_boundary=10 # ldxh reg, [reg+off] -$ op=69 dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=0807 -$ op=69 dst=1 src=1 off=0003 r1 =400000000 imm=00 : ok r1=0403 +$ op=69 dst=1 src=1 off=0007 r1 =400000000 imm=00 : err +$ op=69 dst=1 src=1 off=0003 r1 =400000000 imm=00 : err $ op=69 dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=0100 # ldxw reg, [reg+off] $ op=61 dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=03020100 -$ op=61 dst=1 src=1 off=0001 r1 =400000000 imm=00 : ok r1=04030201 -$ op=61 dst=1 src=1 off=0002 r1 =400000000 imm=00 : ok r1=05040302 -$ op=61 dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=10090807 -$ op=61 dst=1 src=1 off=0008 r1 =400000000 imm=00 : ok r1=11100908 +$ op=61 dst=1 src=1 off=0001 r1 =400000000 imm=00 : err +$ op=61 dst=1 src=1 off=0002 r1 =400000000 imm=00 : err +$ op=61 dst=1 src=1 off=0007 r1 =400000000 imm=00 : err +$ op=61 dst=1 src=1 off=0008 r1 =400000000 imm=00 : err # ldxdw reg, [reg+off] -$ op=79 dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=0706050403020100 -$ op=79 dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=1413121110090807 +$ op=79 dst=1 src=1 off=0000 r1 =400000000 imm=00 : err +$ op=79 dst=1 src=1 off=0007 r1 =400000000 imm=00 : err diff --git a/src/flamenco/vm/instr_test/v2/load.instr b/src/flamenco/vm/instr_test/v2/load.instr index ef46e422c3a..608cb9b5656 100644 --- a/src/flamenco/vm/instr_test/v2/load.instr +++ b/src/flamenco/vm/instr_test/v2/load.instr @@ -5,7 +5,7 @@ $ op=2c dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=00 $ op=2c dst=1 src=2 off=0007 r2 =400000000 imm=01 : ok r1=07 $ op=2c dst=1 src=3 off=000f r3 =400000000 imm=02 : ok r1=15 $ op=2c dst=1 src=3 off=0010 r3 =400000000 imm=02 : err -$ op=2c dst=1 src=3 off=1000 r3 =400000000 imm=02 : err +$ op=2c dst=1 src=3 off=1000 r3 =400000000 imm=02 : err $ op=2c dst=2 src=4 off=7fff r4 =3ffff8002 imm=03 : ok r2=01 $ op=2c dst=2 src=5 off=8000 r5 =400008003 imm=04 : ok r2=03 $ op=2c dst=2 src=6 off=ffff r6 =400000008 imm=05 : ok r2=07 @@ -115,8 +115,8 @@ region_boundary=09 region_boundary=10 # ldxh reg, [reg+off] -$ op=3c dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=0807 -$ op=3c dst=1 src=1 off=0003 r1 =400000000 imm=00 : ok r1=0403 +$ op=3c dst=1 src=1 off=0007 r1 =400000000 imm=00 : err +$ op=3c dst=1 src=1 off=0003 r1 =400000000 imm=00 : err $ op=3c dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=0100 $ op=69 dst=1 src=1 off=0007 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 $ op=69 dst=1 src=1 off=0003 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 @@ -124,10 +124,10 @@ $ op=69 dst=1 src=1 off=0000 r1 =400000000 imm=00 : vfy # invalid ix - removed S # ldxw reg, [reg+off] $ op=8c dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=03020100 -$ op=8c dst=1 src=1 off=0001 r1 =400000000 imm=00 : ok r1=04030201 -$ op=8c dst=1 src=1 off=0002 r1 =400000000 imm=00 : ok r1=05040302 -$ op=8c dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=10090807 -$ op=8c dst=1 src=1 off=0008 r1 =400000000 imm=00 : ok r1=11100908 +$ op=8c dst=1 src=1 off=0001 r1 =400000000 imm=00 : err +$ op=8c dst=1 src=1 off=0002 r1 =400000000 imm=00 : err +$ op=8c dst=1 src=1 off=0007 r1 =400000000 imm=00 : err +$ op=8c dst=1 src=1 off=0008 r1 =400000000 imm=00 : err $ op=61 dst=1 src=1 off=0000 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 $ op=61 dst=1 src=1 off=0001 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 $ op=61 dst=1 src=1 off=0002 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 @@ -135,8 +135,8 @@ $ op=61 dst=1 src=1 off=0007 r1 =400000000 imm=00 : vfy # invalid ix - removed S $ op=61 dst=1 src=1 off=0008 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 # ldxdw reg, [reg+off] -$ op=9c dst=1 src=1 off=0000 r1 =400000000 imm=00 : ok r1=0706050403020100 -$ op=9c dst=1 src=1 off=0007 r1 =400000000 imm=00 : ok r1=1413121110090807 +$ op=9c dst=1 src=1 off=0000 r1 =400000000 imm=00 : err +$ op=9c dst=1 src=1 off=0007 r1 =400000000 imm=00 : err $ op=79 dst=1 src=1 off=0000 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 $ op=79 dst=1 src=1 off=0007 r1 =400000000 imm=00 : vfy # invalid ix - removed SIMD-0173 diff --git a/src/flamenco/vm/syscall/fd_vm_cpi.h b/src/flamenco/vm/syscall/fd_vm_cpi.h index 89fc8804c9f..345cc650b15 100644 --- a/src/flamenco/vm/syscall/fd_vm_cpi.h +++ b/src/flamenco/vm/syscall/fd_vm_cpi.h @@ -171,10 +171,7 @@ struct fd_vm_cpi_caller_account { uchar * serialized_data; /* NULL if direct mapping */ ulong serialized_data_len; ulong vm_data_vaddr; - union { - ulong * translated; /* Set if direct mapping false */ - ulong vaddr; /* Set if direct mapping */ - } ref_to_len_in_vm; + ulong * ref_to_len_in_vm; }; typedef struct fd_vm_cpi_caller_account fd_vm_cpi_caller_account_t; diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c b/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c index 984a6042e3a..6743338f964 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c @@ -299,6 +299,47 @@ fd_vm_syscall_cpi_check_instruction( fd_vm_t const * vm, return FD_VM_SUCCESS; } +/* https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L1134-L1169 */ +static inline int +fd_vm_cpi_update_caller_account_region( fd_vm_t * vm, + ulong instr_acc_idx, + fd_vm_cpi_caller_account_t * caller_account, + fd_borrowed_account_t * borrowed_account ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1141-L1148 */ + ulong address_space_reserved_for_account; + if( vm->stricter_abi_and_runtime_constraints && vm->is_deprecated ) { + address_space_reserved_for_account = caller_account->orig_data_len; + } else { + address_space_reserved_for_account = fd_ulong_sat_add( caller_account->orig_data_len, MAX_PERMITTED_DATA_INCREASE ); + } + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1159-L1164 */ + if( address_space_reserved_for_account > 0UL ) { + /* Note that we don't special-case direct mapping here, as Agave does, + because we do not create regions using CoW upon resize like Agave does. + + Therefore we do not need the logic in the Agave code to create a new + region, as we have already created all the regions for each account. + + Therefore we do not have equivalents of Agave's + modify_memory_region_of_account and create_memory_region_of_account + functions, but we instead inline this logic directly below. */ + fd_vm_acc_region_meta_t * acc_region_meta = &vm->acc_region_metas[instr_acc_idx]; + fd_vm_input_region_t * region = &vm->input_mem_regions[acc_region_meta->region_idx + 1UL]; + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1159-L1165 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L23-L35 */ + region->region_sz = (uint)fd_borrowed_account_get_data_len( borrowed_account ); + + int err; + int is_writable = fd_borrowed_account_can_data_be_changed( borrowed_account, &err ); + + region->is_writable = (uchar)is_writable && ( err == FD_EXECUTOR_INSTR_SUCCESS ); + } + + return FD_VM_SUCCESS; +} + /********************************************************************** CROSS PROGRAM INVOCATION HELPERS **********************************************************************/ @@ -355,10 +396,10 @@ fd_vm_syscall_cpi_check_authorized_program( fd_pubkey_t const * program_id /* Helper functions to get the absolute vaddrs of the serialized accounts pubkey, lamports and owner. For the accounts not owned by the deprecated loader, all of these offsets into the accounts metadata region - are static from fd_vm_acc_region_meta->metadata_region_offset. + are static from region->padding. For accounts owned by the deprecated loader, the unaligned serializer is used, which means only the pubkey - and lamports offsets are static from the metadata_region_offset. The owner is serialized into the region + and lamports offsets are static from the region->padding. The owner is serialized into the region immediately following the account data region (if present) at a fixed offset. */ #define VM_SERIALIZED_PUBKEY_OFFSET (8UL) @@ -369,13 +410,15 @@ fd_vm_syscall_cpi_check_authorized_program( fd_pubkey_t const * program_id #define VM_SERIALIZED_UNALIGNED_LAMPORTS_OFFSET (35UL) static inline -ulong serialized_pubkey_vaddr( fd_vm_t * vm, fd_vm_acc_region_meta_t * acc_region_meta ) { - return FD_VM_MEM_MAP_INPUT_REGION_START + acc_region_meta->metadata_region_offset + +ulong serialized_pubkey_vaddr( fd_vm_t * vm, ulong region_idx ) { +fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ]; + return FD_VM_MEM_MAP_INPUT_REGION_START + region->vaddr_offset + region->padding + (vm->is_deprecated ? VM_SERIALIZED_UNALIGNED_PUBKEY_OFFSET : VM_SERIALIZED_PUBKEY_OFFSET); } static inline -ulong serialized_owner_vaddr( fd_vm_t * vm, fd_vm_acc_region_meta_t * acc_region_meta ) { +ulong serialized_owner_vaddr( fd_vm_t * vm, ulong region_idx ) { +fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ]; if ( vm->is_deprecated ) { /* For deprecated loader programs, the owner is serialized into the start of the region following the account data region (if present) at a fixed offset. @@ -383,17 +426,20 @@ ulong serialized_owner_vaddr( fd_vm_t * vm, fd_vm_acc_region_meta_t * acc_region serialized into the same fixed offset following the account's metadata region. */ - return FD_VM_MEM_MAP_INPUT_REGION_START + vm->input_mem_regions[ - acc_region_meta->has_data_region ? acc_region_meta->region_idx+1U : acc_region_meta->region_idx - ].vaddr_offset; + if( region->address_space_reserved > 0UL ) { + fd_vm_input_region_t * metadata_region = &vm->input_mem_regions[ region_idx+1U ]; + return FD_VM_MEM_MAP_INPUT_REGION_START + metadata_region->vaddr_offset + metadata_region->padding; + } else { + return FD_VM_MEM_MAP_INPUT_REGION_START + region->vaddr_offset + region->padding; + } } - - return FD_VM_MEM_MAP_INPUT_REGION_START + acc_region_meta->metadata_region_offset + VM_SERIALIZED_OWNER_OFFSET; + return FD_VM_MEM_MAP_INPUT_REGION_START + region->vaddr_offset + region->padding + VM_SERIALIZED_OWNER_OFFSET; } static inline -ulong serialized_lamports_vaddr( fd_vm_t * vm, fd_vm_acc_region_meta_t * acc_region_meta ) { - return FD_VM_MEM_MAP_INPUT_REGION_START + acc_region_meta->metadata_region_offset + +ulong serialized_lamports_vaddr( fd_vm_t * vm, ulong region_idx ) { +fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ]; + return FD_VM_MEM_MAP_INPUT_REGION_START + region->vaddr_offset + region->padding + (vm->is_deprecated ? VM_SERIALIZED_UNALIGNED_LAMPORTS_OFFSET : VM_SERIALIZED_LAMPORTS_OFFSET); } @@ -407,7 +453,7 @@ ulong vm_syscall_cpi_acc_info_rc_refcell_as_ptr( ulong rc_refcell_vaddr ) { return (ulong) &(((fd_vm_rc_refcell_t *)rc_refcell_vaddr)->payload); } -/* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L327 +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L310-L316 */ FD_FN_CONST static inline ulong vm_syscall_cpi_data_len_vaddr_c( ulong acct_info_vaddr, ulong data_len_haddr, ulong acct_info_haddr ) { @@ -449,6 +495,7 @@ ulong vm_syscall_cpi_data_len_vaddr_c( ulong acct_info_vaddr, ulong data_len_had #define VM_SYSCALL_CPI_ACC_INFO_LAMPORTS( vm, acc_info, decl ) \ ulong * decl = FD_VM_MEM_HADDR_ST( vm, acc_info->lamports_addr, alignof(ulong), sizeof(ulong) ); +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L304 */ #define VM_SYSCALL_CPI_ACC_INFO_DATA_VADDR( vm, acc_info, decl ) \ ulong decl = acc_info->data_addr; #define VM_SYSCALL_CPI_ACC_INFO_DATA( vm, acc_info, decl ) \ @@ -527,18 +574,25 @@ ulong vm_syscall_cpi_data_len_vaddr_c( ulong acct_info_vaddr, ulong data_len_had /* Extract the vaddr embedded in the RefCell */ \ ulong decl = *FD_EXPAND_THEN_CONCAT2(decl, _hptr_); +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L184-L195 */ #define VM_SYSCALL_CPI_ACC_INFO_LAMPORTS( vm, acc_info, decl ) \ ulong FD_EXPAND_THEN_CONCAT2(decl, _vaddr_) = \ *((ulong const *)FD_VM_MEM_HADDR_LD( vm, vm_syscall_cpi_acc_info_rc_refcell_as_ptr( acc_info->lamports_box_addr ), FD_VM_RC_REFCELL_ALIGN, sizeof(ulong) )); \ ulong * decl = FD_VM_MEM_HADDR_ST( vm, FD_EXPAND_THEN_CONCAT2(decl, _vaddr_), alignof(ulong), sizeof(ulong) ); +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L184-L195 */ #define VM_SYSCALL_CPI_ACC_INFO_DATA_VADDR( vm, acc_info, decl ) \ + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && acc_info->data_box_addr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { \ + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); \ + return FD_VM_SYSCALL_ERR_INVALID_POINTER; \ + } \ /* Translate the vaddr to the slice */ \ fd_vm_vec_t const * FD_EXPAND_THEN_CONCAT2(decl, _hptr_) = \ FD_VM_MEM_HADDR_LD( vm, vm_syscall_cpi_acc_info_rc_refcell_as_ptr( acc_info->data_box_addr ), FD_VM_RC_REFCELL_ALIGN, sizeof(fd_vm_vec_t) ); \ /* Extract the vaddr embedded in the slice */ \ ulong decl = FD_EXPAND_THEN_CONCAT2(decl, _hptr_)->addr; +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L212-L221 */ #define VM_SYSCALL_CPI_ACC_INFO_DATA_LEN_VADDR( vm, acc_info, decl ) \ ulong decl = fd_ulong_sat_add( vm_syscall_cpi_acc_info_rc_refcell_as_ptr( acc_info->data_box_addr ), sizeof(ulong) ); diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c b/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c index fbf1ff9dcd0..5b942128a62 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c @@ -14,7 +14,7 @@ and links to the source have been provided. */ -/* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L23 +/* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L21-L38 This is used for checking that the account info pointers given by the user match up with the addresses in the serialized account metadata. @@ -89,7 +89,7 @@ VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( fd_vm_t * vm, VM_SYSCALL_CPI_ACC_META_IS_SIGNER( cpi_acct_meta ) ); /* Use USHORT_MAX to indicate account not found - https://github.com/anza-xyz/agave/blob/v2.1.14/program-runtime/src/invoke_context.rs#L366-L370 */ + https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/invoke_context.rs#L395-L397 */ int idx_in_txn = fd_exec_txn_ctx_find_index_of_account( vm->instr_ctx->txn_ctx, pubkey ); int idx_in_caller = fd_exec_instr_ctx_find_idx_of_instr_account( vm->instr_ctx, pubkey ); @@ -108,7 +108,7 @@ VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( fd_vm_t * vm, /* fd_vm_syscall_cpi_update_callee_acc_{rust/c} corresponds to solana_bpf_loader_program::syscalls::cpi::update_callee_account: -https://github.com/solana-labs/solana/blob/dbf06e258ae418097049e845035d7d5502fe1327/programs/bpf_loader/src/syscalls/cpi.rs#L1302 +https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1067-L1132 (the copy of the account stored in the instruction context's borrowed accounts cache) @@ -135,7 +135,7 @@ VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC( fd_vm_t * vm, /* Borrow the callee account. TODO: Agave borrows before this function call. Consider refactoring to borrow the account at the same place as Agave. - https://github.com/anza-xyz/agave/blob/v2.1.14/programs/bpf_loader/src/syscalls/cpi.rs#L893 */ + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L817 */ fd_guarded_borrowed_account_t callee_acc = {0}; err = fd_exec_instr_ctx_try_borrow_instr_account( vm->instr_ctx, instr_acc_idx, &callee_acc ); if( FD_UNLIKELY( err ) ) { @@ -143,6 +143,7 @@ VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC( fd_vm_t * vm, return FD_VM_SUCCESS; } + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1087-L1089 */ if( fd_borrowed_account_get_lamports( &callee_acc )!=*(caller_account->lamports) ) { err = fd_borrowed_account_set_lamports( &callee_acc, *(caller_account->lamports) ); if( FD_UNLIKELY( err ) ) { @@ -151,78 +152,74 @@ VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC( fd_vm_t * vm, } } - if( !vm->direct_mapping ) { - /* Get the account data */ - /* Update the account data, if the account data can be changed */ - /* FIXME: double-check these permissions, especially the callee_acc_idx */ + /* With stricter_abi_and_runtime_constraints enabled, we validate account + length changes and update the associated borrowed account with any + changed made. If direct mapping is also enabled, we skip actually copying + the data back to the borrowed account, as it is already updated in-place. + + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1091-L1113 */ + if( vm->stricter_abi_and_runtime_constraints ) { + ulong prev_len = fd_borrowed_account_get_data_len( &callee_acc ); + ulong post_len = *caller_account->ref_to_len_in_vm; + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1094-L1109 */ + if( FD_UNLIKELY( prev_len!=post_len ) ) { + ulong address_space_reserved_for_account; + if( vm->is_deprecated ) { + address_space_reserved_for_account = caller_account->orig_data_len; + } else { + address_space_reserved_for_account = fd_ulong_sat_add( caller_account->orig_data_len, MAX_PERMITTED_DATA_INCREASE ); + } - if( fd_borrowed_account_can_data_be_resized( &callee_acc, caller_account->serialized_data_len, &err ) && - fd_borrowed_account_can_data_be_changed( &callee_acc, &err ) ) { - /* We must ignore the errors here, as they are informational and do not mean the result is invalid. */ - /* TODO: not pass informational errors like this? */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1103-L1105 */ + if( FD_UNLIKELY( post_len>address_space_reserved_for_account ) ) { + FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC ); + return -1; + } - err = fd_borrowed_account_set_data_from_slice( &callee_acc, caller_account->serialized_data, caller_account->serialized_data_len ); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1106 */ + err = fd_borrowed_account_set_data_length( &callee_acc, post_len ); if( FD_UNLIKELY( err ) ) { FD_VM_ERR_FOR_LOG_INSTR( vm, err ); return -1; } - } else if( FD_UNLIKELY( caller_account->serialized_data_len!=fd_borrowed_account_get_data_len( &callee_acc ) || - memcmp( fd_borrowed_account_get_data( &callee_acc ), caller_account->serialized_data, caller_account->serialized_data_len ) ) ) { - FD_VM_ERR_FOR_LOG_INSTR( vm, err ); - return -1; } - } else { /* Direct mapping enabled */ - ulong * ref_to_len = FD_VM_MEM_HADDR_ST( vm, caller_account->ref_to_len_in_vm.vaddr, alignof(ulong), sizeof(ulong) ); - ulong orig_len = caller_account->orig_data_len; - ulong prev_len = fd_borrowed_account_get_data_len( &callee_acc ); - ulong post_len = *ref_to_len; - + /* Without direct mapping, we need to copy the account data from the VM's + serialized buffer back to the borrowed account. With direct mapping, + data is modified in-place so no copy is needed. + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1110-L1112 */ int err; - if( fd_borrowed_account_can_data_be_resized( &callee_acc, post_len, &err ) && - fd_borrowed_account_can_data_be_changed( &callee_acc, &err ) ) { - - ulong realloc_bytes_used = fd_ulong_sat_sub( post_len, orig_len ); - - if( FD_UNLIKELY( vm->is_deprecated && realloc_bytes_used ) ) { - FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC ); + if( !vm->direct_mapping && fd_borrowed_account_can_data_be_changed( &callee_acc, &err ) ) { + err = fd_borrowed_account_set_data_from_slice( &callee_acc, caller_account->serialized_data, caller_account->serialized_data_len ); + if( FD_UNLIKELY( err ) ) { + FD_VM_ERR_FOR_LOG_INSTR( vm, err ); return -1; } + } + } else { + /* Direct mapping is not enabled, so we need to copy the account data + from the VM's serialized buffer back to the borrowed account. - err = fd_borrowed_account_set_data_length( &callee_acc, post_len ); + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1114-L1121 */ + int err; + if( fd_borrowed_account_can_data_be_resized( &callee_acc, caller_account->serialized_data_len, &err ) && + fd_borrowed_account_can_data_be_changed( &callee_acc, &err ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1116 */ + err = fd_borrowed_account_set_data_from_slice( &callee_acc, caller_account->serialized_data, caller_account->serialized_data_len ); if( FD_UNLIKELY( err ) ) { FD_VM_ERR_FOR_LOG_INSTR( vm, err ); return -1; } - - - if( realloc_bytes_used ) { - /* We need to get the relevant data slice. However, we know that the - current length currently exceeds the original length for the account - data. This means that all of the additional bytes must exist in the - account data resizing region. As an invariant, original_len must be - equal to the length of the account data region. This means we can - smartly look up the right region and don't need to worry about - multiple region access.We just need to load in the bytes from - (original len, post_len]. */ - uchar const * realloc_data = FD_VM_MEM_SLICE_HADDR_LD( vm, caller_account->vm_data_vaddr+orig_len, alignof(uchar), realloc_bytes_used ); - - uchar * data = NULL; - ulong dlen = 0UL; - err = fd_borrowed_account_get_data_mut( &callee_acc, &data, &dlen ); - if( FD_UNLIKELY( err ) ) { - FD_VM_ERR_FOR_LOG_INSTR( vm, err ); - return -1; - } - fd_memcpy( data+orig_len, realloc_data, realloc_bytes_used ); - } - - } else if( FD_UNLIKELY( prev_len!=post_len ) ) { + } else if( FD_UNLIKELY( caller_account->serialized_data_len!=fd_borrowed_account_get_data_len( &callee_acc ) || + memcmp( fd_borrowed_account_get_data( &callee_acc ), caller_account->serialized_data, caller_account->serialized_data_len ) ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1117-L1119 */ FD_VM_ERR_FOR_LOG_INSTR( vm, err ); return -1; } } + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1124-L1129 */ if( FD_UNLIKELY( memcmp( fd_borrowed_account_get_owner( &callee_acc ), caller_account->owner, sizeof(fd_pubkey_t) ) ) ) { err = fd_borrowed_account_set_owner( &callee_acc, caller_account->owner ); if( FD_UNLIKELY( err ) ) { @@ -237,7 +234,7 @@ VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC( fd_vm_t * vm, /* fd_vm_syscall_cpi_translate_and_update_accounts_ mirrors the behaviour of solana_bpf_loader_program::syscalls::cpi::translate_and_update_accounts: -https://github.com/solana-labs/solana/blob/dbf06e258ae418097049e845035d7d5502fe1327/programs/bpf_loader/src/syscalls/cpi.rs#L954-L1085 +https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L767-L892 It translates the caller accounts to the host address space, and then calls fd_vm_syscall_cpi_update_callee_acc to update the callee borrowed account with any changes @@ -283,7 +280,7 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( A borrowed account will always have non-NULL meta (if the account doesn't exist, `fd_executor_setup_accounts_for_txn()` will set its meta up) */ - /* https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L878-L881 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L817 */ fd_guarded_borrowed_account_t callee_acct = {0}; FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( vm->instr_ctx, instruction_accounts[i].index_in_caller, &callee_acct ); @@ -306,22 +303,12 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( uint found = 0; for( ushort j=0; juc, acct_addr->uc, sizeof(fd_pubkey_t) ) != 0 ) { continue; } - - /* The following error is practically unreachable because - essentially the same check is performed in - prepare_instruction(). So a missing account error would have - been returned then and there. Hence, we are skipping this - duplicate check here. - https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L914-L923 - */ - - /* The next iteration will overwrite this if it turns out that we do not need to preserve this for update_caller(). */ @@ -339,7 +326,7 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( account_info_keys array is set up. We replicate the check for clarity and also to guard against accidental violation of the assumed invariant in the future. - https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L926-L928 + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L846-L849 */ if( FD_UNLIKELY( j >= account_infos_length ) ) { FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); @@ -348,35 +335,32 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( /* The following implements the checks in from_account_info which is invoked as do_translate() in translate_and_update_accounts() - https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L931 + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L850-L861 */ ////// BEGIN from_account_info fd_vm_acc_region_meta_t * acc_region_meta = &vm->acc_region_metas[index_in_caller]; - if( FD_LIKELY( vm->direct_mapping ) ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.7/programs/bpf_loader/src/syscalls/cpi.rs#L116 - */ - ulong expected_pubkey_vaddr = serialized_pubkey_vaddr( vm, acc_region_meta ); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L138 */ + if( FD_LIKELY( vm->stricter_abi_and_runtime_constraints ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L139-L144 */ + ulong expected_pubkey_vaddr = serialized_pubkey_vaddr( vm, acc_region_meta->region_idx ); /* Max msg_sz: 40 + 18 + 18 = 76 < 127 */ VM_SYSCALL_CPI_CHECK_ACCOUNT_INFO_POINTER_FIELD_MAX_54(vm, account_infos[j].pubkey_addr, expected_pubkey_vaddr, "key"); - /* https://github.com/anza-xyz/agave/blob/v2.1.7/programs/bpf_loader/src/syscalls/cpi.rs#L122 - */ - ulong expected_owner_vaddr = serialized_owner_vaddr( vm, acc_region_meta ); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L145-L150 */ + ulong expected_owner_vaddr = serialized_owner_vaddr( vm, acc_region_meta->region_idx ); /* Max msg_sz: 42 + 18 + 18 = 78 < 127 */ VM_SYSCALL_CPI_CHECK_ACCOUNT_INFO_POINTER_FIELD_MAX_54(vm, account_infos[j].owner_addr, expected_owner_vaddr, "owner"); } - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L134 - */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L155-L175 */ VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_VADDR( vm, (account_infos + j), lamports_vaddr ); - if( FD_LIKELY( vm->direct_mapping ) ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.7/programs/bpf_loader/src/syscalls/cpi.rs#L140 - Check that the account's lamports Rc> is - not stored in the account. - Because a refcell is only present if the Rust SDK is used, we - only need to check this for the Rust ABI. - */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L162-L173 */ + if( FD_LIKELY( vm->stricter_abi_and_runtime_constraints ) ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L163-L165 + Check that the account's lamports Rc> is not + stored in the account region. Because a refcell is only present if + the Rust SDK is used, we only need to check this for the Rust ABI. */ #ifdef VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR( vm, (account_infos + j), lamports_rc_vaddr ) if ( FD_UNLIKELY( lamports_rc_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { @@ -385,107 +369,111 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( } #endif - /* https://github.com/anza-xyz/agave/blob/v2.1.7/programs/bpf_loader/src/syscalls/cpi.rs#L144 - */ - ulong expected_lamports_vaddr = serialized_lamports_vaddr( vm, acc_region_meta ); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L167-L172 */ + ulong expected_lamports_vaddr = serialized_lamports_vaddr( vm, acc_region_meta->region_idx ); /* Max msg_sz: 45 + 18 + 18 = 81 < 127 */ VM_SYSCALL_CPI_CHECK_ACCOUNT_INFO_POINTER_FIELD_MAX_54(vm, lamports_vaddr, expected_lamports_vaddr, "lamports"); } - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L151 + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L153-L175 */ VM_SYSCALL_CPI_ACC_INFO_LAMPORTS( vm, (account_infos + j), lamports_haddr ); caller_account->lamports = lamports_haddr; - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L154 + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L177-L181 */ caller_account->owner = FD_VM_MEM_HADDR_ST( vm, (account_infos + j)->owner_addr, alignof(uchar), sizeof(fd_pubkey_t) ); - if( FD_LIKELY( vm->direct_mapping ) ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L161 - Check that the account's data Rc> is not stored in - the account. - Because a refcell is only present if the Rust SDK is used, we - only need to check this for the Rust ABI. - */ - #ifdef VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR - VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR( vm, (account_infos + j), data_rc_vaddr ) - if( FD_UNLIKELY( data_rc_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); - return FD_VM_SYSCALL_ERR_INVALID_POINTER; - } - #endif - } - - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L166 + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L190-L203 */ VM_SYSCALL_CPI_ACC_INFO_DATA_VADDR( vm, (account_infos + j), data_vaddr ); - if( FD_LIKELY( vm->direct_mapping ) ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.7/programs/bpf_loader/src/syscalls/cpi.rs#L172 */ - ulong expected_data_region_vaddr = FD_VM_MEM_MAP_INPUT_REGION_START + - vm->input_mem_regions[acc_region_meta->region_idx].vaddr_offset; - /* Max msg_sz: 41 + 18 + 18 = 77 < 127 */ - VM_SYSCALL_CPI_CHECK_ACCOUNT_INFO_POINTER_FIELD_MAX_54(vm, data_vaddr, expected_data_region_vaddr, "data"); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L196-L203 */ + if( vm->stricter_abi_and_runtime_constraints ) { + fd_vm_input_region_t * region = &vm->input_mem_regions[ acc_region_meta->region_idx ]; + ulong expected_data_vaddr = FD_VM_MEM_MAP_INPUT_REGION_START + + region->vaddr_offset + region->address_space_reserved; + VM_SYSCALL_CPI_CHECK_ACCOUNT_INFO_POINTER_FIELD_MAX_54(vm, data_vaddr, expected_data_vaddr, "data"); } - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L180 + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L205-L210 */ VM_SYSCALL_CPI_SET_ACC_INFO_DATA_GET_LEN( vm, (account_infos + j), data_vaddr ); FD_VM_CU_UPDATE( vm, data_vaddr_len / FD_VM_CPI_BYTES_PER_UNIT ); - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L187 - https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L331 - */ #ifdef VM_SYSCALL_CPI_ACC_INFO_DATA_LEN_VADDR - /* Rust ABI */ + /* Rust ABI + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L212-L221 */ VM_SYSCALL_CPI_ACC_INFO_DATA_LEN_VADDR( vm, (account_infos + j), data_len_vaddr ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && data_len_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_SYSCALL_ERR_INVALID_POINTER; + } (void)acct_infos_va; #else - /* C ABI */ + /* C ABI + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L310-L316 */ ulong data_len_vaddr = vm_syscall_cpi_data_len_vaddr_c( fd_ulong_sat_add( acct_infos_va, fd_ulong_sat_mul( j, VM_SYSCALL_CPI_ACC_INFO_SIZE ) ), (ulong)&((account_infos + j)->data_sz), (ulong)(account_infos + j) ); #endif - if( FD_LIKELY( vm->direct_mapping ) ) { - #ifdef VM_SYSCALL_CPI_ACC_INFO_DATA_LEN_VADDR - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L193 - */ - if( FD_UNLIKELY( data_len_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); - return FD_VM_SYSCALL_ERR_INVALID_POINTER; - } - #endif - caller_account->ref_to_len_in_vm.vaddr = data_len_vaddr; - } - if( FD_UNLIKELY( !vm->direct_mapping ) ) { - ulong * data_len = FD_VM_MEM_HADDR_ST( - vm, - data_len_vaddr, - 1UL, - sizeof(ulong) - ); - caller_account->ref_to_len_in_vm.translated = data_len; - } - caller_account->vm_data_vaddr = data_vaddr; - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L228 - */ - caller_account->serialized_data = NULL; - if( FD_UNLIKELY( !vm->direct_mapping ) ) { + /* Rust ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L222 + C ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L317 */ + ulong * data_len = FD_VM_MEM_HADDR_ST( vm, data_len_vaddr, 1UL, sizeof(ulong) ); + caller_account->ref_to_len_in_vm = data_len; + + /* Rust ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L226 + C ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L324 */ + caller_account->vm_data_vaddr = data_vaddr; + + /* Rust ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L224-L230 + C ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L302-L308 + + Both ABIs call CallerAccount::get_serialized_data: + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L90-L123 + + With both stricter_abi_and_runtime_constraints and direct_mapping, + account data is modified in-place so we don't track the + serialized_data pointer. + + With stricter_abi only (no direct_mapping), data was copied into the input + region buffer. We don't apply the extra memory translation checks, as + we have checked the data pointer is valid above. So instead we add + the vaddr to the start of the input region address space - copying + this logic from Agave. + + In legacy mode, we translate the data pointer directly, as it just + maps to a location in the single input region. */ + if( vm->stricter_abi_and_runtime_constraints && vm->direct_mapping ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L97-L99 */ + caller_account->serialized_data = NULL; + caller_account->serialized_data_len = 0UL; + } else if( vm->stricter_abi_and_runtime_constraints ) { + /* Skip translation checks here, following the Agave logic: + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L99-L115 */ + uchar * serialization_ptr = (uchar *)FD_VM_MEM_SLICE_HADDR_ST( vm, FD_VM_MEM_MAP_INPUT_REGION_START, alignof(uchar), 1UL ); + caller_account->serialized_data = serialization_ptr + fd_ulong_sat_sub( data_vaddr, FD_VM_MEM_MAP_INPUT_REGION_START ); + caller_account->serialized_data_len = data_vaddr_len; + } else { + /* https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L115-L122 */ VM_SYSCALL_CPI_ACC_INFO_DATA( vm, (account_infos + j), data_haddr ); (void)data_haddr_vm_addr; caller_account->serialized_data = data_haddr; caller_account->serialized_data_len = data_haddr_len; } + /* Rust ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L237 + C ABI: https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L322 */ caller_account->orig_data_len = acc_region_meta->original_data_len; ////// END from_account_info // TODO We should be able to cache the results of translation and reuse them in the update function. - /* Update the callee account to reflect any changes the caller has made */ + /* Update the callee account to reflect any changes the caller has made + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L863-L873 */ int err = VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC( vm, caller_account, (uchar)index_in_caller ); if( FD_UNLIKELY( err ) ) { /* errors are propagated in the function itself. */ @@ -494,7 +482,7 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( } if( !found ) { - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L966 */ + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L882-L887 */ FD_BASE58_ENCODE_32_BYTES( account_key->uc, id_b58 ); fd_log_collector_msg_many( vm->instr_ctx, 2, "Instruction references an unknown account ", 42UL, id_b58, id_b58_len ); FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_MISSING_ACC ); @@ -507,7 +495,7 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( /* fd_vm_cpi_update_caller_acc_{rust/c} mirrors the behaviour of solana_bpf_loader_program::syscalls::cpi::update_caller_account: -https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L1291 +https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1171-L1268 This method should be called after a CPI instruction execution has returned. It updates the given caller account info with any changes the callee @@ -526,198 +514,130 @@ TODO: error codes #define VM_SYSCALL_CPI_UPDATE_CALLER_ACC_FUNC FD_EXPAND_THEN_CONCAT2(fd_vm_cpi_update_caller_acc_, VM_SYSCALL_CPI_ABI) static int VM_SYSCALL_CPI_UPDATE_CALLER_ACC_FUNC( fd_vm_t * vm, - VM_SYSCALL_CPI_ACC_INFO_T const * caller_acc_info, - fd_vm_cpi_caller_account_t const * caller_account, - uchar instr_acc_idx, + VM_SYSCALL_CPI_ACC_INFO_T const * caller_acc_info FD_FN_UNUSED, + fd_vm_cpi_caller_account_t * caller_account, + uchar instr_acc_idx FD_FN_UNUSED, fd_pubkey_t const * pubkey ) { int err; - if( !vm->direct_mapping ) { - /* Look up the borrowed account from the instruction context, which will contain - the callee's changes. - TODO: Agave borrows before entering this function. We should consider doing the same. - https://github.com/anza-xyz/agave/blob/v2.1.14/programs/bpf_loader/src/syscalls/cpi.rs#L1168-L1169 */ - fd_guarded_borrowed_account_t borrowed_callee_acc = {0}; - err = fd_exec_instr_ctx_try_borrow_instr_account_with_key( vm->instr_ctx, pubkey, &borrowed_callee_acc ); - if( FD_UNLIKELY( err && ( err != FD_ACC_MGR_ERR_UNKNOWN_ACCOUNT ) ) ) { - return 1; - } - - fd_txn_account_t * callee_acc = borrowed_callee_acc.acct; - /* Update the caller account lamports with the value from the callee */ - *(caller_account->lamports) = fd_txn_account_get_lamports( callee_acc ); - - /* Update the caller account owner with the value from the callee */ - fd_pubkey_t const * updated_owner = fd_txn_account_get_owner( callee_acc ); - if( updated_owner ) *caller_account->owner = *updated_owner; - else fd_memset( caller_account->owner, 0, sizeof(fd_pubkey_t) ); - - /* Update the caller account data with the value from the callee */ - VM_SYSCALL_CPI_ACC_INFO_DATA( vm, caller_acc_info, caller_acc_data ); - - ulong const updated_data_len = fd_txn_account_get_data_len( callee_acc ); - - if( updated_data_lenref_to_len_in_vm.translated; - if( *ref_to_len != updated_data_len ) { - ulong max_increase = (vm->direct_mapping && vm->is_deprecated) ? 0UL : MAX_PERMITTED_DATA_INCREASE; - // https://github.com/anza-xyz/agave/blob/7f3a6cf6d3c2dcc81bb38e49a5c9ef998a6f4dd9/programs/bpf_loader/src/syscalls/cpi.rs#L1387-L1397 - if( FD_UNLIKELY( updated_data_len>fd_ulong_sat_add( caller_account->orig_data_len, max_increase ) ) ) { - FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC); - return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; - } - - /* FIXME: do we need to zero the memory that was previously used, if the new data_len is smaller? - https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L1361 - I don't think we do but need to double-check. */ - - /* https://github.com/anza-xyz/agave/blob/v2.1.6/programs/bpf_loader/src/syscalls/cpi.rs#L1453 - */ - caller_acc_data = FD_VM_MEM_SLICE_HADDR_ST( vm, caller_acc_data_vm_addr, alignof(uchar), updated_data_len ); - - *ref_to_len = updated_data_len; - - /* Update the serialized len field - https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L1437 */ - ulong * caller_len = FD_VM_MEM_HADDR_ST( vm, fd_ulong_sat_sub(caller_acc_data_vm_addr, sizeof(ulong)), alignof(ulong), sizeof(ulong) ); - *caller_len = updated_data_len; + /* Look up the borrowed account from the instruction context, which will contain + the callee's changes. + TODO: Agave borrows before entering this function. We should consider doing the same. + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1033-L1034 */ + fd_guarded_borrowed_account_t borrowed_callee_acc = {0}; + err = fd_exec_instr_ctx_try_borrow_instr_account_with_key( vm->instr_ctx, pubkey, &borrowed_callee_acc ); + if( FD_UNLIKELY( err && ( err != FD_ACC_MGR_ERR_UNKNOWN_ACCOUNT ) ) ) { + return 1; + } - /* FIXME return instruction error account data size too small in the same scenarios solana does - https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L1534 */ - } + fd_txn_account_t * callee_acc = borrowed_callee_acc.acct; + /* Update the caller account lamports with the value from the callee + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1191 */ + *(caller_account->lamports) = fd_txn_account_get_lamports( callee_acc ); + + /* Update the caller account owner with the value from the callee + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1192 */ + fd_pubkey_t const * updated_owner = fd_txn_account_get_owner( callee_acc ); + if( updated_owner ) *caller_account->owner = *updated_owner; + else fd_memset( caller_account->owner, 0, sizeof(fd_pubkey_t) ); + + /* Update the caller account data with the value from the callee + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1194-L1195 */ + ulong prev_len = *caller_account->ref_to_len_in_vm; + ulong post_len = fd_txn_account_get_data_len( callee_acc ); + + /* Calculate the address space reserved for the account. With stricter_abi_and_runtime_constraints + and deprecated loader, the reserved space equals original length (no realloc space). + Otherwise, we add MAX_PERMITTED_DATA_INCREASE for reallocation. + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1197-L1204 */ + ulong address_space_reserved_for_account; + if( vm->stricter_abi_and_runtime_constraints && vm->is_deprecated ) { + address_space_reserved_for_account = caller_account->orig_data_len; + } else { + address_space_reserved_for_account = fd_ulong_sat_add( caller_account->orig_data_len, MAX_PERMITTED_DATA_INCREASE ); + } - fd_memcpy( caller_acc_data, fd_txn_account_get_data( callee_acc ), updated_data_len ); - } else { /* Direct mapping enabled */ - - /* Look up the borrowed account from the instruction context, which will - contain the callee's changes. - TODO: Agave borrows before entering this function. We should consider doing the same. - https://github.com/anza-xyz/agave/blob/v2.1.14/programs/bpf_loader/src/syscalls/cpi.rs#L1168-L1169 */ - fd_guarded_borrowed_account_t borrowed_callee_acc = {0}; - err = fd_exec_instr_ctx_try_borrow_instr_account_with_key( vm->instr_ctx, pubkey, &borrowed_callee_acc ); - if( FD_UNLIKELY( err && ( err != FD_ACC_MGR_ERR_UNKNOWN_ACCOUNT ) ) ) { - return 1; - } + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1206-L1216 */ + if( post_len > address_space_reserved_for_account && + ( vm->stricter_abi_and_runtime_constraints || prev_len != post_len ) ) { + ulong max_increase = fd_ulong_sat_sub( address_space_reserved_for_account, caller_account->orig_data_len ); + fd_log_collector_printf_dangerous_max_127( vm->instr_ctx, "Account data size realloc limited to %lu in inner instructions", max_increase ); + FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC ); + return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; + } - fd_txn_account_t * callee_acc = borrowed_callee_acc.acct; + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1218-L1252 */ + if( prev_len != post_len ) { - /* Update the caller account lamports with the value from the callee */ - *(caller_account->lamports) = fd_txn_account_get_lamports( callee_acc ); + /* Without direct mapping, we need to adjust the serialized data buffer + when the length changes. - /* Update the caller account owner with the value from the callee */ - fd_pubkey_t const * updated_owner = fd_txn_account_get_owner( callee_acc ); - if( updated_owner ) { - *caller_account->owner = *updated_owner; - } else { - fd_memset( caller_account->owner, 0, sizeof(fd_pubkey_t) ); - } + With direct mapping, data is mapped in-place so no buffer manipulation + is needed. - /* Make sure that the capacity of the borrowed account is sized up in case - it was shrunk in the CPI. It needs to be sized up in order to fit within - the originally delinated regions when the account data was serialized. - https://github.com/anza-xyz/agave/blob/36323b6dcd3e29e4d6fe6d73d716a3f33927148b/programs/bpf_loader/src/syscalls/cpi.rs#L1311 */ - VM_SYSCALL_CPI_ACC_INFO_METADATA( vm, caller_acc_info, caller_acc_data ); - ulong original_len = caller_account->orig_data_len; - - uchar zero_all_mapped_spare_capacity = 0; - /* This case can only be triggered if the original length is more than 0 */ - if( fd_txn_account_get_data_len( callee_acc )stricter_abi_and_runtime_constraints && vm->direct_mapping ) ) { - /* Update the account data region if an account data region exists. We - know that one exists iff the original len was non-zero. */ - ulong acc_region_idx = vm->acc_region_metas[instr_acc_idx].region_idx; - if( original_len && vm->input_mem_regions[ acc_region_idx ].haddr!=(ulong)fd_txn_account_get_data_mut( callee_acc ) ) { - vm->input_mem_regions[ acc_region_idx ].haddr = (ulong)fd_txn_account_get_data_mut( callee_acc ); - zero_all_mapped_spare_capacity = 1; - } + /* If the account has shrunk, zero out memory that was previously used + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1222-L1230 */ + if( post_len < prev_len ) { + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1227-L1228 */ + if( caller_account->serialized_data_len < post_len ) { + FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_ACC_DATA_TOO_SMALL ); + return FD_EXECUTOR_INSTR_ERR_ACC_DATA_TOO_SMALL; + } - ulong prev_len = caller_acc_data_len; - ulong post_len = fd_txn_account_get_data_len( callee_acc ); - - /* Do additional handling in the case where the data size has changed in - the course of the callee's CPI. */ - if( prev_len!=post_len ) { - /* There is an illegal data overflow if the post len is greater than the - original data len + the max resizing limit (10KiB). Can't resize the - account if the deprecated loader is being used */ - ulong max_increase = vm->is_deprecated ? 0UL : MAX_PERMITTED_DATA_INCREASE; - if( FD_UNLIKELY( post_len>fd_ulong_sat_add( (ulong)original_len, max_increase ) ) ) { - FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC); - return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC; + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1225-L1229 */ + fd_memset( caller_account->serialized_data + post_len, 0, caller_account->serialized_data_len - post_len ); } - /* There is additonal handling in the case where the account is larger - than it was previously, but it has still grown since it was initially - serialized. To handle this, we need to just zero out the now unused - space in the account resizing region. */ - if( post_lenoriginal_len ) { - ulong dirty_realloc_start = fd_ulong_max( post_len, original_len ); - ulong dirty_realloc_len = fd_ulong_sat_sub( prev_len, dirty_realloc_start ); - /* We don't have to worry about multiple region writes here since we - know the amount to zero out is located in the account's data - resizing region. We intentionally write to the pointer despite - loading it in because we assume that the permissions were changed - in the callee. */ - uchar * dirty_region = FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, caller_acc_data_vm_addr + dirty_realloc_start, - alignof(uchar), dirty_realloc_len ); - fd_memset( dirty_region, 0, dirty_realloc_len ); - } - - /* Because the account data length changed from before to after the - CPI we must update the fields appropriately. */ - ulong * ref_to_len = FD_VM_MEM_HADDR_ST( vm, caller_account->ref_to_len_in_vm.vaddr, alignof(ulong), sizeof(ulong) ); - *ref_to_len = post_len; - ulong * serialized_len_ptr = FD_VM_MEM_HADDR_ST( vm, fd_ulong_sat_sub( caller_acc_data_vm_addr, sizeof(ulong) ), alignof(ulong), sizeof(ulong) ); - *serialized_len_ptr = post_len; - } - /* We need to zero out the end of the account data buffer if the account - shrunk in size. This is because the bytes are accessible from within - the VM but should be equal to zero to prevent undefined behavior. If - prev_len > post_len, then dlen should be equal to original_len. */ - ulong spare_len = fd_ulong_sat_sub( fd_ulong_if( zero_all_mapped_spare_capacity, original_len, prev_len ), post_len ); - if( FD_UNLIKELY( spare_len ) ) { - if( fd_txn_account_get_data_len( callee_acc )>spare_len ) { - memset( fd_txn_account_get_data_mut( callee_acc ) + fd_txn_account_get_data_len( callee_acc ) - spare_len, 0, spare_len ); + /* Set caller_account.serialized_data to post_len. + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1231-L1238 */ + if( vm->stricter_abi_and_runtime_constraints ) { + /* Calculate the serialized data pointer from the input region base, + as described above. + + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L99-L115 */ + uchar * serialization_ptr = (uchar *)FD_VM_MEM_SLICE_HADDR_ST( vm, FD_VM_MEM_MAP_INPUT_REGION_START, alignof(uchar), 1UL ); + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1234 */ + caller_account->serialized_data = serialization_ptr + fd_ulong_sat_sub( caller_account->vm_data_vaddr, FD_VM_MEM_MAP_INPUT_REGION_START ); + caller_account->serialized_data_len = post_len; + } else { + /* Translate the data pointer directly from the VM address, if + stricter_abi_and_runtime_constraints (or direct mapping) is not + enabled. + + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L115-L122 */ + caller_account->serialized_data = (uchar *)FD_VM_MEM_SLICE_HADDR_ST( vm, caller_account->vm_data_vaddr, alignof(uchar), post_len ); + caller_account->serialized_data_len = post_len; } } - ulong realloc_bytes_used = fd_ulong_sat_sub( post_len, original_len ); - if( realloc_bytes_used && !vm->is_deprecated ) { - /* We intentionally do a load in the case where we are writing to because - we want to ignore the write checks. We load from the first byte of the - resizing region */ - ulong resizing_idx = vm->acc_region_metas[ instr_acc_idx ].region_idx; - if( vm->acc_region_metas[ instr_acc_idx ].has_data_region ) { - resizing_idx++; - } - uchar * to_slice = (uchar*)vm->input_mem_regions[ resizing_idx ].haddr; - uchar * from_slice = fd_txn_account_get_data_mut( callee_acc ) + original_len; + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1240-L1241 */ + *caller_account->ref_to_len_in_vm = post_len; - fd_memcpy( to_slice, from_slice, realloc_bytes_used ); - } + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1243-L1251 */ + ulong * caller_len = FD_VM_MEM_HADDR_ST( vm, fd_ulong_sat_sub(caller_account->vm_data_vaddr, sizeof(ulong)), alignof(ulong), sizeof(ulong) ); + *caller_len = post_len; } + /* Without direct mapping, copy the updated account data from the callee's + account back to the caller's serialized data buffer. With direct mapping, + data was modified in-place so no copy is needed. + + https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1254-L1265 */ + if( !(vm->stricter_abi_and_runtime_constraints && vm->direct_mapping) ) { + fd_memcpy( caller_account->serialized_data, fd_txn_account_get_data( callee_acc ), post_len ); + } + + return FD_VM_SUCCESS; } /* fd_vm_syscall_cpi_{rust/c} is the entrypoint for the sol_invoke_signed_{rust/c} syscalls. The bulk of the high-level logic mirrors Solana's cpi_common entrypoint function at -https://github.com/solana-labs/solana/blob/2afde1b028ed4593da5b6c735729d8994c4bfac6/programs/bpf_loader/src/syscalls/cpi.rs#L1060 +https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L964-L1065 The only differences should be in the order of the error checks, which does not affect consensus. 100-foot flow: @@ -878,12 +798,18 @@ VM_SYSCALL_CPI_ENTRYPOINT( void * _vm, } /* Translate account infos ******************************************/ - /* Direct mapping check - https://github.com/anza-xyz/agave/blob/v2.1.0/programs/bpf_loader/src/syscalls/cpi.rs#L805-L814 */ + + /* With stricter_abi_and_runtime_constraints, verify that the account_infos array + is not inside the input region. This prevents programs from passing pointers to + the serialized account data region as account_infos, which would allow them to + bypass pointer validation checks. + https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L735-L744 */ ulong acc_info_total_sz = fd_ulong_sat_mul( acct_info_cnt, VM_SYSCALL_CPI_ACC_INFO_SIZE ); - if( FD_UNLIKELY( vm->direct_mapping && fd_ulong_sat_add( acct_infos_va, acc_info_total_sz ) >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); - return FD_VM_SYSCALL_ERR_INVALID_POINTER; + if( vm->stricter_abi_and_runtime_constraints ) { + if( FD_UNLIKELY( fd_ulong_sat_add( acct_infos_va, acc_info_total_sz ) >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_SYSCALL_ERR_INVALID_POINTER; + } } /* This is the equivalent of translate_slice in translate_account_infos: @@ -891,20 +817,10 @@ VM_SYSCALL_CPI_ENTRYPOINT( void * _vm, VM_SYSCALL_CPI_ACC_INFO_T const * acc_infos = FD_VM_MEM_SLICE_HADDR_LD( vm, acct_infos_va, VM_SYSCALL_CPI_ACC_INFO_ALIGN, acc_info_total_sz ); /* Right after translating, Agave checks the number of account infos: - https://github.com/anza-xyz/agave/blob/838c1952595809a31520ff1603a13f2c9123aa51/programs/bpf_loader/src/syscalls/cpi.rs#L822 */ - if( FD_FEATURE_ACTIVE_BANK( vm->instr_ctx->txn_ctx->bank, loosen_cpi_size_restriction ) ) { - if( FD_UNLIKELY( acct_info_cnt > get_cpi_max_account_infos( vm->instr_ctx->txn_ctx ) ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_MAX_INSTRUCTION_ACCOUNT_INFOS_EXCEEDED ); - return FD_VM_SYSCALL_ERR_MAX_INSTRUCTION_ACCOUNT_INFOS_EXCEEDED; - } - } else { - ulong adjusted_len = fd_ulong_sat_mul( acct_info_cnt, sizeof( fd_pubkey_t ) ); - if ( FD_UNLIKELY( adjusted_len > FD_VM_MAX_CPI_INSTRUCTION_SIZE ) ) { - /* "Cap the number of account_infos a caller can pass to approximate - maximum that accounts that could be passed in an instruction" */ - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_TOO_MANY_ACCOUNTS ); - return FD_VM_SYSCALL_ERR_TOO_MANY_ACCOUNTS; - } + https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L752 */ + if( FD_UNLIKELY( acct_info_cnt > get_cpi_max_account_infos( vm->instr_ctx->txn_ctx ) ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_MAX_INSTRUCTION_ACCOUNT_INFOS_EXCEEDED ); + return FD_VM_SYSCALL_ERR_MAX_INSTRUCTION_ACCOUNT_INFOS_EXCEEDED; } fd_pubkey_t const * acct_info_keys[ FD_CPI_MAX_ACCOUNT_INFOS ]; @@ -916,7 +832,10 @@ VM_SYSCALL_CPI_ENTRYPOINT( void * _vm, acct_info_keys[ acct_idx ] = FD_VM_MEM_HADDR_LD( vm, acc_infos[ acct_idx ].pubkey_addr, alignof(uchar), sizeof(fd_pubkey_t) ); } - /* Update the callee accounts with any changes made by the caller prior to this CPI execution */ + /* translate_and_update_accounts ************************************************************ + Update the callee accounts with any changes made by the caller prior to this CPI execution + + https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L767-L892 */ fd_vm_cpi_caller_account_t caller_accounts[ 256 ]; ushort callee_account_keys[256]; ushort caller_accounts_to_update[256]; @@ -954,43 +873,6 @@ VM_SYSCALL_CPI_ENTRYPOINT( void * _vm, /* Errors are propagated in fd_execute_instr. */ if( FD_UNLIKELY( err_exec ) ) return err_exec; - /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/syscalls/cpi.rs#L1128-L1145 */ - /* Update all account permissions before updating the account data updates. - We have inlined the anza function update_caller_account_perms here. - TODO: consider factoring this out */ - if( vm->direct_mapping ) { - for( ulong i=0UL; iinstr_ctx->instr, acc_instr_idx ) ) { - - /* Borrow the callee account - https://github.com/anza-xyz/agave/blob/v2.1.14/programs/bpf_loader/src/syscalls/cpi.rs#L1154-L1155 */ - fd_guarded_borrowed_account_t callee_acc = {0}; - FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( vm->instr_ctx, acc_instr_idx, &callee_acc ); - - /* https://github.com/anza-xyz/agave/blob/v2.1.14/programs/bpf_loader/src/syscalls/cpi.rs#L1298 */ - uchar is_writable = !!fd_borrowed_account_can_data_be_changed( &callee_acc, &err ); - /* Lookup memory regions for the account data and the realloc region. */ - ulong data_region_idx = vm->acc_region_metas[ acc_instr_idx ].has_data_region ? vm->acc_region_metas[ acc_instr_idx ].region_idx : 0; - ulong realloc_region_idx = vm->acc_region_metas[ acc_instr_idx ].has_resizing_region ? vm->acc_region_metas[ acc_instr_idx ].region_idx : 0; - if( data_region_idx && realloc_region_idx ) { - realloc_region_idx++; - } - - if( data_region_idx ) { - vm->input_mem_regions[ data_region_idx ].is_writable = is_writable; - } - if( FD_LIKELY( realloc_region_idx ) ) { /* Unless is deprecated loader */ - vm->input_mem_regions[ realloc_region_idx ].is_writable = is_writable; - } - } - } - } - /* Update the caller accounts with any changes made by the callee during CPI execution */ for( ulong i=0UL; istricter_abi_and_runtime_constraints ) { + for( ulong i=0UL; iinstr_ctx->instr->accounts[ callee_account_keys[i] ].index_in_transaction; + fd_pubkey_t const * callee = &vm->instr_ctx->txn_ctx->account_keys[ idx_in_txn ]; + err = fd_exec_instr_ctx_try_borrow_instr_account_with_key( vm->instr_ctx, callee, &borrowed_callee_acc ); + if( FD_UNLIKELY( err && ( err != FD_ACC_MGR_ERR_UNKNOWN_ACCOUNT ) ) ) { + return 1; + } + + /* https://github.com/anza-xyz/agave/blob/v3.0.4/syscalls/src/cpi.rs#L1052-L1058 */ + err = fd_vm_cpi_update_caller_account_region( vm, (ulong)callee_account_keys[i], caller_accounts + i, &borrowed_callee_acc ); + if( FD_UNLIKELY( err ) ) { + return err; + } + } + } + return FD_VM_SUCCESS; } diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_macros.h b/src/flamenco/vm/syscall/fd_vm_syscall_macros.h index 8e4a66d35ce..58df54dc05a 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_macros.h +++ b/src/flamenco/vm/syscall/fd_vm_syscall_macros.h @@ -87,33 +87,34 @@ typedef struct fd_vm_haddr_query fd_vm_haddr_query_t; except for a check on the validity of the size of a load. It only checks that the specific vaddr that is being translated is valid. */ -#define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \ - fd_vm_t const * _vm = (vm); \ - uchar _is_multi = 0; \ - ulong _vaddr = (vaddr); \ - ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \ - int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \ - if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \ - FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { \ - FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - if ( FD_UNLIKELY( _sigbus ) ) { \ - FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - (void const *)_haddr; \ +#define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \ + fd_vm_t * _vm = (vm); \ + ulong _vaddr = (vaddr); \ + ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \ + int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \ + if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \ + FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + if( FD_UNLIKELY( (!_haddr) ) ) { \ + _vm->segv_vaddr = _vaddr; \ + _vm->segv_access_len = (sz); \ + _vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; \ + FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + if ( FD_UNLIKELY( _sigbus ) ) { \ + FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + (void const *)_haddr; \ })) -#define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ - fd_vm_t const * _vm = (vm); \ - uchar _is_multi = 0; \ - ulong _vaddr = (vaddr); \ - ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL, &_is_multi ); \ - (void const *)_haddr; \ +#define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ + fd_vm_t const * _vm = (vm); \ + ulong _vaddr = (vaddr); \ + ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \ + (void const *)_haddr; \ })) @@ -122,18 +123,20 @@ typedef struct fd_vm_haddr_query fd_vm_haddr_query_t; })) static inline void * -FD_VM_MEM_HADDR_ST_( fd_vm_t const *vm, ulong vaddr, ulong align, ulong sz, int *err ) { - fd_vm_t const * _vm = (vm); - uchar _is_multi = 0; - ulong _vaddr = (vaddr); - ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); - int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); +FD_VM_MEM_HADDR_ST_( fd_vm_t *vm, ulong vaddr, ulong align, ulong sz, int *err ) { + fd_vm_t * _vm = (vm); + ulong _vaddr = (vaddr); + ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL ); + int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); if ( FD_UNLIKELY( sz > LONG_MAX ) ) { FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); *err = FD_VM_SYSCALL_ERR_SEGFAULT; return 0; } - if( FD_UNLIKELY( (!_haddr) | _is_multi) ) { + if( FD_UNLIKELY( (!_haddr) ) ) { + vm->segv_vaddr = vaddr; + vm->segv_access_len = (sz); + vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); *err = FD_VM_SYSCALL_ERR_SEGFAULT; return 0; @@ -154,33 +157,34 @@ FD_VM_MEM_HADDR_ST_( fd_vm_t const *vm, ulong vaddr, ulong align, ulong sz, int ret; \ })) -#define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ - fd_vm_t const * _vm = (vm); \ - uchar _is_multi = 0; \ - ulong _vaddr = (vaddr); \ - ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL, &_is_multi ); \ - (void const *)_haddr; \ +#define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ + fd_vm_t * _vm = (vm); \ + ulong _vaddr = (vaddr); \ + ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL ); \ + (void const *)_haddr; \ })) -#define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ - fd_vm_t const * _vm = (vm); \ - uchar _is_multi = 0; \ - ulong _vaddr = (vaddr); \ - ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL, &_is_multi ); \ - int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \ - if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \ - FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - if( FD_UNLIKELY( (!_haddr) | _is_multi ) ) { \ - FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - if ( FD_UNLIKELY( _sigbus ) ) { \ - FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \ - return FD_VM_SYSCALL_ERR_SEGFAULT; \ - } \ - (void *)_haddr; \ +#define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \ + fd_vm_t * _vm = (vm); \ + ulong _vaddr = (vaddr); \ + ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL ); \ + int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \ + if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \ + FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + if( FD_UNLIKELY( !_haddr ) ) { \ + _vm->segv_vaddr = _vaddr; \ + _vm->segv_access_len = (sz); \ + _vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; \ + FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + if ( FD_UNLIKELY( _sigbus ) ) { \ + FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \ + return FD_VM_SYSCALL_ERR_SEGFAULT; \ + } \ + (void *)_haddr; \ })) diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_runtime.c b/src/flamenco/vm/syscall/fd_vm_syscall_runtime.c index b39596ac256..72ca0755fb8 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_runtime.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_runtime.c @@ -26,6 +26,11 @@ fd_vm_syscall_sol_get_clock_sysvar( /**/ void * _vm, FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, sizeof(fd_sol_sysvar_clock_t) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + fd_vm_haddr_query_t var_query = { .vaddr = out_vaddr, .align = FD_VM_ALIGN_RUST_SYSVAR_CLOCK, @@ -57,6 +62,11 @@ fd_vm_syscall_sol_get_epoch_schedule_sysvar( /**/ void * _vm, FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, sizeof(fd_epoch_schedule_t) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + fd_vm_haddr_query_t var_query = { .vaddr = out_vaddr, .align = FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE, @@ -95,6 +105,11 @@ fd_vm_syscall_sol_get_rent_sysvar( /**/ void * _vm, FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, sizeof(fd_rent_t) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + fd_vm_haddr_query_t var_query = { .vaddr = out_vaddr, .align = FD_VM_ALIGN_RUST_SYSVAR_RENT, @@ -127,6 +142,11 @@ fd_vm_syscall_sol_get_last_restart_slot_sysvar( /**/ void * _vm, FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, sizeof(fd_sol_sysvar_last_restart_slot_t) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + fd_vm_haddr_query_t var_query = { .vaddr = out_vaddr, .align = FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT, @@ -167,6 +187,11 @@ fd_vm_syscall_sol_get_sysvar( /**/ void * _vm, ulong sysvar_buf_cost = sz / FD_VM_CPI_BYTES_PER_UNIT; FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, fd_ulong_max( sysvar_buf_cost, FD_VM_MEM_OP_BASE_COST ) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/sysvar.rs#L207-L211 */ fd_vm_haddr_query_t var_query = { .vaddr = out_vaddr, @@ -569,6 +594,11 @@ fd_vm_syscall_sol_get_epoch_rewards_sysvar( /**/ void * _vm, FD_VM_CU_UPDATE( vm, fd_ulong_sat_add( FD_VM_SYSVAR_BASE_COST, sizeof(fd_sysvar_epoch_rewards_t) ) ); + if( FD_UNLIKELY( vm->stricter_abi_and_runtime_constraints && out_vaddr>=FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_POINTER ); + return FD_VM_ERR_INVAL; + } + uchar * out = FD_VM_MEM_HADDR_ST( vm, out_vaddr, FD_VM_ALIGN_RUST_SYSVAR_EPOCH_REWARDS, sizeof(fd_sysvar_epoch_rewards_t) ); fd_sysvar_epoch_rewards_t epoch_rewards; diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_util.c b/src/flamenco/vm/syscall/fd_vm_syscall_util.c index a30e9d72c75..7207c46bc47 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_util.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_util.c @@ -2,11 +2,6 @@ #include "../../../ballet/base64/fd_base64.h" #include "../../../ballet/utf8/fd_utf8.h" -#include "../../runtime/sysvar/fd_sysvar.h" -#include "../../runtime/sysvar/fd_sysvar_clock.h" -#include "../../runtime/sysvar/fd_sysvar_epoch_schedule.h" -#include "../../runtime/context/fd_exec_txn_ctx.h" -#include "../../runtime/context/fd_exec_instr_ctx.h" int fd_vm_syscall_abort( /**/ void * _vm, @@ -349,196 +344,19 @@ fd_vm_memmove( fd_vm_t * vm, return FD_VM_SUCCESS; } - if( !vm->direct_mapping ) { - /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L188-L192 */ - fd_vm_haddr_query_t dst_ref_mut_query = { - .vaddr = dst_vaddr, - .align = FD_VM_ALIGN_RUST_U8, - .sz = sz, - .is_slice = 1, - }; - - fd_vm_haddr_query_t * queries[] = { &dst_ref_mut_query }; - FD_VM_TRANSLATE_MUT( vm, queries ); - - void const * src = FD_VM_MEM_HADDR_LD( vm, src_vaddr, FD_VM_ALIGN_RUST_U8, sz ); - memmove( dst_ref_mut_query.haddr, src, sz ); - } else { - /* If the src and dst vaddrs overlap and src_vaddr < dst_vaddr, Agave iterates through input regions backwards - to maintain correct memmove behavior for overlapping cases. Although this logic should only apply to the src and dst - vaddrs being in the input data region (since that is the only possible case you could have overlapping, chunked-up memmoves), - Agave will iterate backwards in ANY region. If it eventually reaches the end of a region after iterating backwards and - hits an access violation, the bytes from [region_begin, start_vaddr] will still be written to, causing fuzzing mismatches. - In this case, if we didn't have the reverse flag, we would have thrown an access violation before any bytes were copied. - The same logic applies to memmoves that go past the high end of a region - reverse iteration logic would throw an access - violation before any bytes were copied, while the current logic would copy the bytes until the end of the region. - https://github.com/anza-xyz/agave/blob/v2.1.0/programs/bpf_loader/src/syscalls/mem_ops.rs#L184 */ - uchar reverse = !!( dst_vaddr >= src_vaddr && dst_vaddr - src_vaddr < sz ); - - /* In reverse calculations, start from the rightmost vaddr that will be accessed (note the - 1). */ - ulong dst_vaddr_begin = reverse ? fd_ulong_sat_add( dst_vaddr, sz - 1UL ) : dst_vaddr; - ulong src_vaddr_begin = reverse ? fd_ulong_sat_add( src_vaddr, sz - 1UL ) : src_vaddr; - - /* Find the correct src and dst haddrs to start operating from. If the src or dst vaddrs - belong to the input data region (4), keep track of region statistics to memmove in chunks. */ - ulong dst_region = FD_VADDR_TO_REGION( dst_vaddr_begin ); - uchar dst_is_input_mem_region = ( dst_region==FD_VM_INPUT_REGION ); - ulong dst_offset = dst_vaddr_begin & FD_VM_OFFSET_MASK; - ulong dst_region_idx = 0UL; - ulong dst_bytes_rem_in_cur_region; - uchar * dst_haddr; - if( dst_is_input_mem_region ) { - FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED( vm, dst_offset, dst_region_idx, dst_haddr ); - if( FD_UNLIKELY( !vm->input_mem_regions[ dst_region_idx ].is_writable ) ) { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - if( FD_UNLIKELY( reverse ) ) { - /* Bytes remaining between region begin and current position (+ 1 for inclusive region beginning). */ - dst_bytes_rem_in_cur_region = fd_ulong_sat_sub( dst_offset + 1UL, vm->input_mem_regions[ dst_region_idx ].vaddr_offset ); - } else { - /* Bytes remaining between current position and region end. */ - dst_bytes_rem_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ dst_region_idx ].region_sz, ( dst_offset - vm->input_mem_regions[ dst_region_idx ].vaddr_offset ) ); - } - } else { - dst_haddr = (uchar*)FD_VM_MEM_HADDR_ST_NO_SZ_CHECK( vm, dst_vaddr_begin, FD_VM_ALIGN_RUST_U8 ); - - if( FD_UNLIKELY( reverse ) ) { - /* Bytes remaining is minimum of the offset from the beginning of the current - region (+1 for inclusive region beginning) and the number of storable bytes in the region. */ - dst_bytes_rem_in_cur_region = fd_ulong_min( vm->region_st_sz[ dst_region ], dst_offset + 1UL ); - - } else { - /* Bytes remaining is the number of writable bytes left in the region */ - dst_bytes_rem_in_cur_region = fd_ulong_sat_sub( vm->region_st_sz[ dst_region ], dst_offset ); - } - } - - /* Logic for src vaddr translation is similar to above excluding any writable checks. */ - ulong src_region = FD_VADDR_TO_REGION( src_vaddr_begin ); - uchar src_is_input_mem_region = ( src_region==FD_VM_INPUT_REGION ); - ulong src_offset = src_vaddr_begin & FD_VM_OFFSET_MASK; - ulong src_region_idx = 0UL; - ulong src_bytes_rem_in_cur_region; - uchar * src_haddr; - if( src_is_input_mem_region ) { - FD_VM_MEM_HADDR_AND_REGION_IDX_FROM_INPUT_REGION_CHECKED( vm, src_offset, src_region_idx, src_haddr ); - if( FD_UNLIKELY( reverse ) ) { - src_bytes_rem_in_cur_region = fd_ulong_sat_sub( src_offset + 1UL, vm->input_mem_regions[ src_region_idx ].vaddr_offset ); - } else { - src_bytes_rem_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ src_region_idx ].region_sz, ( src_offset - vm->input_mem_regions[ src_region_idx ].vaddr_offset ) ); - } - } else { - src_haddr = (uchar*)FD_VM_MEM_HADDR_LD_NO_SZ_CHECK( vm, src_vaddr_begin, FD_VM_ALIGN_RUST_U8 ); - - if( FD_UNLIKELY( reverse ) ) { - src_bytes_rem_in_cur_region = fd_ulong_min( vm->region_ld_sz[ src_region ], src_offset + 1UL ); - - } else { - src_bytes_rem_in_cur_region = fd_ulong_sat_sub( vm->region_ld_sz[ src_region ], src_offset ); - } - } + /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L188-L192 */ + fd_vm_haddr_query_t dst_ref_mut_query = { + .vaddr = dst_vaddr, + .align = FD_VM_ALIGN_RUST_U8, + .sz = sz, + .is_slice = 1, + }; - /* Short circuit: if the number of copyable bytes stays within all memory regions, - just memmove and return. This is a majority case in mainnet, devnet, and testnet. - Someone would have to be very crafty and clever to construct a transaction that - deploys and invokes a custom program that does not fall into this branch. */ - if( FD_LIKELY( sz<=dst_bytes_rem_in_cur_region && sz<=src_bytes_rem_in_cur_region ) ) { - if( FD_UNLIKELY( reverse ) ) { - /* In the reverse iteration case, the haddrs point to the end of the region here. Since the - above checks guarantee that there are enough bytes left in the src and dst regions to do - a direct memmove, we can just subtract (sz-1) from the haddrs, memmove, and return. */ - memmove( dst_haddr - sz + 1UL, src_haddr - sz + 1UL, sz ); - } else { - /* In normal iteration, the haddrs correspond to the correct starting point for the memcpy, - so no further translation has to be done. */ - memmove( dst_haddr, src_haddr, sz ); - } - return FD_VM_SUCCESS; - } + fd_vm_haddr_query_t * queries[] = { &dst_ref_mut_query }; + FD_VM_TRANSLATE_MUT( vm, queries ); - /* Copy over the bytes from each region in chunks. */ - while( sz>0UL ) { - /* End of region case */ - if( FD_UNLIKELY( src_bytes_rem_in_cur_region==0UL ) ) { - /* Same as above, except no writable checks. */ - if( FD_LIKELY( !reverse && - src_is_input_mem_region && - src_region_idx+1ULinput_mem_regions_cnt ) ) { - if( FD_UNLIKELY( vm->input_mem_regions[ src_region_idx+1UL ].is_acct_data != vm->input_mem_regions[ src_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - src_region_idx++; - src_haddr = (uchar*)vm->input_mem_regions[ src_region_idx ].haddr; - } else if( FD_LIKELY( reverse && src_region_idx>0UL ) ) { - if( FD_UNLIKELY( vm->input_mem_regions[ src_region_idx-1UL ].is_acct_data != vm->input_mem_regions[ src_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - src_region_idx--; - src_haddr = (uchar*)vm->input_mem_regions[ src_region_idx ].haddr + vm->input_mem_regions[ src_region_idx ].region_sz - 1UL; - } else { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - src_bytes_rem_in_cur_region = vm->input_mem_regions[ src_region_idx ].region_sz; - } - if( FD_UNLIKELY( dst_bytes_rem_in_cur_region==0UL ) ) { - /* Only proceed if: - - We are in the input memory region - - There are remaining input memory regions to copy from (for both regular and reverse iteration orders) - - The next input memory region is writable - Fail otherwise. */ - if( FD_LIKELY( !reverse && - dst_is_input_mem_region && - dst_region_idx+1ULinput_mem_regions_cnt && - vm->input_mem_regions[ dst_region_idx+1UL ].is_writable ) ) { - if( FD_UNLIKELY( vm->input_mem_regions[ dst_region_idx+1UL ].is_acct_data != vm->input_mem_regions[ dst_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - /* In normal iteration, we move the haddr to the beginning of the next region. */ - dst_region_idx++; - dst_haddr = (uchar*)vm->input_mem_regions[ dst_region_idx ].haddr; - } else if( FD_LIKELY( reverse && - dst_region_idx>0UL && - vm->input_mem_regions[ dst_region_idx-1UL ].is_writable ) ) { - if( FD_UNLIKELY( vm->input_mem_regions[ dst_region_idx-1UL ].is_acct_data != vm->input_mem_regions[ dst_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - /* Note that when reverse iterating, we set the haddr to the END of the PREVIOUS region. */ - dst_region_idx--; - dst_haddr = (uchar*)vm->input_mem_regions[ dst_region_idx ].haddr + vm->input_mem_regions[ dst_region_idx ].region_sz - 1UL; - } else { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - dst_bytes_rem_in_cur_region = vm->input_mem_regions[ dst_region_idx ].region_sz; - } - - /* Number of bytes to operate on in this iteration is the min of: - - number of bytes left to copy - - bytes left in the current src region - - bytes left in the current dst region */ - ulong num_bytes_to_copy = fd_ulong_min( sz, fd_ulong_min( src_bytes_rem_in_cur_region, dst_bytes_rem_in_cur_region ) ); - if( FD_UNLIKELY( reverse ) ) { - memmove( dst_haddr - num_bytes_to_copy + 1UL, src_haddr - num_bytes_to_copy + 1UL, num_bytes_to_copy ); - dst_haddr -= num_bytes_to_copy; - src_haddr -= num_bytes_to_copy; - } else { - memmove( dst_haddr, src_haddr, num_bytes_to_copy ); - dst_haddr += num_bytes_to_copy; - src_haddr += num_bytes_to_copy; - } - - /* Update size trackers */ - sz -= num_bytes_to_copy; - src_bytes_rem_in_cur_region -= num_bytes_to_copy; - dst_bytes_rem_in_cur_region -= num_bytes_to_copy; - } - } + void const * src = FD_VM_MEM_HADDR_LD( vm, src_vaddr, FD_VM_ALIGN_RUST_U8, sz ); + memmove( dst_ref_mut_query.haddr, src, sz ); return FD_VM_SUCCESS; } @@ -604,173 +422,37 @@ fd_vm_syscall_sol_memcmp( /**/ void * _vm, doesn't provide strong enough guarantees about the return value (it only promises the sign). */ - if( !vm->direct_mapping ) { - uchar const * m0 = (uchar const *)FD_VM_MEM_SLICE_HADDR_LD( vm, m0_vaddr, FD_VM_ALIGN_RUST_U8, sz ); - uchar const * m1 = (uchar const *)FD_VM_MEM_SLICE_HADDR_LD( vm, m1_vaddr, FD_VM_ALIGN_RUST_U8, sz ); - - /* Silly that this doesn't use r0 to return ... slower, more edge - case, different from libc style memcmp, harder to callers to use, - etc ... probably too late to do anything about it now ... sigh */ - - /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L121-L125 */ - fd_vm_haddr_query_t cmp_result_ref_mut_query = { - .vaddr = out_vaddr, - .align = FD_VM_ALIGN_RUST_I32, - .sz = 4UL, - .is_slice = 0, - }; - - fd_vm_haddr_query_t * queries[] = { &cmp_result_ref_mut_query }; - FD_VM_TRANSLATE_MUT( vm, queries ); - - int out = 0; - for( ulong i=0UL; i 0x4000 + 0x50 (region sz 0x50) - m1's region: m1_addr 0x2000 -> 0x2000 + 0x100 (region sz 0x100) - sz: 0x200 - - Case 1: 0x4000 -> 0x4050 does have the same bytes as 0x2000 -> 0x2050 - Case 2: 0x4000 -> 0x4050 does NOT have the same bytes as 0x2000 -> 0x2050 - - Pre-DM: - This will fail out before any bytes are compared because the memory - translation is done first. - - Post-DM: - For case 1, the memcmp will return an error and the VM will exit because - the memcmp will eventually try to access 0x4051 which is invalid. First - 0x50 bytes are compared, but the next chunk will lead to an invalid - access. - - For case 2, the memcmp will first translate the first 0x50 bytes and will - see that the bytes are not the same. This will lead to the syscall - exiting out successfully without detecting the access violation. - - https://github.com/anza-xyz/agave/blob/v2.0.10/programs/bpf_loader/src/syscalls/mem_ops.rs#L213 - */ - - /* TODO: Refactor to use `FD_VM_TRANSLATE_MUT` macro when direct mapping is rewritten */ - void * _out = FD_VM_MEM_HADDR_ST( vm, out_vaddr, FD_VM_ALIGN_RUST_I32, 4UL ); - int out = 0; - - /* Lookup host address chunks. Try to do a standard memcpy if the regions - do not cross memory regions. The translation logic is different if the - the virtual address region is the input region vs. not. See the comment - in fd_bpf_loader_serialization for more details on how the input - region is different from other regions. The input data region will try - to lookup the number of remaining bytes in the specific data region. If - the memory access is not in the input data region, assume the bytes in - the current region are bound by the size of the remaining bytes in the - region. */ - - ulong m0_region = FD_VADDR_TO_REGION( m0_vaddr ); - ulong m0_offset = m0_vaddr & FD_VM_OFFSET_MASK; - ulong m0_region_idx = 0UL; - ulong m0_bytes_in_cur_region = sz; - uchar * m0_haddr = NULL; - if( m0_region==FD_VM_INPUT_REGION ) { - m0_region_idx = fd_vm_get_input_mem_region_idx( vm, m0_offset ); - m0_haddr = (uchar*)(vm->input_mem_regions[ m0_region_idx ].haddr + m0_offset - vm->input_mem_regions[ m0_region_idx ].vaddr_offset); - m0_bytes_in_cur_region = fd_ulong_min( sz, fd_ulong_sat_sub( vm->input_mem_regions[ m0_region_idx ].region_sz, - ((ulong)m0_haddr - vm->input_mem_regions[ m0_region_idx ].haddr) ) ); - } else { - /* We can safely load a slice of 1 byte here because we know that we will - not ever read more than the number of bytes that are left in the - region. */ - m0_bytes_in_cur_region = fd_ulong_min( sz, vm->region_ld_sz[ m0_region ] - m0_offset ); - m0_haddr = (uchar *)FD_VM_MEM_SLICE_HADDR_LD_SZ_UNCHECKED( vm, m0_vaddr, FD_VM_ALIGN_RUST_U8 ); + uchar const * m0 = (uchar const *)FD_VM_MEM_SLICE_HADDR_LD( vm, m0_vaddr, FD_VM_ALIGN_RUST_U8, sz ); + uchar const * m1 = (uchar const *)FD_VM_MEM_SLICE_HADDR_LD( vm, m1_vaddr, FD_VM_ALIGN_RUST_U8, sz ); + + /* Silly that this doesn't use r0 to return ... slower, more edge + case, different from libc style memcmp, harder to callers to use, + etc ... probably too late to do anything about it now ... sigh */ + + /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L121-L125 */ + fd_vm_haddr_query_t cmp_result_ref_mut_query = { + .vaddr = out_vaddr, + .align = FD_VM_ALIGN_RUST_I32, + .sz = 4UL, + .is_slice = 0, + }; + + fd_vm_haddr_query_t * queries[] = { &cmp_result_ref_mut_query }; + FD_VM_TRANSLATE_MUT( vm, queries ); + + int out = 0; + for( ulong i=0UL; iinput_mem_regions[ m1_region_idx ].haddr + m1_offset - vm->input_mem_regions[ m1_region_idx ].vaddr_offset); - m1_bytes_in_cur_region = fd_ulong_min( sz, fd_ulong_sat_sub( vm->input_mem_regions[ m1_region_idx ].region_sz, - ((ulong)m1_haddr - vm->input_mem_regions[ m1_region_idx ].haddr) ) ); - } else { - m1_bytes_in_cur_region = fd_ulong_min( sz, vm->region_ld_sz[ m1_region ] - m1_offset ); - m1_haddr = (uchar *)FD_VM_MEM_SLICE_HADDR_LD_SZ_UNCHECKED( vm, m1_vaddr, FD_VM_ALIGN_RUST_U8 ); - } + fd_memcpy( cmp_result_ref_mut_query.haddr, &out, 4UL ); /* Sigh ... see note above (and might be unaligned ... double sigh) */ - /* Case where the operation spans multiple regions. Copy over the bytes - from each region while iterating to the next one. */ - /* TODO: An optimization would be to memcmp chunks at once */ - ulong m0_idx = 0UL; - ulong m1_idx = 0UL; - for( ulong i=0UL; i=vm->input_mem_regions_cnt ) ) { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - if( FD_UNLIKELY( vm->input_mem_regions[ m0_region_idx-1UL ].is_acct_data != vm->input_mem_regions[ m0_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - /* Otherwise, query the next input region. */ - m0_haddr = (uchar*)vm->input_mem_regions[ m0_region_idx ].haddr; - m0_idx = 0UL; - m0_bytes_in_cur_region = vm->input_mem_regions[ m0_region_idx ].region_sz; - } - if( FD_UNLIKELY( !m1_bytes_in_cur_region ) ) { - if( FD_UNLIKELY( m1_region!=FD_VM_INPUT_REGION || ++m1_region_idx>=vm->input_mem_regions_cnt ) ) { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - if( FD_UNLIKELY( vm->input_mem_regions[ m1_region_idx-1UL ].is_acct_data != vm->input_mem_regions[ m1_region_idx ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - m1_haddr = (uchar*)vm->input_mem_regions[ m1_region_idx ].haddr; - m1_idx = 0UL; - m1_bytes_in_cur_region = vm->input_mem_regions[ m1_region_idx ].region_sz; - } - - int i0 = (int)m0_haddr[ m0_idx ]; - int i1 = (int)m1_haddr[ m1_idx ]; - if( i0!=i1 ) { - out = i0 - i1; - break; - } - - m0_bytes_in_cur_region--; - m1_bytes_in_cur_region--; - m0_idx++; - m1_idx++; - } - fd_memcpy( _out, &out, 4UL ); /* Sigh ... see note above (and might be unaligned ... double sigh) */ - return FD_VM_SUCCESS; - } + return FD_VM_SUCCESS; } int @@ -792,92 +474,19 @@ fd_vm_syscall_sol_memset( /**/ void * _vm, return FD_VM_SUCCESS; } - ulong region = FD_VADDR_TO_REGION( dst_vaddr ); - ulong offset = dst_vaddr & FD_VM_OFFSET_MASK; - uchar * haddr; - int b = (int)(c & 255UL); - if( !vm->direct_mapping ) { - /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L155-L159 */ - fd_vm_haddr_query_t haddr_query = { - .vaddr = dst_vaddr, - .align = FD_VM_ALIGN_RUST_U8, - .sz = sz, - .is_slice = 1, - }; - - fd_vm_haddr_query_t * queries[] = { &haddr_query }; - FD_VM_TRANSLATE_MUT( vm, queries ); - fd_memset( haddr_query.haddr, b, sz ); - } else if( region!=FD_VM_INPUT_REGION ) { - /* Here we special case non-input region memsets: we try to memset - as many bytes as possible until it reaches an unwritable section. - This is done in order to ensure error-code conformance with - Agave. */ - haddr = (uchar*)FD_VM_MEM_HADDR_ST_FAST( vm, dst_vaddr ); - ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->region_st_sz[ region ], offset ); - ulong bytes_to_set = fd_ulong_min( sz, bytes_in_cur_region ); - fd_memset( haddr, b, bytes_to_set ); - if( FD_UNLIKELY( bytes_to_setinput_mem_regions[ region_idx ].vaddr_offset; - ulong bytes_in_cur_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz, offset_in_cur_region ); - - /* Check that current region is writable */ - if( FD_UNLIKELY( !vm->input_mem_regions[ region_idx ].is_writable ) ) { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } + /* https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mem_ops.rs#L155-L159 */ + fd_vm_haddr_query_t haddr_query = { + .vaddr = dst_vaddr, + .align = FD_VM_ALIGN_RUST_U8, + .sz = sz, + .is_slice = 1, + }; - /* Memset goes into multiple regions. */ - while( sz>0UL ) { - - /* Memset bytes */ - ulong num_bytes_to_set = fd_ulong_min( sz, bytes_in_cur_region ); - fd_memset( haddr, b, num_bytes_to_set ); - sz -= num_bytes_to_set; - - if( !sz ) { - break; - } - - /* If no more regions left, break. */ - if( ++region_idx==vm->input_mem_regions_cnt ) { - break; - } - - /* Check that new region is writable. */ - if( FD_UNLIKELY( !vm->input_mem_regions[ region_idx ].is_writable ) ) { - break; - } - - /* If new region crosses into/out of account region, error out. */ - if( FD_UNLIKELY( vm->input_mem_regions[ region_idx ].is_acct_data != - vm->input_mem_regions[ region_idx-1UL ].is_acct_data ) ) { - FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - - /* Move haddr to next region. */ - haddr = (uchar*)vm->input_mem_regions[ region_idx ].haddr; - bytes_in_cur_region = vm->input_mem_regions[ region_idx ].region_sz; - } + fd_vm_haddr_query_t * queries[] = { &haddr_query }; + FD_VM_TRANSLATE_MUT( vm, queries ); + fd_memset( haddr_query.haddr, b, sz ); - /* If we were not able to successfully set all the bytes, throw an error. */ - if( FD_UNLIKELY( sz>0 ) ) { - FD_VM_ERR_FOR_LOG_EBPF( vm, FD_VM_ERR_EBPF_ACCESS_VIOLATION ); - return FD_VM_SYSCALL_ERR_SEGFAULT; - } - } return FD_VM_SUCCESS; } diff --git a/src/flamenco/vm/syscall/test_vm_syscall_curve.c b/src/flamenco/vm/syscall/test_vm_syscall_curve.c index a57ed10809c..a0ee814c0d6 100644 --- a/src/flamenco/vm/syscall/test_vm_syscall_curve.c +++ b/src/flamenco/vm/syscall/test_vm_syscall_curve.c @@ -83,27 +83,28 @@ main( int argc, fd_features_enable_all( &((fd_exec_txn_ctx_t *)instr_ctx->txn_ctx)->features ); int vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, /* required for FD_FEATURE_ACTIVE */ - /* heap_max */ FD_VM_HEAP_DEFAULT, - /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, - /* rodata */ rodata, - /* rodata_sz */ rodata_sz, - /* text */ NULL, - /* text_cnt */ 0UL, - /* text_off */ 0UL, - /* text_sz */ 0UL, - /* entry_pc */ 0UL, - /* calldests */ NULL, - /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, - /* syscalls */ NULL, - /* trace */ NULL, - /* sha */ sha, - /* mem_regions */ NULL, - /* mem_regions_cnt */ 0UL, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, /* required for FD_FEATURE_ACTIVE */ + /* heap_max */ FD_VM_HEAP_DEFAULT, + /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, + /* rodata */ rodata, + /* rodata_sz */ rodata_sz, + /* text */ NULL, + /* text_cnt */ 0UL, + /* text_off */ 0UL, + /* text_sz */ 0UL, + /* entry_pc */ 0UL, + /* calldests */ NULL, + /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, + /* syscalls */ NULL, + /* trace */ NULL, + /* sha */ sha, + /* mem_regions */ NULL, + /* mem_regions_cnt */ 0UL, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); FD_TEST( vm_ok ); diff --git a/src/flamenco/vm/syscall/test_vm_syscalls.c b/src/flamenco/vm/syscall/test_vm_syscalls.c index 890285057a2..0d5005e554c 100644 --- a/src/flamenco/vm/syscall/test_vm_syscalls.c +++ b/src/flamenco/vm/syscall/test_vm_syscalls.c @@ -6,9 +6,10 @@ static inline void set_memory_region( uchar * mem, ulong sz ) { for( ulong i=0UL static void test_vm_syscall_toggle_direct_mapping( fd_vm_t * vm_ctx, int enable ) { ulong slot = enable ? 0UL : FD_FEATURE_DISABLED; - char const * one_offs[] = { "GJVDwRkUPNdk9QaK4VsU4g1N41QNxhy1hevjf8kz45Mq" }; + char const * one_offs[] = { "9s3RKimHWS44rJcJ9P1rwCmn2TvMqtZQBmz815ZUUHqJ", "CxeBn9PVeeXbmjbNwLv6U4C6svNxnC4JX6mfkvgeMocM" }; fd_features_enable_one_offs( (fd_features_t*)&vm_ctx->instr_ctx->txn_ctx->features, one_offs, 1U, slot ); vm_ctx->direct_mapping = enable; + vm_ctx->stricter_abi_and_runtime_constraints = enable; } static void @@ -216,37 +217,38 @@ main( int argc, uchar input[ input_sz ]; ulong const mem_regions_cnt = 4UL; fd_vm_input_region_t input_mem_regions[ mem_regions_cnt ]; - input_mem_regions[0] = (fd_vm_input_region_t){ .haddr = (ulong)input, .region_sz = 100UL, .is_writable = 1, .vaddr_offset = 0UL }; - input_mem_regions[1] = (fd_vm_input_region_t){ .haddr = (ulong)input + 100UL, .region_sz = 1UL, .is_writable = 1, .vaddr_offset = 100UL }; - input_mem_regions[2] = (fd_vm_input_region_t){ .haddr = (ulong)input + 101UL, .region_sz = 400UL, .is_writable = 1, .vaddr_offset = 101UL }; - input_mem_regions[3] = (fd_vm_input_region_t){ .haddr = (ulong)input + 501UL, .region_sz = 499UL, .is_writable = 1, .vaddr_offset = 501UL }; + input_mem_regions[0] = (fd_vm_input_region_t){ .haddr = (ulong)input, .region_sz = 100UL, .address_space_reserved = 100UL, .is_writable = 1, .vaddr_offset = 0UL }; + input_mem_regions[1] = (fd_vm_input_region_t){ .haddr = (ulong)input + 100UL, .region_sz = 1UL, .address_space_reserved = 1UL, .is_writable = 1, .vaddr_offset = 100UL }; + input_mem_regions[2] = (fd_vm_input_region_t){ .haddr = (ulong)input + 101UL, .region_sz = 400UL, .address_space_reserved = 400UL, .is_writable = 1, .vaddr_offset = 101UL }; + input_mem_regions[3] = (fd_vm_input_region_t){ .haddr = (ulong)input + 501UL, .region_sz = 499UL, .address_space_reserved = 499UL, .is_writable = 1, .vaddr_offset = 501UL }; fd_exec_instr_ctx_t instr_ctx[1]; fd_exec_txn_ctx_t txn_ctx[1]; test_vm_minimal_exec_instr_ctx( instr_ctx, txn_ctx ); int vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ FD_VM_HEAP_DEFAULT, - /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, - /* rodata */ rodata, - /* rodata_sz */ rodata_sz, - /* text */ NULL, - /* text_cnt */ 0UL, - /* text_off */ 0UL, - /* text_sz */ 0UL, - /* entry_pc */ 0UL, - /* calldests */ NULL, - /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, - /* syscalls */ NULL, - /* trace */ NULL, - /* sha */ sha, - /* mem_regions */ input_mem_regions, - /* mem_regions_cnt */ (uint)mem_regions_cnt, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ FD_VM_HEAP_DEFAULT, + /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, + /* rodata */ rodata, + /* rodata_sz */ rodata_sz, + /* text */ NULL, + /* text_cnt */ 0UL, + /* text_off */ 0UL, + /* text_sz */ 0UL, + /* entry_pc */ 0UL, + /* calldests */ NULL, + /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, + /* syscalls */ NULL, + /* trace */ NULL, + /* sha */ sha, + /* mem_regions */ input_mem_regions, + /* mem_regions_cnt */ (uint)mem_regions_cnt, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); FD_TEST( vm_ok ); @@ -312,7 +314,7 @@ main( int argc, (ulong)&input, 1UL, 1000UL, - 0UL, FD_VM_SUCCESS ); + 0UL, FD_VM_SYSCALL_ERR_SEGFAULT ); test_vm_syscall_sol_memset( "test_vm_syscall_sol_memset: memset across multiple input mem regions 2", vm, @@ -320,7 +322,7 @@ main( int argc, (ulong)&input + 50UL, 1UL, 800UL, - 0UL, FD_VM_SUCCESS ); + 0UL, FD_VM_SYSCALL_ERR_SEGFAULT ); input_mem_regions[2].is_writable=0; test_vm_syscall_sol_memset( "test_vm_syscall_sol_memset: memset across multiple input mem regions invalid write", @@ -480,7 +482,7 @@ main( int argc, vm->input_mem_regions[0].haddr + 50UL, vm->input_mem_regions[0].haddr + 450UL, 100UL, - 0UL, FD_VM_SUCCESS ); + 0UL, FD_VM_SYSCALL_ERR_SEGFAULT ); test_vm_syscall_sol_memcpy( "test_vm_syscall_sol_memcpy: memcpy in input overlapping vaddr multiple regions", vm, @@ -588,10 +590,10 @@ main( int argc, test_vm_syscall_sol_memmove( "test_vm_syscall_sol_memmove: memmove in input overlapping vaddr", vm, - FD_VM_MEM_MAP_INPUT_REGION_START + 80UL, - FD_VM_MEM_MAP_INPUT_REGION_START + 120UL, - vm->input_mem_regions[0].haddr + 80UL, - vm->input_mem_regions[0].haddr + 120UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 20UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 30UL, + vm->input_mem_regions[0].haddr + 20UL, + vm->input_mem_regions[0].haddr + 30UL, 50UL, 0UL, FD_VM_SUCCESS ); @@ -602,7 +604,7 @@ main( int argc, vm->input_mem_regions[0].haddr + 50UL, vm->input_mem_regions[0].haddr + 450UL, 100UL, - 0UL, FD_VM_SUCCESS ); + 0UL, FD_VM_SYSCALL_ERR_SEGFAULT ); test_vm_syscall_sol_memmove( "test_vm_syscall_sol_memmove: memmove in input overlapping vaddr multiple regions", vm, @@ -611,7 +613,7 @@ main( int argc, vm->input_mem_regions[0].haddr + 50UL, vm->input_mem_regions[0].haddr + 450UL, 500UL, - 0UL, FD_VM_SUCCESS ); + 0UL, FD_VM_SYSCALL_ERR_SEGFAULT ); // test for memcmp at the heap region test_vm_syscall_sol_memcmp( "test_vm_syscall_sol_memcmp: memcmp at the heap region", @@ -653,24 +655,24 @@ main( int argc, test_vm_syscall_sol_memcmp( "test_vm_syscall_sol_memcmp: memcmp input region equal", vm, FD_VM_MEM_MAP_INPUT_REGION_START, - FD_VM_MEM_MAP_INPUT_REGION_START + 200UL, - FD_VM_MEM_MAP_INPUT_REGION_START + 600UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 48UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 160UL, input_mem_regions[0].haddr, - input_mem_regions[0].haddr + 200UL, - input_mem_regions[0].haddr + 600UL, - 200UL, + input_mem_regions[0].haddr + 48UL, + input_mem_regions[0].haddr + 160UL, + 30UL, 0UL, FD_VM_SUCCESS ); - memset( (void*)input_mem_regions[0].haddr, 0xEE, 200UL ); + memset( (void*)input_mem_regions[0].haddr, 0xEE, 50UL ); test_vm_syscall_sol_memcmp( "test_vm_syscall_sol_memcmp: memcmp input region not equal", vm, FD_VM_MEM_MAP_INPUT_REGION_START, - FD_VM_MEM_MAP_INPUT_REGION_START + 200UL, - FD_VM_MEM_MAP_INPUT_REGION_START + 600UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 48UL, + FD_VM_MEM_MAP_INPUT_REGION_START + 160UL, input_mem_regions[0].haddr, - input_mem_regions[0].haddr + 200UL, - input_mem_regions[0].haddr + 600UL, - 200UL, + input_mem_regions[0].haddr + 48UL, + input_mem_regions[0].haddr + 160UL, + 30UL, 0UL, FD_VM_SUCCESS ); uchar expected_log[ FD_VM_LOG_MAX ]; diff --git a/src/flamenco/vm/test_vm_instr.c b/src/flamenco/vm/test_vm_instr.c index 77853830622..fdb26863ae7 100644 --- a/src/flamenco/vm/test_vm_instr.c +++ b/src/flamenco/vm/test_vm_instr.c @@ -411,10 +411,11 @@ run_input( test_input_t const * input, if( !input->region_boundary_cnt ) { input_region[0] = (fd_vm_input_region_t){ - .vaddr_offset = 0UL, - .haddr = (ulong)input_copy, - .region_sz = (uint)input->input_sz, - .is_writable = 1U, + .vaddr_offset = 0UL, + .haddr = (ulong)input_copy, + .region_sz = (uint)input->input_sz, + .address_space_reserved = input->input_sz, + .is_writable = 1U, }; } else { for( uint i=0; iregion_boundary_cnt; ++i ) { @@ -423,10 +424,11 @@ run_input( test_input_t const * input, ulong cur_offset = prev_offset + prev_sz; input_region[i] = (fd_vm_input_region_t){ - .vaddr_offset = cur_offset, - .haddr = (ulong)input_copy + cur_offset, - .region_sz = input->region_boundary[i] - (uint)cur_offset, - .is_writable = 1U, + .vaddr_offset = cur_offset, + .haddr = (ulong)input_copy + cur_offset, + .region_sz = input->region_boundary[i] - (uint)cur_offset, + .address_space_reserved = input->region_boundary[i] - (uint)cur_offset, + .is_writable = 1U, }; } } @@ -447,27 +449,28 @@ run_input( test_input_t const * input, test_vm_minimal_exec_instr_ctx( instr_ctx, txn_ctx ); int vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ 0UL, - /* entry_cu */ 100UL, - /* rodata */ (uchar const *)text, - /* rodata_sz */ text_cnt * sizeof(ulong), - /* text */ text, - /* text_cnt */ text_cnt, - /* text_off */ 0UL, - /* text_sz */ text_cnt * sizeof(ulong), - /* entry_pc */ 0UL, - /* calldests */ calldests, - /* sbpf_version */ sbpf_version, - /* syscalls */ syscalls, - /* trace */ NULL, - /* sha */ NULL, - /* mem_regions */ input_region, - /* mem_regions_cnt */ input->region_boundary_cnt ? input->region_boundary_cnt : 1, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ 0UL, + /* entry_cu */ 100UL, + /* rodata */ (uchar const *)text, + /* rodata_sz */ text_cnt * sizeof(ulong), + /* text */ text, + /* text_cnt */ text_cnt, + /* text_off */ 0UL, + /* text_sz */ text_cnt * sizeof(ulong), + /* entry_pc */ 0UL, + /* calldests */ calldests, + /* sbpf_version */ sbpf_version, + /* syscalls */ syscalls, + /* trace */ NULL, + /* sha */ NULL, + /* mem_regions */ input_region, + /* mem_regions_cnt */ input->region_boundary_cnt ? input->region_boundary_cnt : 1, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); assert( vm_ok ); diff --git a/src/flamenco/vm/test_vm_interp.c b/src/flamenco/vm/test_vm_interp.c index 19011fccc91..302914d3787 100644 --- a/src/flamenco/vm/test_vm_interp.c +++ b/src/flamenco/vm/test_vm_interp.c @@ -33,27 +33,28 @@ test_program_success( char * test_case_name, FD_TEST( vm ); int vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ FD_VM_HEAP_DEFAULT, - /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, - /* rodata */ (uchar *)text, - /* rodata_sz */ 8UL*text_cnt, - /* text */ text, - /* text_cnt */ text_cnt, - /* text_off */ 0UL, - /* text_sz */ 8UL*text_cnt, - /* entry_pc */ 0UL, - /* calldests */ NULL, - /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, - /* syscalls */ syscalls, - /* trace */ NULL, - /* sha */ sha, - /* mem_regions */ NULL, - /* mem_regions_cnt */ 0UL, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ FD_VM_HEAP_DEFAULT, + /* entry_cu */ FD_VM_COMPUTE_UNIT_LIMIT, + /* rodata */ (uchar *)text, + /* rodata_sz */ 8UL*text_cnt, + /* text */ text, + /* text_cnt */ text_cnt, + /* text_off */ 0UL, + /* text_sz */ 8UL*text_cnt, + /* entry_pc */ 0UL, + /* calldests */ NULL, + /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, + /* syscalls */ syscalls, + /* trace */ NULL, + /* sha */ sha, + /* mem_regions */ NULL, + /* mem_regions_cnt */ 0UL, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); FD_TEST( vm_ok ); @@ -231,27 +232,28 @@ test_0cu_exit( void ) { exit instruction reaches zero. */ int vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ FD_VM_HEAP_DEFAULT, - /* entry_cu */ text_cnt, - /* rodata */ (uchar *)text, - /* rodata_sz */ 8UL*text_cnt, - /* text */ text, - /* text_cnt */ text_cnt, - /* text_off */ 0UL, - /* text_sz */ 8UL*text_cnt, - /* entry_pc */ 0UL, - /* calldests */ NULL, - /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, - /* syscalls */ NULL, - /* trace */ NULL, - /* sha */ sha, - /* mem_regions */ NULL, - /* mem_regions_cnt */ 0UL, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ FD_VM_HEAP_DEFAULT, + /* entry_cu */ text_cnt, + /* rodata */ (uchar *)text, + /* rodata_sz */ 8UL*text_cnt, + /* text */ text, + /* text_cnt */ text_cnt, + /* text_off */ 0UL, + /* text_sz */ 8UL*text_cnt, + /* entry_pc */ 0UL, + /* calldests */ NULL, + /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, + /* syscalls */ NULL, + /* trace */ NULL, + /* sha */ sha, + /* mem_regions */ NULL, + /* mem_regions_cnt */ 0UL, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); FD_TEST( vm_ok ); @@ -263,27 +265,28 @@ test_0cu_exit( void ) { /* Ensure the VM exits with failure if CUs are exhausted. */ vm_ok = !!fd_vm_init( - /* vm */ vm, - /* instr_ctx */ instr_ctx, - /* heap_max */ FD_VM_HEAP_DEFAULT, - /* entry_cu */ text_cnt - 1UL, - /* rodata */ (uchar *)text, - /* rodata_sz */ 8UL*text_cnt, - /* text */ text, - /* text_cnt */ text_cnt, - /* text_off */ 0UL, - /* text_sz */ 8UL*text_cnt, - /* entry_pc */ 0UL, - /* calldests */ NULL, - /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, - /* syscalls */ NULL, - /* trace */ NULL, - /* sha */ sha, - /* mem_regions */ NULL, - /* mem_regions_cnt */ 0UL, - /* mem_regions_accs */ NULL, - /* is_deprecated */ 0, - /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, bpf_account_data_direct_mapping ), + /* vm */ vm, + /* instr_ctx */ instr_ctx, + /* heap_max */ FD_VM_HEAP_DEFAULT, + /* entry_cu */ text_cnt - 1UL, + /* rodata */ (uchar *)text, + /* rodata_sz */ 8UL*text_cnt, + /* text */ text, + /* text_cnt */ text_cnt, + /* text_off */ 0UL, + /* text_sz */ 8UL*text_cnt, + /* entry_pc */ 0UL, + /* calldests */ NULL, + /* sbpf_version */ TEST_VM_DEFAULT_SBPF_VERSION, + /* syscalls */ NULL, + /* trace */ NULL, + /* sha */ sha, + /* mem_regions */ NULL, + /* mem_regions_cnt */ 0UL, + /* mem_regions_accs */ NULL, + /* is_deprecated */ 0, + /* direct mapping */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, account_data_direct_mapping ), + /* stricter_abi_and_runtime_constraints */ FD_FEATURE_ACTIVE( instr_ctx->txn_ctx->slot, &instr_ctx->txn_ctx->features, stricter_abi_and_runtime_constraints ), /* dump_syscall_to_pb */ 0 ); FD_TEST( vm_ok );