diff --git a/.golangci.yml b/.golangci.yml index bccd55d933..08b3c8f45d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ # This file configures github.com/golangci/golangci-lint. run: - timeout: 2m + timeout: 3m tests: true # default is true. Enables skipping of directories: # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 3513084567..c32aa4aa0b 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -122,149 +122,55 @@ func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) return nil } -// unpack sets the unmarshalled value to go format. -// Note the dst here must be settable. -func unpack(t *Type, dst interface{}, src interface{}) error { - var ( - dstVal = reflect.ValueOf(dst).Elem() - srcVal = reflect.ValueOf(src) - ) - tuple, typ := false, t - for { - if typ.T == SliceTy || typ.T == ArrayTy { - typ = typ.Elem - continue - } - tuple = typ.T == TupleTy - break - } - if !tuple { - return set(dstVal, srcVal) - } - - // Dereferences interface or pointer wrapper - dstVal = indirectInterfaceOrPtr(dstVal) - - switch t.T { - case TupleTy: - if dstVal.Kind() != reflect.Struct { - return fmt.Errorf("abi: invalid dst value for unpack, want struct, got %s", dstVal.Kind()) - } - fieldmap, err := mapArgNamesToStructFields(t.TupleRawNames, dstVal) - if err != nil { - return err - } - for i, elem := range t.TupleElems { - fname := fieldmap[t.TupleRawNames[i]] - field := dstVal.FieldByName(fname) - if !field.IsValid() { - return fmt.Errorf("abi: field %s can't found in the given value", t.TupleRawNames[i]) - } - if err := unpack(elem, field.Addr().Interface(), srcVal.Field(i).Interface()); err != nil { - return err - } - } - return nil - case SliceTy: - if dstVal.Kind() != reflect.Slice { - return fmt.Errorf("abi: invalid dst value for unpack, want slice, got %s", dstVal.Kind()) - } - slice := reflect.MakeSlice(dstVal.Type(), srcVal.Len(), srcVal.Len()) - for i := 0; i < slice.Len(); i++ { - if err := unpack(t.Elem, slice.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil { - return err - } - } - dstVal.Set(slice) - case ArrayTy: - if dstVal.Kind() != reflect.Array { - return fmt.Errorf("abi: invalid dst value for unpack, want array, got %s", dstVal.Kind()) - } - array := reflect.New(dstVal.Type()).Elem() - for i := 0; i < array.Len(); i++ { - if err := unpack(t.Elem, array.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil { - return err - } - } - dstVal.Set(array) - } - return nil -} - // unpackAtomic unpacks ( hexdata -> go ) a single value func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interface{}) error { - nonIndexedArgs := arguments.NonIndexed() - if len(nonIndexedArgs) == 0 { - return nil - } - argument := nonIndexedArgs[0] - elem := reflect.ValueOf(v).Elem() + dst := reflect.ValueOf(v).Elem() + src := reflect.ValueOf(marshalledValues) - if elem.Kind() == reflect.Struct && argument.Type.T != TupleTy { - fieldmap, err := mapArgNamesToStructFields([]string{argument.Name}, elem) - if err != nil { - return err - } - field := elem.FieldByName(fieldmap[argument.Name]) - if !field.IsValid() { - return fmt.Errorf("abi: field %s can't be found in the given value", argument.Name) - } - return unpack(&argument.Type, field.Addr().Interface(), marshalledValues) + if dst.Kind() == reflect.Struct && src.Kind() != reflect.Struct { + return set(dst.Field(0), src) } - return unpack(&argument.Type, elem.Addr().Interface(), marshalledValues) + return set(dst, src) } // unpackTuple unpacks ( hexdata -> go ) a batch of values. func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error { - var ( - value = reflect.ValueOf(v).Elem() - typ = value.Type() - kind = value.Kind() - nonIndexedArgs = arguments.NonIndexed() - ) - if err := requireUnpackKind(value, len(nonIndexedArgs), arguments); err != nil { - return err - } + value := reflect.ValueOf(v).Elem() + nonIndexedArgs := arguments.NonIndexed() - // If the interface is a struct, get of abi->struct_field mapping - var abi2struct map[string]string - if kind == reflect.Struct { + switch value.Kind() { + case reflect.Struct: argNames := make([]string, len(nonIndexedArgs)) for i, arg := range nonIndexedArgs { argNames[i] = arg.Name } var err error - if abi2struct, err = mapArgNamesToStructFields(argNames, value); err != nil { + abi2struct, err := mapArgNamesToStructFields(argNames, value) + if err != nil { return err } - } - for i, arg := range nonIndexedArgs { - switch kind { - case reflect.Struct: + for i, arg := range nonIndexedArgs { field := value.FieldByName(abi2struct[arg.Name]) if !field.IsValid() { return fmt.Errorf("abi: field %s can't be found in the given value", arg.Name) } - if err := unpack(&arg.Type, field.Addr().Interface(), marshalledValues[i]); err != nil { + if err := set(field, reflect.ValueOf(marshalledValues[i])); err != nil { return err } - case reflect.Slice, reflect.Array: - if value.Len() < i { - return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len()) - } - v := value.Index(i) - if err := requireAssignable(v, reflect.ValueOf(marshalledValues[i])); err != nil { - return err - } - if err := unpack(&arg.Type, v.Addr().Interface(), marshalledValues[i]); err != nil { + } + case reflect.Slice, reflect.Array: + if value.Len() < len(marshalledValues) { + return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len()) + } + for i := range nonIndexedArgs { + if err := set(value.Index(i), reflect.ValueOf(marshalledValues[i])); err != nil { return err } - default: - return fmt.Errorf("abi:[2] cannot unmarshal tuple in to %v", typ) } + default: + return fmt.Errorf("abi:[2] cannot unmarshal tuple in to %v", value.Type()) } return nil - } // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index edb50cc935..07c2149dd3 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -64,7 +64,7 @@ type SimulatedBackend struct { mu sync.Mutex pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on on request + pendingState *state.StateDB // Currently pending state that will be the active on request events *filters.EventSystem // Event system for filtering log events live @@ -133,11 +133,11 @@ func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber * if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) == 0 { return b.blockchain.State() } - block, err := b.BlockByNumber(ctx, blockNumber) + block, err := b.blockByNumberNoLock(ctx, blockNumber) if err != nil { return nil, err } - return b.blockchain.StateAt(block.Hash()) + return b.blockchain.StateAt(block.Root()) } // CodeAt returns the code associated with a certain account in the blockchain. @@ -244,6 +244,12 @@ func (b *SimulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) ( b.mu.Lock() defer b.mu.Unlock() + return b.blockByNumberNoLock(ctx, number) +} + +// blockByNumberNoLock retrieves a block from the database by number, caching it +// (associated with its hash) if found without Lock. +func (b *SimulatedBackend) blockByNumberNoLock(ctx context.Context, number *big.Int) (*types.Block, error) { if number == nil || number.Cmp(b.pendingBlock.Number()) == 0 { return b.blockchain.CurrentBlock(), nil } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index ff8951a0ee..a39ab26563 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -258,6 +258,16 @@ func TestSimulatedBackend_NonceAt(t *testing.T) { if newNonce != nonce+uint64(1) { t.Errorf("received incorrect nonce. expected 1, got %v", nonce) } + // create some more blocks + sim.Commit() + // Check that we can get data for an older block/state + newNonce, err = sim.NonceAt(bgCtx, testAddr, big.NewInt(1)) + if err != nil { + t.Fatalf("could not get nonce for test addr: %v", err) + } + if newNonce != nonce+uint64(1) { + t.Fatalf("received incorrect nonce. expected 1, got %v", nonce) + } } func TestSimulatedBackend_SendTransaction(t *testing.T) { diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index cab98343da..27014fd99c 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -49,7 +49,7 @@ type TransactOpts struct { Nonce *big.Int // Nonce to use for the transaction execution (nil = use pending state) Signer SignerFn // Method to use for signing the transaction (mandatory) - Value *big.Int // Funds to transfer along along the transaction (nil = 0 = no funds) + Value *big.Int // Funds to transfer along the transaction (nil = 0 = no funds) GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle) FeeCurrency *common.Address // Fee currency to be used for transaction (nil = default currency = Celo Gold) GatewayFeeRecipient *common.Address // Address to which gateway fees should be paid (nil = no gateway fees are paid) diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index dd48501241..b903d28654 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -64,7 +64,7 @@ type tmplField struct { SolKind abi.Type // Raw abi type information } -// tmplStruct is a wrapper around an abi.tuple contains a auto-generated +// tmplStruct is a wrapper around an abi.tuple contains an auto-generated // struct name. type tmplStruct struct { Name string // Auto-generated struct name(before solidity v0.5.11) or raw name. diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 07e7916fdf..095451c63c 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -312,14 +312,14 @@ func TestEventTupleUnpack(t *testing.T) { &[]interface{}{common.Address{}, new(big.Int)}, &[]interface{}{}, jsonEventPledge, - "abi: insufficient number of elements in the list/array for unpack, want 3, got 2", + "abi: insufficient number of arguments for unpack, want 3, got 2", "Can not unpack Pledge event into too short slice", }, { pledgeData1, new(map[string]interface{}), &[]interface{}{}, jsonEventPledge, - "abi: cannot unmarshal tuple into map[string]interface {}", + "abi:[2] cannot unmarshal tuple in to map[string]interface {}", "Can not unpack Pledge event into map", }, { mixedCaseData1, diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 4bb6c4fa09..75f1a00a5a 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -17,6 +17,7 @@ package abi import ( + "errors" "fmt" "math/big" "reflect" @@ -32,14 +33,6 @@ func indirect(v reflect.Value) reflect.Value { return v } -// indirectInterfaceOrPtr recursively dereferences the value until value is not interface. -func indirectInterfaceOrPtr(v reflect.Value) reflect.Value { - if (v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr) && v.Elem().IsValid() { - return indirect(v.Elem()) - } - return v -} - // reflectIntType returns the reflect using the given size and // unsignedness. func reflectIntType(unsigned bool, size int) reflect.Type { @@ -90,7 +83,11 @@ func set(dst, src reflect.Value) error { case srcType.AssignableTo(dstType) && dst.CanSet(): dst.Set(src) case dstType.Kind() == reflect.Slice && srcType.Kind() == reflect.Slice && dst.CanSet(): - setSlice(dst, src) + return setSlice(dst, src) + case dstType.Kind() == reflect.Array: + return setArray(dst, src) + case dstType.Kind() == reflect.Struct: + return setStruct(dst, src) default: return fmt.Errorf("abi: cannot unmarshal %v in to %v", src.Type(), dst.Type()) } @@ -100,33 +97,55 @@ func set(dst, src reflect.Value) error { // setSlice attempts to assign src to dst when slices are not assignable by default // e.g. src: [][]byte -> dst: [][15]byte // setSlice ignores if we cannot copy all of src' elements. -func setSlice(dst, src reflect.Value) { +func setSlice(dst, src reflect.Value) error { slice := reflect.MakeSlice(dst.Type(), src.Len(), src.Len()) for i := 0; i < src.Len(); i++ { - reflect.Copy(slice.Index(i), src.Index(i)) + if src.Index(i).Kind() == reflect.Struct { + if err := set(slice.Index(i), src.Index(i)); err != nil { + return err + } + } else { + // e.g. [][32]uint8 to []common.Hash + if err := set(slice.Index(i), src.Index(i)); err != nil { + return err + } + } + } + if dst.CanSet() { + dst.Set(slice) + return nil } - dst.Set(slice) + return errors.New("Cannot set slice, destination not settable") } -// requireAssignable assures that `dest` is a pointer and it's not an interface. -func requireAssignable(dst, src reflect.Value) error { - if dst.Kind() != reflect.Ptr && dst.Kind() != reflect.Interface { - return fmt.Errorf("abi: cannot unmarshal %v into %v", src.Type(), dst.Type()) +func setArray(dst, src reflect.Value) error { + array := reflect.New(dst.Type()).Elem() + min := src.Len() + if src.Len() > dst.Len() { + min = dst.Len() } - return nil + for i := 0; i < min; i++ { + if err := set(array.Index(i), src.Index(i)); err != nil { + return err + } + } + if dst.CanSet() { + dst.Set(array) + return nil + } + return errors.New("Cannot set array, destination not settable") } -// requireUnpackKind verifies preconditions for unpacking `args` into `kind` -func requireUnpackKind(v reflect.Value, minLength int, args Arguments) error { - switch v.Kind() { - case reflect.Struct: - case reflect.Slice, reflect.Array: - if v.Len() < minLength { - return fmt.Errorf("abi: insufficient number of elements in the list/array for unpack, want %d, got %d", - minLength, v.Len()) +func setStruct(dst, src reflect.Value) error { + for i := 0; i < src.NumField(); i++ { + srcField := src.Field(i) + dstField := dst.Field(i) + if !dstField.IsValid() || !srcField.IsValid() { + return fmt.Errorf("Could not find src field: %v value: %v in destination", srcField.Type().Name(), srcField) + } + if err := set(dstField, srcField); err != nil { + return err } - default: - return fmt.Errorf("abi: cannot unmarshal tuple into %v", v.Type()) } return nil } diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 3e140476df..8d73b49fed 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -98,7 +98,7 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty typ.Elem = &embeddedType typ.stringKind = embeddedType.stringKind + sliced } else if len(intz) == 1 { - // is a array + // is an array typ.T = ArrayTy typ.Elem = &embeddedType typ.Size, err = strconv.Atoi(intz[0]) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index d720db77a5..fb5922fce8 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -120,8 +120,7 @@ var unpackTests = []unpackTest{ { def: `[{"type": "bytes"}]`, enc: "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000", - want: [32]byte{}, - err: "abi: cannot unmarshal []uint8 in to [32]uint8", + want: [32]byte{1}, }, { def: `[{"type": "bytes32"}]`, @@ -135,8 +134,7 @@ var unpackTests = []unpackTest{ want: struct { IntOne *big.Int Intone *big.Int - }{}, - err: "abi: purely underscored output cannot unpack to struct", + }{IntOne: big.NewInt(1)}, }, { def: `[{"name":"int_one","type":"int256"},{"name":"IntOne","type":"int256"}]`, @@ -362,7 +360,7 @@ func TestMethodMultiReturn(t *testing.T) { }, { &[]interface{}{new(int)}, &[]interface{}{}, - "abi: insufficient number of elements in the list/array for unpack, want 2, got 1", + "abi: insufficient number of arguments for unpack, want 2, got 1", "Can not unpack into a slice with wrong types", }} for _, tc := range testCases { diff --git a/accounts/hd_test.go b/accounts/hd_test.go index 054fd21119..8815080cfe 100644 --- a/accounts/hd_test.go +++ b/accounts/hd_test.go @@ -61,7 +61,7 @@ func TestHDPathParsing(t *testing.T) { // Weird inputs just to ensure they work {" m / 44 '\n/\n 52752 \n\n\t' /\n0 ' /\t\t 0", DerivationPath{0x80000000 + 44, 0x80000000 + 52752, 0x80000000 + 0, 0}}, - // Invaid derivation paths + // Invalid derivation paths {"", nil}, // Empty relative derivation path {"m", nil}, // Empty absolute derivation path {"m/", nil}, // Missing last derivation component diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 4032e4deb4..b325854564 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -51,6 +51,10 @@ var ( ErrLocked = accounts.NewAuthNeededError("password or unlock") ErrNoMatch = errors.New("no key for given address or file") ErrDecrypt = errors.New("could not decrypt key with given password") + + // ErrAccountAlreadyExists is returned if an account attempted to import is + // already present in the keystore. + ErrAccountAlreadyExists = errors.New("account alreaady exists") ) // KeyStoreType is the reflect type of a keystore backend. @@ -578,19 +582,25 @@ func (ks *KeyStore) Import(keyJSON []byte, passphrase, newPassphrase string) (ac } ks.importMu.Lock() defer ks.importMu.Unlock() + if ks.cache.hasAddress(key.Address) { - return accounts.Account{}, errors.New("account already exists") + return accounts.Account{ + Address: key.Address, + }, ErrAccountAlreadyExists } return ks.importKey(key, newPassphrase) } // ImportECDSA stores the given key into the key directory, encrypting it with the passphrase. func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (accounts.Account, error) { - key := newKeyFromECDSA(priv) ks.importMu.Lock() defer ks.importMu.Unlock() + + key := newKeyFromECDSA(priv) if ks.cache.hasAddress(key.Address) { - return accounts.Account{}, errors.New("account already exists") + return accounts.Account{ + Address: key.Address, + }, ErrAccountAlreadyExists } return ks.importKey(key, passphrase) } diff --git a/cmd/clef/README.md b/cmd/clef/README.md index ae8a4927d9..5cd3da8ab1 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -9,7 +9,7 @@ Clef can run as a daemon on the same machine, off a usb-stick like [USB armory]( Check out the * [CLI tutorial](tutorial.md) for some concrete examples on how Clef works. -* [Setup docs](docs/setup.md) for infos on how to configure Clef on QubesOS or USB Armory. +* [Setup docs](docs/setup.md) for information on how to configure Clef on QubesOS or USB Armory. * [Data types](datatypes.md) for details on the communication messages between Clef and an external UI. ## Command line flags @@ -45,6 +45,7 @@ GLOBAL OPTIONS: --stdio-ui Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process. --stdio-ui-test Mechanism to test interface between Clef and UI. Requires 'stdio-ui'. --advanced If enabled, issues warnings instead of rejections for suspicious requests. Default off + --suppress-bootwarn If set, does not show the warning during boot --help, -h show help --version, -v print the version ``` @@ -879,7 +880,7 @@ TLDR; Use this method to keep track of signed transactions, instead of using the ### OnSignerStartup / `ui_onSignerStartup` -This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external API, +This method provide the UI with information about what API version the signer uses (both internal and external) as well as build-info and external API, in k/v-form. Example call: diff --git a/cmd/clef/datatypes.md b/cmd/clef/datatypes.md index 5ebf9adc97..dd8cda5846 100644 --- a/cmd/clef/datatypes.md +++ b/cmd/clef/datatypes.md @@ -3,7 +3,7 @@ These data types are defined in the channel between clef and the UI ### SignDataRequest -SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message` +SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to present the user with the contents of the `message` Example: ```json diff --git a/cmd/clef/docs/setup.md b/cmd/clef/docs/setup.md index a25f87ce64..d66a0d9502 100644 --- a/cmd/clef/docs/setup.md +++ b/cmd/clef/docs/setup.md @@ -186,7 +186,7 @@ from other qubes. ## USBArmory -The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 Mhz ARM processor. It is a pocket-size +The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 MHz ARM processor. It is a pocket-size computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network to your computer. Over this new network interface, you can SSH into the device. diff --git a/cmd/clef/intapi_changelog.md b/cmd/clef/intapi_changelog.md index f7e6993cf5..eaeb2e6862 100644 --- a/cmd/clef/intapi_changelog.md +++ b/cmd/clef/intapi_changelog.md @@ -12,7 +12,7 @@ Additional labels for pre-release and build metadata are available as extensions ### 7.0.1 -Added `clef_New` to the internal API calleable from a UI. +Added `clef_New` to the internal API callable from a UI. > `New` creates a new password protected Account. The private key is protected with > the given password. Users are responsible to backup the private key that is stored @@ -161,7 +161,7 @@ UserInputResponse struct { #### 1.2.0 * Add `OnStartup` method, to provide the UI with information about what API version -the signer uses (both internal and external) aswell as build-info and external api. +the signer uses (both internal and external) as well as build-info and external api. Example call: ```json diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 3a538d81ec..ff5b5ca8a6 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -40,7 +40,7 @@ import ( "github.com/celo-org/celo-blockchain/cmd/utils" "github.com/celo-org/celo-blockchain/common" "github.com/celo-org/celo-blockchain/common/hexutil" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/core/types" "github.com/celo-org/celo-blockchain/crypto" "github.com/celo-org/celo-blockchain/internal/ethapi" @@ -83,6 +83,10 @@ var ( Name: "advanced", Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off", } + acceptFlag = cli.BoolFlag{ + Name: "suppress-bootwarn", + Usage: "If set, does not show the warning during boot", + } keystoreFlag = cli.StringFlag{ Name: "keystore", Value: filepath.Join(node.DefaultDataDir(), "keystore"), @@ -197,6 +201,7 @@ The delpw command removes a password for a given address (keyfile). logLevelFlag, keystoreFlag, utils.LightKDFFlag, + acceptFlag, }, Description: ` The newaccount command creates a new keystore-backed account. It is a convenience-method @@ -235,6 +240,7 @@ func init() { stdiouiFlag, testFlag, advancedMode, + acceptFlag, } app.Action = signer app.Commands = []cli.Command{initCommand, @@ -433,8 +439,10 @@ func initialize(c *cli.Context) error { if c.GlobalBool(stdiouiFlag.Name) { logOutput = os.Stderr // If using the stdioui, we can't do the 'confirm'-flow - fmt.Fprint(logOutput, legalWarning) - } else { + if !c.GlobalBool(acceptFlag.Name) { + fmt.Fprint(logOutput, legalWarning) + } + } else if !c.GlobalBool(acceptFlag.Name) { if !confirm(legalWarning) { return fmt.Errorf("aborted by user") } @@ -883,14 +891,14 @@ func testExternalUI(api *core.SignerAPI) { // getPassPhrase retrieves the password associated with clef, either fetched // from a list of preloaded passphrases, or requested interactively from the user. // TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one. -func getPassPhrase(prompt string, confirmation bool) string { - fmt.Println(prompt) - password, err := console.Stdin.PromptPassword("Password: ") +func getPassPhrase(query string, confirmation bool) string { + fmt.Println(query) + password, err := prompt.Stdin.PromptPassword("Password: ") if err != nil { utils.Fatalf("Failed to read password: %v", err) } if confirmation { - confirm, err := console.Stdin.PromptPassword("Repeat password: ") + confirm, err := prompt.Stdin.PromptPassword("Repeat password: ") if err != nil { utils.Fatalf("Failed to read password confirmation: %v", err) } @@ -951,7 +959,7 @@ func GenDoc(ctx *cli.Context) { if data, err := json.MarshalIndent(v, "", " "); err == nil { output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data)) } else { - log.Error("Error generating output", err) + log.Error("Error generating output", "err", err) } } ) diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 0482a041f3..3c93b0bfd9 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -27,7 +27,7 @@ import ( "github.com/celo-org/celo-blockchain/accounts/keystore" "github.com/celo-org/celo-blockchain/common" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/p2p/dnsdisc" "github.com/celo-org/celo-blockchain/p2p/enode" cli "gopkg.in/urfave/cli.v1" @@ -226,7 +226,7 @@ func loadSigningKey(keyfile string) *ecdsa.PrivateKey { if err != nil { exit(fmt.Errorf("failed to read the keyfile at '%s': %v", keyfile, err)) } - password, _ := console.Stdin.PromptPassword("Please enter the password for '" + keyfile + "': ") + password, _ := prompt.Stdin.PromptPassword("Please enter the password for '" + keyfile + "': ") key, err := keystore.DecryptKey(keyjson, password) if err != nil { exit(fmt.Errorf("error decrypting key: %v", err)) diff --git a/cmd/ethkey/utils.go b/cmd/ethkey/utils.go index 445d9d71ef..b31260c7ef 100644 --- a/cmd/ethkey/utils.go +++ b/cmd/ethkey/utils.go @@ -23,7 +23,7 @@ import ( "strings" "github.com/celo-org/celo-blockchain/cmd/utils" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/crypto" "gopkg.in/urfave/cli.v1" ) @@ -31,13 +31,13 @@ import ( // promptPassphrase prompts the user for a passphrase. Set confirmation to true // to require the user to confirm the passphrase. func promptPassphrase(confirmation bool) string { - passphrase, err := console.Stdin.PromptPassword("Password: ") + passphrase, err := prompt.Stdin.PromptPassword("Password: ") if err != nil { utils.Fatalf("Failed to read password: %v", err) } if confirmation { - confirm, err := console.Stdin.PromptPassword("Repeat password: ") + confirm, err := prompt.Stdin.PromptPassword("Repeat password: ") if err != nil { utils.Fatalf("Failed to read password confirmation: %v", err) } diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index e5dc08b087..cd5eca3531 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -162,7 +162,6 @@ func main() { if blob, err = ioutil.ReadFile(*accPassFlag); err != nil { log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) } - // Delete trailing newline in password pass := strings.TrimSuffix(string(blob), "\n") ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP) @@ -170,11 +169,12 @@ func main() { log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err) } acc, err := ks.Import(blob, pass, pass) - if err != nil { + if err != nil && err != keystore.ErrAccountAlreadyExists { log.Crit("Failed to import faucet signer account", "err", err) } - ks.Unlock(acc, pass) - + if err := ks.Unlock(acc, pass); err != nil { + log.Crit("Failed to unlock faucet signer account", "err", err) + } // Assemble and start the faucet light service faucet, err := newFaucet(genesis, *ethPortFlag, enodes, *netFlag, *statsFlag, ks, website.Bytes()) if err != nil { @@ -694,9 +694,12 @@ func authTwitter(url string) (string, string, common.Address, error) { return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } // Twitter's API isn't really friendly with direct links. Still, we don't - // want to do ask read permissions from users, so just load the public posts and - // scrape it for the Ethereum address and profile URL. + // want to do ask read permissions from users, so just load the public posts + // and scrape it for the Ethereum address and profile URL. We need to load + // the mobile page though since the main page loads tweet contents via JS. // #nosec (we don't use faucet) + url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) + //nolint:gosec G107 this function is called with a twitter url res, err := http.Get(url) if err != nil { return "", "", common.Address{}, err diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0a636beb08..598790a306 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -26,7 +26,7 @@ import ( "github.com/celo-org/celo-blockchain/accounts/usbwallet" "github.com/celo-org/celo-blockchain/cmd/utils" "github.com/celo-org/celo-blockchain/common" - "github.com/celo-org/celo-blockchain/console" + prompt2 "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/crypto" "github.com/celo-org/celo-blockchain/log" cli "gopkg.in/urfave/cli.v1" @@ -417,12 +417,12 @@ func getPassPhrase(prompt string, confirmation bool, i int, passwords []string) if prompt != "" { fmt.Println(prompt) } - password, err := console.Stdin.PromptPassword("Password: ") + password, err := prompt2.Stdin.PromptPassword("Password: ") if err != nil { utils.Fatalf("Failed to read password: %v", err) } if confirmation { - confirm, err := console.Stdin.PromptPassword("Repeat password: ") + confirm, err := prompt2.Stdin.PromptPassword("Repeat password: ") if err != nil { utils.Fatalf("Failed to read password confirmation: %v", err) } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index b9542afae9..e1ac309d1c 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -28,7 +28,7 @@ import ( "github.com/celo-org/celo-blockchain/cmd/utils" "github.com/celo-org/celo-blockchain/common" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/core" "github.com/celo-org/celo-blockchain/core/rawdb" "github.com/celo-org/celo-blockchain/core/state" @@ -523,7 +523,7 @@ func removeDB(ctx *cli.Context) error { // confirmAndRemoveDB prompts the user for a last confirmation and removes the // folder if accepted. func confirmAndRemoveDB(database string, kind string) { - confirm, err := console.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) + confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) switch { case err != nil: utils.Fatalf("%v", err) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 3d2b4227f3..730fc612fa 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -73,7 +73,7 @@ at block: 0 ({{niltime}}) // Tests that a console can be attached to a running node via various means. func TestIPCAttachWelcome(t *testing.T) { - // Configure the instance for IPC attachement + // Configure the instance for IPC attachment coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" var ipc string if runtime.GOOS == "windows" { diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index b2912f45c9..ec927b90e0 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -36,7 +36,9 @@ var daoOldGenesis = `{ "nonce" : "0x0000000000000042", "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", "timestamp" : "0x00", - "config" : {} + "config" : { + "homesteadBlock" : 0 + } }` // Genesis block for nodes which actively oppose the DAO fork @@ -48,6 +50,7 @@ var daoNoForkGenesis = `{ "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", "timestamp" : "0x00", "config" : { + "homesteadBlock" : 0, "daoForkBlock" : 314, "daoForkSupport" : false } @@ -62,6 +65,7 @@ var daoProForkGenesis = `{ "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", "timestamp" : "0x00", "config" : { + "homesteadBlock" : 0, "daoForkBlock" : 314, "daoForkSupport" : true } diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index 0c5904661f..054d012c16 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -52,7 +52,7 @@ var customGenesisTests = []struct { "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", "timestamp" : "0xabcdf0", "config" : { - "homesteadBlock" : 314, + "homesteadBlock" : 42, "daoForkBlock" : 141, "daoForkSupport" : true, "istanbul": {} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 3e7887ab62..460f9481b8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -21,7 +21,6 @@ import ( "fmt" "math" "os" - "runtime" godebug "runtime/debug" "sort" "strconv" @@ -32,7 +31,7 @@ import ( "github.com/celo-org/celo-blockchain/accounts/keystore" "github.com/celo-org/celo-blockchain/cmd/utils" "github.com/celo-org/celo-blockchain/common" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/contract_comm/blockchain_parameters" "github.com/celo-org/celo-blockchain/eth" "github.com/celo-org/celo-blockchain/eth/downloader" @@ -42,7 +41,7 @@ import ( "github.com/celo-org/celo-blockchain/log" "github.com/celo-org/celo-blockchain/metrics" "github.com/celo-org/celo-blockchain/node" - "github.com/elastic/gosigar" + gopsutil "github.com/shirou/gopsutil/mem" cli "gopkg.in/urfave/cli.v1" ) @@ -262,7 +261,7 @@ func init() { } app.After = func(ctx *cli.Context) error { debug.Exit() - console.Stdin.Close() // Resets terminal mode. + prompt.Stdin.Close() // Resets terminal mode. return nil } } @@ -306,20 +305,16 @@ func prepare(ctx *cli.Context) { ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128)) } // Cap the cache allowance and tune the garbage collector - var mem gosigar.Mem - // Workaround until OpenBSD support lands into gosigar - // Check https://github.com/elastic/gosigar#supported-platforms - if runtime.GOOS != "openbsd" { - if err := mem.Get(); err == nil { - if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 { - log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024) - mem.Total = 2 * 1024 * 1024 * 1024 - } - allowance := int(mem.Total / 1024 / 1024 / 3) - if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance { - log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance)) - } + mem, err := gopsutil.VirtualMemory() + if err == nil { + if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 { + log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024) + mem.Total = 2 * 1024 * 1024 * 1024 + } + allowance := int(mem.Total / 1024 / 1024 / 3) + if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance { + log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) + ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance)) } } // Ensure Go's GC ignores the database cache for trigger percentage diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 86eb440ce6..0279dc954f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1735,13 +1735,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = params.BaklavaNetworkId } cfg.Genesis = core.DefaultBaklavaGenesisBlock() - setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.BaklavaGenesisHash]) + setDNSDiscoveryDefaults(cfg, params.BaklavaGenesisHash) case ctx.GlobalBool(AlfajoresFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = params.AlfajoresNetworkId } cfg.Genesis = core.DefaultAlfajoresGenesisBlock() - setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.AlfajoresGenesisHash]) + setDNSDiscoveryDefaults(cfg, params.AlfajoresGenesisHash) case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1771,18 +1771,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } default: if cfg.NetworkId == params.MainnetNetworkId { - setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.MainnetGenesisHash]) + setDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) } } } // setDNSDiscoveryDefaults configures DNS discovery with the given URL if // no URLs are set. -func setDNSDiscoveryDefaults(cfg *eth.Config, url string) { +func setDNSDiscoveryDefaults(cfg *eth.Config, genesis common.Hash) { if cfg.DiscoveryURLs != nil { - return + return // already set through flags/config + } + + protocol := "all" + if cfg.SyncMode == downloader.LightSync { + protocol = "les" + } + if url := params.KnownDNSNetwork(genesis, protocol); url != "" { + cfg.DiscoveryURLs = []string{url} } - cfg.DiscoveryURLs = []string{url} } // RegisterEthService adds an Ethereum client to the stack. diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 97bdcc29f9..335696e28f 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -38,7 +38,7 @@ import ( "github.com/celo-org/celo-blockchain/cmd/utils" "github.com/celo-org/celo-blockchain/common" "github.com/celo-org/celo-blockchain/common/hexutil" - "github.com/celo-org/celo-blockchain/console" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/crypto" "github.com/celo-org/celo-blockchain/log" "github.com/celo-org/celo-blockchain/p2p" @@ -210,7 +210,7 @@ func initialize() { if *mailServerMode { if len(msPassword) == 0 { - msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ") + msPassword, err = prompt.Stdin.PromptPassword("Please enter the Mail Server password: ") if err != nil { utils.Fatalf("Failed to read Mail Server password: %s", err) } @@ -346,7 +346,7 @@ func configureNode() { if *requestMail { p2pAccept = true if len(msPassword) == 0 { - msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ") + msPassword, err = prompt.Stdin.PromptPassword("Please enter the Mail Server password: ") if err != nil { utils.Fatalf("Failed to read Mail Server password: %s", err) } @@ -355,7 +355,7 @@ func configureNode() { if !*asymmetricMode && !*forwarderMode { if len(symPass) == 0 { - symPass, err = console.Stdin.PromptPassword("Please enter the password for symmetric encryption: ") + symPass, err = prompt.Stdin.PromptPassword("Please enter the password for symmetric encryption: ") if err != nil { utils.Fatalf("Failed to read password: %v", err) } diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index c6878504ad..358a5248c0 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -27,7 +27,7 @@ import ( ) var ( - // ErrBadProDAOExtra is returned if a header doens't support the DAO fork on a + // ErrBadProDAOExtra is returned if a header doesn't support the DAO fork on a // pro-fork client. ErrBadProDAOExtra = errors.New("bad DAO pro-fork extra-data") diff --git a/console/bridge.go b/console/bridge.go index fe265a459c..2fef40d67f 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -26,6 +26,7 @@ import ( "github.com/celo-org/celo-blockchain/accounts/usbwallet" "github.com/celo-org/celo-blockchain/common/hexutil" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/internal/jsre" "github.com/celo-org/celo-blockchain/rpc" "github.com/dop251/goja" @@ -34,13 +35,13 @@ import ( // bridge is a collection of JavaScript utility methods to bride the .js runtime // environment and the Go RPC connection backing the remote method calls. type bridge struct { - client *rpc.Client // RPC client to execute Ethereum requests through - prompter UserPrompter // Input prompter to allow interactive user feedback - printer io.Writer // Output writer to serialize any display strings to + client *rpc.Client // RPC client to execute Ethereum requests through + prompter prompt.UserPrompter // Input prompter to allow interactive user feedback + printer io.Writer // Output writer to serialize any display strings to } // newBridge creates a new JavaScript wrapper around an RPC client. -func newBridge(client *rpc.Client, prompter UserPrompter, printer io.Writer) *bridge { +func newBridge(client *rpc.Client, prompter prompt.UserPrompter, printer io.Writer) *bridge { return &bridge{ client: client, prompter: prompter, @@ -178,14 +179,15 @@ func (b *bridge) readPinAndReopenWallet(call jsre.Call) (goja.Value, error) { // original RPC method (saved in jeth.unlockAccount) with it to actually execute // the RPC call. func (b *bridge) UnlockAccount(call jsre.Call) (goja.Value, error) { - if nArgs := len(call.Arguments); nArgs < 2 { + if len(call.Arguments) < 1 { return nil, fmt.Errorf("usage: unlockAccount(account, [ password, duration ])") } + + account := call.Argument(0) // Make sure we have an account specified to unlock. - if call.Argument(0).ExportType().Kind() != reflect.String { + if goja.IsUndefined(account) || goja.IsNull(account) || account.ExportType().Kind() != reflect.String { return nil, fmt.Errorf("first argument must be the account to unlock") } - account := call.Argument(0) // If password is not given or is the null value, prompt the user for it. var passwd goja.Value @@ -233,10 +235,10 @@ func (b *bridge) Sign(call jsre.Call) (goja.Value, error) { passwd = call.Argument(2) ) - if message.ExportType().Kind() != reflect.String { + if goja.IsUndefined(message) || message.ExportType().Kind() != reflect.String { return nil, fmt.Errorf("first argument must be the message to sign") } - if account.ExportType().Kind() != reflect.String { + if goja.IsUndefined(account) || account.ExportType().Kind() != reflect.String { return nil, fmt.Errorf("second argument must be the account to sign with") } @@ -265,10 +267,11 @@ func (b *bridge) Sleep(call jsre.Call) (goja.Value, error) { if nArgs := len(call.Arguments); nArgs < 1 { return nil, fmt.Errorf("usage: sleep()") } - if !isNumber(call.Argument(0)) { + sleepObj := call.Argument(0) + if goja.IsUndefined(sleepObj) || goja.IsNull(sleepObj) || !isNumber(sleepObj) { return nil, fmt.Errorf("usage: sleep()") } - sleep := call.Argument(0).ToFloat() + sleep := sleepObj.ToFloat() time.Sleep(time.Duration(sleep * float64(time.Second))) return call.VM.ToValue(true), nil } @@ -286,13 +289,13 @@ func (b *bridge) SleepBlocks(call jsre.Call) (goja.Value, error) { return nil, fmt.Errorf("usage: sleepBlocks([, max sleep in seconds])") } if nArgs >= 1 { - if !isNumber(call.Argument(0)) { + if goja.IsNull(call.Argument(0)) || goja.IsUndefined(call.Argument(0)) || !isNumber(call.Argument(0)) { return nil, fmt.Errorf("expected number as first argument") } blocks = call.Argument(0).ToInteger() } if nArgs >= 2 { - if !isNumber(call.Argument(1)) { + if goja.IsNull(call.Argument(1)) || goja.IsUndefined(call.Argument(1)) || !isNumber(call.Argument(1)) { return nil, fmt.Errorf("expected number as second argument") } sleep = call.Argument(1).ToInteger() diff --git a/console/bridge_test.go b/console/bridge_test.go new file mode 100644 index 0000000000..dd5ce76919 --- /dev/null +++ b/console/bridge_test.go @@ -0,0 +1,48 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package console + +import ( + "testing" + + "github.com/celo-org/celo-blockchain/internal/jsre" + "github.com/dop251/goja" +) + +// TestUndefinedAsParam ensures that personal functions can receive +// `undefined` as a parameter. +func TestUndefinedAsParam(t *testing.T) { + b := bridge{} + call := jsre.Call{} + call.Arguments = []goja.Value{goja.Undefined()} + + b.UnlockAccount(call) + b.Sign(call) + b.Sleep(call) +} + +// TestNullAsParam ensures that personal functions can receive +// `null` as a parameter. +func TestNullAsParam(t *testing.T) { + b := bridge{} + call := jsre.Call{} + call.Arguments = []goja.Value{goja.Null()} + + b.UnlockAccount(call) + b.Sign(call) + b.Sleep(call) +} diff --git a/console/console.go b/console/console.go index c130df3903..0a36b83282 100644 --- a/console/console.go +++ b/console/console.go @@ -28,6 +28,7 @@ import ( "strings" "syscall" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/internal/jsre" "github.com/celo-org/celo-blockchain/internal/jsre/deps" "github.com/celo-org/celo-blockchain/internal/web3ext" @@ -52,26 +53,26 @@ const DefaultPrompt = "> " // Config is the collection of configurations to fine tune the behavior of the // JavaScript console. type Config struct { - DataDir string // Data directory to store the console history at - DocRoot string // Filesystem path from where to load JavaScript files from - Client *rpc.Client // RPC client to execute Ethereum requests through - Prompt string // Input prompt prefix string (defaults to DefaultPrompt) - Prompter UserPrompter // Input prompter to allow interactive user feedback (defaults to TerminalPrompter) - Printer io.Writer // Output writer to serialize any display strings to (defaults to os.Stdout) - Preload []string // Absolute paths to JavaScript files to preload + DataDir string // Data directory to store the console history at + DocRoot string // Filesystem path from where to load JavaScript files from + Client *rpc.Client // RPC client to execute Ethereum requests through + Prompt string // Input prompt prefix string (defaults to DefaultPrompt) + Prompter prompt.UserPrompter // Input prompter to allow interactive user feedback (defaults to TerminalPrompter) + Printer io.Writer // Output writer to serialize any display strings to (defaults to os.Stdout) + Preload []string // Absolute paths to JavaScript files to preload } // Console is a JavaScript interpreted runtime environment. It is a fully fledged // JavaScript console attached to a running node via an external or in-process RPC // client. type Console struct { - client *rpc.Client // RPC client to execute Ethereum requests through - jsre *jsre.JSRE // JavaScript runtime environment running the interpreter - prompt string // Input prompt prefix string - prompter UserPrompter // Input prompter to allow interactive user feedback - histPath string // Absolute path to the console scrollback history - history []string // Scroll history maintained by the console - printer io.Writer // Output writer to serialize any display strings to + client *rpc.Client // RPC client to execute Ethereum requests through + jsre *jsre.JSRE // JavaScript runtime environment running the interpreter + prompt string // Input prompt prefix string + prompter prompt.UserPrompter // Input prompter to allow interactive user feedback + histPath string // Absolute path to the console scrollback history + history []string // Scroll history maintained by the console + printer io.Writer // Output writer to serialize any display strings to } // New initializes a JavaScript interpreted runtime environment and sets defaults @@ -79,7 +80,7 @@ type Console struct { func New(config Config) (*Console, error) { // Handle unset config values gracefully if config.Prompter == nil { - config.Prompter = Stdin + config.Prompter = prompt.Stdin } if config.Prompt == "" { config.Prompt = DefaultPrompt diff --git a/console/console_test.go b/console/console_test.go index 577307a82e..d02f517ede 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/celo-org/celo-blockchain/common" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/core" "github.com/celo-org/celo-blockchain/eth" "github.com/celo-org/celo-blockchain/internal/jsre" @@ -66,10 +67,10 @@ func (p *hookedPrompter) PromptPassword(prompt string) (string, error) { func (p *hookedPrompter) PromptConfirm(prompt string) (bool, error) { return false, errors.New("not implemented") } -func (p *hookedPrompter) SetHistory(history []string) {} -func (p *hookedPrompter) AppendHistory(command string) {} -func (p *hookedPrompter) ClearHistory() {} -func (p *hookedPrompter) SetWordCompleter(completer WordCompleter) {} +func (p *hookedPrompter) SetHistory(history []string) {} +func (p *hookedPrompter) AppendHistory(command string) {} +func (p *hookedPrompter) ClearHistory() {} +func (p *hookedPrompter) SetWordCompleter(completer prompt.WordCompleter) {} // tester is a console test environment for the console tests to operate on. type tester struct { diff --git a/console/prompter.go b/console/prompt/prompter.go similarity index 99% rename from console/prompter.go rename to console/prompt/prompter.go index 65675061a7..810b6c3e14 100644 --- a/console/prompter.go +++ b/console/prompt/prompter.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package console +package prompt import ( "fmt" diff --git a/core/blockchain.go b/core/blockchain.go index 8c38639576..e8e29f2049 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -191,11 +191,10 @@ type BlockChain struct { txLookupCache *lru.Cache // Cache for the most recent transaction lookup data. futureBlocks *lru.Cache // future blocks are blocks added for later processing - quit chan struct{} // blockchain quit channel - running int32 // running must be called atomically - // procInterrupt must be atomically called - procInterrupt int32 // interrupt signaler for block processing + quit chan struct{} // blockchain quit channel wg sync.WaitGroup // chain processing wait group for shutting down + running int32 // 0 if chain is running, 1 when stopped + procInterrupt int32 // interrupt signaler for block processing engine consensus.Engine validator Validator // Block and state validator interface @@ -246,7 +245,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.processor = NewStateProcessor(chainConfig, bc, engine) var err error - bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) + bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) if err != nil { return nil, err } @@ -346,10 +345,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par return bc, nil } -func (bc *BlockChain) getProcInterrupt() bool { - return atomic.LoadInt32(&bc.procInterrupt) == 1 -} - // GetVMConfig returns the block chain VM config. func (bc *BlockChain) GetVMConfig() *vm.Config { return &bc.vmConfig @@ -891,8 +886,7 @@ func (bc *BlockChain) Stop() { // Unsubscribe all subscriptions registered from blockchain bc.scope.Close() close(bc.quit) - atomic.StoreInt32(&bc.procInterrupt, 1) - + bc.StopInsert() bc.wg.Wait() // Ensure that the entirety of the state snapshot is journalled to disk. @@ -937,6 +931,18 @@ func (bc *BlockChain) Stop() { log.Info("Blockchain stopped") } +// StopInsert interrupts all insertion methods, causing them to return +// errInsertionInterrupted as soon as possible. Insertion is permanently disabled after +// calling this method. +func (bc *BlockChain) StopInsert() { + atomic.StoreInt32(&bc.procInterrupt, 1) +} + +// insertStopped returns true after StopInsert has been called. +func (bc *BlockChain) insertStopped() bool { + return atomic.LoadInt32(&bc.procInterrupt) == 1 +} + func (bc *BlockChain) procFutureBlocks() { blocks := make([]*types.Block, 0, bc.futureBlocks.Len()) for _, hash := range bc.futureBlocks.Keys() { @@ -1026,7 +1032,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ size = 0 ) // updateHead updates the head fast sync block if the inserted blocks are better - // and returns a indicator whether the inserted blocks are canonical. + // and returns an indicator whether the inserted blocks are canonical. updateHead := func(head *types.Block) bool { bc.chainmu.Lock() @@ -1065,7 +1071,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ var deleted []*numberHash for i, block := range blockChain { // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&bc.procInterrupt) == 1 { + if bc.insertStopped() { return 0, errInsertionInterrupted } // Short circuit insertion if it is required(used in testing only) @@ -1212,7 +1218,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ batch := bc.db.NewBatch() for i, block := range blockChain { // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&bc.procInterrupt) == 1 { + if bc.insertStopped() { return 0, errInsertionInterrupted } // Short circuit if the owner header is unknown @@ -1696,8 +1702,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // No validation errors for the first block (or chain prefix skipped) for ; block != nil && err == nil || err == ErrKnownBlock; block, err = it.next() { // If the chain is terminating, stop processing blocks - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - log.Debug("Premature abort during blocks processing") + if bc.insertStopped() { + log.Debug("Abort during block processing") break } // If the header is a banned one, straight out abort @@ -1984,8 +1990,8 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i blocks, memory = blocks[:0], 0 // If the chain is terminating, stop processing blocks - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - log.Debug("Premature abort during blocks processing") + if bc.insertStopped() { + log.Debug("Abort during blocks processing") return 0, nil } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 9784331077..87e963accc 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1409,7 +1409,7 @@ func TestEIP161AccountRemoval(t *testing.T) { t.Error("account should not exist") } - // account musn't be created post eip 161 + // account mustn't be created post eip 161 if _, err := blockchain.InsertChain(types.Blocks{blocks[2]}); err != nil { t.Fatal(err) } diff --git a/core/error.go b/core/error.go index f456c23330..1b8fd7cb44 100644 --- a/core/error.go +++ b/core/error.go @@ -29,7 +29,7 @@ var ( ErrNoGenesis = errors.New("genesis not found in chain") ) -// List of evm-call-message pre-checking errors. All state transtion messages will +// List of evm-call-message pre-checking errors. All state transition messages will // be pre-checked before execution. If any invalidation detected, the corresponding // error should be returned which is defined here. // diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index fc4f3ab46e..683328e95d 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -27,7 +27,7 @@ import ( "strings" "github.com/celo-org/celo-blockchain/common" - "github.com/celo-org/celo-blockchain/core" + "github.com/celo-org/celo-blockchain/core/types" "github.com/celo-org/celo-blockchain/log" "github.com/celo-org/celo-blockchain/params" ) @@ -44,6 +44,18 @@ var ( ErrLocalIncompatibleOrStale = errors.New("local incompatible or needs update") ) +// Blockchain defines all necessary method to build a forkID. +type Blockchain interface { + // Config retrieves the chain's fork configuration. + Config() *params.ChainConfig + + // Genesis retrieves the chain's genesis block. + Genesis() *types.Block + + // CurrentHeader retrieves the current head header of the canonical chain. + CurrentHeader() *types.Header +} + // ID is a fork identifier as defined by EIP-2124. type ID struct { Hash [4]byte // CRC32 checksum of the genesis block and passed fork block numbers @@ -54,7 +66,7 @@ type ID struct { type Filter func(id ID) error // NewID calculates the Ethereum fork ID from the chain config and head. -func NewID(chain *core.BlockChain) ID { +func NewID(chain Blockchain) ID { return newID( chain.Config(), chain.Genesis().Hash(), @@ -85,7 +97,7 @@ func newID(config *params.ChainConfig, genesis common.Hash, head uint64) ID { // NewFilter creates a filter that returns if a fork ID should be rejected or not // based on the local chain's status. -func NewFilter(chain *core.BlockChain) Filter { +func NewFilter(chain Blockchain) Filter { return newFilter( chain.Config(), chain.Genesis().Hash(), @@ -223,7 +235,7 @@ func gatherForks(config *params.ChainConfig) []uint64 { forks = append(forks, rule.Uint64()) } } - // Sort the fork block numbers to permit chronologival XOR + // Sort the fork block numbers to permit chronological XOR for i := 0; i < len(forks); i++ { for j := i + 1; j < len(forks); j++ { if forks[i] > forks[j] { diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index a700d4ccca..8652c91d04 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -125,7 +125,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool } } } - // process runs in parallell + // process runs in parallel nThreadsAlive := int32(threads) process := func() { defer func() { diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 900b34e9d7..ba02e16e58 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -425,7 +425,7 @@ func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]str // flatten pushes all data from this point downwards, flattening everything into // a single diff at the bottom. Since usually the lowermost diff is the largest, -// the flattening bulds up from there in reverse. +// the flattening builds up from there in reverse. func (dl *diffLayer) flatten() snapshot { // If the parent is not diff, we're the first in line, return unmodified parent, ok := dl.parent.(*diffLayer) diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index d0576cb7fb..101ff08c20 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -26,7 +26,7 @@ import ( "github.com/celo-org/celo-blockchain/ethdb" ) -// Iterator is a iterator to step over all the accounts or the specific +// Iterator is an iterator to step over all the accounts or the specific // storage in a snapshot which may or may not be composed of multiple layers. type Iterator interface { // Next steps the iterator forward one element, returning false if exhausted, @@ -47,7 +47,7 @@ type Iterator interface { Release() } -// AccountIterator is a iterator to step over all the accounts in a snapshot, +// AccountIterator is an iterator to step over all the accounts in a snapshot, // which may or may not be composed of multiple layers. type AccountIterator interface { Iterator @@ -57,7 +57,7 @@ type AccountIterator interface { Account() []byte } -// StorageIterator is a iterator to step over the specific storage in a snapshot, +// StorageIterator is an iterator to step over the specific storage in a snapshot, // which may or may not be composed of multiple layers. type StorageIterator interface { Iterator @@ -250,7 +250,7 @@ type diffStorageIterator struct { func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { // Create the storage for this account even it's marked // as destructed. The iterator is for the new one which - // just has the same adddress as the deleted one. + // just has the same address as the deleted one. hashes, destructed := dl.StorageList(account) index := sort.Search(len(hashes), func(i int) bool { return bytes.Compare(seek[:], hashes[i][:]) <= 0 diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 9ef1ee3887..89b557570d 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -238,7 +238,7 @@ func (fi *fastIterator) next(idx int) bool { fi.iterators = append(fi.iterators[:idx], fi.iterators[idx+1:]...) return len(fi.iterators) > 0 } - // If there's noone left to cascade into, return + // If there's no one left to cascade into, return if idx == len(fi.iterators)-1 { return true } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index f575a092dc..cb31448f33 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -330,7 +330,7 @@ func (t *Tree) Cap(root common.Hash, layers int) error { remove(root) } } - // If the disk layer was modified, regenerate all the cummulative blooms + // If the disk layer was modified, regenerate all the cumulative blooms if persisted != nil { var rebloom func(root common.Hash) rebloom = func(root common.Hash) { diff --git a/core/state/snapshot/wipe.go b/core/state/snapshot/wipe.go index f1727cb348..42d821c041 100644 --- a/core/state/snapshot/wipe.go +++ b/core/state/snapshot/wipe.go @@ -94,7 +94,7 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int it := db.NewIterator(prefix, nil) for it.Next() { - // Skip any keys with the correct prefix but wrong lenth (trie nodes) + // Skip any keys with the correct prefix but wrong length (trie nodes) key := it.Key() if !bytes.HasPrefix(key, prefix) { break diff --git a/core/tx_pool.go b/core/tx_pool.go index a7cdcfabe7..fc5dfb16ac 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -77,7 +77,7 @@ var ( // maximum allowance of the current block. ErrGasLimit = errors.New("exceeds block gas limit") - // ErrNegativeValue is a sanity error to ensure noone is able to specify a + // ErrNegativeValue is a sanity error to ensure no one is able to specify a // transaction with a negative value. ErrNegativeValue = errors.New("negative value") @@ -1151,13 +1151,7 @@ func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirt } // Check for pending transactions for every account that sent new ones promoted := pool.promoteExecutables(promoteAddrs) - for _, tx := range promoted { - addr, _ := types.Sender(pool.signer, tx) - if _, ok := events[addr]; !ok { - events[addr] = newTxSortedMap() - } - events[addr].Put(tx) - } + // If a new block appeared, validate the pool of pending transactions. This will // remove any transaction that has been included in the block or was invalidated // because of another transaction (e.g. higher gas price). @@ -1176,6 +1170,13 @@ func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirt pool.mu.Unlock() // Notify subsystems for newly added transactions + for _, tx := range promoted { + addr, _ := types.Sender(pool.signer, tx) + if _, ok := events[addr]; !ok { + events[addr] = newTxSortedMap() + } + events[addr].Put(tx) + } if len(events) > 0 { var txs []*types.Transaction for _, set := range events { diff --git a/core/vm/contract.go b/core/vm/contract.go index 8e35c92b45..b0eab8b2e5 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -84,7 +84,7 @@ func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uin func (c *Contract) validJumpdest(dest *big.Int) bool { udest := dest.Uint64() - // PC cannot go beyond len(code) and certainly can't be bigger than 63bits. + // PC cannot go beyond len(code) and certainly can't be bigger than 63 bits. // Don't bother checking for JUMPDEST in that case. if dest.BitLen() >= 63 || udest >= uint64(len(c.Code)) { return false @@ -93,6 +93,25 @@ func (c *Contract) validJumpdest(dest *big.Int) bool { if OpCode(c.Code[udest]) != JUMPDEST { return false } + return c.isCode(udest) +} + +func (c *Contract) validJumpSubdest(udest uint64) bool { + // PC cannot go beyond len(code) and certainly can't be bigger than 63 bits. + // Don't bother checking for BEGINSUB in that case. + if int64(udest) < 0 || udest >= uint64(len(c.Code)) { + return false + } + // Only BEGINSUBs allowed for destinations + if OpCode(c.Code[udest]) != BEGINSUB { + return false + } + return c.isCode(udest) +} + +// isCode returns true if the provided PC location is an actual opcode, as +// opposed to a data-segment following a PUSHN operation. +func (c *Contract) isCode(udest uint64) bool { // Do we have a contract hash already? if c.CodeHash != (common.Hash{}) { // Does parent context have the analysis? @@ -103,6 +122,8 @@ func (c *Contract) validJumpdest(dest *big.Int) bool { analysis = codeBitmap(c.Code) c.jumpdests[c.CodeHash] = analysis } + // Also stash it in current contract for faster access + c.analysis = analysis return analysis.codeSegment(udest) } // We don't have the code hash, most likely a piece of initcode not already diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 3fac116b7d..8b54a4cbf8 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -861,7 +861,7 @@ func (c *blake2F) Run(input []byte, caller common.Address, evm *EVM, gas uint64) if err != nil { return nil, gas, err } - // Make sure the input is valid (correct lenth and final flag) + // Make sure the input is valid (correct length and final flag) if len(input) != blake2FInputLength { return nil, gas, errBlake2FInvalidInputLength } diff --git a/core/vm/eips.go b/core/vm/eips.go index 5871afa1e5..735fe1a025 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -32,6 +32,8 @@ func EnableEIP(eipNum int, jt *JumpTable) error { enable1884(jt) case 1344: enable1344(jt) + case 2315: + enable2315(jt) default: return fmt.Errorf("undefined eip %d", eipNum) } @@ -96,3 +98,34 @@ func enable2200(jt *JumpTable) { // jt[SLOAD].constantGas = params.SloadGasEIP2200 jt[SSTORE].dynamicGas = gasSStoreEIP2200 } + +// enable2315 applies EIP-2315 (Simple Subroutines) +// - Adds opcodes that jump to and return from subroutines +func enable2315(jt *JumpTable) { + // New opcode + jt[BEGINSUB] = operation{ + execute: opBeginSub, + constantGas: GasQuickStep, + minStack: minStack(0, 0), + maxStack: maxStack(0, 0), + valid: true, + } + // New opcode + jt[JUMPSUB] = operation{ + execute: opJumpSub, + constantGas: GasSlowStep, + minStack: minStack(1, 0), + maxStack: maxStack(1, 0), + jumps: true, + valid: true, + } + // New opcode + jt[RETURNSUB] = operation{ + execute: opReturnSub, + constantGas: GasFastStep, + minStack: minStack(0, 0), + maxStack: maxStack(0, 0), + valid: true, + jumps: true, + } +} diff --git a/core/vm/errors.go b/core/vm/errors.go index b9c77aed79..4055a88111 100644 --- a/core/vm/errors.go +++ b/core/vm/errors.go @@ -23,6 +23,9 @@ import ( // List evm execution errors var ( + // ErrInvalidSubroutineEntry means that a BEGINSUB was reached via iteration, + // as opposed to from a JUMPSUB instruction + ErrInvalidSubroutineEntry = errors.New("invalid subroutine entry") ErrOutOfGas = errors.New("out of gas") ErrCodeStoreOutOfGas = errors.New("contract creation code storage out of gas") ErrDepth = errors.New("max call depth exceeded") @@ -43,6 +46,8 @@ var ( ErrWriteProtection = errors.New("write protection") ErrReturnDataOutOfBounds = errors.New("return data out of bounds") ErrGasUintOverflow = errors.New("gas uint64 overflow") + ErrInvalidRetsub = errors.New("invalid retsub") + ErrReturnStackExceeded = errors.New("return stack limit reached") ) // ErrStackUnderflow wraps an evm error when the items on the stack less diff --git a/core/vm/gas.go b/core/vm/gas.go index 59ce3b39f4..bda326cdc7 100644 --- a/core/vm/gas.go +++ b/core/vm/gas.go @@ -39,7 +39,7 @@ func callGas(isEip150 bool, availableGas, base uint64, callCost *big.Int) (uint6 availableGas = availableGas - base gas := availableGas - availableGas/64 // If the bit length exceeds 64 bit we know that the newly calculated "gas" for EIP150 - // is smaller than the requested amount. Therefor we return the new gas instead + // is smaller than the requested amount. Therefore we return the new gas instead // of returning an error. if !callCost.IsUint64() || gas < callCost.Uint64() { return gas, nil diff --git a/core/vm/gen_structlog.go b/core/vm/gen_structlog.go index d7354c3d79..831bccc87e 100644 --- a/core/vm/gen_structlog.go +++ b/core/vm/gen_structlog.go @@ -23,6 +23,7 @@ func (s StructLog) MarshalJSON() ([]byte, error) { Memory hexutil.Bytes `json:"memory"` MemorySize int `json:"memSize"` Stack []*math.HexOrDecimal256 `json:"stack"` + ReturnStack []math.HexOrDecimal64 `json:"returnStack"` Storage map[common.Hash]common.Hash `json:"-"` Depth int `json:"depth"` RefundCounter uint64 `json:"refund"` @@ -43,6 +44,12 @@ func (s StructLog) MarshalJSON() ([]byte, error) { enc.Stack[k] = (*math.HexOrDecimal256)(v) } } + if s.ReturnStack != nil { + enc.ReturnStack = make([]math.HexOrDecimal64, len(s.ReturnStack)) + for k, v := range s.ReturnStack { + enc.ReturnStack[k] = math.HexOrDecimal64(v) + } + } enc.Storage = s.Storage enc.Depth = s.Depth enc.RefundCounter = s.RefundCounter @@ -62,6 +69,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { Memory *hexutil.Bytes `json:"memory"` MemorySize *int `json:"memSize"` Stack []*math.HexOrDecimal256 `json:"stack"` + ReturnStack []math.HexOrDecimal64 `json:"returnStack"` Storage map[common.Hash]common.Hash `json:"-"` Depth *int `json:"depth"` RefundCounter *uint64 `json:"refund"` @@ -95,6 +103,12 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { s.Stack[k] = (*big.Int)(v) } } + if dec.ReturnStack != nil { + s.ReturnStack = make([]uint64, len(dec.ReturnStack)) + for k, v := range dec.ReturnStack { + s.ReturnStack[k] = uint64(v) + } + } if dec.Storage != nil { s.Storage = dec.Storage } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 78dc5ec20c..b409a7270d 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -654,6 +654,39 @@ func opJumpdest(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ( return nil, nil } +func opBeginSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + return nil, ErrInvalidSubroutineEntry +} + +func opJumpSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + if len(callContext.rstack.data) >= 1023 { + return nil, ErrReturnStackExceeded + } + pos := callContext.stack.pop() + if !pos.IsUint64() { + return nil, ErrInvalidJump + } + posU64 := pos.Uint64() + if !callContext.contract.validJumpSubdest(posU64) { + return nil, ErrInvalidJump + } + callContext.rstack.push(*pc) + *pc = posU64 + 1 + interpreter.intPool.put(pos) + return nil, nil +} + +func opReturnSub(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { + if len(callContext.rstack.data) == 0 { + return nil, ErrInvalidRetsub + } + // Other than the check that the return stack is not empty, there is no + // need to validate the pc from 'returns', since we only ever push valid + //values onto it via jumpsub. + *pc = callContext.rstack.pop() + 1 + return nil, nil +} + func opPc(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) { callContext.stack.push(interpreter.intPool.get().SetUint64(*pc)) return nil, nil diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index c61a575419..d433d3d9da 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -94,6 +94,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu var ( env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) stack = newstack() + rstack = newReturnStack() pc = uint64(0) evmInterpreter = env.interpreter.(*EVMInterpreter) ) @@ -109,7 +110,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu expected := new(big.Int).SetBytes(common.Hex2Bytes(test.Expected)) stack.push(x) stack.push(y) - opFn(&pc, evmInterpreter, &callCtx{nil, stack, nil}) + opFn(&pc, evmInterpreter, &callCtx{nil, stack, rstack, nil}) actual := stack.pop() if actual.Cmp(expected) != 0 { @@ -211,10 +212,10 @@ func TestSAR(t *testing.T) { // getResult is a convenience function to generate the expected values func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( - env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter.(*EVMInterpreter) + env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) + stack, rstack = newstack(), newReturnStack() + pc = uint64(0) + interpreter = env.interpreter.(*EVMInterpreter) ) interpreter.intPool = poolOfIntPools.get() result := make([]TwoOperandTestcase, len(args)) @@ -223,7 +224,7 @@ func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcas y := new(big.Int).SetBytes(common.Hex2Bytes(param.y)) stack.push(x) stack.push(y) - opFn(&pc, interpreter, &callCtx{nil, stack, nil}) + opFn(&pc, interpreter, &callCtx{nil, stack, rstack, nil}) actual := stack.pop() result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} } @@ -263,7 +264,7 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) - stack = newstack() + stack, rstack = newstack(), newReturnStack() evmInterpreter = NewEVMInterpreter(env, &env.vmConfig) ) @@ -281,7 +282,7 @@ func opBenchmark(bench *testing.B, op executionFunc, args ...string) { a := new(big.Int).SetBytes(arg) stack.push(a) } - op(&pc, evmInterpreter, &callCtx{nil, stack, nil}) + op(&pc, evmInterpreter, &callCtx{nil, stack, rstack, nil}) stack.pop() } poolOfIntPools.put(evmInterpreter.intPool) @@ -498,7 +499,7 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) - stack = newstack() + stack, rstack = newstack(), newReturnStack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, &env.vmConfig) ) @@ -509,12 +510,12 @@ func TestOpMstore(t *testing.T) { pc := uint64(0) v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" stack.pushN(new(big.Int).SetBytes(common.Hex2Bytes(v)), big.NewInt(0)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) if got := common.Bytes2Hex(mem.GetCopy(0, 32)); got != v { t.Fatalf("Mstore fail, got %v, expected %v", got, v) } stack.pushN(big.NewInt(0x1), big.NewInt(0)) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) if common.Bytes2Hex(mem.GetCopy(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { t.Fatalf("Mstore failed to overwrite previous value") } @@ -524,7 +525,7 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) - stack = newstack() + stack, rstack = newstack(), newReturnStack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, &env.vmConfig) ) @@ -539,7 +540,7 @@ func BenchmarkOpMstore(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(value, memStart) - opMstore(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opMstore(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) } poolOfIntPools.put(evmInterpreter.intPool) } @@ -547,7 +548,7 @@ func BenchmarkOpMstore(bench *testing.B) { func BenchmarkOpSHA3(bench *testing.B) { var ( env = NewEVM(Context{}, nil, params.IstanbulTestChainConfig, Config{}) - stack = newstack() + stack, rstack = newstack(), newReturnStack() mem = NewMemory() evmInterpreter = NewEVMInterpreter(env, &env.vmConfig) ) @@ -560,7 +561,7 @@ func BenchmarkOpSHA3(bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { stack.pushN(big.NewInt(32), start) - opSha3(&pc, evmInterpreter, &callCtx{mem, stack, nil}) + opSha3(&pc, evmInterpreter, &callCtx{mem, stack, rstack, nil}) } poolOfIntPools.put(evmInterpreter.intPool) } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index e7d08ee81b..bb81763a64 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -67,6 +67,7 @@ type Interpreter interface { type callCtx struct { memory *Memory stack *Stack + rstack *ReturnStack contract *Contract } @@ -167,12 +168,14 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } var ( - op OpCode // current opcode - mem = NewMemory() // bound memory - stack = newstack() // local stack + op OpCode // current opcode + mem = NewMemory() // bound memory + stack = newstack() // local stack + returns = newReturnStack() // local returns stack callContext = &callCtx{ memory: mem, stack: stack, + rstack: returns, contract: contract, } // For optimisation reason we're using uint64 as the program counter. @@ -195,9 +198,9 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer func() { if err != nil { if !logged { - in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, mem, stack, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pcCopy, op, gasCopy, cost, mem, stack, returns, contract, in.evm.depth, err) } else { - in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, mem, stack, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureFault(in.evm, pcCopy, op, gasCopy, cost, mem, stack, returns, contract, in.evm.depth, err) } } }() @@ -282,7 +285,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } if in.cfg.Debug { - in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, mem, stack, contract, in.evm.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pc, op, gasCopy, cost, mem, stack, returns, contract, in.evm.depth, err) logged = true } diff --git a/core/vm/logger.go b/core/vm/logger.go index 4f3a67a788..dd93ce3ea3 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "math/big" + "strings" "time" "github.com/celo-org/celo-blockchain/common" @@ -63,6 +64,7 @@ type StructLog struct { Memory []byte `json:"memory"` MemorySize int `json:"memSize"` Stack []*big.Int `json:"stack"` + ReturnStack []uint64 `json:"returnStack"` Storage map[common.Hash]common.Hash `json:"-"` Depth int `json:"depth"` RefundCounter uint64 `json:"refund"` @@ -72,6 +74,7 @@ type StructLog struct { // overrides for gencodec type structLogMarshaling struct { Stack []*math.HexOrDecimal256 + ReturnStack []math.HexOrDecimal64 Gas math.HexOrDecimal64 GasCost math.HexOrDecimal64 Memory hexutil.Bytes @@ -99,8 +102,8 @@ func (s *StructLog) ErrorString() string { // if you need to retain them beyond the current call. type Tracer interface { CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error - CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error - CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error + CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error + CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error } @@ -137,7 +140,7 @@ func (l *StructLogger) CaptureStart(from common.Address, to common.Address, crea // CaptureState logs a new structured log message and pushes it out to the environment // // CaptureState also tracks SSTORE ops to track dirty values. -func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { // check if already accumulated the specified number of logs if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { return ErrTraceLimitReached @@ -177,8 +180,13 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui if !l.cfg.DisableStorage { storage = l.changedValues[contract.Address()].Copy() } + var rstack []uint64 + if !l.cfg.DisableStack && rStack != nil { + rstck := make([]uint64, len(rStack.data)) + copy(rstck, rStack.data) + } // create a new snapshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, env.StateDB.GetRefund(), err} + log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, rstack, storage, depth, env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) return nil @@ -186,7 +194,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui // CaptureFault implements the Tracer interface to trace an execution fault // while running an opcode. -func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { +func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { return nil } @@ -227,6 +235,12 @@ func WriteTrace(writer io.Writer, logs []StructLog) { fmt.Fprintf(writer, "%08d %x\n", len(log.Stack)-i-1, math.PaddedBigBytes(log.Stack[i], 32)) } } + if len(log.ReturnStack) > 0 { + fmt.Fprintln(writer, "ReturnStack:") + for i := len(log.Stack) - 1; i >= 0; i-- { + fmt.Fprintf(writer, "%08d 0x%x (%d)\n", len(log.Stack)-i-1, log.ReturnStack[i], log.ReturnStack[i]) + } + } if len(log.Memory) > 0 { fmt.Fprintln(writer, "Memory:") fmt.Fprint(writer, hex.Dump(log.Memory)) @@ -254,3 +268,79 @@ func WriteLogs(writer io.Writer, logs []*types.Log) { fmt.Fprintln(writer) } } + +type mdLogger struct { + out io.Writer + cfg *LogConfig +} + +// NewMarkdownLogger creates a logger which outputs information in a format adapted +// for human readability, and is also a valid markdown table +func NewMarkdownLogger(cfg *LogConfig, writer io.Writer) *mdLogger { + l := &mdLogger{writer, cfg} + if l.cfg == nil { + l.cfg = &LogConfig{} + } + return l +} + +func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { + if !create { + fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + from.String(), to.String(), + input, gas, value) + } else { + fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + from.String(), to.String(), + input, gas, value) + } + + fmt.Fprintf(t.out, ` +| Pc | Op | Cost | Stack | RStack | +|-------|-------------|------|-----------|-----------| +`) + return nil +} + +func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { + fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, op, cost) + + if !t.cfg.DisableStack { // format stack + var a []string + for _, elem := range stack.data { + a = append(a, fmt.Sprintf("%d", elem)) + } + b := fmt.Sprintf("[%v]", strings.Join(a, ",")) + fmt.Fprintf(t.out, "%10v |", b) + } + if !t.cfg.DisableStack { // format return stack + var a []string + for _, elem := range rStack.data { + a = append(a, fmt.Sprintf("%2d", elem)) + } + b := fmt.Sprintf("[%v]", strings.Join(a, ",")) + fmt.Fprintf(t.out, "%10v |", b) + } + fmt.Fprintln(t.out, "") + if err != nil { + fmt.Fprintf(t.out, "Error: %v\n", err) + } + return nil +} + +func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { + + fmt.Fprintf(t.out, "\nError: at pc=%d, op=%v: %v\n", pc, op, err) + + return nil +} + +func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) error { + fmt.Fprintf(t.out, ` +Output: 0x%x +Consumed gas: %d +Error: %v +`, + output, gasUsed, err) + return nil +} diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index 9701416c76..12b9011548 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -46,7 +46,7 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create } // CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { +func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { log := StructLog{ Pc: pc, Op: op, @@ -63,12 +63,13 @@ func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint } if !l.cfg.DisableStack { log.Stack = stack.Data() + log.ReturnStack = rStack.data } return l.encoder.Encode(log) } // CaptureFault outputs state information on the logger. -func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { +func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, rStack *ReturnStack, contract *Contract, depth int, err error) error { return nil } diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 85513de6d9..c8579a01a7 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -54,12 +54,13 @@ func TestStoreCapture(t *testing.T) { logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() + rstack = newReturnStack() contract = NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 0) ) stack.push(big.NewInt(1)) stack.push(big.NewInt(0)) var index common.Hash - logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil) + logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, rstack, contract, 0, nil) if len(logger.changedValues[contract.Address()]) == 0 { t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()])) } diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 975283c126..0d9d0e27d6 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -105,18 +105,21 @@ const ( // 0x50 range - 'storage' and execution. const ( - POP OpCode = 0x50 + iota - MLOAD - MSTORE - MSTORE8 - SLOAD - SSTORE - JUMP - JUMPI - PC - MSIZE - GAS - JUMPDEST + POP OpCode = 0x50 + MLOAD OpCode = 0x51 + MSTORE OpCode = 0x52 + MSTORE8 OpCode = 0x53 + SLOAD OpCode = 0x54 + SSTORE OpCode = 0x55 + JUMP OpCode = 0x56 + JUMPI OpCode = 0x57 + PC OpCode = 0x58 + MSIZE OpCode = 0x59 + GAS OpCode = 0x5a + JUMPDEST OpCode = 0x5b + BEGINSUB OpCode = 0x5c + RETURNSUB OpCode = 0x5d + JUMPSUB OpCode = 0x5e ) // 0x60 range. @@ -293,6 +296,10 @@ var opCodeToString = map[OpCode]string{ GAS: "GAS", JUMPDEST: "JUMPDEST", + BEGINSUB: "BEGINSUB", + JUMPSUB: "JUMPSUB", + RETURNSUB: "RETURNSUB", + // 0x60 range - push. PUSH1: "PUSH1", PUSH2: "PUSH2", @@ -455,6 +462,9 @@ var stringToOp = map[string]OpCode{ "MSIZE": MSIZE, "GAS": GAS, "JUMPDEST": JUMPDEST, + "BEGINSUB": BEGINSUB, + "RETURNSUB": RETURNSUB, + "JUMPSUB": JUMPSUB, "PUSH1": PUSH1, "PUSH2": PUSH2, "PUSH3": PUSH3, diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 3540003371..7518c677e4 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -18,8 +18,10 @@ package runtime import ( "math/big" + "os" "strings" "testing" + "time" "github.com/celo-org/celo-blockchain/accounts/abi" "github.com/celo-org/celo-blockchain/common" @@ -357,3 +359,99 @@ func BenchmarkSimpleLoop(b *testing.B) { Execute(code, nil, nil) } } + +type stepCounter struct { + inner *vm.JSONLogger + steps int +} + +func (s *stepCounter) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { + return nil +} + +func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { + s.steps++ + // Enable this for more output + //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) + return nil +} + +func (s *stepCounter) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { + return nil +} + +func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error { + return nil +} + +func TestJumpSub1024Limit(t *testing.T) { + state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + address := common.HexToAddress("0x0a") + // Code is + // 0 beginsub + // 1 push 0 + // 3 jumpsub + // + // The code recursively calls itself. It should error when the returns-stack + // grows above 1023 + state.SetCode(address, []byte{ + byte(vm.PUSH1), 3, + byte(vm.JUMPSUB), + byte(vm.BEGINSUB), + byte(vm.PUSH1), 3, + byte(vm.JUMPSUB), + }) + tracer := stepCounter{inner: vm.NewJSONLogger(nil, os.Stdout)} + // Enable 2315 + _, _, err := Call(address, nil, &Config{State: state, + GasLimit: 20000, + ChainConfig: params.IstanbulTestChainConfig, + EVMConfig: vm.Config{ + ExtraEips: []int{2315}, + Debug: true, + //Tracer: vm.NewJSONLogger(nil, os.Stdout), + Tracer: &tracer, + }}) + exp := "return stack limit reached" + if err.Error() != exp { + t.Fatalf("expected %v, got %v", exp, err) + } + if exp, got := 2048, tracer.steps; exp != got { + t.Fatalf("expected %d steps, got %d", exp, got) + } +} + +func TestReturnSubShallow(t *testing.T) { + state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + address := common.HexToAddress("0x0a") + // The code does returnsub without having anything on the returnstack. + // It should not panic, but just fail after one step + state.SetCode(address, []byte{ + byte(vm.PUSH1), 5, + byte(vm.JUMPSUB), + byte(vm.RETURNSUB), + byte(vm.PC), + byte(vm.BEGINSUB), + byte(vm.RETURNSUB), + byte(vm.PC), + }) + tracer := stepCounter{} + + // Enable 2315 + _, _, err := Call(address, nil, &Config{State: state, + GasLimit: 10000, + ChainConfig: params.IstanbulTestChainConfig, + EVMConfig: vm.Config{ + ExtraEips: []int{2315}, + Debug: true, + Tracer: &tracer, + }}) + + exp := "invalid retsub" + if err.Error() != exp { + t.Fatalf("expected %v, got %v", exp, err) + } + if exp, got := 4, tracer.steps; exp != got { + t.Fatalf("expected %d steps, got %d", exp, got) + } +} diff --git a/core/vm/stack.go b/core/vm/stack.go index c9c3d07f4b..0171ad0dbd 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -86,3 +86,22 @@ func (st *Stack) Print() { } fmt.Println("#############") } + +// ReturnStack is an object for basic return stack operations. +type ReturnStack struct { + data []uint64 +} + +func newReturnStack() *ReturnStack { + return &ReturnStack{data: make([]uint64, 0, 1024)} +} + +func (st *ReturnStack) push(d uint64) { + st.data = append(st.data, d) +} + +func (st *ReturnStack) pop() (ret uint64) { + ret = st.data[len(st.data)-1] + st.data = st.data[:len(st.data)-1] + return +} diff --git a/core/vm/testdata/precompiles/blake2F.json b/core/vm/testdata/precompiles/blake2F.json new file mode 100644 index 0000000000..0fce477ba6 --- /dev/null +++ b/core/vm/testdata/precompiles/blake2F.json @@ -0,0 +1,32 @@ +[ + { + "Input": "0000000048c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "Expected": "08c9bcf367e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d282e6ad7f520e511f6c3e2b8c68059b9442be0454267ce079217e1319cde05b", + "Name": "vector 4", + "NoBenchmark": false + }, + { + "Input": "0000000c48c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "Expected": "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d17d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923", + "Name": "vector 5", + "NoBenchmark": false + }, + { + "Input": "0000000c48c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000", + "Expected": "75ab69d3190a562c51aef8d88f1c2775876944407270c42c9844252c26d2875298743e7f6d5ea2f2d3e8d226039cd31b4e426ac4f2d3d666a610c2116fde4735", + "Name": "vector 6", + "NoBenchmark": false + }, + { + "Input": "0000000148c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "Expected": "b63a380cb2897d521994a85234ee2c181b5f844d2c624c002677e9703449d2fba551b3a8333bcdf5f2f7e08993d53923de3d64fcc68c034e717b9293fed7a421", + "Name": "vector 7", + "NoBenchmark": false + }, + { + "Input": "007A120048c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "Expected": "6d2ce9e534d50e18ff866ae92d70cceba79bbcd14c63819fe48752c8aca87a4bb7dcc230d22a4047f0486cfcfb50a17b24b2899eb8fca370f22240adb5170189", + "Name": "vector 8", + "NoBenchmark": false + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/bn256Add.json b/core/vm/testdata/precompiles/bn256Add.json new file mode 100644 index 0000000000..e211547c69 --- /dev/null +++ b/core/vm/testdata/precompiles/bn256Add.json @@ -0,0 +1,98 @@ +[ + { + "Input": "18b18acfb4c2c30276db5411368e7185b311dd124691610c5d3b74034e093dc9063c909c4720840cb5134cb9f59fa749755796819658d32efc0d288198f3726607c2b7f58a84bd6145f00c9c2bc0bb1a187f20ff2c92963a88019e7c6a014eed06614e20c147e940f2d70da3f74c9a17df361706a4485c742bd6788478fa17d7", + "Expected": "2243525c5efd4b9c3d3c45ac0ca3fe4dd85e830a4ce6b65fa1eeaee202839703301d1d33be6da8e509df21cc35964723180eed7532537db9ae5e7d48f195c915", + "Name": "chfast1", + "NoBenchmark": false + }, + { + "Input": "2243525c5efd4b9c3d3c45ac0ca3fe4dd85e830a4ce6b65fa1eeaee202839703301d1d33be6da8e509df21cc35964723180eed7532537db9ae5e7d48f195c91518b18acfb4c2c30276db5411368e7185b311dd124691610c5d3b74034e093dc9063c909c4720840cb5134cb9f59fa749755796819658d32efc0d288198f37266", + "Expected": "2bd3e6d0f3b142924f5ca7b49ce5b9d54c4703d7ae5648e61d02268b1a0a9fb721611ce0a6af85915e2f1d70300909ce2e49dfad4a4619c8390cae66cefdb204", + "Name": "chfast2", + "NoBenchmark": false + }, + { + "Input": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio1", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio2", + "NoBenchmark": false + }, + { + "Input": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio3", + "NoBenchmark": false + }, + { + "Input": "", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio4", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio5", + "NoBenchmark": false + }, + { + "Input": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Expected": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Name": "cdetrio6", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Name": "cdetrio7", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Expected": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Name": "cdetrio8", + "NoBenchmark": false + }, + { + "Input": "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Name": "cdetrio9", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Name": "cdetrio10", + "NoBenchmark": false + }, + { + "Input": "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + "Expected": "030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd315ed738c0e0a7c92e7845f96b2ae9c0a68a6a449e3538fc7ff3ebf7a5a18a2c4", + "Name": "cdetrio11", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd315ed738c0e0a7c92e7845f96b2ae9c0a68a6a449e3538fc7ff3ebf7a5a18a2c4", + "Name": "cdetrio12", + "NoBenchmark": false + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d98", + "Expected": "15bf2bb17880144b5d1cd2b1f46eff9d617bffd1ca57c37fb5a49bd84e53cf66049c797f9ce0d17083deb32b5e36f2ea2a212ee036598dd7624c168993d1355f", + "Name": "cdetrio13", + "NoBenchmark": false + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa92e83f8d734803fc370eba25ed1f6b8768bd6d83887b87165fc2434fe11a830cb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Expected": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "Name": "cdetrio14", + "NoBenchmark": false + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/bn256Pairing.json b/core/vm/testdata/precompiles/bn256Pairing.json new file mode 100644 index 0000000000..9474b1aa88 --- /dev/null +++ b/core/vm/testdata/precompiles/bn256Pairing.json @@ -0,0 +1,86 @@ +[ + { + "Input": "1c76476f4def4bb94541d57ebba1193381ffa7aa76ada664dd31c16024c43f593034dd2920f673e204fee2811c678745fc819b55d3e9d294e45c9b03a76aef41209dd15ebff5d46c4bd888e51a93cf99a7329636c63514396b4a452003a35bf704bf11ca01483bfa8b34b43561848d28905960114c8ac04049af4b6315a416782bb8324af6cfc93537a2ad1a445cfd0ca2a71acd7ac41fadbf933c2a51be344d120a2a4cf30c1bf9845f20c6fe39e07ea2cce61f0c9bb048165fe5e4de877550111e129f1cf1097710d41c4ac70fcdfa5ba2023c6ff1cbeac322de49d1b6df7c2032c61a830e3c17286de9462bf242fca2883585b93870a73853face6a6bf411198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "jeff1", + "NoBenchmark": false + }, + { + "Input": "2eca0c7238bf16e83e7a1e6c5d49540685ff51380f309842a98561558019fc0203d3260361bb8451de5ff5ecd17f010ff22f5c31cdf184e9020b06fa5997db841213d2149b006137fcfb23036606f848d638d576a120ca981b5b1a5f9300b3ee2276cf730cf493cd95d64677bbb75fc42db72513a4c1e387b476d056f80aa75f21ee6226d31426322afcda621464d0611d226783262e21bb3bc86b537e986237096df1f82dff337dd5972e32a8ad43e28a78a96a823ef1cd4debe12b6552ea5f06967a1237ebfeca9aaae0d6d0bab8e28c198c5a339ef8a2407e31cdac516db922160fa257a5fd5b280642ff47b65eca77e626cb685c84fa6d3b6882a283ddd1198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "jeff2", + "NoBenchmark": false + }, + { + "Input": "0f25929bcb43d5a57391564615c9e70a992b10eafa4db109709649cf48c50dd216da2f5cb6be7a0aa72c440c53c9bbdfec6c36c7d515536431b3a865468acbba2e89718ad33c8bed92e210e81d1853435399a271913a6520736a4729cf0d51eb01a9e2ffa2e92599b68e44de5bcf354fa2642bd4f26b259daa6f7ce3ed57aeb314a9a87b789a58af499b314e13c3d65bede56c07ea2d418d6874857b70763713178fb49a2d6cd347dc58973ff49613a20757d0fcc22079f9abd10c3baee245901b9e027bd5cfc2cb5db82d4dc9677ac795ec500ecd47deee3b5da006d6d049b811d7511c78158de484232fc68daf8a45cf217d1c2fae693ff5871e8752d73b21198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "jeff3", + "NoBenchmark": false + }, + { + "Input": "2f2ea0b3da1e8ef11914acf8b2e1b32d99df51f5f4f206fc6b947eae860eddb6068134ddb33dc888ef446b648d72338684d678d2eb2371c61a50734d78da4b7225f83c8b6ab9de74e7da488ef02645c5a16a6652c3c71a15dc37fe3a5dcb7cb122acdedd6308e3bb230d226d16a105295f523a8a02bfc5e8bd2da135ac4c245d065bbad92e7c4e31bf3757f1fe7362a63fbfee50e7dc68da116e67d600d9bf6806d302580dc0661002994e7cd3a7f224e7ddc27802777486bf80f40e4ca3cfdb186bac5188a98c45e6016873d107f5cd131f3a3e339d0375e58bd6219347b008122ae2b09e539e152ec5364e7e2204b03d11d3caa038bfc7cd499f8176aacbee1f39e4e4afc4bc74790a4a028aff2c3d2538731fb755edefd8cb48d6ea589b5e283f150794b6736f670d6a1033f9b46c6f5204f50813eb85c8dc4b59db1c5d39140d97ee4d2b36d99bc49974d18ecca3e7ad51011956051b464d9e27d46cc25e0764bb98575bd466d32db7b15f582b2d5c452b36aa394b789366e5e3ca5aabd415794ab061441e51d01e94640b7e3084a07e02c78cf3103c542bc5b298669f211b88da1679b0b64a63b7e0e7bfe52aae524f73a55be7fe70c7e9bfc94b4cf0da1213d2149b006137fcfb23036606f848d638d576a120ca981b5b1a5f9300b3ee2276cf730cf493cd95d64677bbb75fc42db72513a4c1e387b476d056f80aa75f21ee6226d31426322afcda621464d0611d226783262e21bb3bc86b537e986237096df1f82dff337dd5972e32a8ad43e28a78a96a823ef1cd4debe12b6552ea5f", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "jeff4", + "NoBenchmark": false + }, + { + "Input": "20a754d2071d4d53903e3b31a7e98ad6882d58aec240ef981fdf0a9d22c5926a29c853fcea789887315916bbeb89ca37edb355b4f980c9a12a94f30deeed30211213d2149b006137fcfb23036606f848d638d576a120ca981b5b1a5f9300b3ee2276cf730cf493cd95d64677bbb75fc42db72513a4c1e387b476d056f80aa75f21ee6226d31426322afcda621464d0611d226783262e21bb3bc86b537e986237096df1f82dff337dd5972e32a8ad43e28a78a96a823ef1cd4debe12b6552ea5f1abb4a25eb9379ae96c84fff9f0540abcfc0a0d11aeda02d4f37e4baf74cb0c11073b3ff2cdbb38755f8691ea59e9606696b3ff278acfc098fa8226470d03869217cee0a9ad79a4493b5253e2e4e3a39fc2df38419f230d341f60cb064a0ac290a3d76f140db8418ba512272381446eb73958670f00cf46f1d9e64cba057b53c26f64a8ec70387a13e41430ed3ee4a7db2059cc5fc13c067194bcc0cb49a98552fd72bd9edb657346127da132e5b82ab908f5816c826acb499e22f2412d1a2d70f25929bcb43d5a57391564615c9e70a992b10eafa4db109709649cf48c50dd2198a1f162a73261f112401aa2db79c7dab1533c9935c77290a6ce3b191f2318d198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "jeff5", + "NoBenchmark": false + }, + { + "Input": "1c76476f4def4bb94541d57ebba1193381ffa7aa76ada664dd31c16024c43f593034dd2920f673e204fee2811c678745fc819b55d3e9d294e45c9b03a76aef41209dd15ebff5d46c4bd888e51a93cf99a7329636c63514396b4a452003a35bf704bf11ca01483bfa8b34b43561848d28905960114c8ac04049af4b6315a416782bb8324af6cfc93537a2ad1a445cfd0ca2a71acd7ac41fadbf933c2a51be344d120a2a4cf30c1bf9845f20c6fe39e07ea2cce61f0c9bb048165fe5e4de877550111e129f1cf1097710d41c4ac70fcdfa5ba2023c6ff1cbeac322de49d1b6df7c103188585e2364128fe25c70558f1560f4f9350baf3959e603cc91486e110936198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000000", + "Name": "jeff6", + "NoBenchmark": false + }, + { + "Input": "", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "empty_data", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000000", + "Name": "one_point", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "two_point_match_2", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "two_point_match_3", + "NoBenchmark": false + }, + { + "Input": "105456a333e6d636854f987ea7bb713dfd0ae8371a72aea313ae0c32c0bf10160cf031d41b41557f3e7e3ba0c51bebe5da8e6ecd855ec50fc87efcdeac168bcc0476be093a6d2b4bbf907172049874af11e1b6267606e00804d3ff0037ec57fd3010c68cb50161b7d1d96bb71edfec9880171954e56871abf3d93cc94d745fa114c059d74e5b6c4ec14ae5864ebe23a71781d86c29fb8fb6cce94f70d3de7a2101b33461f39d9e887dbb100f170a2345dde3c07e256d1dfa2b657ba5cd030427000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000021a2c3013d2ea92e13c800cde68ef56a294b883f6ac35d25f587c09b1b3c635f7290158a80cd3d66530f74dc94c94adb88f5cdb481acca997b6e60071f08a115f2f997f3dbd66a7afe07fe7862ce239edba9e05c5afff7f8a1259c9733b2dfbb929d1691530ca701b4a106054688728c9972c8512e9789e9567aae23e302ccd75", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "two_point_match_4", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed275dc4a288d1afb3cbb1ac09187524c7db36395df7be3b99e673b13a075a65ec1d9befcd05a5323e6da4d435f3b617cdb3af83285c2df711ef39c01571827f9d", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "ten_point_match_1", + "NoBenchmark": false + }, + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002203e205db4f19b37b60121b83a7333706db86431c6d835849957ed8c3928ad7927dc7234fd11d3e8c36c59277c3e6f149d5cd3cfa9a62aee49f8130962b4b3b9195e8aa5b7827463722b8c153931579d3505566b4edf48d498e185f0509de15204bb53b8977e5f92a0bc372742c4830944a59b4fe6b1c0466e2a6dad122b5d2e030644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd31a76dae6d3272396d0cbe61fced2bc532edac647851e3ac53ce1cc9c7e645a83198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c21800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "ten_point_match_2", + "NoBenchmark": false + }, + { + "Input": "105456a333e6d636854f987ea7bb713dfd0ae8371a72aea313ae0c32c0bf10160cf031d41b41557f3e7e3ba0c51bebe5da8e6ecd855ec50fc87efcdeac168bcc0476be093a6d2b4bbf907172049874af11e1b6267606e00804d3ff0037ec57fd3010c68cb50161b7d1d96bb71edfec9880171954e56871abf3d93cc94d745fa114c059d74e5b6c4ec14ae5864ebe23a71781d86c29fb8fb6cce94f70d3de7a2101b33461f39d9e887dbb100f170a2345dde3c07e256d1dfa2b657ba5cd030427000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000021a2c3013d2ea92e13c800cde68ef56a294b883f6ac35d25f587c09b1b3c635f7290158a80cd3d66530f74dc94c94adb88f5cdb481acca997b6e60071f08a115f2f997f3dbd66a7afe07fe7862ce239edba9e05c5afff7f8a1259c9733b2dfbb929d1691530ca701b4a106054688728c9972c8512e9789e9567aae23e302ccd75", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "ten_point_match_3", + "NoBenchmark": false + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/bn256ScalarMul.json b/core/vm/testdata/precompiles/bn256ScalarMul.json new file mode 100644 index 0000000000..eae26a860f --- /dev/null +++ b/core/vm/testdata/precompiles/bn256ScalarMul.json @@ -0,0 +1,110 @@ +[ + { + "Input": "2bd3e6d0f3b142924f5ca7b49ce5b9d54c4703d7ae5648e61d02268b1a0a9fb721611ce0a6af85915e2f1d70300909ce2e49dfad4a4619c8390cae66cefdb20400000000000000000000000000000000000000000000000011138ce750fa15c2", + "Expected": "070a8d6a982153cae4be29d434e8faef8a47b274a053f5a4ee2a6c9c13c31e5c031b8ce914eba3a9ffb989f9cdd5b0f01943074bf4f0f315690ec3cec6981afc", + "Name": "chfast1", + "NoBenchmark": false + }, + { + "Input": "070a8d6a982153cae4be29d434e8faef8a47b274a053f5a4ee2a6c9c13c31e5c031b8ce914eba3a9ffb989f9cdd5b0f01943074bf4f0f315690ec3cec6981afc30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd46", + "Expected": "025a6f4181d2b4ea8b724290ffb40156eb0adb514c688556eb79cdea0752c2bb2eff3f31dea215f1eb86023a133a996eb6300b44da664d64251d05381bb8a02e", + "Name": "chfast2", + "NoBenchmark": false + }, + { + "Input": "025a6f4181d2b4ea8b724290ffb40156eb0adb514c688556eb79cdea0752c2bb2eff3f31dea215f1eb86023a133a996eb6300b44da664d64251d05381bb8a02e183227397098d014dc2822db40c0ac2ecbc0b548b438e5469e10460b6c3e7ea3", + "Expected": "14789d0d4a730b354403b5fac948113739e276c23e0258d8596ee72f9cd9d3230af18a63153e0ec25ff9f2951dd3fa90ed0197bfef6e2a1a62b5095b9d2b4a27", + "Name": "chfast3", + "NoBenchmark": false + }, + { + "Input": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f6ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "Expected": "2cde5879ba6f13c0b5aa4ef627f159a3347df9722efce88a9afbb20b763b4c411aa7e43076f6aee272755a7f9b84832e71559ba0d2e0b17d5f9f01755e5b0d11", + "Name": "cdetrio1", + "NoBenchmark": false + }, + { + "Input": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f630644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000000", + "Expected": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe3163511ddc1c3f25d396745388200081287b3fd1472d8339d5fecb2eae0830451", + "Name": "cdetrio2", + "NoBenchmark": true + }, + { + "Input": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f60000000000000000000000000000000100000000000000000000000000000000", + "Expected": "1051acb0700ec6d42a88215852d582efbaef31529b6fcbc3277b5c1b300f5cf0135b2394bb45ab04b8bd7611bd2dfe1de6a4e6e2ccea1ea1955f577cd66af85b", + "Name": "cdetrio3", + "NoBenchmark": true + }, + { + "Input": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f60000000000000000000000000000000000000000000000000000000000000009", + "Expected": "1dbad7d39dbc56379f78fac1bca147dc8e66de1b9d183c7b167351bfe0aeab742cd757d51289cd8dbd0acf9e673ad67d0f0a89f912af47ed1be53664f5692575", + "Name": "cdetrio4", + "NoBenchmark": true + }, + { + "Input": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f60000000000000000000000000000000000000000000000000000000000000001", + "Expected": "1a87b0584ce92f4593d161480614f2989035225609f08058ccfa3d0f940febe31a2f3c951f6dadcc7ee9007dff81504b0fcd6d7cf59996efdc33d92bf7f9f8f6", + "Name": "cdetrio5", + "NoBenchmark": true + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7cffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "Expected": "29e587aadd7c06722aabba753017c093f70ba7eb1f1c0104ec0564e7e3e21f6022b1143f6a41008e7755c71c3d00b6b915d386de21783ef590486d8afa8453b1", + "Name": "cdetrio6", + "NoBenchmark": false + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000000", + "Expected": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa92e83f8d734803fc370eba25ed1f6b8768bd6d83887b87165fc2434fe11a830cb", + "Name": "cdetrio7", + "NoBenchmark": true + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c0000000000000000000000000000000100000000000000000000000000000000", + "Expected": "221a3577763877920d0d14a91cd59b9479f83b87a653bb41f82a3f6f120cea7c2752c7f64cdd7f0e494bff7b60419f242210f2026ed2ec70f89f78a4c56a1f15", + "Name": "cdetrio8", + "NoBenchmark": true + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c0000000000000000000000000000000000000000000000000000000000000009", + "Expected": "228e687a379ba154554040f8821f4e41ee2be287c201aa9c3bc02c9dd12f1e691e0fd6ee672d04cfd924ed8fdc7ba5f2d06c53c1edc30f65f2af5a5b97f0a76a", + "Name": "cdetrio9", + "NoBenchmark": true + }, + { + "Input": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c0000000000000000000000000000000000000000000000000000000000000001", + "Expected": "17c139df0efee0f766bc0204762b774362e4ded88953a39ce849a8a7fa163fa901e0559bacb160664764a357af8a9fe70baa9258e0b959273ffc5718c6d4cc7c", + "Name": "cdetrio10", + "NoBenchmark": true + }, + { + "Input": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d98ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "Expected": "00a1a234d08efaa2616607e31eca1980128b00b415c845ff25bba3afcb81dc00242077290ed33906aeb8e42fd98c41bcb9057ba03421af3f2d08cfc441186024", + "Name": "cdetrio11", + "NoBenchmark": false + }, + { + "Input": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d9830644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000000", + "Expected": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b8692929ee761a352600f54921df9bf472e66217e7bb0cee9032e00acc86b3c8bfaf", + "Name": "cdetrio12", + "NoBenchmark": true + }, + { + "Input": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d980000000000000000000000000000000100000000000000000000000000000000", + "Expected": "1071b63011e8c222c5a771dfa03c2e11aac9666dd097f2c620852c3951a4376a2f46fe2f73e1cf310a168d56baa5575a8319389d7bfa6b29ee2d908305791434", + "Name": "cdetrio13", + "NoBenchmark": true + }, + { + "Input": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d980000000000000000000000000000000000000000000000000000000000000009", + "Expected": "19f75b9dd68c080a688774a6213f131e3052bd353a304a189d7a2ee367e3c2582612f545fb9fc89fde80fd81c68fc7dcb27fea5fc124eeda69433cf5c46d2d7f", + "Name": "cdetrio14", + "NoBenchmark": true + }, + { + "Input": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d980000000000000000000000000000000000000000000000000000000000000001", + "Expected": "039730ea8dff1254c0fee9c0ea777d29a9c710b7e616683f194f18c43b43b869073a5ffcc6fc7a28c30723d6e58ce577356982d65b833a5a5c15bf9024b43d98", + "Name": "cdetrio15", + "NoBenchmark": true + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/ecRecover.json b/core/vm/testdata/precompiles/ecRecover.json new file mode 100644 index 0000000000..ed07ac3ad9 --- /dev/null +++ b/core/vm/testdata/precompiles/ecRecover.json @@ -0,0 +1,32 @@ +[ + { + "Input": "a8b53bdf3306a35a7103ab5504a0c9b492295564b6202b1942a84ef300107281000000000000000000000000000000000000000000000000000000000000001b307835653165303366353363653138623737326363623030393366663731663366353366356337356237346463623331613835616138623838393262346538621122334455667788991011121314151617181920212223242526272829303132", + "Expected": "", + "Name": "CallEcrecoverUnrecoverableKey", + "NoBenchmark": false + }, + { + "Input": "18c547e4f7b0f325ad1e56f57e26c745b09a3e503d86e00e5255ff7f715d3d1c000000000000000000000000000000000000000000000000000000000000001c73b1693892219d736caba55bdb67216e485557ea6b6af75f37096c9aa6a5a75feeb940b1d03b21e36b0e47e79769f095fe2ab855bd91e3a38756b7d75a9c4549", + "Expected": "000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "Name": "ValidKey", + "NoBenchmark": false + }, + { + "Input": "18c547e4f7b0f325ad1e56f57e26c745b09a3e503d86e00e5255ff7f715d3d1c100000000000000000000000000000000000000000000000000000000000001c73b1693892219d736caba55bdb67216e485557ea6b6af75f37096c9aa6a5a75feeb940b1d03b21e36b0e47e79769f095fe2ab855bd91e3a38756b7d75a9c4549", + "Expected": "", + "Name": "InvalidHighV-bits-1", + "NoBenchmark": false + }, + { + "Input": "18c547e4f7b0f325ad1e56f57e26c745b09a3e503d86e00e5255ff7f715d3d1c000000000000000000000000000000000000001000000000000000000000001c73b1693892219d736caba55bdb67216e485557ea6b6af75f37096c9aa6a5a75feeb940b1d03b21e36b0e47e79769f095fe2ab855bd91e3a38756b7d75a9c4549", + "Expected": "", + "Name": "InvalidHighV-bits-2", + "NoBenchmark": false + }, + { + "Input": "18c547e4f7b0f325ad1e56f57e26c745b09a3e503d86e00e5255ff7f715d3d1c000000000000000000000000000000000000001000000000000000000000011c73b1693892219d736caba55bdb67216e485557ea6b6af75f37096c9aa6a5a75feeb940b1d03b21e36b0e47e79769f095fe2ab855bd91e3a38756b7d75a9c4549", + "Expected": "", + "Name": "InvalidHighV-bits-3", + "NoBenchmark": false + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/fail-blake2f.json b/core/vm/testdata/precompiles/fail-blake2f.json new file mode 100644 index 0000000000..70835aa5b0 --- /dev/null +++ b/core/vm/testdata/precompiles/fail-blake2f.json @@ -0,0 +1,22 @@ +[ + { + "Input": "", + "ExpectedError": "invalid input length", + "Name": "vector 0: empty input" + }, + { + "Input": "00000c48c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "ExpectedError": "invalid input length", + "Name": "vector 1: less than 213 bytes input" + }, + { + "Input": "000000000c48c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000001", + "ExpectedError": "invalid input length", + "Name": "vector 2: more than 213 bytes input" + }, + { + "Input": "0000000c48c9bdf267e6096a3ba7ca8485ae67bb2bf894fe72f36e3cf1361d5f3af54fa5d182e6ad7f520e511f6c3e2b8c68059b6bbd41fbabd9831f79217e1319cde05b61626300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000002", + "ExpectedError": "invalid final flag", + "Name": "vector 3: malformed final block indicator flag" + } +] \ No newline at end of file diff --git a/core/vm/testdata/precompiles/modexp.json b/core/vm/testdata/precompiles/modexp.json new file mode 100644 index 0000000000..fe70a7f817 --- /dev/null +++ b/core/vm/testdata/precompiles/modexp.json @@ -0,0 +1,104 @@ +[ + { + "Input": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002003fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2efffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f", + "Expected": "0000000000000000000000000000000000000000000000000000000000000001", + "Name": "eip_example1", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000020fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2efffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f", + "Expected": "0000000000000000000000000000000000000000000000000000000000000000", + "Name": "eip_example2", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000040e09ad9675465c53a109fac66a445c91b292d2bb2c5268addb30cd82f80fcb0033ff97c80a5fc6f39193ae969c6ede6710a6b7ac27078a06d90ef1c72e5c85fb502fc9e1f6beb81516545975218075ec2af118cd8798df6e08a147c60fd6095ac2bb02c2908cf4dd7c81f11c289e4bce98f3553768f392a80ce22bf5c4f4a248c6b", + "Expected": "60008f1614cc01dcfb6bfb09c625cf90b47d4468db81b5f8b7a39d42f332eab9b2da8f2d95311648a8f243f4bb13cfb3d8f7f2a3c014122ebb3ed41b02783adc", + "Name": "nagydani-1-square", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000040e09ad9675465c53a109fac66a445c91b292d2bb2c5268addb30cd82f80fcb0033ff97c80a5fc6f39193ae969c6ede6710a6b7ac27078a06d90ef1c72e5c85fb503fc9e1f6beb81516545975218075ec2af118cd8798df6e08a147c60fd6095ac2bb02c2908cf4dd7c81f11c289e4bce98f3553768f392a80ce22bf5c4f4a248c6b", + "Expected": "4834a46ba565db27903b1c720c9d593e84e4cbd6ad2e64b31885d944f68cd801f92225a8961c952ddf2797fa4701b330c85c4b363798100b921a1a22a46a7fec", + "Name": "nagydani-1-qube", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000040e09ad9675465c53a109fac66a445c91b292d2bb2c5268addb30cd82f80fcb0033ff97c80a5fc6f39193ae969c6ede6710a6b7ac27078a06d90ef1c72e5c85fb5010001fc9e1f6beb81516545975218075ec2af118cd8798df6e08a147c60fd6095ac2bb02c2908cf4dd7c81f11c289e4bce98f3553768f392a80ce22bf5c4f4a248c6b", + "Expected": "c36d804180c35d4426b57b50c5bfcca5c01856d104564cd513b461d3c8b8409128a5573e416d0ebe38f5f736766d9dc27143e4da981dfa4d67f7dc474cbee6d2", + "Name": "nagydani-1-pow0x10001", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080cad7d991a00047dd54d3399b6b0b937c718abddef7917c75b6681f40cc15e2be0003657d8d4c34167b2f0bbbca0ccaa407c2a6a07d50f1517a8f22979ce12a81dcaf707cc0cebfc0ce2ee84ee7f77c38b9281b9822a8d3de62784c089c9b18dcb9a2a5eecbede90ea788a862a9ddd9d609c2c52972d63e289e28f6a590ffbf5102e6d893b80aeed5e6e9ce9afa8a5d5675c93a32ac05554cb20e9951b2c140e3ef4e433068cf0fb73bc9f33af1853f64aa27a0028cbf570d7ac9048eae5dc7b28c87c31e5810f1e7fa2cda6adf9f1076dbc1ec1238560071e7efc4e9565c49be9e7656951985860a558a754594115830bcdb421f741408346dd5997bb01c287087", + "Expected": "981dd99c3b113fae3e3eaa9435c0dc96779a23c12a53d1084b4f67b0b053a27560f627b873e3f16ad78f28c94f14b6392def26e4d8896c5e3c984e50fa0b3aa44f1da78b913187c6128baa9340b1e9c9a0fd02cb78885e72576da4a8f7e5a113e173a7a2889fde9d407bd9f06eb05bc8fc7b4229377a32941a02bf4edcc06d70", + "Name": "nagydani-2-square", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080cad7d991a00047dd54d3399b6b0b937c718abddef7917c75b6681f40cc15e2be0003657d8d4c34167b2f0bbbca0ccaa407c2a6a07d50f1517a8f22979ce12a81dcaf707cc0cebfc0ce2ee84ee7f77c38b9281b9822a8d3de62784c089c9b18dcb9a2a5eecbede90ea788a862a9ddd9d609c2c52972d63e289e28f6a590ffbf5103e6d893b80aeed5e6e9ce9afa8a5d5675c93a32ac05554cb20e9951b2c140e3ef4e433068cf0fb73bc9f33af1853f64aa27a0028cbf570d7ac9048eae5dc7b28c87c31e5810f1e7fa2cda6adf9f1076dbc1ec1238560071e7efc4e9565c49be9e7656951985860a558a754594115830bcdb421f741408346dd5997bb01c287087", + "Expected": "d89ceb68c32da4f6364978d62aaa40d7b09b59ec61eb3c0159c87ec3a91037f7dc6967594e530a69d049b64adfa39c8fa208ea970cfe4b7bcd359d345744405afe1cbf761647e32b3184c7fbe87cee8c6c7ff3b378faba6c68b83b6889cb40f1603ee68c56b4c03d48c595c826c041112dc941878f8c5be828154afd4a16311f", + "Name": "nagydani-2-qube", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000080cad7d991a00047dd54d3399b6b0b937c718abddef7917c75b6681f40cc15e2be0003657d8d4c34167b2f0bbbca0ccaa407c2a6a07d50f1517a8f22979ce12a81dcaf707cc0cebfc0ce2ee84ee7f77c38b9281b9822a8d3de62784c089c9b18dcb9a2a5eecbede90ea788a862a9ddd9d609c2c52972d63e289e28f6a590ffbf51010001e6d893b80aeed5e6e9ce9afa8a5d5675c93a32ac05554cb20e9951b2c140e3ef4e433068cf0fb73bc9f33af1853f64aa27a0028cbf570d7ac9048eae5dc7b28c87c31e5810f1e7fa2cda6adf9f1076dbc1ec1238560071e7efc4e9565c49be9e7656951985860a558a754594115830bcdb421f741408346dd5997bb01c287087", + "Expected": "ad85e8ef13fd1dd46eae44af8b91ad1ccae5b7a1c92944f92a19f21b0b658139e0cabe9c1f679507c2de354bf2c91ebd965d1e633978a830d517d2f6f8dd5fd58065d58559de7e2334a878f8ec6992d9b9e77430d4764e863d77c0f87beede8f2f7f2ab2e7222f85cc9d98b8467f4bb72e87ef2882423ebdb6daf02dddac6db2", + "Name": "nagydani-2-pow0x10001", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000100c9130579f243e12451760976261416413742bd7c91d39ae087f46794062b8c239f2a74abf3918605a0e046a7890e049475ba7fbb78f5de6490bd22a710cc04d30088179a919d86c2da62cf37f59d8f258d2310d94c24891be2d7eeafaa32a8cb4b0cfe5f475ed778f45907dc8916a73f03635f233f7a77a00a3ec9ca6761a5bbd558a2318ecd0caa1c5016691523e7e1fa267dd35e70c66e84380bdcf7c0582f540174e572c41f81e93da0b757dff0b0fe23eb03aa19af0bdec3afb474216febaacb8d0381e631802683182b0fe72c28392539850650b70509f54980241dc175191a35d967288b532a7a8223ce2440d010615f70df269501944d4ec16fe4a3cb02d7a85909174757835187cb52e71934e6c07ef43b4c46fc30bbcd0bc72913068267c54a4aabebb493922492820babdeb7dc9b1558fcf7bd82c37c82d3147e455b623ab0efa752fe0b3a67ca6e4d126639e645a0bf417568adbb2a6a4eef62fa1fa29b2a5a43bebea1f82193a7dd98eb483d09bb595af1fa9c97c7f41f5649d976aee3e5e59e2329b43b13bea228d4a93f16ba139ccb511de521ffe747aa2eca664f7c9e33da59075cc335afcd2bf3ae09765f01ab5a7c3e3938ec168b74724b5074247d200d9970382f683d6059b94dbc336603d1dfee714e4b447ac2fa1d99ecb4961da2854e03795ed758220312d101e1e3d87d5313a6d052aebde75110363d", + "Expected": "affc7507ea6d84751ec6b3f0d7b99dbcc263f33330e450d1b3ff0bc3d0874320bf4edd57debd587306988157958cb3cfd369cc0c9c198706f635c9e0f15d047df5cb44d03e2727f26b083c4ad8485080e1293f171c1ed52aef5993a5815c35108e848c951cf1e334490b4a539a139e57b68f44fee583306f5b85ffa57206b3ee5660458858534e5386b9584af3c7f67806e84c189d695e5eb96e1272d06ec2df5dc5fabc6e94b793718c60c36be0a4d031fc84cd658aa72294b2e16fc240aef70cb9e591248e38bd49c5a554d1afa01f38dab72733092f7555334bbef6c8c430119840492380aa95fa025dcf699f0a39669d812b0c6946b6091e6e235337b6f8", + "Name": "nagydani-3-square", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000100c9130579f243e12451760976261416413742bd7c91d39ae087f46794062b8c239f2a74abf3918605a0e046a7890e049475ba7fbb78f5de6490bd22a710cc04d30088179a919d86c2da62cf37f59d8f258d2310d94c24891be2d7eeafaa32a8cb4b0cfe5f475ed778f45907dc8916a73f03635f233f7a77a00a3ec9ca6761a5bbd558a2318ecd0caa1c5016691523e7e1fa267dd35e70c66e84380bdcf7c0582f540174e572c41f81e93da0b757dff0b0fe23eb03aa19af0bdec3afb474216febaacb8d0381e631802683182b0fe72c28392539850650b70509f54980241dc175191a35d967288b532a7a8223ce2440d010615f70df269501944d4ec16fe4a3cb03d7a85909174757835187cb52e71934e6c07ef43b4c46fc30bbcd0bc72913068267c54a4aabebb493922492820babdeb7dc9b1558fcf7bd82c37c82d3147e455b623ab0efa752fe0b3a67ca6e4d126639e645a0bf417568adbb2a6a4eef62fa1fa29b2a5a43bebea1f82193a7dd98eb483d09bb595af1fa9c97c7f41f5649d976aee3e5e59e2329b43b13bea228d4a93f16ba139ccb511de521ffe747aa2eca664f7c9e33da59075cc335afcd2bf3ae09765f01ab5a7c3e3938ec168b74724b5074247d200d9970382f683d6059b94dbc336603d1dfee714e4b447ac2fa1d99ecb4961da2854e03795ed758220312d101e1e3d87d5313a6d052aebde75110363d", + "Expected": "1b280ecd6a6bf906b806d527c2a831e23b238f89da48449003a88ac3ac7150d6a5e9e6b3be4054c7da11dd1e470ec29a606f5115801b5bf53bc1900271d7c3ff3cd5ed790d1c219a9800437a689f2388ba1a11d68f6a8e5b74e9a3b1fac6ee85fc6afbac599f93c391f5dc82a759e3c6c0ab45ce3f5d25d9b0c1bf94cf701ea6466fc9a478dacc5754e593172b5111eeba88557048bceae401337cd4c1182ad9f700852bc8c99933a193f0b94cf1aedbefc48be3bc93ef5cb276d7c2d5462ac8bb0c8fe8923a1db2afe1c6b90d59c534994a6a633f0ead1d638fdc293486bb634ff2c8ec9e7297c04241a61c37e3ae95b11d53343d4ba2b4cc33d2cfa7eb705e", + "Name": "nagydani-3-qube", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000100c9130579f243e12451760976261416413742bd7c91d39ae087f46794062b8c239f2a74abf3918605a0e046a7890e049475ba7fbb78f5de6490bd22a710cc04d30088179a919d86c2da62cf37f59d8f258d2310d94c24891be2d7eeafaa32a8cb4b0cfe5f475ed778f45907dc8916a73f03635f233f7a77a00a3ec9ca6761a5bbd558a2318ecd0caa1c5016691523e7e1fa267dd35e70c66e84380bdcf7c0582f540174e572c41f81e93da0b757dff0b0fe23eb03aa19af0bdec3afb474216febaacb8d0381e631802683182b0fe72c28392539850650b70509f54980241dc175191a35d967288b532a7a8223ce2440d010615f70df269501944d4ec16fe4a3cb010001d7a85909174757835187cb52e71934e6c07ef43b4c46fc30bbcd0bc72913068267c54a4aabebb493922492820babdeb7dc9b1558fcf7bd82c37c82d3147e455b623ab0efa752fe0b3a67ca6e4d126639e645a0bf417568adbb2a6a4eef62fa1fa29b2a5a43bebea1f82193a7dd98eb483d09bb595af1fa9c97c7f41f5649d976aee3e5e59e2329b43b13bea228d4a93f16ba139ccb511de521ffe747aa2eca664f7c9e33da59075cc335afcd2bf3ae09765f01ab5a7c3e3938ec168b74724b5074247d200d9970382f683d6059b94dbc336603d1dfee714e4b447ac2fa1d99ecb4961da2854e03795ed758220312d101e1e3d87d5313a6d052aebde75110363d", + "Expected": "37843d7c67920b5f177372fa56e2a09117df585f81df8b300fba245b1175f488c99476019857198ed459ed8d9799c377330e49f4180c4bf8e8f66240c64f65ede93d601f957b95b83efdee1e1bfde74169ff77002eaf078c71815a9220c80b2e3b3ff22c2f358111d816ebf83c2999026b6de50bfc711ff68705d2f40b753424aefc9f70f08d908b5a20276ad613b4ab4309a3ea72f0c17ea9df6b3367d44fb3acab11c333909e02e81ea2ed404a712d3ea96bba87461720e2d98723e7acd0520ac1a5212dbedcd8dc0c1abf61d4719e319ff4758a774790b8d463cdfe131d1b2dcfee52d002694e98e720cb6ae7ccea353bc503269ba35f0f63bf8d7b672a76", + "Name": "nagydani-3-pow0x10001", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000200db34d0e438249c0ed685c949cc28776a05094e1c48691dc3f2dca5fc3356d2a0663bd376e4712839917eb9a19c670407e2c377a2de385a3ff3b52104f7f1f4e0c7bf7717fb913896693dc5edbb65b760ef1b00e42e9d8f9af17352385e1cd742c9b006c0f669995cb0bb21d28c0aced2892267637b6470d8cee0ab27fc5d42658f6e88240c31d6774aa60a7ebd25cd48b56d0da11209f1928e61005c6eb709f3e8e0aaf8d9b10f7d7e296d772264dc76897ccdddadc91efa91c1903b7232a9e4c3b941917b99a3bc0c26497dedc897c25750af60237aa67934a26a2bc491db3dcc677491944bc1f51d3e5d76b8d846a62db03dedd61ff508f91a56d71028125035c3a44cbb041497c83bf3e4ae2a9613a401cc721c547a2afa3b16a2969933d3626ed6d8a7428648f74122fd3f2a02a20758f7f693892c8fd798b39abac01d18506c45e71432639e9f9505719ee822f62ccbf47f6850f096ff77b5afaf4be7d772025791717dbe5abf9b3f40cff7d7aab6f67e38f62faf510747276e20a42127e7500c444f9ed92baf65ade9e836845e39c4316d9dce5f8e2c8083e2c0acbb95296e05e51aab13b6b8f53f06c9c4276e12b0671133218cc3ea907da3bd9a367096d9202128d14846cc2e20d56fc8473ecb07cecbfb8086919f3971926e7045b853d85a69d026195c70f9f7a823536e2a8f4b3e12e94d9b53a934353451094b8102df3143a0057457d75e8c708b6337a6f5a4fd1a06727acf9fb93e2993c62f3378b37d56c85e7b1e00f0145ebf8e4095bd723166293c60b6ac1252291ef65823c9e040ddad14969b3b340a4ef714db093a587c37766d68b8d6b5016e741587e7e6bf7e763b44f0247e64bae30f994d248bfd20541a333e5b225ef6a61199e301738b1e688f70ec1d7fb892c183c95dc543c3e12adf8a5e8b9ca9d04f9445cced3ab256f29e998e69efaa633a7b60e1db5a867924ccab0a171d9d6e1098dfa15acde9553de599eaa56490c8f411e4985111f3d40bddfc5e301edb01547b01a886550a61158f7e2033c59707789bf7c854181d0c2e2a42a93cf09209747d7082e147eb8544de25c3eb14f2e35559ea0c0f5877f2f3fc92132c0ae9da4e45b2f6c866a224ea6d1f28c05320e287750fbc647368d41116e528014cc1852e5531d53e4af938374daba6cee4baa821ed07117253bb3601ddd00d59a3d7fb2ef1f5a2fbba7c429f0cf9a5b3462410fd833a69118f8be9c559b1000cc608fd877fb43f8e65c2d1302622b944462579056874b387208d90623fcdaf93920ca7a9e4ba64ea208758222ad868501cc2c345e2d3a5ea2a17e5069248138c8a79c0251185d29ee73e5afab5354769142d2bf0cb6712727aa6bf84a6245fcdae66e4938d84d1b9dd09a884818622080ff5f98942fb20acd7e0c916c2d5ea7ce6f7e173315384518f", + "Expected": "8a5aea5f50dcc03dc7a7a272b5aeebc040554dbc1ffe36753c4fc75f7ed5f6c2cc0de3a922bf96c78bf0643a73025ad21f45a4a5cadd717612c511ab2bff1190fe5f1ae05ba9f8fe3624de1de2a817da6072ddcdb933b50216811dbe6a9ca79d3a3c6b3a476b079fd0d05f04fb154e2dd3e5cb83b148a006f2bcbf0042efb2ae7b916ea81b27aac25c3bf9a8b6d35440062ad8eae34a83f3ffa2cc7b40346b62174a4422584f72f95316f6b2bee9ff232ba9739301c97c99a9ded26c45d72676eb856ad6ecc81d36a6de36d7f9dafafee11baa43a4b0d5e4ecffa7b9b7dcefd58c397dd373e6db4acd2b2c02717712e6289bed7c813b670c4a0c6735aa7f3b0f1ce556eae9fcc94b501b2c8781ba50a8c6220e8246371c3c7359fe4ef9da786ca7d98256754ca4e496be0a9174bedbecb384bdf470779186d6a833f068d2838a88d90ef3ad48ff963b67c39cc5a3ee123baf7bf3125f64e77af7f30e105d72c4b9b5b237ed251e4c122c6d8c1405e736299c3afd6db16a28c6a9cfa68241e53de4cd388271fe534a6a9b0dbea6171d170db1b89858468885d08fecbd54c8e471c3e25d48e97ba450b96d0d87e00ac732aaa0d3ce4309c1064bd8a4c0808a97e0143e43a24cfa847635125cd41c13e0574487963e9d725c01375db99c31da67b4cf65eff555f0c0ac416c727ff8d438ad7c42030551d68c2e7adda0abb1ca7c10", + "Name": "nagydani-4-square", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000200db34d0e438249c0ed685c949cc28776a05094e1c48691dc3f2dca5fc3356d2a0663bd376e4712839917eb9a19c670407e2c377a2de385a3ff3b52104f7f1f4e0c7bf7717fb913896693dc5edbb65b760ef1b00e42e9d8f9af17352385e1cd742c9b006c0f669995cb0bb21d28c0aced2892267637b6470d8cee0ab27fc5d42658f6e88240c31d6774aa60a7ebd25cd48b56d0da11209f1928e61005c6eb709f3e8e0aaf8d9b10f7d7e296d772264dc76897ccdddadc91efa91c1903b7232a9e4c3b941917b99a3bc0c26497dedc897c25750af60237aa67934a26a2bc491db3dcc677491944bc1f51d3e5d76b8d846a62db03dedd61ff508f91a56d71028125035c3a44cbb041497c83bf3e4ae2a9613a401cc721c547a2afa3b16a2969933d3626ed6d8a7428648f74122fd3f2a02a20758f7f693892c8fd798b39abac01d18506c45e71432639e9f9505719ee822f62ccbf47f6850f096ff77b5afaf4be7d772025791717dbe5abf9b3f40cff7d7aab6f67e38f62faf510747276e20a42127e7500c444f9ed92baf65ade9e836845e39c4316d9dce5f8e2c8083e2c0acbb95296e05e51aab13b6b8f53f06c9c4276e12b0671133218cc3ea907da3bd9a367096d9202128d14846cc2e20d56fc8473ecb07cecbfb8086919f3971926e7045b853d85a69d026195c70f9f7a823536e2a8f4b3e12e94d9b53a934353451094b8103df3143a0057457d75e8c708b6337a6f5a4fd1a06727acf9fb93e2993c62f3378b37d56c85e7b1e00f0145ebf8e4095bd723166293c60b6ac1252291ef65823c9e040ddad14969b3b340a4ef714db093a587c37766d68b8d6b5016e741587e7e6bf7e763b44f0247e64bae30f994d248bfd20541a333e5b225ef6a61199e301738b1e688f70ec1d7fb892c183c95dc543c3e12adf8a5e8b9ca9d04f9445cced3ab256f29e998e69efaa633a7b60e1db5a867924ccab0a171d9d6e1098dfa15acde9553de599eaa56490c8f411e4985111f3d40bddfc5e301edb01547b01a886550a61158f7e2033c59707789bf7c854181d0c2e2a42a93cf09209747d7082e147eb8544de25c3eb14f2e35559ea0c0f5877f2f3fc92132c0ae9da4e45b2f6c866a224ea6d1f28c05320e287750fbc647368d41116e528014cc1852e5531d53e4af938374daba6cee4baa821ed07117253bb3601ddd00d59a3d7fb2ef1f5a2fbba7c429f0cf9a5b3462410fd833a69118f8be9c559b1000cc608fd877fb43f8e65c2d1302622b944462579056874b387208d90623fcdaf93920ca7a9e4ba64ea208758222ad868501cc2c345e2d3a5ea2a17e5069248138c8a79c0251185d29ee73e5afab5354769142d2bf0cb6712727aa6bf84a6245fcdae66e4938d84d1b9dd09a884818622080ff5f98942fb20acd7e0c916c2d5ea7ce6f7e173315384518f", + "Expected": "5a2664252aba2d6e19d9600da582cdd1f09d7a890ac48e6b8da15ae7c6ff1856fc67a841ac2314d283ffa3ca81a0ecf7c27d89ef91a5a893297928f5da0245c99645676b481b7e20a566ee6a4f2481942bee191deec5544600bb2441fd0fb19e2ee7d801ad8911c6b7750affec367a4b29a22942c0f5f4744a4e77a8b654da2a82571037099e9c6d930794efe5cdca73c7b6c0844e386bdca8ea01b3d7807146bb81365e2cdc6475f8c23e0ff84463126189dc9789f72bbce2e3d2d114d728a272f1345122de23df54c922ec7a16e5c2a8f84da8871482bd258c20a7c09bbcd64c7a96a51029bbfe848736a6ba7bf9d931a9b7de0bcaf3635034d4958b20ae9ab3a95a147b0421dd5f7ebff46c971010ebfc4adbbe0ad94d5498c853e7142c450d8c71de4b2f84edbf8acd2e16d00c8115b150b1c30e553dbb82635e781379fe2a56360420ff7e9f70cc64c00aba7e26ed13c7c19622865ae07248daced36416080f35f8cc157a857ed70ea4f347f17d1bee80fa038abd6e39b1ba06b97264388b21364f7c56e192d4b62d9b161405f32ab1e2594e86243e56fcf2cb30d21adef15b9940f91af681da24328c883d892670c6aa47940867a81830a82b82716895db810df1b834640abefb7db2092dd92912cb9a735175bc447be40a503cf22dfe565b4ed7a3293ca0dfd63a507430b323ee248ec82e843b673c97ad730728cebc", + "Name": "nagydani-4-qube", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000200db34d0e438249c0ed685c949cc28776a05094e1c48691dc3f2dca5fc3356d2a0663bd376e4712839917eb9a19c670407e2c377a2de385a3ff3b52104f7f1f4e0c7bf7717fb913896693dc5edbb65b760ef1b00e42e9d8f9af17352385e1cd742c9b006c0f669995cb0bb21d28c0aced2892267637b6470d8cee0ab27fc5d42658f6e88240c31d6774aa60a7ebd25cd48b56d0da11209f1928e61005c6eb709f3e8e0aaf8d9b10f7d7e296d772264dc76897ccdddadc91efa91c1903b7232a9e4c3b941917b99a3bc0c26497dedc897c25750af60237aa67934a26a2bc491db3dcc677491944bc1f51d3e5d76b8d846a62db03dedd61ff508f91a56d71028125035c3a44cbb041497c83bf3e4ae2a9613a401cc721c547a2afa3b16a2969933d3626ed6d8a7428648f74122fd3f2a02a20758f7f693892c8fd798b39abac01d18506c45e71432639e9f9505719ee822f62ccbf47f6850f096ff77b5afaf4be7d772025791717dbe5abf9b3f40cff7d7aab6f67e38f62faf510747276e20a42127e7500c444f9ed92baf65ade9e836845e39c4316d9dce5f8e2c8083e2c0acbb95296e05e51aab13b6b8f53f06c9c4276e12b0671133218cc3ea907da3bd9a367096d9202128d14846cc2e20d56fc8473ecb07cecbfb8086919f3971926e7045b853d85a69d026195c70f9f7a823536e2a8f4b3e12e94d9b53a934353451094b81010001df3143a0057457d75e8c708b6337a6f5a4fd1a06727acf9fb93e2993c62f3378b37d56c85e7b1e00f0145ebf8e4095bd723166293c60b6ac1252291ef65823c9e040ddad14969b3b340a4ef714db093a587c37766d68b8d6b5016e741587e7e6bf7e763b44f0247e64bae30f994d248bfd20541a333e5b225ef6a61199e301738b1e688f70ec1d7fb892c183c95dc543c3e12adf8a5e8b9ca9d04f9445cced3ab256f29e998e69efaa633a7b60e1db5a867924ccab0a171d9d6e1098dfa15acde9553de599eaa56490c8f411e4985111f3d40bddfc5e301edb01547b01a886550a61158f7e2033c59707789bf7c854181d0c2e2a42a93cf09209747d7082e147eb8544de25c3eb14f2e35559ea0c0f5877f2f3fc92132c0ae9da4e45b2f6c866a224ea6d1f28c05320e287750fbc647368d41116e528014cc1852e5531d53e4af938374daba6cee4baa821ed07117253bb3601ddd00d59a3d7fb2ef1f5a2fbba7c429f0cf9a5b3462410fd833a69118f8be9c559b1000cc608fd877fb43f8e65c2d1302622b944462579056874b387208d90623fcdaf93920ca7a9e4ba64ea208758222ad868501cc2c345e2d3a5ea2a17e5069248138c8a79c0251185d29ee73e5afab5354769142d2bf0cb6712727aa6bf84a6245fcdae66e4938d84d1b9dd09a884818622080ff5f98942fb20acd7e0c916c2d5ea7ce6f7e173315384518f", + "Expected": "bed8b970c4a34849fc6926b08e40e20b21c15ed68d18f228904878d4370b56322d0da5789da0318768a374758e6375bfe4641fca5285ec7171828922160f48f5ca7efbfee4d5148612c38ad683ae4e3c3a053d2b7c098cf2b34f2cb19146eadd53c86b2d7ccf3d83b2c370bfb840913ee3879b1057a6b4e07e110b6bcd5e958bc71a14798c91d518cc70abee264b0d25a4110962a764b364ac0b0dd1ee8abc8426d775ec0f22b7e47b32576afaf1b5a48f64573ed1c5c29f50ab412188d9685307323d990802b81dacc06c6e05a1e901830ba9fcc67688dc29c5e27bde0a6e845ca925f5454b6fb3747edfaa2a5820838fb759eadf57f7cb5cec57fc213ddd8a4298fa079c3c0f472b07fb15aa6a7f0a3780bd296ff6a62e58ef443870b02260bd4fd2bbc98255674b8e1f1f9f8d33c7170b0ebbea4523b695911abbf26e41885344823bd0587115fdd83b721a4e8457a31c9a84b3d3520a07e0e35df7f48e5a9d534d0ec7feef1ff74de6a11e7f93eab95175b6ce22c68d78a642ad642837897ec11349205d8593ac19300207572c38d29ca5dfa03bc14cdbc32153c80e5cc3e739403d34c75915e49beb43094cc6dcafb3665b305ddec9286934ae66ec6b777ca528728c851318eb0f207b39f1caaf96db6eeead6b55ed08f451939314577d42bcc9f97c0b52d0234f88fd07e4c1d7780fdebc025cfffcb572cb27a8c33963", + "Name": "nagydani-4-pow0x10001", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000400c5a1611f8be90071a43db23cc2fe01871cc4c0e8ab5743f6378e4fef77f7f6db0095c0727e20225beb665645403453e325ad5f9aeb9ba99bf3c148f63f9c07cf4fe8847ad5242d6b7d4499f93bd47056ddab8f7dee878fc2314f344dbee2a7c41a5d3db91eff372c730c2fdd3a141a4b61999e36d549b9870cf2f4e632c4d5df5f024f81c028000073a0ed8847cfb0593d36a47142f578f05ccbe28c0c06aeb1b1da027794c48db880278f79ba78ae64eedfea3c07d10e0562668d839749dc95f40467d15cf65b9cfc52c7c4bcef1cda3596dd52631aac942f146c7cebd46065131699ce8385b0db1874336747ee020a5698a3d1a1082665721e769567f579830f9d259cec1a836845109c21cf6b25da572512bf3c42fd4b96e43895589042ab60dd41f497db96aec102087fe784165bb45f942859268fd2ff6c012d9d00c02ba83eace047cc5f7b2c392c2955c58a49f0338d6fc58749c9db2155522ac17914ec216ad87f12e0ee95574613942fa615898c4d9e8a3be68cd6afa4e7a003dedbdf8edfee31162b174f965b20ae752ad89c967b3068b6f722c16b354456ba8e280f987c08e0a52d40a2e8f3a59b94d590aeef01879eb7a90b3ee7d772c839c85519cbeaddc0c193ec4874a463b53fcaea3271d80ebfb39b33489365fc039ae549a17a9ff898eea2f4cb27b8dbee4c17b998438575b2b8d107e4a0d66ba7fca85b41a58a8d51f191a35c856dfbe8aef2b00048a694bbccff832d23c8ca7a7ff0b6c0b3011d00b97c86c0628444d267c951d9e4fb8f83e154b8f74fb51aa16535e498235c5597dac9606ed0be3173a3836baa4e7d756ffe1e2879b415d3846bccd538c05b847785699aefde3e305decb600cd8fb0e7d8de5efc26971a6ad4e6d7a2d91474f1023a0ac4b78dc937da0ce607a45974d2cac1c33a2631ff7fe6144a3b2e5cf98b531a9627dea92c1dc82204d09db0439b6a11dd64b484e1263aa45fd9539b6020b55e3baece3986a8bffc1003406348f5c61265099ed43a766ee4f93f5f9c5abbc32a0fd3ac2b35b87f9ec26037d88275bd7dd0a54474995ee34ed3727f3f97c48db544b1980193a4b76a8a3ddab3591ce527f16d91882e67f0103b5cda53f7da54d489fc4ac08b6ab358a5a04aa9daa16219d50bd672a7cb804ed769d218807544e5993f1c27427104b349906a0b654df0bf69328afd3013fbe430155339c39f236df5557bf92f1ded7ff609a8502f49064ec3d1dbfb6c15d3a4c11a4f8acd12278cbf68acd5709463d12e3338a6eddb8c112f199645e23154a8e60879d2a654e3ed9296aa28f134168619691cd2c6b9e2eba4438381676173fc63c2588a3c5910dc149cf3760f0aa9fa9c3f5faa9162b0bf1aac9dd32b706a60ef53cbdb394b6b40222b5bc80eea82ba8958386672564cae3794f977871ab62337cf02e30049201ec12937e7ce79d0f55d9c810e20acf52212aca1d3888949e0e4830aad88d804161230eb89d4d329cc83570fe257217d2119134048dd2ed167646975fc7d77136919a049ea74cf08ddd2b896890bb24a0ba18094a22baa351bf29ad96c66bbb1a598f2ca391749620e62d61c3561a7d3653ccc8892c7b99baaf76bf836e2991cb06d6bc0514568ff0d1ec8bb4b3d6984f5eaefb17d3ea2893722375d3ddb8e389a8eef7d7d198f8e687d6a513983df906099f9a2d23f4f9dec6f8ef2f11fc0a21fac45353b94e00486f5e17d386af42502d09db33cf0cf28310e049c07e88682aeeb00cb833c5174266e62407a57583f1f88b304b7c6e0c84bbe1c0fd423072d37a5bd0aacf764229e5c7cd02473460ba3645cd8e8ae144065bf02d0dd238593d8e230354f67e0b2f23012c23274f80e3ee31e35e2606a4a3f31d94ab755e6d163cff52cbb36b6d0cc67ffc512aeed1dce4d7a0d70ce82f2baba12e8d514dc92a056f994adfb17b5b9712bd5186f27a2fda1f7039c5df2c8587fdc62f5627580c13234b55be4df3056050e2d1ef3218f0dd66cb05265fe1acfb0989d8213f2c19d1735a7cf3fa65d88dad5af52dc2bba22b7abf46c3bc77b5091baab9e8f0ddc4d5e581037de91a9f8dcbc69309be29cc815cf19a20a7585b8b3073edf51fc9baeb3e509b97fa4ecfd621e0fd57bd61cac1b895c03248ff12bdbc57509250df3517e8a3fe1d776836b34ab352b973d932ef708b14f7418f9eceb1d87667e61e3e758649cb083f01b133d37ab2f5afa96d6c84bcacf4efc3851ad308c1e7d9113624fce29fab460ab9d2a48d92cdb281103a5250ad44cb2ff6e67ac670c02fdafb3e0f1353953d6d7d5646ca1568dea55275a050ec501b7c6250444f7219f1ba7521ba3b93d089727ca5f3bbe0d6c1300b423377004954c5628fdb65770b18ced5c9b23a4a5a6d6ef25fe01b4ce278de0bcc4ed86e28a0a68818ffa40970128cf2c38740e80037984428c1bd5113f40ff47512ee6f4e4d8f9b8e8e1b3040d2928d003bd1c1329dc885302fbce9fa81c23b4dc49c7c82d29b52957847898676c89aa5d32b5b0e1c0d5a2b79a19d67562f407f19425687971a957375879d90c5f57c857136c17106c9ab1b99d80e69c8c954ed386493368884b55c939b8d64d26f643e800c56f90c01079d7c534e3b2b7ae352cefd3016da55f6a85eb803b85e2304915fd2001f77c74e28746293c46e4f5f0fd49cf988aafd0026b8e7a3bab2da5cdce1ea26c2e29ec03f4807fac432662b2d6c060be1c7be0e5489de69d0a6e03a4b9117f9244b34a0f1ecba89884f781c6320412413a00c4980287409a2a78c2cd7e65cecebbe4ec1c28cac4dd95f6998e78fc6f1392384331c9436aa10e10e2bf8ad2c4eafbcf276aa7bae64b74428911b3269c749338b0fc5075ad", + "Expected": "d61fe4e3f32ac260915b5b03b78a86d11bfc41d973fce5b0cc59035cf8289a8a2e3878ea15fa46565b0d806e2f85b53873ea20ed653869b688adf83f3ef444535bf91598ff7e80f334fb782539b92f39f55310cc4b35349ab7b278346eda9bc37c0d8acd3557fae38197f412f8d9e57ce6a76b7205c23564cab06e5615be7c6f05c3d05ec690cba91da5e89d55b152ff8dd2157dc5458190025cf94b1ad98f7cbe64e9482faba95e6b33844afc640892872b44a9932096508f4a782a4805323808f23e54b6ff9b841dbfa87db3505ae4f687972c18ea0f0d0af89d36c1c2a5b14560c153c3fee406f5cf15cfd1c0bb45d767426d465f2f14c158495069d0c5955a00150707862ecaae30624ebacdd8ac33e4e6aab3ff90b6ba445a84689386b9e945d01823a65874444316e83767290fcff630d2477f49d5d8ffdd200e08ee1274270f86ed14c687895f6caf5ce528bd970c20d2408a9ba66216324c6a011ac4999098362dbd98a038129a2d40c8da6ab88318aa3046cb660327cc44236d9e5d2163bd0959062195c51ed93d0088b6f92051fc99050ece2538749165976233697ab4b610385366e5ce0b02ad6b61c168ecfbedcdf74278a38de340fd7a5fead8e588e294795f9b011e2e60377a89e25c90e145397cdeabc60fd32444a6b7642a611a83c464d8b8976666351b4865c37b02e6dc21dbcdf5f930341707b618cc0f03c3122646b3385c9df9f2ec730eec9d49e7dfc9153b6e6289da8c4f0ebea9ccc1b751948e3bb7171c9e4d57423b0eeeb79095c030cb52677b3f7e0b45c30f645391f3f9c957afa549c4e0b2465b03c67993cd200b1af01035962edbc4c9e89b31c82ac121987d6529dafdeef67a132dc04b6dc68e77f22862040b75e2ceb9ff16da0fca534e6db7bd12fa7b7f51b6c08c1e23dfcdb7acbd2da0b51c87ffbced065a612e9b1c8bba9b7e2d8d7a2f04fcc4aaf355b60d764879a76b5e16762d5f2f55d585d0c8e82df6940960cddfb72c91dfa71f6b4e1c6ca25dfc39a878e998a663c04fe29d5e83b9586d047b4d7ff70a9f0d44f127e7d741685ca75f11629128d916a0ffef4be586a30c4b70389cc746e84ebf177c01ee8a4511cfbb9d1ecf7f7b33c7dd8177896e10bbc82f838dcd6db7ac67de62bf46b6a640fb580c5d1d2708f3862e3d2b645d0d18e49ef088053e3a220adc0e033c2afcfe61c90e32151152eb3caaf746c5e377d541cafc6cbb0cc0fa48b5caf1728f2e1957f5addfc234f1a9d89e40d49356c9172d0561a695fce6dab1d412321bbf407f63766ffd7b6b3d79bcfa07991c5a9709849c1008689e3b47c50d613980bec239fb64185249d055b30375ccb4354d71fe4d05648fbf6c80634dfc3575f2f24abb714c1e4c95e8896763bf4316e954c7ad19e5780ab7a040ca6fb9271f90a8b22ae738daf6cb", + "Name": "nagydani-5-square", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000400c5a1611f8be90071a43db23cc2fe01871cc4c0e8ab5743f6378e4fef77f7f6db0095c0727e20225beb665645403453e325ad5f9aeb9ba99bf3c148f63f9c07cf4fe8847ad5242d6b7d4499f93bd47056ddab8f7dee878fc2314f344dbee2a7c41a5d3db91eff372c730c2fdd3a141a4b61999e36d549b9870cf2f4e632c4d5df5f024f81c028000073a0ed8847cfb0593d36a47142f578f05ccbe28c0c06aeb1b1da027794c48db880278f79ba78ae64eedfea3c07d10e0562668d839749dc95f40467d15cf65b9cfc52c7c4bcef1cda3596dd52631aac942f146c7cebd46065131699ce8385b0db1874336747ee020a5698a3d1a1082665721e769567f579830f9d259cec1a836845109c21cf6b25da572512bf3c42fd4b96e43895589042ab60dd41f497db96aec102087fe784165bb45f942859268fd2ff6c012d9d00c02ba83eace047cc5f7b2c392c2955c58a49f0338d6fc58749c9db2155522ac17914ec216ad87f12e0ee95574613942fa615898c4d9e8a3be68cd6afa4e7a003dedbdf8edfee31162b174f965b20ae752ad89c967b3068b6f722c16b354456ba8e280f987c08e0a52d40a2e8f3a59b94d590aeef01879eb7a90b3ee7d772c839c85519cbeaddc0c193ec4874a463b53fcaea3271d80ebfb39b33489365fc039ae549a17a9ff898eea2f4cb27b8dbee4c17b998438575b2b8d107e4a0d66ba7fca85b41a58a8d51f191a35c856dfbe8aef2b00048a694bbccff832d23c8ca7a7ff0b6c0b3011d00b97c86c0628444d267c951d9e4fb8f83e154b8f74fb51aa16535e498235c5597dac9606ed0be3173a3836baa4e7d756ffe1e2879b415d3846bccd538c05b847785699aefde3e305decb600cd8fb0e7d8de5efc26971a6ad4e6d7a2d91474f1023a0ac4b78dc937da0ce607a45974d2cac1c33a2631ff7fe6144a3b2e5cf98b531a9627dea92c1dc82204d09db0439b6a11dd64b484e1263aa45fd9539b6020b55e3baece3986a8bffc1003406348f5c61265099ed43a766ee4f93f5f9c5abbc32a0fd3ac2b35b87f9ec26037d88275bd7dd0a54474995ee34ed3727f3f97c48db544b1980193a4b76a8a3ddab3591ce527f16d91882e67f0103b5cda53f7da54d489fc4ac08b6ab358a5a04aa9daa16219d50bd672a7cb804ed769d218807544e5993f1c27427104b349906a0b654df0bf69328afd3013fbe430155339c39f236df5557bf92f1ded7ff609a8502f49064ec3d1dbfb6c15d3a4c11a4f8acd12278cbf68acd5709463d12e3338a6eddb8c112f199645e23154a8e60879d2a654e3ed9296aa28f134168619691cd2c6b9e2eba4438381676173fc63c2588a3c5910dc149cf3760f0aa9fa9c3f5faa9162b0bf1aac9dd32b706a60ef53cbdb394b6b40222b5bc80eea82ba8958386672564cae3794f977871ab62337cf03e30049201ec12937e7ce79d0f55d9c810e20acf52212aca1d3888949e0e4830aad88d804161230eb89d4d329cc83570fe257217d2119134048dd2ed167646975fc7d77136919a049ea74cf08ddd2b896890bb24a0ba18094a22baa351bf29ad96c66bbb1a598f2ca391749620e62d61c3561a7d3653ccc8892c7b99baaf76bf836e2991cb06d6bc0514568ff0d1ec8bb4b3d6984f5eaefb17d3ea2893722375d3ddb8e389a8eef7d7d198f8e687d6a513983df906099f9a2d23f4f9dec6f8ef2f11fc0a21fac45353b94e00486f5e17d386af42502d09db33cf0cf28310e049c07e88682aeeb00cb833c5174266e62407a57583f1f88b304b7c6e0c84bbe1c0fd423072d37a5bd0aacf764229e5c7cd02473460ba3645cd8e8ae144065bf02d0dd238593d8e230354f67e0b2f23012c23274f80e3ee31e35e2606a4a3f31d94ab755e6d163cff52cbb36b6d0cc67ffc512aeed1dce4d7a0d70ce82f2baba12e8d514dc92a056f994adfb17b5b9712bd5186f27a2fda1f7039c5df2c8587fdc62f5627580c13234b55be4df3056050e2d1ef3218f0dd66cb05265fe1acfb0989d8213f2c19d1735a7cf3fa65d88dad5af52dc2bba22b7abf46c3bc77b5091baab9e8f0ddc4d5e581037de91a9f8dcbc69309be29cc815cf19a20a7585b8b3073edf51fc9baeb3e509b97fa4ecfd621e0fd57bd61cac1b895c03248ff12bdbc57509250df3517e8a3fe1d776836b34ab352b973d932ef708b14f7418f9eceb1d87667e61e3e758649cb083f01b133d37ab2f5afa96d6c84bcacf4efc3851ad308c1e7d9113624fce29fab460ab9d2a48d92cdb281103a5250ad44cb2ff6e67ac670c02fdafb3e0f1353953d6d7d5646ca1568dea55275a050ec501b7c6250444f7219f1ba7521ba3b93d089727ca5f3bbe0d6c1300b423377004954c5628fdb65770b18ced5c9b23a4a5a6d6ef25fe01b4ce278de0bcc4ed86e28a0a68818ffa40970128cf2c38740e80037984428c1bd5113f40ff47512ee6f4e4d8f9b8e8e1b3040d2928d003bd1c1329dc885302fbce9fa81c23b4dc49c7c82d29b52957847898676c89aa5d32b5b0e1c0d5a2b79a19d67562f407f19425687971a957375879d90c5f57c857136c17106c9ab1b99d80e69c8c954ed386493368884b55c939b8d64d26f643e800c56f90c01079d7c534e3b2b7ae352cefd3016da55f6a85eb803b85e2304915fd2001f77c74e28746293c46e4f5f0fd49cf988aafd0026b8e7a3bab2da5cdce1ea26c2e29ec03f4807fac432662b2d6c060be1c7be0e5489de69d0a6e03a4b9117f9244b34a0f1ecba89884f781c6320412413a00c4980287409a2a78c2cd7e65cecebbe4ec1c28cac4dd95f6998e78fc6f1392384331c9436aa10e10e2bf8ad2c4eafbcf276aa7bae64b74428911b3269c749338b0fc5075ad", + "Expected": "5f9c70ec884926a89461056ad20ac4c30155e817f807e4d3f5bb743d789c83386762435c3627773fa77da5144451f2a8aad8adba88e0b669f5377c5e9bad70e45c86fe952b613f015a9953b8a5de5eaee4566acf98d41e327d93a35bd5cef4607d025e58951167957df4ff9b1627649d3943805472e5e293d3efb687cfd1e503faafeb2840a3e3b3f85d016051a58e1c9498aab72e63b748d834b31eb05d85dcde65e27834e266b85c75cc4ec0135135e0601cb93eeeb6e0010c8ceb65c4c319623c5e573a2c8c9fbbf7df68a930beb412d3f4dfd146175484f45d7afaa0d2e60684af9b34730f7c8438465ad3e1d0c3237336722f2aa51095bd5759f4b8ab4dda111b684aa3dac62a761722e7ae43495b7709933512c81c4e3c9133a51f7ce9f2b51fcec064f65779666960b4e45df3900f54311f5613e8012dd1b8efd359eda31a778264c72aa8bb419d862734d769076bce2810011989a45374e5c5d8729fec21427f0bf397eacbb4220f603cf463a4b0c94efd858ffd9768cd60d6ce68d755e0fbad007ce5c2223d70c7018345a102e4ab3c60a13a9e7794303156d4c2063e919f2153c13961fb324c80b240742f47773a7a8e25b3e3fb19b00ce839346c6eb3c732fbc6b888df0b1fe0a3d07b053a2e9402c267b2d62f794d8a2840526e3ade15ce2264496ccd7519571dfde47f7a4bb16292241c20b2be59f3f8fb4f6383f232d838c5a22d8c95b6834d9d2ca493f5a505ebe8899503b0e8f9b19e6e2dd81c1628b80016d02097e0134de51054c4e7674824d4d758760fc52377d2cad145e259aa2ffaf54139e1a66b1e0c1c191e32ac59474c6b526f5b3ba07d3e5ec286eddf531fcd5292869be58c9f22ef91026159f7cf9d05ef66b4299f4da48cc1635bf2243051d342d378a22c83390553e873713c0454ce5f3234397111ac3fe3207b86f0ed9fc025c81903e1748103692074f83824fda6341be4f95ff00b0a9a208c267e12fa01825054cc0513629bf3dbb56dc5b90d4316f87654a8be18227978ea0a8a522760cad620d0d14fd38920fb7321314062914275a5f99f677145a6979b156bd82ecd36f23f8e1273cc2759ecc0b2c69d94dad5211d1bed939dd87ed9e07b91d49713a6e16ade0a98aea789f04994e318e4ff2c8a188cd8d43aeb52c6daa3bc29b4af50ea82a247c5cd67b573b34cbadcc0a376d3bbd530d50367b42705d870f2e27a8197ef46070528bfe408360faa2ebb8bf76e9f388572842bcb119f4d84ee34ae31f5cc594f23705a49197b181fb78ed1ec99499c690f843a4d0cf2e226d118e9372271054fbabdcc5c92ae9fefaef0589cd0e722eaf30c1703ec4289c7fd81beaa8a455ccee5298e31e2080c10c366a6fcf56f7d13582ad0bcad037c612b710fc595b70fbefaaca23623b60c6c39b11beb8e5843b6b3dac60f", + "Name": "nagydani-5-qube", + "NoBenchmark": false + }, + { + "Input": "000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000400c5a1611f8be90071a43db23cc2fe01871cc4c0e8ab5743f6378e4fef77f7f6db0095c0727e20225beb665645403453e325ad5f9aeb9ba99bf3c148f63f9c07cf4fe8847ad5242d6b7d4499f93bd47056ddab8f7dee878fc2314f344dbee2a7c41a5d3db91eff372c730c2fdd3a141a4b61999e36d549b9870cf2f4e632c4d5df5f024f81c028000073a0ed8847cfb0593d36a47142f578f05ccbe28c0c06aeb1b1da027794c48db880278f79ba78ae64eedfea3c07d10e0562668d839749dc95f40467d15cf65b9cfc52c7c4bcef1cda3596dd52631aac942f146c7cebd46065131699ce8385b0db1874336747ee020a5698a3d1a1082665721e769567f579830f9d259cec1a836845109c21cf6b25da572512bf3c42fd4b96e43895589042ab60dd41f497db96aec102087fe784165bb45f942859268fd2ff6c012d9d00c02ba83eace047cc5f7b2c392c2955c58a49f0338d6fc58749c9db2155522ac17914ec216ad87f12e0ee95574613942fa615898c4d9e8a3be68cd6afa4e7a003dedbdf8edfee31162b174f965b20ae752ad89c967b3068b6f722c16b354456ba8e280f987c08e0a52d40a2e8f3a59b94d590aeef01879eb7a90b3ee7d772c839c85519cbeaddc0c193ec4874a463b53fcaea3271d80ebfb39b33489365fc039ae549a17a9ff898eea2f4cb27b8dbee4c17b998438575b2b8d107e4a0d66ba7fca85b41a58a8d51f191a35c856dfbe8aef2b00048a694bbccff832d23c8ca7a7ff0b6c0b3011d00b97c86c0628444d267c951d9e4fb8f83e154b8f74fb51aa16535e498235c5597dac9606ed0be3173a3836baa4e7d756ffe1e2879b415d3846bccd538c05b847785699aefde3e305decb600cd8fb0e7d8de5efc26971a6ad4e6d7a2d91474f1023a0ac4b78dc937da0ce607a45974d2cac1c33a2631ff7fe6144a3b2e5cf98b531a9627dea92c1dc82204d09db0439b6a11dd64b484e1263aa45fd9539b6020b55e3baece3986a8bffc1003406348f5c61265099ed43a766ee4f93f5f9c5abbc32a0fd3ac2b35b87f9ec26037d88275bd7dd0a54474995ee34ed3727f3f97c48db544b1980193a4b76a8a3ddab3591ce527f16d91882e67f0103b5cda53f7da54d489fc4ac08b6ab358a5a04aa9daa16219d50bd672a7cb804ed769d218807544e5993f1c27427104b349906a0b654df0bf69328afd3013fbe430155339c39f236df5557bf92f1ded7ff609a8502f49064ec3d1dbfb6c15d3a4c11a4f8acd12278cbf68acd5709463d12e3338a6eddb8c112f199645e23154a8e60879d2a654e3ed9296aa28f134168619691cd2c6b9e2eba4438381676173fc63c2588a3c5910dc149cf3760f0aa9fa9c3f5faa9162b0bf1aac9dd32b706a60ef53cbdb394b6b40222b5bc80eea82ba8958386672564cae3794f977871ab62337cf010001e30049201ec12937e7ce79d0f55d9c810e20acf52212aca1d3888949e0e4830aad88d804161230eb89d4d329cc83570fe257217d2119134048dd2ed167646975fc7d77136919a049ea74cf08ddd2b896890bb24a0ba18094a22baa351bf29ad96c66bbb1a598f2ca391749620e62d61c3561a7d3653ccc8892c7b99baaf76bf836e2991cb06d6bc0514568ff0d1ec8bb4b3d6984f5eaefb17d3ea2893722375d3ddb8e389a8eef7d7d198f8e687d6a513983df906099f9a2d23f4f9dec6f8ef2f11fc0a21fac45353b94e00486f5e17d386af42502d09db33cf0cf28310e049c07e88682aeeb00cb833c5174266e62407a57583f1f88b304b7c6e0c84bbe1c0fd423072d37a5bd0aacf764229e5c7cd02473460ba3645cd8e8ae144065bf02d0dd238593d8e230354f67e0b2f23012c23274f80e3ee31e35e2606a4a3f31d94ab755e6d163cff52cbb36b6d0cc67ffc512aeed1dce4d7a0d70ce82f2baba12e8d514dc92a056f994adfb17b5b9712bd5186f27a2fda1f7039c5df2c8587fdc62f5627580c13234b55be4df3056050e2d1ef3218f0dd66cb05265fe1acfb0989d8213f2c19d1735a7cf3fa65d88dad5af52dc2bba22b7abf46c3bc77b5091baab9e8f0ddc4d5e581037de91a9f8dcbc69309be29cc815cf19a20a7585b8b3073edf51fc9baeb3e509b97fa4ecfd621e0fd57bd61cac1b895c03248ff12bdbc57509250df3517e8a3fe1d776836b34ab352b973d932ef708b14f7418f9eceb1d87667e61e3e758649cb083f01b133d37ab2f5afa96d6c84bcacf4efc3851ad308c1e7d9113624fce29fab460ab9d2a48d92cdb281103a5250ad44cb2ff6e67ac670c02fdafb3e0f1353953d6d7d5646ca1568dea55275a050ec501b7c6250444f7219f1ba7521ba3b93d089727ca5f3bbe0d6c1300b423377004954c5628fdb65770b18ced5c9b23a4a5a6d6ef25fe01b4ce278de0bcc4ed86e28a0a68818ffa40970128cf2c38740e80037984428c1bd5113f40ff47512ee6f4e4d8f9b8e8e1b3040d2928d003bd1c1329dc885302fbce9fa81c23b4dc49c7c82d29b52957847898676c89aa5d32b5b0e1c0d5a2b79a19d67562f407f19425687971a957375879d90c5f57c857136c17106c9ab1b99d80e69c8c954ed386493368884b55c939b8d64d26f643e800c56f90c01079d7c534e3b2b7ae352cefd3016da55f6a85eb803b85e2304915fd2001f77c74e28746293c46e4f5f0fd49cf988aafd0026b8e7a3bab2da5cdce1ea26c2e29ec03f4807fac432662b2d6c060be1c7be0e5489de69d0a6e03a4b9117f9244b34a0f1ecba89884f781c6320412413a00c4980287409a2a78c2cd7e65cecebbe4ec1c28cac4dd95f6998e78fc6f1392384331c9436aa10e10e2bf8ad2c4eafbcf276aa7bae64b74428911b3269c749338b0fc5075ad", + "Expected": "5a0eb2bdf0ac1cae8e586689fa16cd4b07dfdedaec8a110ea1fdb059dd5253231b6132987598dfc6e11f86780428982d50cf68f67ae452622c3b336b537ef3298ca645e8f89ee39a26758206a5a3f6409afc709582f95274b57b71fae5c6b74619ae6f089a5393c5b79235d9caf699d23d88fb873f78379690ad8405e34c19f5257d596580c7a6a7206a3712825afe630c76b31cdb4a23e7f0632e10f14f4e282c81a66451a26f8df2a352b5b9f607a7198449d1b926e27036810368e691a74b91c61afa73d9d3b99453e7c8b50fd4f09c039a2f2feb5c419206694c31b92df1d9586140cb3417b38d0c503c7b508cc2ed12e813a1c795e9829eb39ee78eeaf360a169b491a1d4e419574e712402de9d48d54c1ae5e03739b7156615e8267e1fb0a897f067afd11fb33f6e24182d7aaaaa18fe5bc1982f20d6b871e5a398f0f6f718181d31ec225cfa9a0a70124ed9a70031bdf0c1c7829f708b6e17d50419ef361cf77d99c85f44607186c8d683106b8bd38a49b5d0fb503b397a83388c5678dcfcc737499d84512690701ed621a6f0172aecf037184ddf0f2453e4053024018e5ab2e30d6d5363b56e8b41509317c99042f517247474ab3abc848e00a07f69c254f46f2a05cf6ed84e5cc906a518fdcfdf2c61ce731f24c5264f1a25fc04934dc28aec112134dd523f70115074ca34e3807aa4cb925147f3a0ce152d323bd8c675ace446d0fd1ae30c4b57f0eb2c23884bc18f0964c0114796c5b6d080c3d89175665fbf63a6381a6a9da39ad070b645c8bb1779506da14439a9f5b5d481954764ea114fac688930bc68534d403cff4210673b6a6ff7ae416b7cd41404c3d3f282fcd193b86d0f54d0006c2a503b40d5c3930da980565b8f9630e9493a79d1c03e74e5f93ac8e4dc1a901ec5e3b3e57049124c7b72ea345aa359e782285d9e6a5c144a378111dd02c40855ff9c2be9b48425cb0b2fd62dc8678fd151121cf26a65e917d65d8e0dacfae108eb5508b601fb8ffa370be1f9a8b749a2d12eeab81f41079de87e2d777994fa4d28188c579ad327f9957fb7bdecec5c680844dd43cb57cf87aeb763c003e65011f73f8c63442df39a92b946a6bd968a1c1e4d5fa7d88476a68bd8e20e5b70a99259c7d3f85fb1b65cd2e93972e6264e74ebf289b8b6979b9b68a85cd5b360c1987f87235c3c845d62489e33acf85d53fa3561fe3a3aee18924588d9c6eba4edb7a4d106b31173e42929f6f0c48c80ce6a72d54eca7c0fe870068b7a7c89c63cdda593f5b32d3cb4ea8a32c39f00ab449155757172d66763ed9527019d6de6c9f2416aa6203f4d11c9ebee1e1d3845099e55504446448027212616167eb36035726daa7698b075286f5379cd3e93cb3e0cf4f9cb8d017facbb5550ed32d5ec5400ae57e47e2bf78d1eaeff9480cc765ceff39db500", + "Name": "nagydani-5-pow0x10001", + "NoBenchmark": false + } +] \ No newline at end of file diff --git a/crypto/bls12381/arithmetic_x86_adx.go b/crypto/bls12381/arithmetic_x86_adx.go new file mode 100644 index 0000000000..9c30741e6a --- /dev/null +++ b/crypto/bls12381/arithmetic_x86_adx.go @@ -0,0 +1,24 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build amd64,blsadx + +package bls12381 + +// enableADX is true if the ADX/BMI2 instruction set was requested for the BLS +// implementation. The system may still fall back to plain ASM if the necessary +// instructions are unavailable on the CPU. +const enableADX = true diff --git a/crypto/bls12381/arithmetic_x86_noadx.go b/crypto/bls12381/arithmetic_x86_noadx.go new file mode 100644 index 0000000000..eaac4b45d7 --- /dev/null +++ b/crypto/bls12381/arithmetic_x86_noadx.go @@ -0,0 +1,24 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build amd64,blsasm + +package bls12381 + +// enableADX is true if the ADX/BMI2 instruction set was requested for the BLS +// implementation. The system may still fall back to plain ASM if the necessary +// instructions are unavailable on the CPU. +const enableADX = false diff --git a/eth/backend.go b/eth/backend.go index 34c387c3af..2ab6c63a69 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -74,7 +74,7 @@ type Ethereum struct { blockchain *core.BlockChain protocolManager *ProtocolManager lesServer LesServer - dialCandiates enode.Iterator + dialCandidates enode.Iterator // DB interfaces chainDb ethdb.Database // Block chain database @@ -249,7 +249,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.APIBackend = &EthAPIBackend{ctx.ExtRPCEnabled(), eth} - eth.dialCandiates, err = eth.setupDiscovery(&ctx.Config.P2P) + eth.dialCandidates, err = eth.setupDiscovery(&ctx.Config.P2P) if err != nil { return nil, err } @@ -588,7 +588,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { for i, vsn := range istanbul.ProtocolVersions { protos[i] = s.protocolManager.makeProtocol(vsn) protos[i].Attributes = []enr.Entry{s.currentEthEntry()} - protos[i].DialCandidates = s.dialCandiates + protos[i].DialCandidates = s.dialCandidates } if s.lesServer != nil { protos = append(protos, s.lesServer.Protocols()...) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 683de09d3a..47cdc36783 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -771,7 +771,7 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) // If we're doing a light sync, ensure the floor doesn't go below the CHT, as // all headers before that point will be missing. if !d.Mode.SyncFullBlockChain() { - // If we dont know the current CHT position, find it + // If we don't know the current CHT position, find it if d.genesis == 0 { header := d.lightchain.CurrentHeader() for header != nil { @@ -1319,7 +1319,7 @@ func (d *Downloader) fetchReceipts(from uint64) error { // - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) // - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks // - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) -// - kind: textual label of the type being downloaded to display in log mesages +// - kind: textual label of the type being downloaded to display in log messages func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, expire func() map[string]int, pending func() int, inFlight func() bool, throttle func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, error), fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 207b3b566c..08faff947e 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -1040,7 +1040,7 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { } } -// Tests that a peer advertising an high TD doesn't get to stall the downloader +// Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) } func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 7fc99c6bac..91ca343027 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -155,7 +155,7 @@ func (p *peerConnection) FetchHeaders(from uint64, count int) error { } p.headerStarted = time.Now() - // Issue the header retrieval request (absolut upwards without gaps) + // Issue the header retrieval request (absolute upwards without gaps) go p.peer.RequestHeadersByNumber(from, count, 0, false) return nil diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index c6c4cd8e52..b9f074a150 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -515,7 +515,7 @@ func (f *TxFetcher) loop() { // Schedule a new transaction retrieval f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) - // No idea if we sheduled something or not, trigger the timer if needed + // No idea if we scheduled something or not, trigger the timer if needed // TODO(karalabe): this is kind of lame, can't we dump it into scheduleFetches somehow? f.rescheduleTimeout(timeoutTimer, timeoutTrigger) diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index cfe837d18d..16c365e522 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -30,7 +30,7 @@ import ( ) var ( - // testTxs is a set of transactions to use during testing that have meaninful hashes. + // testTxs is a set of transactions to use during testing that have meaningful hashes. testTxs = []*types.Transaction{ types.NewTransaction(5577006791947779410, common.Address{0x0f}, new(big.Int), 0, new(big.Int), nil, nil, nil, nil), types.NewTransaction(15352856648520921629, common.Address{0xbb}, new(big.Int), 0, new(big.Int), nil, nil, nil, nil), @@ -449,7 +449,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { }) } -// Tests that non-returned transactions are either re-sheduled from a +// Tests that non-returned transactions are either re-scheduled from a // different peer, or self if they are after the cutoff point. func TestTransactionFetcherMissingRescheduling(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ diff --git a/eth/sync.go b/eth/sync.go index 6fc652219c..054084c591 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -202,7 +202,6 @@ func (cs *chainSyncer) loop() { cs.pm.txFetcher.Start() defer cs.pm.blockFetcher.Stop() defer cs.pm.txFetcher.Stop() - defer cs.pm.downloader.Terminate() // The force timer lowers the peer count threshold down to one when it fires. // This ensures we'll always start sync even if there aren't enough peers. @@ -225,8 +224,13 @@ func (cs *chainSyncer) loop() { cs.forced = true case <-cs.pm.quitSync: + // Disable all insertion on the blockchain. This needs to happen before + // terminating the downloader because the downloader waits for blockchain + // inserts, and these can take a long time to finish. + cs.pm.blockchain.StopInsert() + cs.pm.downloader.Terminate() if cs.doneCh != nil { - cs.pm.downloader.Terminate() // Double term is fine, Cancel would block until queue is emptied + // Wait for the current sync to end. <-cs.doneCh } return @@ -240,7 +244,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { return nil // Sync already running. } - // Ensure we're at mininum peer count. + // Ensure we're at minimum peer count. minPeers := defaultMinSyncPeers if cs.forced { minPeers = 1 diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index b8f1702ecc..4922aca4b0 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -541,7 +541,7 @@ func (jst *Tracer) CaptureStart(from common.Address, to common.Address, create b } // CaptureState implements the Tracer interface to trace a single step of VM execution. -func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { +func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { if jst.err == nil { // Initialize the context if it wasn't done yet if !jst.inited { @@ -580,7 +580,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost // CaptureFault implements the Tracer interface to trace an execution fault // while running an opcode. -func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { +func (jst *Tracer) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, rStack *vm.ReturnStack, contract *vm.Contract, depth int, err error) error { if jst.err == nil { // Apart from the error, everything matches the previous invocation jst.errorValue = new(string) diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 95add0c649..4c7c0cfa7c 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -169,10 +169,10 @@ func TestHaltBetweenSteps(t *testing.T) { env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.IstanbulTestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) timeout := errors.New("stahp") tracer.Stop(timeout) - tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) + tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, nil, contract, 0, nil) if _, err := tracer.GetResult(); err.Error() != timeout.Error() { t.Errorf("Expected timeout error, got %v", err) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index c3834c4b54..bf5c2f71b2 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -248,6 +248,9 @@ func (db *Database) meter(refresh time.Duration) { merr error ) + timer := time.NewTimer(refresh) + defer timer.Stop() + // Iterate ad infinitum and collect the stats for i := 1; errc == nil && merr == nil; i++ { // Retrieve the database stats @@ -399,7 +402,8 @@ func (db *Database) meter(refresh time.Duration) { select { case errc = <-db.quitChan: // Quit requesting, stop hammering the database - case <-time.After(refresh): + case <-timer.C: + timer.Reset(refresh) // Timeout, gather a new set of stats } } diff --git a/go.mod b/go.mod index 4fd5279cf1..9fec059b6a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87 - github.com/elastic/gosigar v0.10.5 github.com/fatih/color v1.3.0 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff @@ -55,6 +54,7 @@ require ( github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect github.com/shopspring/decimal v1.2.0 + github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect @@ -70,7 +70,7 @@ require ( golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce - gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9 + gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc gopkg.in/urfave/cli.v1 v1.20.0 gopkg.in/yaml.v2 v2.2.7 // indirect gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index 7acac49d52..e8895d28e7 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,6 @@ github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmak github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87 h1:OMbqMXf9OAXzH1dDH82mQMrddBE8LIIwDtxeK4wE1/A= github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= -github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= @@ -193,6 +191,8 @@ github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubr github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible h1:+gAR1bMhuoQnZMTWFIvp7ukynULPsteLzG+siZKLtD8= +github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= @@ -269,8 +269,8 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9 h1:ITeyKbRetrVzqR3U1eY+ywgp7IBspGd1U/bkwd1gWu4= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc h1:17cdygvFw3DEyNMh81Bk687W74d5pcC5qEKQICv9N6g= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index d4ccc58329..0edfccec5a 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -77,7 +77,7 @@ func (tt *TestCmd) Run(name string, args ...string) { } } -// InputLine writes the given text to the childs stdin. +// InputLine writes the given text to the child's stdin. // This method can also be called from an expect template, e.g.: // // geth.expect(`Passphrase: {{.InputLine "password"}}`) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index bc0851c747..b0d112de14 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -292,7 +292,7 @@ func (s *PrivateAccountAPI) NewAccount(password string) (common.Address, error) return common.Address{}, err } -// fetchKeystore retrives the encrypted keystore from the account manager. +// fetchKeystore retrieves the encrypted keystore from the account manager. func fetchKeystore(am *accounts.Manager) *keystore.KeyStore { return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) } diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index da4ad9e9e6..2ca5606466 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3056,7 +3056,7 @@ ContractFactory.prototype.at = function (address, callback) { var contract = new Contract(this.eth, this.abi, address); // this functions are not part of prototype, - // because we dont want to spoil the interface + // because we don't want to spoil the interface addFunctionsToContract(contract); addEventsToContract(contract); @@ -13623,7 +13623,7 @@ module.exports = BigNumber; // jshint ignore:line },{}],"web3":[function(require,module,exports){ var Web3 = require('./lib/web3'); -// dont override global variable +// don't override global variable if (typeof window !== 'undefined' && typeof window.Web3 === 'undefined') { window.Web3 = Web3; } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 8431722e00..2b1cc4a242 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -919,10 +919,6 @@ web3._extend({ new web3._extend.Property({ name: 'gatewayFeeCache', getter: 'les_gatewayFeeCache' - }), - new web3._extend.Property({ - name: 'serverPoolEntries', - getter: 'les_serverPoolEntries' }) ] }); diff --git a/les/api.go b/les/api.go index c039e21ccc..66bf8d3b6b 100644 --- a/les/api.go +++ b/les/api.go @@ -424,7 +424,3 @@ func (api *PrivateLightClientAPI) SuggestGatewayFee() (*GatewayFeeInformation, e } return bestGatewayFeeInfo, nil } - -func (api *PrivateLightClientAPI) ServerPoolEntries() ([]*poolEntryInfo, error) { - return api.le.serverPool.Info(), nil -} diff --git a/les/client.go b/les/client.go index 11e94294d9..ea989cba95 100644 --- a/les/client.go +++ b/les/client.go @@ -52,17 +52,18 @@ import ( type LightEthereum struct { lesCommons - peers *serverPeerSet - reqDist *requestDistributor - retriever *retrieveManager - odr *LesOdr - relay *lesTxRelay - handler *clientHandler - txPool *light.TxPool - blockchain *light.LightChain - serverPool *serverPool - chainreader *LightChainReader - valueTracker *lpc.ValueTracker + peers *serverPeerSet + reqDist *requestDistributor + retriever *retrieveManager + odr *LesOdr + relay *lesTxRelay + handler *clientHandler + txPool *light.TxPool + blockchain *light.LightChain + serverPool *serverPool + chainreader *LightChainReader + valueTracker *lpc.ValueTracker + dialCandidates enode.Iterator bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports @@ -122,11 +123,19 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { engine: eth.CreateConsensusEngine(ctx, chainConfig, config, nil, false, chainDb), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), - serverPool: newServerPool(chainDb, config.UltraLightServers), valueTracker: lpc.NewValueTracker(lespayDb, &mclock.System{}, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)), } peers.subscribe((*vtSubscription)(leth.valueTracker)) - leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) + + dnsdisc, err := leth.setupDiscovery(&ctx.Config.P2P) + if err != nil { + return nil, err + } + leth.serverPool = newServerPool(lespayDb, []byte("serverpool:"), leth.valueTracker, dnsdisc, time.Second, nil, &mclock.System{}, config.UltraLightServers) + peers.subscribe(leth.serverPool) + leth.dialCandidates = leth.serverPool.dialIterator + + leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.getTimeout) leth.relay = newLesTxRelay(peers, leth.retriever) leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.retriever) @@ -171,12 +180,6 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.chtIndexer.Start(leth.blockchain) } - // TODO mcortesi (needs etherbase & gatewayFee?) - leth.handler = newClientHandler(syncMode, config.UltraLightServers, config.UltraLightFraction, checkpoint, leth, config.GatewayFee) - if leth.handler.ulc != nil { - log.Warn("Ultra light client is enabled", "trustedNodes", len(leth.handler.ulc.keys), "minTrustedFraction", leth.handler.ulc.fraction) - leth.blockchain.DisableCheckFreq() - } // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -195,7 +198,12 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if istanbul, isIstanbul := leth.engine.(*istanbulBackend.Backend); isIstanbul { istanbul.SetChain(leth.chainreader, nil, nil) } - + // TODO mcortesi (needs etherbase & gatewayFee?) + leth.handler = newClientHandler(syncMode, config.UltraLightServers, config.UltraLightFraction, checkpoint, leth, config.GatewayFee) + if leth.handler.ulc != nil { + log.Warn("Ultra light client is enabled", "trustedNodes", len(leth.handler.ulc.keys), "minTrustedFraction", leth.handler.ulc.fraction) + leth.blockchain.DisableCheckFreq() + } return leth, nil } @@ -303,7 +311,7 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { return p.Info() } return nil - }) + }, s.dialCandidates) } // Start implements node.Service, starting all internal goroutines needed by the @@ -311,15 +319,12 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { func (s *LightEthereum) Start(srvr *p2p.Server) error { log.Warn("Light client mode is an experimental feature") + s.serverPool.start() // Start bloom request workers. s.wg.Add(bloomServiceThreads) s.startBloomHandlers(params.BloomBitsBlocksClient) s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.config.NetworkId) - - // clients are searching for the first advertised protocol in the list - protocolVersion := AdvertiseProtocolVersions[0] - s.serverPool.start(srvr, lesTopic(s.blockchain.Genesis().Hash(), protocolVersion)) return nil } @@ -331,6 +336,8 @@ func (s *LightEthereum) GetRandomPeerEtherbase() common.Address { // Ethereum protocol. func (s *LightEthereum) Stop() error { close(s.closeCh) + s.serverPool.stop() + s.valueTracker.Stop() s.peers.close() s.reqDist.close() s.odr.Stop() @@ -346,8 +353,6 @@ func (s *LightEthereum) Stop() error { s.txPool.Stop() s.engine.Close() s.eventMux.Stop() - s.serverPool.stop() - s.valueTracker.Stop() s.chainDb.Close() s.wg.Wait() log.Info("Light ethereum stopped") diff --git a/les/client_handler.go b/les/client_handler.go index ec364b0a2f..e6459a5b9b 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -136,7 +136,7 @@ func newClientHandler(syncMode downloader.SyncMode, ulcServers []string, ulcFrac if checkpoint != nil { height = (checkpoint.SectionIndex+1)*params.CHTFrequency - 1 } - handler.fetcher = newLightFetcher(handler) + handler.fetcher = newLightFetcher(handler, backend.serverPool.getTimeout) // TODO mcortesi lightest boolean handler.downloader = downloader.New(height, backend.chainDb, nil, backend.eventMux, nil, backend.blockchain, handler.removePeer) handler.backend.peers.subscribe((*downloaderPeerNotify)(handler)) @@ -160,14 +160,9 @@ func (h *clientHandler) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) } peer := newServerPeer(int(version), h.backend.config.NetworkId, trusted, p, newMeteredMsgWriter(rw, int(version))) defer peer.close() - peer.poolEntry = h.backend.serverPool.connect(peer, peer.Node()) - if peer.poolEntry == nil { - return p2p.DiscRequested - } h.wg.Add(1) defer h.wg.Done() err := h.handle(peer) - h.backend.serverPool.disconnect(peer.poolEntry) return err } @@ -214,11 +209,6 @@ func (h *clientHandler) handle(p *serverPeer) error { h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) - // pool entry can be nil during the unit test. - if p.poolEntry != nil { - h.backend.serverPool.registered(p.poolEntry) - } - // Loop until we receive a RequestEtherbase response or timeout. go func() { maxRequests := 10 diff --git a/les/clientpool.go b/les/clientpool.go index f7c1558a71..9e618687d0 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -585,7 +585,7 @@ func (f *clientPool) addBalance(id enode.ID, amount int64, meta string) (uint64, if !c.priority && pb.value > 0 { // The capacity should be adjusted based on the requirement, // but we have no idea about the new capacity, need a second - // call to udpate it. + // call to update it. c.priority = true f.priorityConnected += c.capacity c.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) }) diff --git a/les/commons.go b/les/commons.go index 3790d1650e..3417327e34 100644 --- a/les/commons.go +++ b/les/commons.go @@ -78,7 +78,7 @@ type NodeInfo struct { } // makeProtocols creates protocol descriptors for the given LES versions. -func (c *lesCommons) makeProtocols(versions []uint, runPeer func(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) error, peerInfo func(id enode.ID) interface{}) []p2p.Protocol { +func (c *lesCommons) makeProtocols(versions []uint, runPeer func(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) error, peerInfo func(id enode.ID) interface{}, dialCandidates enode.Iterator) []p2p.Protocol { protos := make([]p2p.Protocol, len(versions)) for i, version := range versions { version := version @@ -90,7 +90,8 @@ func (c *lesCommons) makeProtocols(versions []uint, runPeer func(version uint, p Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error { return runPeer(version, peer, rw) }, - PeerInfo: peerInfo, + PeerInfo: peerInfo, + DialCandidates: dialCandidates, } } return protos diff --git a/les/distributor.go b/les/distributor.go index c8d6ee59ff..083c4ade50 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -180,12 +180,11 @@ func (d *requestDistributor) loop() { type selectPeerItem struct { peer distPeer req *distReq - weight int64 + weight uint64 } -// Weight implements wrsItem interface -func (sp selectPeerItem) Weight() int64 { - return sp.weight +func selectPeerWeight(i interface{}) uint64 { + return i.(selectPeerItem).weight } // nextRequest returns the next possible request from any peer, along with the @@ -220,9 +219,9 @@ func (d *requestDistributor) nextRequest() (distPeer, *distReq, time.Duration) { wait, bufRemain := peer.waitBefore(cost) if wait == 0 { if sel == nil { - sel = utils.NewWeightedRandomSelect() + sel = utils.NewWeightedRandomSelect(selectPeerWeight) } - sel.Update(selectPeerItem{peer: peer, req: req, weight: int64(bufRemain*1000000) + 1}) + sel.Update(selectPeerItem{peer: peer, req: req, weight: uint64(bufRemain*1000000) + 1}) } else { if bestWait == 0 || wait < bestWait { bestWait = wait diff --git a/les/enr_entry.go b/les/enr_entry.go index 64889f3711..120cc42800 100644 --- a/les/enr_entry.go +++ b/les/enr_entry.go @@ -17,6 +17,9 @@ package les import ( + "github.com/celo-org/celo-blockchain/p2p" + "github.com/celo-org/celo-blockchain/p2p/dnsdisc" + "github.com/celo-org/celo-blockchain/p2p/enode" "github.com/celo-org/celo-blockchain/rlp" ) @@ -30,3 +33,12 @@ type lesEntry struct { func (e lesEntry) ENRKey() string { return "les" } + +// setupDiscovery creates the node discovery source for the eth protocol. +func (eth *LightEthereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error) { + if /*cfg.NoDiscovery || */ len(eth.config.DiscoveryURLs) == 0 { + return nil, nil + } + client := dnsdisc.NewClient(dnsdisc.Config{}) + return client.NewIterator(eth.config.DiscoveryURLs...) +} diff --git a/les/fetcher.go b/les/fetcher.go index c5bfddfc71..eed3661cc7 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -41,8 +41,9 @@ const ( // ODR system to ensure that we only request data related to a certain block from peers who have already processed // and announced that block. type lightFetcher struct { - handler *clientHandler - chain *light.LightChain + handler *clientHandler + chain *light.LightChain + softRequestTimeout func() time.Duration lock sync.Mutex // lock protects access to the fetcher's internal state variables except sent requests maxConfirmedTd *big.Int @@ -110,18 +111,19 @@ type fetchResponse struct { } // newLightFetcher creates a new light fetcher -func newLightFetcher(h *clientHandler) *lightFetcher { +func newLightFetcher(h *clientHandler, softRequestTimeout func() time.Duration) *lightFetcher { f := &lightFetcher{ - handler: h, - chain: h.backend.blockchain, - peers: make(map[*serverPeer]*fetcherPeerInfo), - deliverChn: make(chan fetchResponse, 100), - requested: make(map[uint64]fetchRequest), - timeoutChn: make(chan uint64), - requestTrigger: make(chan struct{}, 1), - syncDone: make(chan *serverPeer), - closeCh: make(chan struct{}), - maxConfirmedTd: big.NewInt(0), + handler: h, + chain: h.backend.blockchain, + peers: make(map[*serverPeer]*fetcherPeerInfo), + deliverChn: make(chan fetchResponse, 100), + requested: make(map[uint64]fetchRequest), + timeoutChn: make(chan uint64), + requestTrigger: make(chan struct{}, 1), + syncDone: make(chan *serverPeer), + closeCh: make(chan struct{}), + maxConfirmedTd: big.NewInt(0), + softRequestTimeout: softRequestTimeout, } h.backend.peers.subscribe(f) @@ -164,7 +166,7 @@ func (f *lightFetcher) syncLoop() { f.lock.Unlock() } else { go func() { - time.Sleep(softRequestTimeout) + time.Sleep(f.softRequestTimeout()) f.reqMu.Lock() req, ok := f.requested[reqID] if ok { @@ -188,7 +190,6 @@ func (f *lightFetcher) syncLoop() { } f.reqMu.Unlock() if ok { - f.handler.backend.serverPool.adjustResponseTime(req.peer.poolEntry, time.Duration(mclock.Now()-req.sent), true) req.peer.Log().Debug("Fetching data timed out hard") go f.handler.removePeer(req.peer.id) } @@ -202,9 +203,6 @@ func (f *lightFetcher) syncLoop() { delete(f.requested, resp.reqID) } f.reqMu.Unlock() - if ok { - f.handler.backend.serverPool.adjustResponseTime(req.peer.poolEntry, time.Duration(mclock.Now()-req.sent), req.timeout) - } f.lock.Lock() if !ok || !(f.syncing || f.processResponse(req, resp)) { resp.peer.Log().Debug("Failed processing response") @@ -891,12 +889,10 @@ func (f *lightFetcher) checkUpdateStats(p *serverPeer, newEntry *updateStatsEntr fp.firstUpdateStats = newEntry } for fp.firstUpdateStats != nil && fp.firstUpdateStats.time <= now-mclock.AbsTime(blockDelayTimeout) { - f.handler.backend.serverPool.adjustBlockDelay(p.poolEntry, blockDelayTimeout) fp.firstUpdateStats = fp.firstUpdateStats.next } if fp.confirmedTd != nil { for fp.firstUpdateStats != nil && fp.firstUpdateStats.td.Cmp(fp.confirmedTd) <= 0 { - f.handler.backend.serverPool.adjustBlockDelay(p.poolEntry, time.Duration(now-fp.firstUpdateStats.time)) fp.firstUpdateStats = fp.firstUpdateStats.next } } diff --git a/les/lespay/client/fillset.go b/les/lespay/client/fillset.go new file mode 100644 index 0000000000..2a17edfb64 --- /dev/null +++ b/les/lespay/client/fillset.go @@ -0,0 +1,107 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "sync" + + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +// FillSet tries to read nodes from an input iterator and add them to a node set by +// setting the specified node state flag(s) until the size of the set reaches the target. +// Note that other mechanisms (like other FillSet instances reading from different inputs) +// can also set the same flag(s) and FillSet will always care about the total number of +// nodes having those flags. +type FillSet struct { + lock sync.Mutex + cond *sync.Cond + ns *nodestate.NodeStateMachine + input enode.Iterator + closed bool + flags nodestate.Flags + count, target int +} + +// NewFillSet creates a new FillSet +func NewFillSet(ns *nodestate.NodeStateMachine, input enode.Iterator, flags nodestate.Flags) *FillSet { + fs := &FillSet{ + ns: ns, + input: input, + flags: flags, + } + fs.cond = sync.NewCond(&fs.lock) + + ns.SubscribeState(flags, func(n *enode.Node, oldState, newState nodestate.Flags) { + fs.lock.Lock() + if oldState.Equals(flags) { + fs.count-- + } + if newState.Equals(flags) { + fs.count++ + } + if fs.target > fs.count { + fs.cond.Signal() + } + fs.lock.Unlock() + }) + + go fs.readLoop() + return fs +} + +// readLoop keeps reading nodes from the input and setting the specified flags for them +// whenever the node set size is under the current target +func (fs *FillSet) readLoop() { + for { + fs.lock.Lock() + for fs.target <= fs.count && !fs.closed { + fs.cond.Wait() + } + + fs.lock.Unlock() + if !fs.input.Next() { + return + } + fs.ns.SetState(fs.input.Node(), fs.flags, nodestate.Flags{}, 0) + } +} + +// SetTarget sets the current target for node set size. If the previous target was not +// reached and FillSet was still waiting for the next node from the input then the next +// incoming node will be added to the set regardless of the target. This ensures that +// all nodes coming from the input are eventually added to the set. +func (fs *FillSet) SetTarget(target int) { + fs.lock.Lock() + defer fs.lock.Unlock() + + fs.target = target + if fs.target > fs.count { + fs.cond.Signal() + } +} + +// Close shuts FillSet down and closes the input iterator +func (fs *FillSet) Close() { + fs.lock.Lock() + defer fs.lock.Unlock() + + fs.closed = true + fs.input.Close() + fs.cond.Signal() +} diff --git a/les/lespay/client/fillset_test.go b/les/lespay/client/fillset_test.go new file mode 100644 index 0000000000..fb47920a58 --- /dev/null +++ b/les/lespay/client/fillset_test.go @@ -0,0 +1,113 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "math/rand" + "testing" + "time" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +type testIter struct { + waitCh chan struct{} + nodeCh chan *enode.Node + node *enode.Node +} + +func (i *testIter) Next() bool { + i.waitCh <- struct{}{} + i.node = <-i.nodeCh + return i.node != nil +} + +func (i *testIter) Node() *enode.Node { + return i.node +} + +func (i *testIter) Close() {} + +func (i *testIter) push() { + var id enode.ID + rand.Read(id[:]) + i.nodeCh <- enode.SignNull(new(enr.Record), id) +} + +func (i *testIter) waiting(timeout time.Duration) bool { + select { + case <-i.waitCh: + return true + case <-time.After(timeout): + return false + } +} + +func TestFillSet(t *testing.T) { + ns := nodestate.NewNodeStateMachine(nil, nil, &mclock.Simulated{}, testSetup) + iter := &testIter{ + waitCh: make(chan struct{}), + nodeCh: make(chan *enode.Node), + } + fs := NewFillSet(ns, iter, sfTest1) + ns.Start() + + expWaiting := func(i int, push bool) { + for ; i > 0; i-- { + if !iter.waiting(time.Second * 10) { + t.Fatalf("FillSet not waiting for new nodes") + } + if push { + iter.push() + } + } + } + + expNotWaiting := func() { + if iter.waiting(time.Millisecond * 100) { + t.Fatalf("FillSet unexpectedly waiting for new nodes") + } + } + + expNotWaiting() + fs.SetTarget(3) + expWaiting(3, true) + expNotWaiting() + fs.SetTarget(100) + expWaiting(2, true) + expWaiting(1, false) + // lower the target before the previous one has been filled up + fs.SetTarget(0) + iter.push() + expNotWaiting() + fs.SetTarget(10) + expWaiting(4, true) + expNotWaiting() + // remove all previosly set flags + ns.ForEach(sfTest1, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { + ns.SetState(node, nodestate.Flags{}, sfTest1, 0) + }) + // now expect FillSet to fill the set up again with 10 new nodes + expWaiting(10, true) + expNotWaiting() + + fs.Close() + ns.Stop() +} diff --git a/les/lespay/client/queueiterator.go b/les/lespay/client/queueiterator.go new file mode 100644 index 0000000000..4ce31b47de --- /dev/null +++ b/les/lespay/client/queueiterator.go @@ -0,0 +1,123 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "sync" + + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +// QueueIterator returns nodes from the specified selectable set in the same order as +// they entered the set. +type QueueIterator struct { + lock sync.Mutex + cond *sync.Cond + + ns *nodestate.NodeStateMachine + queue []*enode.Node + nextNode *enode.Node + waitCallback func(bool) + fifo, closed bool +} + +// NewQueueIterator creates a new QueueIterator. Nodes are selectable if they have all the required +// and none of the disabled flags set. When a node is selected the selectedFlag is set which also +// disables further selectability until it is removed or times out. +func NewQueueIterator(ns *nodestate.NodeStateMachine, requireFlags, disableFlags nodestate.Flags, fifo bool, waitCallback func(bool)) *QueueIterator { + qi := &QueueIterator{ + ns: ns, + fifo: fifo, + waitCallback: waitCallback, + } + qi.cond = sync.NewCond(&qi.lock) + + ns.SubscribeState(requireFlags.Or(disableFlags), func(n *enode.Node, oldState, newState nodestate.Flags) { + oldMatch := oldState.HasAll(requireFlags) && oldState.HasNone(disableFlags) + newMatch := newState.HasAll(requireFlags) && newState.HasNone(disableFlags) + if newMatch == oldMatch { + return + } + + qi.lock.Lock() + defer qi.lock.Unlock() + + if newMatch { + qi.queue = append(qi.queue, n) + } else { + id := n.ID() + for i, qn := range qi.queue { + if qn.ID() == id { + copy(qi.queue[i:len(qi.queue)-1], qi.queue[i+1:]) + qi.queue = qi.queue[:len(qi.queue)-1] + break + } + } + } + qi.cond.Signal() + }) + return qi +} + +// Next moves to the next selectable node. +func (qi *QueueIterator) Next() bool { + qi.lock.Lock() + if !qi.closed && len(qi.queue) == 0 { + if qi.waitCallback != nil { + qi.waitCallback(true) + } + for !qi.closed && len(qi.queue) == 0 { + qi.cond.Wait() + } + if qi.waitCallback != nil { + qi.waitCallback(false) + } + } + if qi.closed { + qi.nextNode = nil + qi.lock.Unlock() + return false + } + // Move to the next node in queue. + if qi.fifo { + qi.nextNode = qi.queue[0] + copy(qi.queue[:len(qi.queue)-1], qi.queue[1:]) + qi.queue = qi.queue[:len(qi.queue)-1] + } else { + qi.nextNode = qi.queue[len(qi.queue)-1] + qi.queue = qi.queue[:len(qi.queue)-1] + } + qi.lock.Unlock() + return true +} + +// Close ends the iterator. +func (qi *QueueIterator) Close() { + qi.lock.Lock() + qi.closed = true + qi.lock.Unlock() + qi.cond.Signal() +} + +// Node returns the current node. +func (qi *QueueIterator) Node() *enode.Node { + qi.lock.Lock() + defer qi.lock.Unlock() + + return qi.nextNode +} diff --git a/les/lespay/client/queueiterator_test.go b/les/lespay/client/queueiterator_test.go new file mode 100644 index 0000000000..3aab0c75e9 --- /dev/null +++ b/les/lespay/client/queueiterator_test.go @@ -0,0 +1,106 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "testing" + "time" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +func testNodeID(i int) enode.ID { + return enode.ID{42, byte(i % 256), byte(i / 256)} +} + +func testNodeIndex(id enode.ID) int { + if id[0] != 42 { + return -1 + } + return int(id[1]) + int(id[2])*256 +} + +func testNode(i int) *enode.Node { + return enode.SignNull(new(enr.Record), testNodeID(i)) +} + +func TestQueueIteratorFIFO(t *testing.T) { + testQueueIterator(t, true) +} + +func TestQueueIteratorLIFO(t *testing.T) { + testQueueIterator(t, false) +} + +func testQueueIterator(t *testing.T, fifo bool) { + ns := nodestate.NewNodeStateMachine(nil, nil, &mclock.Simulated{}, testSetup) + qi := NewQueueIterator(ns, sfTest2, sfTest3.Or(sfTest4), fifo, nil) + ns.Start() + for i := 1; i <= iterTestNodeCount; i++ { + ns.SetState(testNode(i), sfTest1, nodestate.Flags{}, 0) + } + next := func() int { + ch := make(chan struct{}) + go func() { + qi.Next() + close(ch) + }() + select { + case <-ch: + case <-time.After(time.Second * 5): + t.Fatalf("Iterator.Next() timeout") + } + node := qi.Node() + ns.SetState(node, sfTest4, nodestate.Flags{}, 0) + return testNodeIndex(node.ID()) + } + exp := func(i int) { + n := next() + if n != i { + t.Errorf("Wrong item returned by iterator (expected %d, got %d)", i, n) + } + } + explist := func(list []int) { + for i := range list { + if fifo { + exp(list[i]) + } else { + exp(list[len(list)-1-i]) + } + } + } + + ns.SetState(testNode(1), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(2), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(3), sfTest2, nodestate.Flags{}, 0) + explist([]int{1, 2, 3}) + ns.SetState(testNode(4), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(5), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(6), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(5), sfTest3, nodestate.Flags{}, 0) + explist([]int{4, 6}) + ns.SetState(testNode(1), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(2), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(3), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(2), sfTest3, nodestate.Flags{}, 0) + ns.SetState(testNode(2), nodestate.Flags{}, sfTest3, 0) + explist([]int{1, 3, 2}) + ns.Stop() +} diff --git a/les/lespay/client/valuetracker.go b/les/lespay/client/valuetracker.go index 4bfeac94fe..32d7afb781 100644 --- a/les/lespay/client/valuetracker.go +++ b/les/lespay/client/valuetracker.go @@ -213,6 +213,15 @@ func (vt *ValueTracker) StatsExpirer() *utils.Expirer { return &vt.statsExpirer } +// StatsExpirer returns the current expiration factor so that other values can be expired +// with the same rate as the service value statistics. +func (vt *ValueTracker) StatsExpFactor() utils.ExpirationFactor { + vt.statsExpLock.RLock() + defer vt.statsExpLock.RUnlock() + + return vt.statsExpFactor +} + // loadFromDb loads the value tracker's state from the database and converts saved // request basket index mapping if it does not match the specified index to name mapping. func (vt *ValueTracker) loadFromDb(mapping []string) error { @@ -500,16 +509,3 @@ func (vt *ValueTracker) RequestStats() []RequestStatsItem { } return res } - -// TotalServiceValue returns the total service value provided by the given node (as -// a function of the weights which are calculated from the request timeout value). -func (vt *ValueTracker) TotalServiceValue(nv *NodeValueTracker, weights ResponseTimeWeights) float64 { - vt.statsExpLock.RLock() - expFactor := vt.statsExpFactor - vt.statsExpLock.RUnlock() - - nv.lock.Lock() - defer nv.lock.Unlock() - - return nv.rtStats.Value(weights, expFactor) -} diff --git a/les/lespay/client/wrsiterator.go b/les/lespay/client/wrsiterator.go new file mode 100644 index 0000000000..8d40e64a4d --- /dev/null +++ b/les/lespay/client/wrsiterator.go @@ -0,0 +1,128 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "sync" + + "github.com/celo-org/celo-blockchain/les/utils" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +// WrsIterator returns nodes from the specified selectable set with a weighted random +// selection. Selection weights are provided by a callback function. +type WrsIterator struct { + lock sync.Mutex + cond *sync.Cond + + ns *nodestate.NodeStateMachine + wrs *utils.WeightedRandomSelect + nextNode *enode.Node + closed bool +} + +// NewWrsIterator creates a new WrsIterator. Nodes are selectable if they have all the required +// and none of the disabled flags set. When a node is selected the selectedFlag is set which also +// disables further selectability until it is removed or times out. +func NewWrsIterator(ns *nodestate.NodeStateMachine, requireFlags, disableFlags nodestate.Flags, weightField nodestate.Field) *WrsIterator { + wfn := func(i interface{}) uint64 { + n := ns.GetNode(i.(enode.ID)) + if n == nil { + return 0 + } + wt, _ := ns.GetField(n, weightField).(uint64) + return wt + } + + w := &WrsIterator{ + ns: ns, + wrs: utils.NewWeightedRandomSelect(wfn), + } + w.cond = sync.NewCond(&w.lock) + + ns.SubscribeField(weightField, func(n *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) { + if state.HasAll(requireFlags) && state.HasNone(disableFlags) { + w.lock.Lock() + w.wrs.Update(n.ID()) + w.lock.Unlock() + w.cond.Signal() + } + }) + + ns.SubscribeState(requireFlags.Or(disableFlags), func(n *enode.Node, oldState, newState nodestate.Flags) { + oldMatch := oldState.HasAll(requireFlags) && oldState.HasNone(disableFlags) + newMatch := newState.HasAll(requireFlags) && newState.HasNone(disableFlags) + if newMatch == oldMatch { + return + } + + w.lock.Lock() + if newMatch { + w.wrs.Update(n.ID()) + } else { + w.wrs.Remove(n.ID()) + } + w.lock.Unlock() + w.cond.Signal() + }) + return w +} + +// Next selects the next node. +func (w *WrsIterator) Next() bool { + w.nextNode = w.chooseNode() + return w.nextNode != nil +} + +func (w *WrsIterator) chooseNode() *enode.Node { + w.lock.Lock() + defer w.lock.Unlock() + + for { + for !w.closed && w.wrs.IsEmpty() { + w.cond.Wait() + } + if w.closed { + return nil + } + // Choose the next node at random. Even though w.wrs is guaranteed + // non-empty here, Choose might return nil if all items have weight + // zero. + if c := w.wrs.Choose(); c != nil { + id := c.(enode.ID) + w.wrs.Remove(id) + return w.ns.GetNode(id) + } + } + +} + +// Close ends the iterator. +func (w *WrsIterator) Close() { + w.lock.Lock() + w.closed = true + w.lock.Unlock() + w.cond.Signal() +} + +// Node returns the current node. +func (w *WrsIterator) Node() *enode.Node { + w.lock.Lock() + defer w.lock.Unlock() + return w.nextNode +} diff --git a/les/lespay/client/wrsiterator_test.go b/les/lespay/client/wrsiterator_test.go new file mode 100644 index 0000000000..2cbf4d405f --- /dev/null +++ b/les/lespay/client/wrsiterator_test.go @@ -0,0 +1,103 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "reflect" + "testing" + "time" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/p2p/nodestate" +) + +var ( + testSetup = &nodestate.Setup{} + sfTest1 = testSetup.NewFlag("test1") + sfTest2 = testSetup.NewFlag("test2") + sfTest3 = testSetup.NewFlag("test3") + sfTest4 = testSetup.NewFlag("test4") + sfiTestWeight = testSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) +) + +const iterTestNodeCount = 6 + +func TestWrsIterator(t *testing.T) { + ns := nodestate.NewNodeStateMachine(nil, nil, &mclock.Simulated{}, testSetup) + w := NewWrsIterator(ns, sfTest2, sfTest3.Or(sfTest4), sfiTestWeight) + ns.Start() + for i := 1; i <= iterTestNodeCount; i++ { + ns.SetState(testNode(i), sfTest1, nodestate.Flags{}, 0) + ns.SetField(testNode(i), sfiTestWeight, uint64(1)) + } + next := func() int { + ch := make(chan struct{}) + go func() { + w.Next() + close(ch) + }() + select { + case <-ch: + case <-time.After(time.Second * 5): + t.Fatalf("Iterator.Next() timeout") + } + node := w.Node() + ns.SetState(node, sfTest4, nodestate.Flags{}, 0) + return testNodeIndex(node.ID()) + } + set := make(map[int]bool) + expset := func() { + for len(set) > 0 { + n := next() + if !set[n] { + t.Errorf("Item returned by iterator not in the expected set (got %d)", n) + } + delete(set, n) + } + } + + ns.SetState(testNode(1), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(2), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(3), sfTest2, nodestate.Flags{}, 0) + set[1] = true + set[2] = true + set[3] = true + expset() + ns.SetState(testNode(4), sfTest2, nodestate.Flags{}, 0) + ns.SetState(testNode(5), sfTest2.Or(sfTest3), nodestate.Flags{}, 0) + ns.SetState(testNode(6), sfTest2, nodestate.Flags{}, 0) + set[4] = true + set[6] = true + expset() + ns.SetField(testNode(2), sfiTestWeight, uint64(0)) + ns.SetState(testNode(1), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(2), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(3), nodestate.Flags{}, sfTest4, 0) + set[1] = true + set[3] = true + expset() + ns.SetField(testNode(2), sfiTestWeight, uint64(1)) + ns.SetState(testNode(2), nodestate.Flags{}, sfTest2, 0) + ns.SetState(testNode(1), nodestate.Flags{}, sfTest4, 0) + ns.SetState(testNode(2), sfTest2, sfTest4, 0) + ns.SetState(testNode(3), nodestate.Flags{}, sfTest4, 0) + set[1] = true + set[2] = true + set[3] = true + expset() + ns.Stop() +} diff --git a/les/metrics.go b/les/metrics.go index 2be84d6c0c..a321f97acd 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -113,7 +113,12 @@ var ( requestRTT = metrics.NewRegisteredTimer("les/client/req/rtt", nil) requestSendDelay = metrics.NewRegisteredTimer("les/client/req/sendDelay", nil) - clientDiscoveredNodesCounter = metrics.NewRegisteredCounter("les/client/discovered", nil) // Counter for discovered nodes + serverSelectableGauge = metrics.NewRegisteredGauge("les/client/serverPool/selectable", nil) + serverDialedMeter = metrics.NewRegisteredMeter("les/client/serverPool/dialed", nil) + serverConnectedGauge = metrics.NewRegisteredGauge("les/client/serverPool/connected", nil) + sessionValueMeter = metrics.NewRegisteredMeter("les/client/serverPool/sessionValue", nil) + totalValueGauge = metrics.NewRegisteredGauge("les/client/serverPool/totalValue", nil) + suggestedTimeoutGauge = metrics.NewRegisteredGauge("les/client/serverPool/timeout", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/les/peer.go b/les/peer.go index 22ee370840..329491dc79 100644 --- a/les/peer.go +++ b/les/peer.go @@ -150,7 +150,7 @@ func (p *peerCommons) isFrozen() bool { return atomic.LoadUint32(&p.frozen) != 0 } -// canQueue returns an indicator whether the peer can queue a operation. +// canQueue returns an indicator whether the peer can queue an operation. func (p *peerCommons) canQueue() bool { return p.sendQueue.CanQueue() && !p.isFrozen() } @@ -340,7 +340,6 @@ type serverPeer struct { checkpointNumber uint64 // The block height which the checkpoint is registered. checkpoint params.TrustedCheckpoint // The advertised checkpoint sent by server. - poolEntry *poolEntry // Statistic for server peer. fcServer *flowcontrol.ServerNode // Client side mirror token bucket. vtLock sync.Mutex valueTracker *lpc.ValueTracker diff --git a/les/protocol.go b/les/protocol.go index a7df7eb1d4..41b73460aa 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -138,7 +138,6 @@ func init() { } requestMapping[uint32(code)] = rm } - } type errCode int diff --git a/les/retrieve.go b/les/retrieve.go index 4517a5f85f..ed6d47d8e5 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -24,22 +24,20 @@ import ( "sync" "time" - "github.com/celo-org/celo-blockchain/common/mclock" "github.com/celo-org/celo-blockchain/light" ) var ( retryQueue = time.Millisecond * 100 - softRequestTimeout = time.Millisecond * 500 hardRequestTimeout = time.Second * 10 ) // retrieveManager is a layer on top of requestDistributor which takes care of // matching replies by request ID and handles timeouts and resends if necessary. type retrieveManager struct { - dist *requestDistributor - peers *serverPeerSet - serverPool peerSelector + dist *requestDistributor + peers *serverPeerSet + softRequestTimeout func() time.Duration lock sync.RWMutex sentReqs map[uint64]*sentReq @@ -48,11 +46,6 @@ type retrieveManager struct { // validatorFunc is a function that processes a reply message type validatorFunc func(distPeer, *Msg) error -// peerSelector receives feedback info about response times and timeouts -type peerSelector interface { - adjustResponseTime(*poolEntry, time.Duration, bool) -} - // sentReq represents a request sent and tracked by retrieveManager type sentReq struct { rm *retrieveManager @@ -99,12 +92,12 @@ const ( ) // newRetrieveManager creates the retrieve manager -func newRetrieveManager(peers *serverPeerSet, dist *requestDistributor, serverPool peerSelector) *retrieveManager { +func newRetrieveManager(peers *serverPeerSet, dist *requestDistributor, srto func() time.Duration) *retrieveManager { return &retrieveManager{ - peers: peers, - dist: dist, - serverPool: serverPool, - sentReqs: make(map[uint64]*sentReq), + peers: peers, + dist: dist, + sentReqs: make(map[uint64]*sentReq), + softRequestTimeout: srto, } } @@ -325,8 +318,7 @@ func (r *sentReq) tryRequest() { return } - reqSent := mclock.Now() - srto, hrto := false, false + hrto := false r.lock.RLock() s, ok := r.sentTo[p] @@ -338,11 +330,7 @@ func (r *sentReq) tryRequest() { defer func() { // send feedback to server pool and remove peer if hard timeout happened pp, ok := p.(*serverPeer) - if ok && r.rm.serverPool != nil { - respTime := time.Duration(mclock.Now() - reqSent) - r.rm.serverPool.adjustResponseTime(pp.poolEntry, respTime, srto) - } - if hrto { + if hrto && ok { pp.Log().Debug("Request timed out hard") if r.rm.peers != nil { r.rm.peers.unregister(pp.id) @@ -363,8 +351,7 @@ func (r *sentReq) tryRequest() { } r.eventsCh <- reqPeerEvent{event, p} return - case <-time.After(softRequestTimeout): - srto = true + case <-time.After(r.rm.softRequestTimeout()): r.eventsCh <- reqPeerEvent{rpSoftTimeout, p} } diff --git a/les/server.go b/les/server.go index 44b49a345c..003d77dcb9 100644 --- a/les/server.go +++ b/les/server.go @@ -175,7 +175,7 @@ func (s *LesServer) Protocols() []p2p.Protocol { return p.Info() } return nil - }) + }, nil) // Add "les" ENR entries. for i := range ps { ps[i].Attributes = []enr.Entry{&lesEntry{}} diff --git a/les/serverpool.go b/les/serverpool.go index deaf0ac646..ebd84e6995 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2020 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,969 +17,457 @@ package les import ( - "crypto/ecdsa" - "fmt" - "io" - "math" + "errors" "math/rand" - "net" - "strconv" + "reflect" "sync" + "sync/atomic" "time" "github.com/celo-org/celo-blockchain/common/mclock" - "github.com/celo-org/celo-blockchain/crypto" "github.com/celo-org/celo-blockchain/ethdb" + lpc "github.com/celo-org/celo-blockchain/les/lespay/client" "github.com/celo-org/celo-blockchain/les/utils" "github.com/celo-org/celo-blockchain/log" - "github.com/celo-org/celo-blockchain/p2p" - "github.com/celo-org/celo-blockchain/p2p/discv5" "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" + "github.com/celo-org/celo-blockchain/p2p/nodestate" "github.com/celo-org/celo-blockchain/rlp" ) const ( - // After a connection has been ended or timed out, there is a waiting period - // before it can be selected for connection again. - // waiting period = base delay * (1 + random(1)) - // base delay = shortRetryDelay for the first shortRetryCnt times after a - // successful connection, after that longRetryDelay is applied - shortRetryCnt = 5 - shortRetryDelay = time.Second * 5 - longRetryDelay = time.Minute * 10 - // maxNewEntries is the maximum number of newly discovered (never connected) nodes. - // If the limit is reached, the least recently discovered one is thrown out. - maxNewEntries = 1000 - // maxKnownEntries is the maximum number of known (already connected) nodes. - // If the limit is reached, the least recently connected one is thrown out. - // (not that unlike new entries, known entries are persistent) - maxKnownEntries = 1000 - // target for simultaneously connected servers - targetServerCount = 5 - // target for servers selected from the known table - // (we leave room for trying new ones if there is any) - targetKnownSelect = 3 - // after dialTimeout, consider the server unavailable and adjust statistics - dialTimeout = time.Second * 30 - // targetConnTime is the minimum expected connection duration before a server - // drops a client without any specific reason - targetConnTime = time.Minute * 10 - // new entry selection weight calculation based on most recent discovery time: - // unity until discoverExpireStart, then exponential decay with discoverExpireConst - discoverExpireStart = time.Minute * 20 - discoverExpireConst = time.Minute * 20 - // known entry selection weight is dropped by a factor of exp(-failDropLn) after - // each unsuccessful connection (restored after a successful one) - failDropLn = 0.1 - // known node connection success and quality statistics have a long term average - // and a short term value which is adjusted exponentially with a factor of - // pstatRecentAdjust with each dial/connection and also returned exponentially - // to the average with the time constant pstatReturnToMeanTC - pstatReturnToMeanTC = time.Hour - // node address selection weight is dropped by a factor of exp(-addrFailDropLn) after - // each unsuccessful connection (restored after a successful one) - addrFailDropLn = math.Ln2 - // responseScoreTC and delayScoreTC are exponential decay time constants for - // calculating selection chances from response times and block delay times - responseScoreTC = time.Millisecond * 100 - delayScoreTC = time.Second * 5 - timeoutPow = 10 - // initStatsWeight is used to initialize previously unknown peers with good - // statistics to give a chance to prove themselves - initStatsWeight = 1 + minTimeout = time.Millisecond * 500 // minimum request timeout suggested by the server pool + timeoutRefresh = time.Second * 5 // recalculate timeout if older than this + dialCost = 10000 // cost of a TCP dial (used for known node selection weight calculation) + dialWaitStep = 1.5 // exponential multiplier of redial wait time when no value was provided by the server + queryCost = 500 // cost of a UDP pre-negotiation query + queryWaitStep = 1.02 // exponential multiplier of redial wait time when no value was provided by the server + waitThreshold = time.Hour * 2000 // drop node if waiting time is over the threshold + nodeWeightMul = 1000000 // multiplier constant for node weight calculation + nodeWeightThreshold = 100 // minimum weight for keeping a node in the the known (valuable) set + minRedialWait = 10 // minimum redial wait time in seconds + preNegLimit = 5 // maximum number of simultaneous pre-negotiation queries + maxQueryFails = 100 // number of consecutive UDP query failures before we print a warning ) -// connReq represents a request for peer connection. -type connReq struct { - p *serverPeer - node *enode.Node - result chan *poolEntry -} - -// disconnReq represents a request for peer disconnection. -type disconnReq struct { - entry *poolEntry - stopped bool - done chan struct{} -} - -// registerReq represents a request for peer registration. -type registerReq struct { - entry *poolEntry - done chan struct{} -} - -// serverPool implements a pool for storing and selecting newly discovered and already -// known light server nodes. It received discovered nodes, stores statistics about -// known nodes and takes care of always having enough good quality servers connected. +// serverPool provides a node iterator for dial candidates. The output is a mix of newly discovered +// nodes, a weighted random selection of known (previously valuable) nodes and trusted/paid nodes. type serverPool struct { - db ethdb.Database - dbKey []byte - server *p2p.Server - connWg sync.WaitGroup - - topic discv5.Topic - - discSetPeriod chan time.Duration - discNodes chan *enode.Node - discLookups chan bool - - trustedNodes map[enode.ID]*enode.Node - entries map[enode.ID]*poolEntry - timeout, enableRetry chan *poolEntry - adjustStats chan poolStatAdjust - - knownQueue, newQueue poolEntryQueue - knownSelect, newSelect *utils.WeightedRandomSelect - knownSelected, newSelected int - fastDiscover bool - connCh chan *connReq - disconnCh chan *disconnReq - registerCh chan *registerReq - - closeCh chan struct{} - wg sync.WaitGroup -} - -// newServerPool creates a new serverPool instance -func newServerPool(db ethdb.Database, ulcServers []string) *serverPool { - pool := &serverPool{ - db: db, - entries: make(map[enode.ID]*poolEntry), - timeout: make(chan *poolEntry, 1), - adjustStats: make(chan poolStatAdjust, 100), - enableRetry: make(chan *poolEntry, 1), - connCh: make(chan *connReq), - disconnCh: make(chan *disconnReq), - registerCh: make(chan *registerReq), - closeCh: make(chan struct{}), - knownSelect: utils.NewWeightedRandomSelect(), - newSelect: utils.NewWeightedRandomSelect(), - fastDiscover: true, - trustedNodes: parseTrustedNodes(ulcServers), - } - - pool.knownQueue = newPoolEntryQueue(maxKnownEntries, pool.removeEntry) - pool.newQueue = newPoolEntryQueue(maxNewEntries, pool.removeEntry) - return pool -} - -func (pool *serverPool) start(server *p2p.Server, topic discv5.Topic) { - pool.server = server - pool.topic = topic - pool.dbKey = append([]byte("serverPool/"), []byte(topic)...) - pool.loadNodes() - pool.connectToTrustedNodes() - - if pool.server.DiscV5 != nil { - pool.discSetPeriod = make(chan time.Duration, 1) - pool.discNodes = make(chan *enode.Node, 100) - pool.discLookups = make(chan bool, 100) - go pool.discoverNodes() - } - pool.checkDial() - pool.wg.Add(1) - go pool.eventLoop() - - // Inject the bootstrap nodes as initial dial candiates. - pool.wg.Add(1) - go func() { - defer pool.wg.Done() - for _, n := range server.BootstrapNodes { - select { - case pool.discNodes <- n: - case <-pool.closeCh: - return + clock mclock.Clock + unixTime func() int64 + db ethdb.KeyValueStore + + ns *nodestate.NodeStateMachine + vt *lpc.ValueTracker + mixer *enode.FairMix + mixSources []enode.Iterator + dialIterator enode.Iterator + validSchemes enr.IdentityScheme + trustedURLs []string + fillSet *lpc.FillSet + queryFails uint32 + + timeoutLock sync.RWMutex + timeout time.Duration + timeWeights lpc.ResponseTimeWeights + timeoutRefreshed mclock.AbsTime +} + +// nodeHistory keeps track of dial costs which determine node weight together with the +// service value calculated by lpc.ValueTracker. +type nodeHistory struct { + dialCost utils.ExpiredValue + redialWaitStart, redialWaitEnd int64 // unix time (seconds) +} + +type nodeHistoryEnc struct { + DialCost utils.ExpiredValue + RedialWaitStart, RedialWaitEnd uint64 +} + +// queryFunc sends a pre-negotiation query and blocks until a response arrives or timeout occurs. +// It returns 1 if the remote node has confirmed that connection is possible, 0 if not +// possible and -1 if no response arrived (timeout). +type queryFunc func(*enode.Node) int + +var ( + serverPoolSetup = &nodestate.Setup{Version: 1} + sfHasValue = serverPoolSetup.NewPersistentFlag("hasValue") + sfQueried = serverPoolSetup.NewFlag("queried") + sfCanDial = serverPoolSetup.NewFlag("canDial") + sfDialing = serverPoolSetup.NewFlag("dialed") + sfWaitDialTimeout = serverPoolSetup.NewFlag("dialTimeout") + sfConnected = serverPoolSetup.NewFlag("connected") + sfRedialWait = serverPoolSetup.NewFlag("redialWait") + sfAlwaysConnect = serverPoolSetup.NewFlag("alwaysConnect") + sfDisableSelection = nodestate.MergeFlags(sfQueried, sfCanDial, sfDialing, sfConnected, sfRedialWait) + + sfiNodeHistory = serverPoolSetup.NewPersistentField("nodeHistory", reflect.TypeOf(nodeHistory{}), + func(field interface{}) ([]byte, error) { + if n, ok := field.(nodeHistory); ok { + ne := nodeHistoryEnc{ + DialCost: n.dialCost, + RedialWaitStart: uint64(n.redialWaitStart), + RedialWaitEnd: uint64(n.redialWaitEnd), + } + enc, err := rlp.EncodeToBytes(&ne) + return enc, err + } else { + return nil, errors.New("invalid field type") } - } - }() -} - -func (pool *serverPool) stop() { - close(pool.closeCh) - pool.wg.Wait() -} - -// discoverNodes wraps SearchTopic, converting result nodes to enode.Node. -func (pool *serverPool) discoverNodes() { - ch := make(chan *discv5.Node) - go func() { - pool.server.DiscV5.SearchTopic(pool.topic, pool.discSetPeriod, ch, pool.discLookups) - close(ch) - }() - for n := range ch { - pubkey, err := decodePubkey64(n.ID[:]) - if err != nil { - continue - } - clientDiscoveredNodesCounter.Inc(1) - pool.discNodes <- enode.NewV4(pubkey, n.IP, int(n.TCP), int(n.UDP)) - } -} - -// connect should be called upon any incoming connection. If the connection has been -// dialed by the server pool recently, the appropriate pool entry is returned. -// Otherwise, the connection should be rejected. -// Note that whenever a connection has been accepted and a pool entry has been returned, -// disconnect should also always be called. -func (pool *serverPool) connect(p *serverPeer, node *enode.Node) *poolEntry { - log.Debug("Connect new entry", "enode", p.id) - req := &connReq{p: p, node: node, result: make(chan *poolEntry, 1)} - select { - case pool.connCh <- req: - case <-pool.closeCh: - return nil - } - return <-req.result -} - -// registered should be called after a successful handshake -func (pool *serverPool) registered(entry *poolEntry) { - log.Debug("Registered new entry", "enode", entry.node.ID()) - req := ®isterReq{entry: entry, done: make(chan struct{})} - select { - case pool.registerCh <- req: - case <-pool.closeCh: - return - } - <-req.done -} - -// disconnect should be called when ending a connection. Service quality statistics -// can be updated optionally (not updated if no registration happened, in this case -// only connection statistics are updated, just like in case of timeout) -func (pool *serverPool) disconnect(entry *poolEntry) { - stopped := false - select { - case <-pool.closeCh: - stopped = true - default: - } - log.Debug("Disconnected old entry", "enode", entry.node.ID()) - req := &disconnReq{entry: entry, stopped: stopped, done: make(chan struct{})} - - // Block until disconnection request is served. - pool.disconnCh <- req - <-req.done -} - -const ( - pseBlockDelay = iota - pseResponseTime - pseResponseTimeout + }, + func(enc []byte) (interface{}, error) { + var ne nodeHistoryEnc + err := rlp.DecodeBytes(enc, &ne) + n := nodeHistory{ + dialCost: ne.DialCost, + redialWaitStart: int64(ne.RedialWaitStart), + redialWaitEnd: int64(ne.RedialWaitEnd), + } + return n, err + }, + ) + sfiNodeWeight = serverPoolSetup.NewField("nodeWeight", reflect.TypeOf(uint64(0))) + sfiConnectedStats = serverPoolSetup.NewField("connectedStats", reflect.TypeOf(lpc.ResponseTimeStats{})) ) -// poolStatAdjust records are sent to adjust peer block delay/response time statistics -type poolStatAdjust struct { - adjustType int - entry *poolEntry - time time.Duration -} - -// adjustBlockDelay adjusts the block announce delay statistics of a node -func (pool *serverPool) adjustBlockDelay(entry *poolEntry, time time.Duration) { - if entry == nil { - return - } - pool.adjustStats <- poolStatAdjust{pseBlockDelay, entry, time} -} - -// adjustResponseTime adjusts the request response time statistics of a node -func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, timeout bool) { - if entry == nil { - return - } - if timeout { - pool.adjustStats <- poolStatAdjust{pseResponseTimeout, entry, time} - } else { - pool.adjustStats <- poolStatAdjust{pseResponseTime, entry, time} - } -} - -// eventLoop handles pool events and mutex locking for all internal functions -func (pool *serverPool) eventLoop() { - defer pool.wg.Done() - lookupCnt := 0 - var convTime mclock.AbsTime - if pool.discSetPeriod != nil { - pool.discSetPeriod <- time.Millisecond * 100 - } - - // disconnect updates service quality statistics depending on the connection time - // and disconnection initiator. - disconnect := func(req *disconnReq, stopped bool) { - // Handle peer disconnection requests. - entry := req.entry - if entry.state == psRegistered { - connAdjust := float64(mclock.Now()-entry.regTime) / float64(targetConnTime) - if connAdjust > 1 { - connAdjust = 1 - } - if stopped { - // disconnect requested by ourselves. - entry.connectStats.add(1, connAdjust) - } else { - // disconnect requested by server side. - entry.connectStats.add(connAdjust, 1) - } - } - entry.state = psNotConnected +// newServerPool creates a new server pool +func newServerPool(db ethdb.KeyValueStore, dbKey []byte, vt *lpc.ValueTracker, discovery enode.Iterator, mixTimeout time.Duration, query queryFunc, clock mclock.Clock, trustedURLs []string) *serverPool { + s := &serverPool{ + db: db, + clock: clock, + unixTime: func() int64 { return time.Now().Unix() }, + validSchemes: enode.ValidSchemes, + trustedURLs: trustedURLs, + vt: vt, + ns: nodestate.NewNodeStateMachine(db, []byte(string(dbKey)+"ns:"), clock, serverPoolSetup), + } + s.recalTimeout() + s.mixer = enode.NewFairMix(mixTimeout) + knownSelector := lpc.NewWrsIterator(s.ns, sfHasValue, sfDisableSelection, sfiNodeWeight) + alwaysConnect := lpc.NewQueueIterator(s.ns, sfAlwaysConnect, sfDisableSelection, true, nil) + s.mixSources = append(s.mixSources, knownSelector) + s.mixSources = append(s.mixSources, alwaysConnect) + if discovery != nil { + s.mixSources = append(s.mixSources, discovery) + } + + iter := enode.Iterator(s.mixer) + if query != nil { + iter = s.addPreNegFilter(iter, query) + } + s.dialIterator = enode.Filter(iter, func(node *enode.Node) bool { + s.ns.SetState(node, sfDialing, sfCanDial, 0) + s.ns.SetState(node, sfWaitDialTimeout, nodestate.Flags{}, time.Second*10) + return true + }) - if entry.knownSelected { - pool.knownSelected-- - } else { - pool.newSelected-- + s.ns.SubscribeState(nodestate.MergeFlags(sfWaitDialTimeout, sfConnected), func(n *enode.Node, oldState, newState nodestate.Flags) { + if oldState.Equals(sfWaitDialTimeout) && newState.IsEmpty() { + // dial timeout, no connection + s.setRedialWait(n, dialCost, dialWaitStep) + s.ns.SetState(n, nodestate.Flags{}, sfDialing, 0) } - pool.setRetryDial(entry) - pool.connWg.Done() - close(req.done) - } - - for { - select { - case entry := <-pool.timeout: - if !entry.removed { - pool.checkDialTimeout(entry) - } - - case entry := <-pool.enableRetry: - if !entry.removed { - entry.delayedRetry = false - pool.updateCheckDial(entry) - } + }) - case adj := <-pool.adjustStats: - switch adj.adjustType { - case pseBlockDelay: - adj.entry.delayStats.add(float64(adj.time), 1) - case pseResponseTime: - adj.entry.responseStats.add(float64(adj.time), 1) - adj.entry.timeoutStats.add(0, 1) - case pseResponseTimeout: - adj.entry.timeoutStats.add(1, 1) + s.ns.AddLogMetrics(sfHasValue, sfDisableSelection, "selectable", nil, nil, serverSelectableGauge) + s.ns.AddLogMetrics(sfDialing, nodestate.Flags{}, "dialed", serverDialedMeter, nil, nil) + s.ns.AddLogMetrics(sfConnected, nodestate.Flags{}, "connected", nil, nil, serverConnectedGauge) + return s +} + +// addPreNegFilter installs a node filter mechanism that performs a pre-negotiation query. +// Nodes that are filtered out and does not appear on the output iterator are put back +// into redialWait state. +func (s *serverPool) addPreNegFilter(input enode.Iterator, query queryFunc) enode.Iterator { + s.fillSet = lpc.NewFillSet(s.ns, input, sfQueried) + s.ns.SubscribeState(sfQueried, func(n *enode.Node, oldState, newState nodestate.Flags) { + if newState.Equals(sfQueried) { + fails := atomic.LoadUint32(&s.queryFails) + if fails == maxQueryFails { + log.Warn("UDP pre-negotiation query does not seem to work") } - - case node := <-pool.discNodes: - if pool.trustedNodes[node.ID()] == nil { - entry := pool.findOrNewNode(node) - pool.updateCheckDial(entry) + if fails > maxQueryFails { + fails = maxQueryFails } - - case conv := <-pool.discLookups: - if conv { - if lookupCnt == 0 { - convTime = mclock.Now() - } - lookupCnt++ - if pool.fastDiscover && (lookupCnt == 50 || time.Duration(mclock.Now()-convTime) > time.Minute) { - pool.fastDiscover = false - if pool.discSetPeriod != nil { - pool.discSetPeriod <- time.Minute - } - } + if rand.Intn(maxQueryFails*2) < int(fails) { + // skip pre-negotiation with increasing chance, max 50% + // this ensures that the client can operate even if UDP is not working at all + s.ns.SetState(n, sfCanDial, nodestate.Flags{}, time.Second*10) + // set canDial before resetting queried so that FillSet will not read more + // candidates unnecessarily + s.ns.SetState(n, nodestate.Flags{}, sfQueried, 0) + return } - - case req := <-pool.connCh: - if pool.trustedNodes[req.p.ID()] != nil { - // ignore trusted nodes - req.result <- &poolEntry{trusted: true} - } else { - // Handle peer connection requests. - entry := pool.entries[req.p.ID()] - if entry == nil { - entry = pool.findOrNewNode(req.node) - } - if entry.state == psConnected || entry.state == psRegistered { - req.result <- nil - continue + go func() { + q := query(n) + if q == -1 { + atomic.AddUint32(&s.queryFails, 1) + } else { + atomic.StoreUint32(&s.queryFails, 0) } - pool.connWg.Add(1) - entry.peer = req.p - entry.state = psConnected - addr := &poolEntryAddress{ - ip: req.node.IP(), - port: uint16(req.node.TCP()), - lastSeen: mclock.Now(), + if q == 1 { + s.ns.SetState(n, sfCanDial, nodestate.Flags{}, time.Second*10) + } else { + s.setRedialWait(n, queryCost, queryWaitStep) } - entry.lastConnected = addr - entry.addr = make(map[string]*poolEntryAddress) - entry.addr[addr.strKey()] = addr - entry.addrSelect = *utils.NewWeightedRandomSelect() - entry.addrSelect.Update(addr) - req.result <- entry - } - - case req := <-pool.registerCh: - if req.entry.trusted { - continue - } - // Handle peer registration requests. - entry := req.entry - entry.state = psRegistered - entry.regTime = mclock.Now() - if !entry.known { - pool.newQueue.remove(entry) - entry.known = true - } - pool.knownQueue.setLatest(entry) - entry.shortRetry = shortRetryCnt - pool.saveNodes() - close(req.done) - - case req := <-pool.disconnCh: - if req.entry.trusted { - continue - } - // Handle peer disconnection requests. - disconnect(req, req.stopped) - - case <-pool.closeCh: - if pool.discSetPeriod != nil { - close(pool.discSetPeriod) - } - - // Spawn a goroutine to close the disconnCh after all connections are disconnected. - go func() { - pool.connWg.Wait() - close(pool.disconnCh) + s.ns.SetState(n, nodestate.Flags{}, sfQueried, 0) }() - - // Handle all remaining disconnection requests before exit. - for req := range pool.disconnCh { - disconnect(req, true) - } - pool.saveNodes() - return - } - } -} - -func (pool *serverPool) findOrNewNode(node *enode.Node) *poolEntry { - now := mclock.Now() - entry := pool.entries[node.ID()] - if entry == nil { - log.Debug("Discovered new entry", "id", node.ID()) - entry = &poolEntry{ - node: node, - addr: make(map[string]*poolEntryAddress), - addrSelect: *utils.NewWeightedRandomSelect(), - shortRetry: shortRetryCnt, } - pool.entries[node.ID()] = entry - // initialize previously unknown peers with good statistics to give a chance to prove themselves - entry.connectStats.add(1, initStatsWeight) - entry.delayStats.add(0, initStatsWeight) - entry.responseStats.add(0, initStatsWeight) - entry.timeoutStats.add(0, initStatsWeight) - } - entry.lastDiscovered = now - addr := &poolEntryAddress{ip: node.IP(), port: uint16(node.TCP())} - if a, ok := entry.addr[addr.strKey()]; ok { - addr = a - } else { - entry.addr[addr.strKey()] = addr - } - addr.lastSeen = now - entry.addrSelect.Update(addr) - if !entry.known { - pool.newQueue.setLatest(entry) - } - return entry -} - -// loadNodes loads known nodes and their statistics from the database -func (pool *serverPool) loadNodes() { - enc, err := pool.db.Get(pool.dbKey) - if err != nil { - return - } - var list []*poolEntry - err = rlp.DecodeBytes(enc, &list) - if err != nil { - log.Debug("Failed to decode node list", "err", err) - return - } - for _, e := range list { - log.Debug("Loaded server stats", "id", e.node.ID(), "fails", e.lastConnected.fails, - "conn", fmt.Sprintf("%v/%v", e.connectStats.avg, e.connectStats.weight), - "delay", fmt.Sprintf("%v/%v", time.Duration(e.delayStats.avg), e.delayStats.weight), - "response", fmt.Sprintf("%v/%v", time.Duration(e.responseStats.avg), e.responseStats.weight), - "timeout", fmt.Sprintf("%v/%v", e.timeoutStats.avg, e.timeoutStats.weight)) - pool.entries[e.node.ID()] = e - if pool.trustedNodes[e.node.ID()] == nil { - pool.knownQueue.setLatest(e) - pool.knownSelect.Update((*knownEntry)(e)) + }) + return lpc.NewQueueIterator(s.ns, sfCanDial, nodestate.Flags{}, false, func(waiting bool) { + if waiting { + s.fillSet.SetTarget(preNegLimit) + } else { + s.fillSet.SetTarget(0) } - } + }) } -// connectToTrustedNodes adds trusted server nodes as static trusted peers. -// -// Note: trusted nodes are not handled by the server pool logic, they are not -// added to either the known or new selection pools. They are connected/reconnected -// by p2p.Server whenever possible. -func (pool *serverPool) connectToTrustedNodes() { - //connect to trusted nodes - for _, node := range pool.trustedNodes { - pool.server.AddTrustedPeer(node, p2p.ExplicitStaticPurpose) - pool.server.AddPeer(node, p2p.ExplicitStaticPurpose) - log.Debug("Added trusted node", "id", node.ID().String()) +// start starts the server pool. Note that NodeStateMachine should be started first. +func (s *serverPool) start() { + s.ns.Start() + for _, iter := range s.mixSources { + // add sources to mixer at startup because the mixer instantly tries to read them + // which should only happen after NodeStateMachine has been started + s.mixer.AddSource(iter) } -} - -// parseTrustedNodes returns valid and parsed enodes -func parseTrustedNodes(trustedNodes []string) map[enode.ID]*enode.Node { - nodes := make(map[enode.ID]*enode.Node) - - for _, node := range trustedNodes { - node, err := enode.Parse(enode.ValidSchemes, node) - if err != nil { - log.Warn("Trusted node URL invalid", "enode", node, "err", err) - continue + for _, url := range s.trustedURLs { + if node, err := enode.Parse(s.validSchemes, url); err == nil { + s.ns.SetState(node, sfAlwaysConnect, nodestate.Flags{}, 0) + } else { + log.Error("Invalid trusted server URL", "url", url, "error", err) } - nodes[node.ID()] = node - } - return nodes -} - -// saveNodes saves known nodes and their statistics into the database. Nodes are -// ordered from least to most recently connected. -func (pool *serverPool) saveNodes() { - nodes := pool.knownQueue.list() - log.Debug("Saving serverPool nodes", "length", len(nodes)) - enc, err := rlp.EncodeToBytes(nodes) - if err == nil { - pool.db.Put(pool.dbKey, enc) - } -} - -// removeEntry removes a pool entry when the entry count limit is reached. -// Note that it is called by the new/known queues from which the entry has already -// been removed so removing it from the queues is not necessary. -func (pool *serverPool) removeEntry(entry *poolEntry) { - pool.newSelect.Remove((*discoveredEntry)(entry)) - pool.knownSelect.Remove((*knownEntry)(entry)) - entry.removed = true - delete(pool.entries, entry.node.ID()) -} - -// setRetryDial starts the timer which will enable dialing a certain node again -func (pool *serverPool) setRetryDial(entry *poolEntry) { - delay := longRetryDelay - if entry.shortRetry > 0 { - entry.shortRetry-- - delay = shortRetryDelay } - delay += time.Duration(rand.Int63n(int64(delay) + 1)) - entry.delayedRetry = true - go func() { - select { - case <-pool.closeCh: - case <-time.After(delay): - select { - case <-pool.closeCh: - case pool.enableRetry <- entry: + unixTime := s.unixTime() + s.ns.ForEach(sfHasValue, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { + s.calculateWeight(node) + if n, ok := s.ns.GetField(node, sfiNodeHistory).(nodeHistory); ok && n.redialWaitEnd > unixTime { + wait := n.redialWaitEnd - unixTime + lastWait := n.redialWaitEnd - n.redialWaitStart + if wait > lastWait { + // if the time until expiration is larger than the last suggested + // waiting time then the system clock was probably adjusted + wait = lastWait } + s.ns.SetState(node, sfRedialWait, nodestate.Flags{}, time.Duration(wait)*time.Second) } - }() -} - -// updateCheckDial is called when an entry can potentially be dialed again. It updates -// its selection weights and checks if new dials can/should be made. -func (pool *serverPool) updateCheckDial(entry *poolEntry) { - pool.newSelect.Update((*discoveredEntry)(entry)) - pool.knownSelect.Update((*knownEntry)(entry)) - pool.checkDial() + }) } -// checkDial checks if new dials can/should be made. It tries to select servers both -// based on good statistics and recent discovery. -func (pool *serverPool) checkDial() { - fillWithKnownSelects := !pool.fastDiscover - for pool.knownSelected < targetKnownSelect { - entry := pool.knownSelect.Choose() - if entry == nil { - fillWithKnownSelects = false - break - } - pool.dial((*poolEntry)(entry.(*knownEntry)), true) +// stop stops the server pool +func (s *serverPool) stop() { + s.dialIterator.Close() + if s.fillSet != nil { + s.fillSet.Close() } - for pool.knownSelected+pool.newSelected < targetServerCount { - entry := pool.newSelect.Choose() - if entry == nil { - break - } - pool.dial((*poolEntry)(entry.(*discoveredEntry)), false) - } - if fillWithKnownSelects { - // no more newly discovered nodes to select and since fast discover period - // is over, we probably won't find more in the near future so select more - // known entries if possible - for pool.knownSelected < targetServerCount { - entry := pool.knownSelect.Choose() - if entry == nil { - break - } - pool.dial((*poolEntry)(entry.(*knownEntry)), true) - } - } -} - -// dial initiates a new connection -func (pool *serverPool) dial(entry *poolEntry, knownSelected bool) { - if pool.server == nil || entry.state != psNotConnected { + s.ns.ForEach(sfConnected, nodestate.Flags{}, func(n *enode.Node, state nodestate.Flags) { + // recalculate weight of connected nodes in order to update hasValue flag if necessary + s.calculateWeight(n) + }) + s.ns.Stop() +} + +// registerPeer implements serverPeerSubscriber +func (s *serverPool) registerPeer(p *serverPeer) { + s.ns.SetState(p.Node(), sfConnected, sfDialing.Or(sfWaitDialTimeout), 0) + nvt := s.vt.Register(p.ID()) + s.ns.SetField(p.Node(), sfiConnectedStats, nvt.RtStats()) + p.setValueTracker(s.vt, nvt) + p.updateVtParams() +} + +// unregisterPeer implements serverPeerSubscriber +func (s *serverPool) unregisterPeer(p *serverPeer) { + s.setRedialWait(p.Node(), dialCost, dialWaitStep) + s.ns.SetState(p.Node(), nodestate.Flags{}, sfConnected, 0) + s.ns.SetField(p.Node(), sfiConnectedStats, nil) + s.vt.Unregister(p.ID()) + p.setValueTracker(nil, nil) +} + +// recalTimeout calculates the current recommended timeout. This value is used by +// the client as a "soft timeout" value. It also affects the service value calculation +// of individual nodes. +func (s *serverPool) recalTimeout() { + // Use cached result if possible, avoid recalculating too frequently. + s.timeoutLock.RLock() + refreshed := s.timeoutRefreshed + s.timeoutLock.RUnlock() + now := s.clock.Now() + if refreshed != 0 && time.Duration(now-refreshed) < timeoutRefresh { return } - entry.state = psDialed - entry.knownSelected = knownSelected - if knownSelected { - pool.knownSelected++ - } else { - pool.newSelected++ - } - addr := entry.addrSelect.Choose().(*poolEntryAddress) - log.Debug("Dialing new peer", "lesaddr", entry.node.ID().String()+"@"+addr.strKey(), "set", len(entry.addr), "known", knownSelected) - entry.dialed = addr - go func() { - pool.server.AddPeer(entry.node, p2p.ExplicitStaticPurpose) - select { - case <-pool.closeCh: - case <-time.After(dialTimeout): - select { - case <-pool.closeCh: - case pool.timeout <- entry: - } - } - }() -} + // Cached result is stale, recalculate a new one. + rts := s.vt.RtStats() -// checkDialTimeout checks if the node is still in dialed state and if so, resets it -// and adjusts connection statistics accordingly. -func (pool *serverPool) checkDialTimeout(entry *poolEntry) { - if entry.state != psDialed { - return - } - log.Debug("Dial timeout", "lesaddr", entry.node.ID().String()+"@"+entry.dialed.strKey()) - entry.state = psNotConnected - if entry.knownSelected { - pool.knownSelected-- - } else { - pool.newSelected-- - } - entry.connectStats.add(0, 1) - entry.dialed.fails++ - pool.setRetryDial(entry) -} + // Add a fake statistic here. It is an easy way to initialize with some + // conservative values when the database is new. As soon as we have a + // considerable amount of real stats this small value won't matter. + rts.Add(time.Second*2, 10, s.vt.StatsExpFactor()) -func (pool *serverPool) Info() []*poolEntryInfo { - entryInfos := make([]*poolEntryInfo, 0) - for _, entry := range pool.entries { - entryInfos = append(entryInfos, &poolEntryInfo{ - Peered: entry.peer != nil, - Node: entry.node, - Known: entry.known, - KnownSelected: entry.knownSelected, - Trusted: entry.trusted, - LastDiscoveredTime: entry.lastDiscovered, - RegisteredTime: entry.regTime, - State: entry.state.String(), - }) + // Use either 10% failure rate timeout or twice the median response time + // as the recommended timeout. + timeout := minTimeout + if t := rts.Timeout(0.1); t > timeout { + timeout = t } - return entryInfos -} - -type poolEntryState int - -const ( - psNotConnected = poolEntryState(iota) - psDialed - psConnected - psRegistered -) - -func (e poolEntryState) String() string { - switch e { - case psNotConnected: - return "NotConnected" - case psDialed: - return "Dialed" - case psConnected: - return "Connected" - case psRegistered: - return "Registered" - default: - return fmt.Sprintf("poolEntryState(%d)", e) + if t := rts.Timeout(0.5) * 2; t > timeout { + timeout = t } -} - -// poolEntry represents a server node and stores its current state and statistics. -type poolEntry struct { - peer *serverPeer - addr map[string]*poolEntryAddress - node *enode.Node - lastConnected, dialed *poolEntryAddress - addrSelect utils.WeightedRandomSelect - - lastDiscovered mclock.AbsTime - known, knownSelected, trusted bool - connectStats, delayStats poolStats - responseStats, timeoutStats poolStats - state poolEntryState - regTime mclock.AbsTime - queueIdx int - removed bool - - delayedRetry bool - shortRetry int -} - -type poolEntryInfo struct { - Peered bool `json:"peered"` - Node *enode.Node `json:"node"` - Known bool `json:"known"` - KnownSelected bool `json:"knownSelected"` - Trusted bool `json:"trusted"` - LastDiscoveredTime mclock.AbsTime `json:"lastDiscoveredTime"` - RegisteredTime mclock.AbsTime `json:"registeredTime"` - State string `json:"state"` -} + s.timeoutLock.Lock() + if s.timeout != timeout { + s.timeout = timeout + s.timeWeights = lpc.TimeoutWeights(s.timeout) -// poolEntryEnc is the RLP encoding of poolEntry. -type poolEntryEnc struct { - Pubkey []byte - IP net.IP - Port uint16 - Fails uint - CStat, DStat, RStat, TStat poolStats -} - -func (e *poolEntry) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &poolEntryEnc{ - Pubkey: encodePubkey64(e.node.Pubkey()), - IP: e.lastConnected.ip, - Port: e.lastConnected.port, - Fails: e.lastConnected.fails, - CStat: e.connectStats, - DStat: e.delayStats, - RStat: e.responseStats, - TStat: e.timeoutStats, - }) -} - -func (e *poolEntry) DecodeRLP(s *rlp.Stream) error { - var entry poolEntryEnc - if err := s.Decode(&entry); err != nil { - return err + suggestedTimeoutGauge.Update(int64(s.timeout / time.Millisecond)) + totalValueGauge.Update(int64(rts.Value(s.timeWeights, s.vt.StatsExpFactor()))) } - pubkey, err := decodePubkey64(entry.Pubkey) - if err != nil { - return err - } - addr := &poolEntryAddress{ip: entry.IP, port: entry.Port, fails: entry.Fails, lastSeen: mclock.Now()} - e.node = enode.NewV4(pubkey, entry.IP, int(entry.Port), int(entry.Port)) - e.addr = make(map[string]*poolEntryAddress) - e.addr[addr.strKey()] = addr - e.addrSelect = *utils.NewWeightedRandomSelect() - e.addrSelect.Update(addr) - e.lastConnected = addr - e.connectStats = entry.CStat - e.delayStats = entry.DStat - e.responseStats = entry.RStat - e.timeoutStats = entry.TStat - e.shortRetry = shortRetryCnt - e.known = true - return nil + s.timeoutRefreshed = now + s.timeoutLock.Unlock() } -func encodePubkey64(pub *ecdsa.PublicKey) []byte { - return crypto.FromECDSAPub(pub)[1:] +// getTimeout returns the recommended request timeout. +func (s *serverPool) getTimeout() time.Duration { + s.recalTimeout() + s.timeoutLock.RLock() + defer s.timeoutLock.RUnlock() + return s.timeout } -func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) { - return crypto.UnmarshalPubkey(append([]byte{0x04}, b...)) +// getTimeoutAndWeight returns the recommended request timeout as well as the +// response time weight which is necessary to calculate service value. +func (s *serverPool) getTimeoutAndWeight() (time.Duration, lpc.ResponseTimeWeights) { + s.recalTimeout() + s.timeoutLock.RLock() + defer s.timeoutLock.RUnlock() + return s.timeout, s.timeWeights } -// discoveredEntry implements wrsItem -type discoveredEntry poolEntry - -// Weight calculates random selection weight for newly discovered entries -func (e *discoveredEntry) Weight() int64 { - if e.state != psNotConnected || e.delayedRetry { - return 0 +// addDialCost adds the given amount of dial cost to the node history and returns the current +// amount of total dial cost +func (s *serverPool) addDialCost(n *nodeHistory, amount int64) uint64 { + logOffset := s.vt.StatsExpirer().LogOffset(s.clock.Now()) + if amount > 0 { + n.dialCost.Add(amount, logOffset) } - t := time.Duration(mclock.Now() - e.lastDiscovered) - if t <= discoverExpireStart { - return 1000000000 + totalDialCost := n.dialCost.Value(logOffset) + if totalDialCost < dialCost { + totalDialCost = dialCost } - return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst))) + return totalDialCost } -// knownEntry implements wrsItem -type knownEntry poolEntry - -// Weight calculates random selection weight for known entries -func (e *knownEntry) Weight() int64 { - if e.state != psNotConnected || !e.known || e.delayedRetry { - return 0 +// serviceValue returns the service value accumulated in this session and in total +func (s *serverPool) serviceValue(node *enode.Node) (sessionValue, totalValue float64) { + nvt := s.vt.GetNode(node.ID()) + if nvt == nil { + return 0, 0 } - return int64(1000000000 * e.connectStats.recentAvg() * math.Exp(-float64(e.lastConnected.fails)*failDropLn-e.responseStats.recentAvg()/float64(responseScoreTC)-e.delayStats.recentAvg()/float64(delayScoreTC)) * math.Pow(1-e.timeoutStats.recentAvg(), timeoutPow)) -} + currentStats := nvt.RtStats() + _, timeWeights := s.getTimeoutAndWeight() + expFactor := s.vt.StatsExpFactor() -// poolEntryAddress is a separate object because currently it is necessary to remember -// multiple potential network addresses for a pool entry. This will be removed after -// the final implementation of v5 discovery which will retrieve signed and serial -// numbered advertisements, making it clear which IP/port is the latest one. -type poolEntryAddress struct { - ip net.IP - port uint16 - lastSeen mclock.AbsTime // last time it was discovered, connected or loaded from db - fails uint // connection failures since last successful connection (persistent) -} - -func (a *poolEntryAddress) Weight() int64 { - t := time.Duration(mclock.Now() - a.lastSeen) - return int64(1000000*math.Exp(-float64(t)/float64(discoverExpireConst)-float64(a.fails)*addrFailDropLn)) + 1 -} - -func (a *poolEntryAddress) strKey() string { - return a.ip.String() + ":" + strconv.Itoa(int(a.port)) -} - -// poolStats implement statistics for a certain quantity with a long term average -// and a short term value which is adjusted exponentially with a factor of -// pstatRecentAdjust with each update and also returned exponentially to the -// average with the time constant pstatReturnToMeanTC -type poolStats struct { - sum, weight, avg, recent float64 - lastRecalc mclock.AbsTime -} - -// init initializes stats with a long term sum/update count pair retrieved from the database -func (s *poolStats) init(sum, weight float64) { - s.sum = sum - s.weight = weight - var avg float64 - if weight > 0 { - avg = s.sum / weight + totalValue = currentStats.Value(timeWeights, expFactor) + if connStats, ok := s.ns.GetField(node, sfiConnectedStats).(lpc.ResponseTimeStats); ok { + diff := currentStats + diff.SubStats(&connStats) + sessionValue = diff.Value(timeWeights, expFactor) + sessionValueMeter.Mark(int64(sessionValue)) } - s.avg = avg - s.recent = avg - s.lastRecalc = mclock.Now() + return } -// recalc recalculates recent value return-to-mean and long term average -func (s *poolStats) recalc() { - now := mclock.Now() - s.recent = s.avg + (s.recent-s.avg)*math.Exp(-float64(now-s.lastRecalc)/float64(pstatReturnToMeanTC)) - if s.sum == 0 { - s.avg = 0 +// updateWeight calculates the node weight and updates the nodeWeight field and the +// hasValue flag. It also saves the node state if necessary. +func (s *serverPool) updateWeight(node *enode.Node, totalValue float64, totalDialCost uint64) { + weight := uint64(totalValue * nodeWeightMul / float64(totalDialCost)) + if weight >= nodeWeightThreshold { + s.ns.SetState(node, sfHasValue, nodestate.Flags{}, 0) + s.ns.SetField(node, sfiNodeWeight, weight) } else { - if s.sum > s.weight*1e30 { - s.avg = 1e30 - } else { - s.avg = s.sum / s.weight - } - } - s.lastRecalc = now -} - -// add updates the stats with a new value -func (s *poolStats) add(value, weight float64) { - s.weight += weight - s.sum += value * weight - s.recalc() -} - -// recentAvg returns the short-term adjusted average -func (s *poolStats) recentAvg() float64 { - s.recalc() - return s.recent -} - -func (s *poolStats) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, []interface{}{math.Float64bits(s.sum), math.Float64bits(s.weight)}) -} - -func (s *poolStats) DecodeRLP(st *rlp.Stream) error { - var stats struct { - SumUint, WeightUint uint64 - } - if err := st.Decode(&stats); err != nil { - return err - } - s.init(math.Float64frombits(stats.SumUint), math.Float64frombits(stats.WeightUint)) - return nil -} - -// poolEntryQueue keeps track of its least recently accessed entries and removes -// them when the number of entries reaches the limit -type poolEntryQueue struct { - queue map[int]*poolEntry // known nodes indexed by their latest lastConnCnt value - newPtr, oldPtr, maxCnt int - removeFromPool func(*poolEntry) -} - -// newPoolEntryQueue returns a new poolEntryQueue -func newPoolEntryQueue(maxCnt int, removeFromPool func(*poolEntry)) poolEntryQueue { - return poolEntryQueue{queue: make(map[int]*poolEntry), maxCnt: maxCnt, removeFromPool: removeFromPool} -} - -// fetchOldest returns and removes the least recently accessed entry -func (q *poolEntryQueue) fetchOldest() *poolEntry { - if len(q.queue) == 0 { - return nil - } - for { - if e := q.queue[q.oldPtr]; e != nil { - delete(q.queue, q.oldPtr) - q.oldPtr++ - return e - } - q.oldPtr++ - } -} - -// remove removes an entry from the queue -func (q *poolEntryQueue) remove(entry *poolEntry) { - if q.queue[entry.queueIdx] == entry { - delete(q.queue, entry.queueIdx) - } -} - -// setLatest adds or updates a recently accessed entry. It also checks if an old entry -// needs to be removed and removes it from the parent pool too with a callback function. -func (q *poolEntryQueue) setLatest(entry *poolEntry) { - if q.queue[entry.queueIdx] == entry { - delete(q.queue, entry.queueIdx) + s.ns.SetState(node, nodestate.Flags{}, sfHasValue, 0) + s.ns.SetField(node, sfiNodeWeight, nil) + } + s.ns.Persist(node) // saved if node history or hasValue changed +} + +// setRedialWait calculates and sets the redialWait timeout based on the service value +// and dial cost accumulated during the last session/attempt and in total. +// The waiting time is raised exponentially if no service value has been received in order +// to prevent dialing an unresponsive node frequently for a very long time just because it +// was useful in the past. It can still be occasionally dialed though and once it provides +// a significant amount of service value again its waiting time is quickly reduced or reset +// to the minimum. +// Note: node weight is also recalculated and updated by this function. +func (s *serverPool) setRedialWait(node *enode.Node, addDialCost int64, waitStep float64) { + n, _ := s.ns.GetField(node, sfiNodeHistory).(nodeHistory) + sessionValue, totalValue := s.serviceValue(node) + totalDialCost := s.addDialCost(&n, addDialCost) + + // if the current dial session has yielded at least the average value/dial cost ratio + // then the waiting time should be reset to the minimum. If the session value + // is below average but still positive then timeout is limited to the ratio of + // average / current service value multiplied by the minimum timeout. If the attempt + // was unsuccessful then timeout is raised exponentially without limitation. + // Note: dialCost is used in the formula below even if dial was not attempted at all + // because the pre-negotiation query did not return a positive result. In this case + // the ratio has no meaning anyway and waitFactor is always raised, though in smaller + // steps because queries are cheaper and therefore we can allow more failed attempts. + unixTime := s.unixTime() + plannedTimeout := float64(n.redialWaitEnd - n.redialWaitStart) // last planned redialWait timeout + var actualWait float64 // actual waiting time elapsed + if unixTime > n.redialWaitEnd { + // the planned timeout has elapsed + actualWait = plannedTimeout } else { - if len(q.queue) == q.maxCnt { - e := q.fetchOldest() - q.remove(e) - q.removeFromPool(e) - } - } - entry.queueIdx = q.newPtr - q.queue[entry.queueIdx] = entry - q.newPtr++ -} - -// traverses the queue and returns it as an array ordered oldest -> newest -func (q *poolEntryQueue) list() []*poolEntry { - if len(q.queue) == 0 { - return nil - } - list := make([]*poolEntry, len(q.queue)) - ptr := q.oldPtr - for i := range list { - queueTraverser: - for { - if e := q.queue[ptr]; e != nil { - ptr++ - list[i] = e - break queueTraverser - } - ptr++ - } - } - return list + // if the node was redialed earlier then we do not raise the planned timeout + // exponentially because that could lead to the timeout rising very high in + // a short amount of time + // Note that in case of an early redial actualWait also includes the dial + // timeout or connection time of the last attempt but it still serves its + // purpose of preventing the timeout rising quicker than linearly as a function + // of total time elapsed without a successful connection. + actualWait = float64(unixTime - n.redialWaitStart) + } + // raise timeout exponentially if the last planned timeout has elapsed + // (use at least the last planned timeout otherwise) + nextTimeout := actualWait * waitStep + if plannedTimeout > nextTimeout { + nextTimeout = plannedTimeout + } + // we reduce the waiting time if the server has provided service value during the + // connection (but never under the minimum) + a := totalValue * dialCost * float64(minRedialWait) + b := float64(totalDialCost) * sessionValue + if a < b*nextTimeout { + nextTimeout = a / b + } + if nextTimeout < minRedialWait { + nextTimeout = minRedialWait + } + wait := time.Duration(float64(time.Second) * nextTimeout) + if wait < waitThreshold { + n.redialWaitStart = unixTime + n.redialWaitEnd = unixTime + int64(nextTimeout) + s.ns.SetField(node, sfiNodeHistory, n) + s.ns.SetState(node, sfRedialWait, nodestate.Flags{}, wait) + s.updateWeight(node, totalValue, totalDialCost) + } else { + // discard known node statistics if waiting time is very long because the node + // hasn't been responsive for a very long time + s.ns.SetField(node, sfiNodeHistory, nil) + s.ns.SetField(node, sfiNodeWeight, nil) + s.ns.SetState(node, nodestate.Flags{}, sfHasValue, 0) + } +} + +// calculateWeight calculates and sets the node weight without altering the node history. +// This function should be called during startup and shutdown only, otherwise setRedialWait +// will keep the weights updated as the underlying statistics are adjusted. +func (s *serverPool) calculateWeight(node *enode.Node) { + n, _ := s.ns.GetField(node, sfiNodeHistory).(nodeHistory) + _, totalValue := s.serviceValue(node) + totalDialCost := s.addDialCost(&n, 0) + s.updateWeight(node, totalValue, totalDialCost) } diff --git a/les/serverpool_test.go b/les/serverpool_test.go new file mode 100644 index 0000000000..c352b583fa --- /dev/null +++ b/les/serverpool_test.go @@ -0,0 +1,352 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "math/rand" + "sync/atomic" + "testing" + "time" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/ethdb" + "github.com/celo-org/celo-blockchain/ethdb/memorydb" + lpc "github.com/celo-org/celo-blockchain/les/lespay/client" + "github.com/celo-org/celo-blockchain/p2p" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" +) + +const ( + spTestNodes = 1000 + spTestTarget = 5 + spTestLength = 10000 + spMinTotal = 40000 + spMaxTotal = 50000 +) + +func testNodeID(i int) enode.ID { + return enode.ID{42, byte(i % 256), byte(i / 256)} +} + +func testNodeIndex(id enode.ID) int { + if id[0] != 42 { + return -1 + } + return int(id[1]) + int(id[2])*256 +} + +type serverPoolTest struct { + db ethdb.KeyValueStore + clock *mclock.Simulated + quit chan struct{} + preNeg, preNegFail bool + vt *lpc.ValueTracker + sp *serverPool + input enode.Iterator + testNodes []spTestNode + trusted []string + waitCount, waitEnded int32 + + cycle, conn, servedConn int + serviceCycles, dialCount int + disconnect map[int][]int +} + +type spTestNode struct { + connectCycles, waitCycles int + nextConnCycle, totalConn int + connected, service bool + peer *serverPeer +} + +func newServerPoolTest(preNeg, preNegFail bool) *serverPoolTest { + nodes := make([]*enode.Node, spTestNodes) + for i := range nodes { + nodes[i] = enode.SignNull(&enr.Record{}, testNodeID(i)) + } + return &serverPoolTest{ + clock: &mclock.Simulated{}, + db: memorydb.New(), + input: enode.CycleNodes(nodes), + testNodes: make([]spTestNode, spTestNodes), + preNeg: preNeg, + preNegFail: preNegFail, + } +} + +func (s *serverPoolTest) beginWait() { + // ensure that dialIterator and the maximal number of pre-neg queries are not all stuck in a waiting state + for atomic.AddInt32(&s.waitCount, 1) > preNegLimit { + atomic.AddInt32(&s.waitCount, -1) + s.clock.Run(time.Second) + } +} + +func (s *serverPoolTest) endWait() { + atomic.AddInt32(&s.waitCount, -1) + atomic.AddInt32(&s.waitEnded, 1) +} + +func (s *serverPoolTest) addTrusted(i int) { + s.trusted = append(s.trusted, enode.SignNull(&enr.Record{}, testNodeID(i)).String()) +} + +func (s *serverPoolTest) start() { + var testQuery queryFunc + if s.preNeg { + testQuery = func(node *enode.Node) int { + idx := testNodeIndex(node.ID()) + n := &s.testNodes[idx] + canConnect := !n.connected && n.connectCycles != 0 && s.cycle >= n.nextConnCycle + if s.preNegFail { + // simulate a scenario where UDP queries never work + s.beginWait() + s.clock.Sleep(time.Second * 5) + s.endWait() + return -1 + } else { + switch idx % 3 { + case 0: + // pre-neg returns true only if connection is possible + if canConnect { + return 1 + } else { + return 0 + } + case 1: + // pre-neg returns true but connection might still fail + return 1 + case 2: + // pre-neg returns true if connection is possible, otherwise timeout (node unresponsive) + if canConnect { + return 1 + } else { + s.beginWait() + s.clock.Sleep(time.Second * 5) + s.endWait() + return -1 + } + } + return -1 + } + } + } + + s.vt = lpc.NewValueTracker(s.db, s.clock, requestList, time.Minute, 1/float64(time.Hour), 1/float64(time.Hour*100), 1/float64(time.Hour*1000)) + s.sp = newServerPool(s.db, []byte("serverpool:"), s.vt, s.input, 0, testQuery, s.clock, s.trusted) + s.sp.validSchemes = enode.ValidSchemesForTesting + s.sp.unixTime = func() int64 { return int64(s.clock.Now()) / int64(time.Second) } + s.disconnect = make(map[int][]int) + s.sp.start() + s.quit = make(chan struct{}) + go func() { + last := int32(-1) + for { + select { + case <-time.After(time.Millisecond * 100): + c := atomic.LoadInt32(&s.waitEnded) + if c == last { + // advance clock if test is stuck (might happen in rare cases) + s.clock.Run(time.Second) + } + last = c + case <-s.quit: + return + } + } + }() +} + +func (s *serverPoolTest) stop() { + close(s.quit) + s.sp.stop() + s.vt.Stop() + for i := range s.testNodes { + n := &s.testNodes[i] + if n.connected { + n.totalConn += s.cycle + } + n.connected = false + n.peer = nil + n.nextConnCycle = 0 + } + s.conn, s.servedConn = 0, 0 +} + +func (s *serverPoolTest) run() { + for count := spTestLength; count > 0; count-- { + if dcList := s.disconnect[s.cycle]; dcList != nil { + for _, idx := range dcList { + n := &s.testNodes[idx] + s.sp.unregisterPeer(n.peer) + n.totalConn += s.cycle + n.connected = false + n.peer = nil + s.conn-- + if n.service { + s.servedConn-- + } + n.nextConnCycle = s.cycle + n.waitCycles + } + delete(s.disconnect, s.cycle) + } + if s.conn < spTestTarget { + s.dialCount++ + s.beginWait() + s.sp.dialIterator.Next() + s.endWait() + dial := s.sp.dialIterator.Node() + id := dial.ID() + idx := testNodeIndex(id) + n := &s.testNodes[idx] + if !n.connected && n.connectCycles != 0 && s.cycle >= n.nextConnCycle { + s.conn++ + if n.service { + s.servedConn++ + } + n.totalConn -= s.cycle + n.connected = true + dc := s.cycle + n.connectCycles + s.disconnect[dc] = append(s.disconnect[dc], idx) + n.peer = &serverPeer{peerCommons: peerCommons{Peer: p2p.NewPeer(id, "", nil)}} + s.sp.registerPeer(n.peer) + if n.service { + s.vt.Served(s.vt.GetNode(id), []lpc.ServedRequest{{ReqType: 0, Amount: 100}}, 0) + } + } + } + s.serviceCycles += s.servedConn + s.clock.Run(time.Second) + s.cycle++ + } +} + +func (s *serverPoolTest) setNodes(count, conn, wait int, service, trusted bool) (res []int) { + for ; count > 0; count-- { + idx := rand.Intn(spTestNodes) + for s.testNodes[idx].connectCycles != 0 || s.testNodes[idx].connected { + idx = rand.Intn(spTestNodes) + } + res = append(res, idx) + s.testNodes[idx] = spTestNode{ + connectCycles: conn, + waitCycles: wait, + service: service, + } + if trusted { + s.addTrusted(idx) + } + } + return +} + +func (s *serverPoolTest) resetNodes() { + for i, n := range s.testNodes { + if n.connected { + n.totalConn += s.cycle + s.sp.unregisterPeer(n.peer) + } + s.testNodes[i] = spTestNode{totalConn: n.totalConn} + } + s.conn, s.servedConn = 0, 0 + s.disconnect = make(map[int][]int) + s.trusted = nil +} + +func (s *serverPoolTest) checkNodes(t *testing.T, nodes []int) { + var sum int + for _, idx := range nodes { + n := &s.testNodes[idx] + if n.connected { + n.totalConn += s.cycle + } + sum += n.totalConn + n.totalConn = 0 + if n.connected { + n.totalConn -= s.cycle + } + } + if sum < spMinTotal || sum > spMaxTotal { + t.Errorf("Total connection amount %d outside expected range %d to %d", sum, spMinTotal, spMaxTotal) + } +} + +func TestServerPool(t *testing.T) { testServerPool(t, false, false) } +func TestServerPoolWithPreNeg(t *testing.T) { testServerPool(t, true, false) } +func TestServerPoolWithPreNegFail(t *testing.T) { testServerPool(t, true, true) } +func testServerPool(t *testing.T, preNeg, fail bool) { + s := newServerPoolTest(preNeg, fail) + nodes := s.setNodes(100, 200, 200, true, false) + s.setNodes(100, 20, 20, false, false) + s.start() + s.run() + s.stop() + s.checkNodes(t, nodes) +} + +func TestServerPoolChangedNodes(t *testing.T) { testServerPoolChangedNodes(t, false) } +func TestServerPoolChangedNodesWithPreNeg(t *testing.T) { testServerPoolChangedNodes(t, true) } +func testServerPoolChangedNodes(t *testing.T, preNeg bool) { + s := newServerPoolTest(preNeg, false) + nodes := s.setNodes(100, 200, 200, true, false) + s.setNodes(100, 20, 20, false, false) + s.start() + s.run() + s.checkNodes(t, nodes) + for i := 0; i < 3; i++ { + s.resetNodes() + nodes := s.setNodes(100, 200, 200, true, false) + s.setNodes(100, 20, 20, false, false) + s.run() + s.checkNodes(t, nodes) + } + s.stop() +} + +func TestServerPoolRestartNoDiscovery(t *testing.T) { testServerPoolRestartNoDiscovery(t, false) } +func TestServerPoolRestartNoDiscoveryWithPreNeg(t *testing.T) { + testServerPoolRestartNoDiscovery(t, true) +} +func testServerPoolRestartNoDiscovery(t *testing.T, preNeg bool) { + s := newServerPoolTest(preNeg, false) + nodes := s.setNodes(100, 200, 200, true, false) + s.setNodes(100, 20, 20, false, false) + s.start() + s.run() + s.stop() + s.checkNodes(t, nodes) + s.input = nil + s.start() + s.run() + s.stop() + s.checkNodes(t, nodes) +} + +func TestServerPoolTrustedNoDiscovery(t *testing.T) { testServerPoolTrustedNoDiscovery(t, false) } +func TestServerPoolTrustedNoDiscoveryWithPreNeg(t *testing.T) { + testServerPoolTrustedNoDiscovery(t, true) +} +func testServerPoolTrustedNoDiscovery(t *testing.T, preNeg bool) { + s := newServerPoolTest(preNeg, false) + trusted := s.setNodes(200, 200, 200, true, true) + s.input = nil + s.start() + s.run() + s.stop() + s.checkNodes(t, trusted) +} diff --git a/les/test_helper.go b/les/test_helper.go index f6a2f93d51..37e2f7b095 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -507,7 +507,7 @@ func newClientServerEnv(t *testing.T, syncMode downloader.SyncMode, blocks int, clock = &mclock.Simulated{} } dist := newRequestDistributor(speers, clock) - rm := newRetrieveManager(speers, dist, nil) + rm := newRetrieveManager(speers, dist, func() time.Duration { return time.Millisecond * 500 }) odr := NewLesOdr(cdb, light.TestClientIndexerConfig, rm) sindexers := testIndexers(sdb, nil, light.TestServerIndexerConfig) diff --git a/les/utils/expiredvalue.go b/les/utils/expiredvalue.go index 774cd168d1..55cf235ac0 100644 --- a/les/utils/expiredvalue.go +++ b/les/utils/expiredvalue.go @@ -63,14 +63,7 @@ func ExpFactor(logOffset Fixed64) ExpirationFactor { // Value calculates the expired value based on a floating point base and integer // power-of-2 exponent. This function should be used by multi-value expired structures. func (e ExpirationFactor) Value(base float64, exp uint64) float64 { - res := base / e.Factor - if exp > e.Exp { - res *= float64(uint64(1) << (exp - e.Exp)) - } - if exp < e.Exp { - res /= float64(uint64(1) << (e.Exp - exp)) - } - return res + return base / e.Factor * math.Pow(2, float64(int64(exp-e.Exp))) } // value calculates the value at the given moment. diff --git a/les/utils/weighted_select.go b/les/utils/weighted_select.go index fbf1f37d62..d6db3c0e65 100644 --- a/les/utils/weighted_select.go +++ b/les/utils/weighted_select.go @@ -16,28 +16,44 @@ package utils -import "math/rand" +import ( + "math/rand" +) -// wrsItem interface should be implemented by any entries that are to be selected from -// a WeightedRandomSelect set. Note that recalculating monotonously decreasing item -// weights on-demand (without constantly calling Update) is allowed -type wrsItem interface { - Weight() int64 +type ( + // WeightedRandomSelect is capable of weighted random selection from a set of items + WeightedRandomSelect struct { + root *wrsNode + idx map[WrsItem]int + wfn WeightFn + } + WrsItem interface{} + WeightFn func(interface{}) uint64 +) + +// NewWeightedRandomSelect returns a new WeightedRandomSelect structure +func NewWeightedRandomSelect(wfn WeightFn) *WeightedRandomSelect { + return &WeightedRandomSelect{root: &wrsNode{maxItems: wrsBranches}, idx: make(map[WrsItem]int), wfn: wfn} } -// WeightedRandomSelect is capable of weighted random selection from a set of items -type WeightedRandomSelect struct { - root *wrsNode - idx map[wrsItem]int +// Update updates an item's weight, adds it if it was non-existent or removes it if +// the new weight is zero. Note that explicitly updating decreasing weights is not necessary. +func (w *WeightedRandomSelect) Update(item WrsItem) { + w.setWeight(item, w.wfn(item)) } -// NewWeightedRandomSelect returns a new WeightedRandomSelect structure -func NewWeightedRandomSelect() *WeightedRandomSelect { - return &WeightedRandomSelect{root: &wrsNode{maxItems: wrsBranches}, idx: make(map[wrsItem]int)} +// Remove removes an item from the set +func (w *WeightedRandomSelect) Remove(item WrsItem) { + w.setWeight(item, 0) +} + +// IsEmpty returns true if the set is empty +func (w *WeightedRandomSelect) IsEmpty() bool { + return w.root.sumWeight == 0 } // setWeight sets an item's weight to a specific value (removes it if zero) -func (w *WeightedRandomSelect) setWeight(item wrsItem, weight int64) { +func (w *WeightedRandomSelect) setWeight(item WrsItem, weight uint64) { idx, ok := w.idx[item] if ok { w.root.setWeight(idx, weight) @@ -58,33 +74,22 @@ func (w *WeightedRandomSelect) setWeight(item wrsItem, weight int64) { } } -// Update updates an item's weight, adds it if it was non-existent or removes it if -// the new weight is zero. Note that explicitly updating decreasing weights is not necessary. -func (w *WeightedRandomSelect) Update(item wrsItem) { - w.setWeight(item, item.Weight()) -} - -// Remove removes an item from the set -func (w *WeightedRandomSelect) Remove(item wrsItem) { - w.setWeight(item, 0) -} - // Choose randomly selects an item from the set, with a chance proportional to its // current weight. If the weight of the chosen element has been decreased since the // last stored value, returns it with a newWeight/oldWeight chance, otherwise just // updates its weight and selects another one -func (w *WeightedRandomSelect) Choose() wrsItem { +func (w *WeightedRandomSelect) Choose() WrsItem { for { if w.root.sumWeight == 0 { return nil } - val := rand.Int63n(w.root.sumWeight) + val := uint64(rand.Int63n(int64(w.root.sumWeight))) choice, lastWeight := w.root.choose(val) - weight := choice.Weight() + weight := w.wfn(choice) if weight != lastWeight { w.setWeight(choice, weight) } - if weight >= lastWeight || rand.Int63n(lastWeight) < weight { + if weight >= lastWeight || uint64(rand.Int63n(int64(lastWeight))) < weight { return choice } } @@ -92,16 +97,16 @@ func (w *WeightedRandomSelect) Choose() wrsItem { const wrsBranches = 8 // max number of branches in the wrsNode tree -// wrsNode is a node of a tree structure that can store wrsItems or further wrsNodes. +// wrsNode is a node of a tree structure that can store WrsItems or further wrsNodes. type wrsNode struct { items [wrsBranches]interface{} - weights [wrsBranches]int64 - sumWeight int64 + weights [wrsBranches]uint64 + sumWeight uint64 level, itemCnt, maxItems int } // insert recursively inserts a new item to the tree and returns the item index -func (n *wrsNode) insert(item wrsItem, weight int64) int { +func (n *wrsNode) insert(item WrsItem, weight uint64) int { branch := 0 for n.items[branch] != nil && (n.level == 0 || n.items[branch].(*wrsNode).itemCnt == n.items[branch].(*wrsNode).maxItems) { branch++ @@ -129,7 +134,7 @@ func (n *wrsNode) insert(item wrsItem, weight int64) int { // setWeight updates the weight of a certain item (which should exist) and returns // the change of the last weight value stored in the tree -func (n *wrsNode) setWeight(idx int, weight int64) int64 { +func (n *wrsNode) setWeight(idx int, weight uint64) uint64 { if n.level == 0 { oldWeight := n.weights[idx] n.weights[idx] = weight @@ -152,12 +157,12 @@ func (n *wrsNode) setWeight(idx int, weight int64) int64 { return diff } -// Choose recursively selects an item from the tree and returns it along with its weight -func (n *wrsNode) choose(val int64) (wrsItem, int64) { +// choose recursively selects an item from the tree and returns it along with its weight +func (n *wrsNode) choose(val uint64) (WrsItem, uint64) { for i, w := range n.weights { if val < w { if n.level == 0 { - return n.items[i].(wrsItem), n.weights[i] + return n.items[i].(WrsItem), n.weights[i] } return n.items[i].(*wrsNode).choose(val) } diff --git a/les/utils/weighted_select_test.go b/les/utils/weighted_select_test.go index e1969e1a61..3e1c0ad987 100644 --- a/les/utils/weighted_select_test.go +++ b/les/utils/weighted_select_test.go @@ -26,17 +26,18 @@ type testWrsItem struct { widx *int } -func (t *testWrsItem) Weight() int64 { +func testWeight(i interface{}) uint64 { + t := i.(*testWrsItem) w := *t.widx if w == -1 || w == t.idx { - return int64(t.idx + 1) + return uint64(t.idx + 1) } return 0 } func TestWeightedRandomSelect(t *testing.T) { testFn := func(cnt int) { - s := NewWeightedRandomSelect() + s := NewWeightedRandomSelect(testWeight) w := -1 list := make([]testWrsItem, cnt) for i := range list { diff --git a/light/lightchain.go b/light/lightchain.go index c8b1ea2f8e..b68edaa835 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -319,10 +319,16 @@ func (lc *LightChain) Stop() { return } close(lc.quit) - atomic.StoreInt32(&lc.procInterrupt, 1) - + lc.StopInsert() lc.wg.Wait() - log.Info("Blockchain manager stopped") + log.Info("Blockchain stopped") +} + +// StopInsert interrupts all insertion methods, causing them to return +// errInsertionInterrupted as soon as possible. Insertion is permanently disabled after +// calling this method. +func (lc *LightChain) StopInsert() { + atomic.StoreInt32(&lc.procInterrupt, 1) } // Rollback is designed to remove a chain of links from the database that aren't diff --git a/log/doc.go b/log/doc.go index bff2f4966a..993743c0fd 100644 --- a/log/doc.go +++ b/log/doc.go @@ -65,7 +65,7 @@ This will output a log line that includes the path context that is attached to t Handlers -The Handler interface defines where log lines are printed to and how they are formated. Handler is a +The Handler interface defines where log lines are printed to and how they are formatted. Handler is a single interface that is inspired by net/http's handler interface: type Handler interface { diff --git a/log/format.go b/log/format.go index 08eedda378..e0d6874c9c 100644 --- a/log/format.go +++ b/log/format.go @@ -78,7 +78,7 @@ type TerminalStringer interface { // a terminal with color-coded level output and terser human friendly timestamp. // This format should only be used for interactive programs or while developing. // -// [LEVEL] [TIME] MESAGE key=value key=value ... +// [LEVEL] [TIME] MESSAGE key=value key=value ... // // Example: // diff --git a/log/handler.go b/log/handler.go index 3c99114dcb..4ad433334e 100644 --- a/log/handler.go +++ b/log/handler.go @@ -117,7 +117,7 @@ func formatCall(format string, c stack.Call) string { } // CallerStackHandler returns a Handler that adds a stack trace to the context -// with key "stack". The stack trace is formated as a space separated list of +// with key "stack". The stack trace is formatted as a space separated list of // call sites inside matching []'s. The most recent call site is listed first. // Each call site is formatted according to format. See the documentation of // package github.com/go-stack/stack for the list of supported formats. diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index 99d44e4002..00d666eef4 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -18,14 +18,22 @@ package metrics -import "github.com/elastic/gosigar" +import ( + "github.com/celo-org/celo-blockchain/log" + "github.com/shirou/gopsutil/cpu" +) // ReadCPUStats retrieves the current CPU stats. func ReadCPUStats(stats *CPUStats) { - global := gosigar.Cpu{} - global.Get() - - stats.GlobalTime = int64(global.User + global.Nice + global.Sys) - stats.GlobalWait = int64(global.Wait) + // passing false to request all cpu times + timeStats, err := cpu.Times(false) + if err != nil { + log.Error("Could not read cpu stats", "err", err) + return + } + // requesting all cpu times will always return an array with only one time stats entry + timeStat := timeStats[0] + stats.GlobalTime = int64((timeStat.User + timeStat.Nice + timeStat.System) * cpu.ClocksPerSec) + stats.GlobalWait = int64((timeStat.Iowait) * cpu.ClocksPerSec) stats.LocalTime = getProcessCPUTime() } diff --git a/metrics/prometheus/collector.go b/metrics/prometheus/collector.go index 60c1579dfe..2e0fe7e600 100644 --- a/metrics/prometheus/collector.go +++ b/metrics/prometheus/collector.go @@ -30,7 +30,7 @@ var ( typeCounterTpl = "# TYPE %s counter\n" typeSummaryTpl = "# TYPE %s summary\n" keyValueTpl = "%s %v\n\n" - keyQuantileTagValueTpl = "%s {quantile=\"%s\"} %v\n\n" + keyQuantileTagValueTpl = "%s {quantile=\"%s\"} %v\n" ) // collector is a collection of byte buffers that aggregate Prometheus reports @@ -39,7 +39,7 @@ type collector struct { buff *bytes.Buffer } -// newCollector createa a new Prometheus metric aggregator. +// newCollector creates a new Prometheus metric aggregator. func newCollector() *collector { return &collector{ buff: &bytes.Buffer{}, @@ -62,9 +62,11 @@ func (c *collector) addHistogram(name string, m metrics.Histogram) { pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} ps := m.Percentiles(pv) c.writeSummaryCounter(name, m.Count()) + c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) for i := range pv { c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) } + c.buff.WriteRune('\n') } func (c *collector) addMeter(name string, m metrics.Meter) { @@ -75,9 +77,11 @@ func (c *collector) addTimer(name string, m metrics.Timer) { pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} ps := m.Percentiles(pv) c.writeSummaryCounter(name, m.Count()) + c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) for i := range pv { c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i]) } + c.buff.WriteRune('\n') } func (c *collector) addResettingTimer(name string, m metrics.ResettingTimer) { @@ -87,9 +91,11 @@ func (c *collector) addResettingTimer(name string, m metrics.ResettingTimer) { ps := m.Percentiles([]float64{50, 95, 99}) val := m.Values() c.writeSummaryCounter(name, len(val)) + c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, mutateKey(name))) c.writeSummaryPercentile(name, "0.50", ps[0]) c.writeSummaryPercentile(name, "0.95", ps[1]) c.writeSummaryPercentile(name, "0.99", ps[2]) + c.buff.WriteRune('\n') } func (c *collector) writeGaugeCounter(name string, value interface{}) { @@ -106,7 +112,6 @@ func (c *collector) writeSummaryCounter(name string, value interface{}) { func (c *collector) writeSummaryPercentile(name, p string, value interface{}) { name = mutateKey(name) - c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, name)) c.buff.WriteString(fmt.Sprintf(keyQuantileTagValueTpl, name, p, value)) } diff --git a/metrics/prometheus/collector_test.go b/metrics/prometheus/collector_test.go new file mode 100644 index 0000000000..b279246395 --- /dev/null +++ b/metrics/prometheus/collector_test.go @@ -0,0 +1,110 @@ +package prometheus + +import ( + "os" + "testing" + "time" + + "github.com/celo-org/celo-blockchain/metrics" +) + +func TestMain(m *testing.M) { + metrics.Enabled = true + os.Exit(m.Run()) +} + +func TestCollector(t *testing.T) { + c := newCollector() + + counter := metrics.NewCounter() + counter.Inc(12345) + c.addCounter("test/counter", counter) + + gauge := metrics.NewGauge() + gauge.Update(23456) + c.addGauge("test/gauge", gauge) + + gaugeFloat64 := metrics.NewGaugeFloat64() + gaugeFloat64.Update(34567.89) + c.addGaugeFloat64("test/gauge_float64", gaugeFloat64) + + histogram := metrics.NewHistogram(&metrics.NilSample{}) + c.addHistogram("test/histogram", histogram) + + meter := metrics.NewMeter() + defer meter.Stop() + meter.Mark(9999999) + c.addMeter("test/meter", meter) + + timer := metrics.NewTimer() + defer timer.Stop() + timer.Update(20 * time.Millisecond) + timer.Update(21 * time.Millisecond) + timer.Update(22 * time.Millisecond) + timer.Update(120 * time.Millisecond) + timer.Update(23 * time.Millisecond) + timer.Update(24 * time.Millisecond) + c.addTimer("test/timer", timer) + + resettingTimer := metrics.NewResettingTimer() + resettingTimer.Update(10 * time.Millisecond) + resettingTimer.Update(11 * time.Millisecond) + resettingTimer.Update(12 * time.Millisecond) + resettingTimer.Update(120 * time.Millisecond) + resettingTimer.Update(13 * time.Millisecond) + resettingTimer.Update(14 * time.Millisecond) + c.addResettingTimer("test/resetting_timer", resettingTimer.Snapshot()) + + emptyResettingTimer := metrics.NewResettingTimer().Snapshot() + c.addResettingTimer("test/empty_resetting_timer", emptyResettingTimer) + + const expectedOutput = `# TYPE test_counter gauge +test_counter 12345 + +# TYPE test_gauge gauge +test_gauge 23456 + +# TYPE test_gauge_float64 gauge +test_gauge_float64 34567.89 + +# TYPE test_histogram_count counter +test_histogram_count 0 + +# TYPE test_histogram summary +test_histogram {quantile="0.5"} 0 +test_histogram {quantile="0.75"} 0 +test_histogram {quantile="0.95"} 0 +test_histogram {quantile="0.99"} 0 +test_histogram {quantile="0.999"} 0 +test_histogram {quantile="0.9999"} 0 + +# TYPE test_meter gauge +test_meter 9999999 + +# TYPE test_timer_count counter +test_timer_count 6 + +# TYPE test_timer summary +test_timer {quantile="0.5"} 2.25e+07 +test_timer {quantile="0.75"} 4.8e+07 +test_timer {quantile="0.95"} 1.2e+08 +test_timer {quantile="0.99"} 1.2e+08 +test_timer {quantile="0.999"} 1.2e+08 +test_timer {quantile="0.9999"} 1.2e+08 + +# TYPE test_resetting_timer_count counter +test_resetting_timer_count 6 + +# TYPE test_resetting_timer summary +test_resetting_timer {quantile="0.50"} 12000000 +test_resetting_timer {quantile="0.95"} 120000000 +test_resetting_timer {quantile="0.99"} 120000000 + +` + exp := c.buff.String() + if exp != expectedOutput { + t.Log("Expected Output:\n", expectedOutput) + t.Log("Actual Output:\n", exp) + t.Fatal("unexpected collector output") + } +} diff --git a/miner/miner.go b/miner/miner.go index 5dbf4cc6c2..b15f094b9e 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -251,6 +251,6 @@ func (miner *Miner) DisablePreseal() { // SubscribePendingLogs starts delivering logs from pending transactions // to the given channel. -func (self *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return self.worker.pendingLogsFeed.Subscribe(ch) +func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { + return miner.worker.pendingLogsFeed.Subscribe(ch) } diff --git a/mobile/doc.go b/mobile/doc.go index 64d47bec2a..20131afc2e 100644 --- a/mobile/doc.go +++ b/mobile/doc.go @@ -24,7 +24,7 @@ // // Since gomobile cannot bridge arbitrary types between Go and Android/iOS, the // exposed APIs need to be manually wrapped into simplified types, with custom -// constructors and getters/setters to ensure that they can be meaninfully used +// constructors and getters/setters to ensure that they can be meaningfully used // from Java/ObjC too. // // With this in mind, please try to limit the scope of this package and only add diff --git a/node/node_test.go b/node/node_test.go index c62431c8b4..ed9178e2eb 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -352,7 +352,7 @@ func TestServiceStartupAbortion(t *testing.T) { } // Tests that even if a registered service fails to shut down cleanly, it does -// not influece the rest of the shutdown invocations. +// not influence the rest of the shutdown invocations. func TestServiceTerminationGuarantee(t *testing.T) { stack, err := New(testNodeConfig()) if err != nil { diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index f4760905b6..fd9c87fc26 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -251,7 +251,7 @@ func (it *randomIterator) Next() bool { return it.cur != nil } -// addTree adds a enrtree:// URL to the iterator. +// addTree adds an enrtree:// URL to the iterator. func (it *randomIterator) addTree(url string) error { le, err := parseLink(url) if err != nil { diff --git a/p2p/nodestate/nodestate.go b/p2p/nodestate/nodestate.go new file mode 100644 index 0000000000..dab56caa39 --- /dev/null +++ b/p2p/nodestate/nodestate.go @@ -0,0 +1,880 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package nodestate + +import ( + "errors" + "reflect" + "sync" + "time" + "unsafe" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/ethdb" + "github.com/celo-org/celo-blockchain/log" + "github.com/celo-org/celo-blockchain/metrics" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" + "github.com/celo-org/celo-blockchain/rlp" +) + +type ( + // NodeStateMachine connects different system components operating on subsets of + // network nodes. Node states are represented by 64 bit vectors with each bit assigned + // to a state flag. Each state flag has a descriptor structure and the mapping is + // created automatically. It is possible to subscribe to subsets of state flags and + // receive a callback if one of the nodes has a relevant state flag changed. + // Callbacks can also modify further flags of the same node or other nodes. State + // updates only return after all immediate effects throughout the system have happened + // (deadlocks should be avoided by design of the implemented state logic). The caller + // can also add timeouts assigned to a certain node and a subset of state flags. + // If the timeout elapses, the flags are reset. If all relevant flags are reset then + // the timer is dropped. State flags with no timeout are persisted in the database + // if the flag descriptor enables saving. If a node has no state flags set at any + // moment then it is discarded. + // + // Extra node fields can also be registered so system components can also store more + // complex state for each node that is relevant to them, without creating a custom + // peer set. Fields can be shared across multiple components if they all know the + // field ID. Subscription to fields is also possible. Persistent fields should have + // an encoder and a decoder function. + NodeStateMachine struct { + started, stopped bool + lock sync.Mutex + clock mclock.Clock + db ethdb.KeyValueStore + dbNodeKey []byte + nodes map[enode.ID]*nodeInfo + offlineCallbackList []offlineCallback + + // Registered state flags or fields. Modifications are allowed + // only when the node state machine has not been started. + setup *Setup + fields []*fieldInfo + saveFlags bitMask + + // Installed callbacks. Modifications are allowed only when the + // node state machine has not been started. + stateSubs []stateSub + + // Testing hooks, only for testing purposes. + saveNodeHook func(*nodeInfo) + } + + // Flags represents a set of flags from a certain setup + Flags struct { + mask bitMask + setup *Setup + } + + // Field represents a field from a certain setup + Field struct { + index int + setup *Setup + } + + // flagDefinition describes a node state flag. Each registered instance is automatically + // mapped to a bit of the 64 bit node states. + // If persistent is true then the node is saved when state machine is shutdown. + flagDefinition struct { + name string + persistent bool + } + + // fieldDefinition describes an optional node field of the given type. The contents + // of the field are only retained for each node as long as at least one of the + // state flags is set. + fieldDefinition struct { + name string + ftype reflect.Type + encode func(interface{}) ([]byte, error) + decode func([]byte) (interface{}, error) + } + + // stateSetup contains the list of flags and fields used by the application + Setup struct { + Version uint + flags []flagDefinition + fields []fieldDefinition + } + + // bitMask describes a node state or state mask. It represents a subset + // of node flags with each bit assigned to a flag index (LSB represents flag 0). + bitMask uint64 + + // StateCallback is a subscription callback which is called when one of the + // state flags that is included in the subscription state mask is changed. + // Note: oldState and newState are also masked with the subscription mask so only + // the relevant bits are included. + StateCallback func(n *enode.Node, oldState, newState Flags) + + // FieldCallback is a subscription callback which is called when the value of + // a specific field is changed. + FieldCallback func(n *enode.Node, state Flags, oldValue, newValue interface{}) + + // nodeInfo contains node state, fields and state timeouts + nodeInfo struct { + node *enode.Node + state bitMask + timeouts []*nodeStateTimeout + fields []interface{} + db, dirty bool + } + + nodeInfoEnc struct { + Enr enr.Record + Version uint + State bitMask + Fields [][]byte + } + + stateSub struct { + mask bitMask + callback StateCallback + } + + nodeStateTimeout struct { + mask bitMask + timer mclock.Timer + } + + fieldInfo struct { + fieldDefinition + subs []FieldCallback + } + + offlineCallback struct { + node *enode.Node + state bitMask + fields []interface{} + } +) + +// offlineState is a special state that is assumed to be set before a node is loaded from +// the database and after it is shut down. +const offlineState = bitMask(1) + +// NewFlag creates a new node state flag +func (s *Setup) NewFlag(name string) Flags { + if s.flags == nil { + s.flags = []flagDefinition{{name: "offline"}} + } + f := Flags{mask: bitMask(1) << uint(len(s.flags)), setup: s} + s.flags = append(s.flags, flagDefinition{name: name}) + return f +} + +// NewPersistentFlag creates a new persistent node state flag +func (s *Setup) NewPersistentFlag(name string) Flags { + if s.flags == nil { + s.flags = []flagDefinition{{name: "offline"}} + } + f := Flags{mask: bitMask(1) << uint(len(s.flags)), setup: s} + s.flags = append(s.flags, flagDefinition{name: name, persistent: true}) + return f +} + +// OfflineFlag returns the system-defined offline flag belonging to the given setup +func (s *Setup) OfflineFlag() Flags { + return Flags{mask: offlineState, setup: s} +} + +// NewField creates a new node state field +func (s *Setup) NewField(name string, ftype reflect.Type) Field { + f := Field{index: len(s.fields), setup: s} + s.fields = append(s.fields, fieldDefinition{ + name: name, + ftype: ftype, + }) + return f +} + +// NewPersistentField creates a new persistent node field +func (s *Setup) NewPersistentField(name string, ftype reflect.Type, encode func(interface{}) ([]byte, error), decode func([]byte) (interface{}, error)) Field { + f := Field{index: len(s.fields), setup: s} + s.fields = append(s.fields, fieldDefinition{ + name: name, + ftype: ftype, + encode: encode, + decode: decode, + }) + return f +} + +// flagOp implements binary flag operations and also checks whether the operands belong to the same setup +func flagOp(a, b Flags, trueIfA, trueIfB, trueIfBoth bool) Flags { + if a.setup == nil { + if a.mask != 0 { + panic("Node state flags have no setup reference") + } + a.setup = b.setup + } + if b.setup == nil { + if b.mask != 0 { + panic("Node state flags have no setup reference") + } + b.setup = a.setup + } + if a.setup != b.setup { + panic("Node state flags belong to a different setup") + } + res := Flags{setup: a.setup} + if trueIfA { + res.mask |= a.mask & ^b.mask + } + if trueIfB { + res.mask |= b.mask & ^a.mask + } + if trueIfBoth { + res.mask |= a.mask & b.mask + } + return res +} + +// And returns the set of flags present in both a and b +func (a Flags) And(b Flags) Flags { return flagOp(a, b, false, false, true) } + +// AndNot returns the set of flags present in a but not in b +func (a Flags) AndNot(b Flags) Flags { return flagOp(a, b, true, false, false) } + +// Or returns the set of flags present in either a or b +func (a Flags) Or(b Flags) Flags { return flagOp(a, b, true, true, true) } + +// Xor returns the set of flags present in either a or b but not both +func (a Flags) Xor(b Flags) Flags { return flagOp(a, b, true, true, false) } + +// HasAll returns true if b is a subset of a +func (a Flags) HasAll(b Flags) bool { return flagOp(a, b, false, true, false).mask == 0 } + +// HasNone returns true if a and b have no shared flags +func (a Flags) HasNone(b Flags) bool { return flagOp(a, b, false, false, true).mask == 0 } + +// Equals returns true if a and b have the same flags set +func (a Flags) Equals(b Flags) bool { return flagOp(a, b, true, true, false).mask == 0 } + +// IsEmpty returns true if a has no flags set +func (a Flags) IsEmpty() bool { return a.mask == 0 } + +// MergeFlags merges multiple sets of state flags +func MergeFlags(list ...Flags) Flags { + if len(list) == 0 { + return Flags{} + } + res := list[0] + for i := 1; i < len(list); i++ { + res = res.Or(list[i]) + } + return res +} + +// String returns a list of the names of the flags specified in the bit mask +func (f Flags) String() string { + if f.mask == 0 { + return "[]" + } + s := "[" + comma := false + for index, flag := range f.setup.flags { + if f.mask&(bitMask(1)< 8*int(unsafe.Sizeof(bitMask(0))) { + panic("Too many node state flags") + } + ns := &NodeStateMachine{ + db: db, + dbNodeKey: dbKey, + clock: clock, + setup: setup, + nodes: make(map[enode.ID]*nodeInfo), + fields: make([]*fieldInfo, len(setup.fields)), + } + stateNameMap := make(map[string]int) + for index, flag := range setup.flags { + if _, ok := stateNameMap[flag.name]; ok { + panic("Node state flag name collision") + } + stateNameMap[flag.name] = index + if flag.persistent { + ns.saveFlags |= bitMask(1) << uint(index) + } + } + fieldNameMap := make(map[string]int) + for index, field := range setup.fields { + if _, ok := fieldNameMap[field.name]; ok { + panic("Node field name collision") + } + ns.fields[index] = &fieldInfo{fieldDefinition: field} + fieldNameMap[field.name] = index + } + return ns +} + +// stateMask checks whether the set of flags belongs to the same setup and returns its internal bit mask +func (ns *NodeStateMachine) stateMask(flags Flags) bitMask { + if flags.setup != ns.setup && flags.mask != 0 { + panic("Node state flags belong to a different setup") + } + return flags.mask +} + +// fieldIndex checks whether the field belongs to the same setup and returns its internal index +func (ns *NodeStateMachine) fieldIndex(field Field) int { + if field.setup != ns.setup { + panic("Node field belongs to a different setup") + } + return field.index +} + +// SubscribeState adds a node state subscription. The callback is called while the state +// machine mutex is not held and it is allowed to make further state updates. All immediate +// changes throughout the system are processed in the same thread/goroutine. It is the +// responsibility of the implemented state logic to avoid deadlocks caused by the callbacks, +// infinite toggling of flags or hazardous/non-deterministic state changes. +// State subscriptions should be installed before loading the node database or making the +// first state update. +func (ns *NodeStateMachine) SubscribeState(flags Flags, callback StateCallback) { + ns.lock.Lock() + defer ns.lock.Unlock() + + if ns.started { + panic("state machine already started") + } + ns.stateSubs = append(ns.stateSubs, stateSub{ns.stateMask(flags), callback}) +} + +// SubscribeField adds a node field subscription. Same rules apply as for SubscribeState. +func (ns *NodeStateMachine) SubscribeField(field Field, callback FieldCallback) { + ns.lock.Lock() + defer ns.lock.Unlock() + + if ns.started { + panic("state machine already started") + } + f := ns.fields[ns.fieldIndex(field)] + f.subs = append(f.subs, callback) +} + +// newNode creates a new nodeInfo +func (ns *NodeStateMachine) newNode(n *enode.Node) *nodeInfo { + return &nodeInfo{node: n, fields: make([]interface{}, len(ns.fields))} +} + +// checkStarted checks whether the state machine has already been started and panics otherwise. +func (ns *NodeStateMachine) checkStarted() { + if !ns.started { + panic("state machine not started yet") + } +} + +// Start starts the state machine, enabling state and field operations and disabling +// further subscriptions. +func (ns *NodeStateMachine) Start() { + ns.lock.Lock() + if ns.started { + panic("state machine already started") + } + ns.started = true + if ns.db != nil { + ns.loadFromDb() + } + ns.lock.Unlock() + ns.offlineCallbacks(true) +} + +// Stop stops the state machine and saves its state if a database was supplied +func (ns *NodeStateMachine) Stop() { + ns.lock.Lock() + for _, node := range ns.nodes { + fields := make([]interface{}, len(node.fields)) + copy(fields, node.fields) + ns.offlineCallbackList = append(ns.offlineCallbackList, offlineCallback{node.node, node.state, fields}) + } + ns.stopped = true + if ns.db != nil { + ns.saveToDb() + ns.lock.Unlock() + } else { + ns.lock.Unlock() + } + ns.offlineCallbacks(false) +} + +// loadFromDb loads persisted node states from the database +func (ns *NodeStateMachine) loadFromDb() { + it := ns.db.NewIterator(ns.dbNodeKey, nil) + for it.Next() { + var id enode.ID + if len(it.Key()) != len(ns.dbNodeKey)+len(id) { + log.Error("Node state db entry with invalid length", "found", len(it.Key()), "expected", len(ns.dbNodeKey)+len(id)) + continue + } + copy(id[:], it.Key()[len(ns.dbNodeKey):]) + ns.decodeNode(id, it.Value()) + } +} + +type dummyIdentity enode.ID + +func (id dummyIdentity) Verify(r *enr.Record, sig []byte) error { return nil } +func (id dummyIdentity) NodeAddr(r *enr.Record) []byte { return id[:] } + +// decodeNode decodes a node database entry and adds it to the node set if successful +func (ns *NodeStateMachine) decodeNode(id enode.ID, data []byte) { + var enc nodeInfoEnc + if err := rlp.DecodeBytes(data, &enc); err != nil { + log.Error("Failed to decode node info", "id", id, "error", err) + return + } + n, _ := enode.New(dummyIdentity(id), &enc.Enr) + node := ns.newNode(n) + node.db = true + + if enc.Version != ns.setup.Version { + log.Debug("Removing stored node with unknown version", "current", ns.setup.Version, "stored", enc.Version) + ns.deleteNode(id) + return + } + if len(enc.Fields) > len(ns.setup.fields) { + log.Error("Invalid node field count", "id", id, "stored", len(enc.Fields)) + return + } + // Resolve persisted node fields + for i, encField := range enc.Fields { + if len(encField) == 0 { + continue + } + if decode := ns.fields[i].decode; decode != nil { + if field, err := decode(encField); err == nil { + node.fields[i] = field + } else { + log.Error("Failed to decode node field", "id", id, "field name", ns.fields[i].name, "error", err) + return + } + } else { + log.Error("Cannot decode node field", "id", id, "field name", ns.fields[i].name) + return + } + } + // It's a compatible node record, add it to set. + ns.nodes[id] = node + node.state = enc.State + fields := make([]interface{}, len(node.fields)) + copy(fields, node.fields) + ns.offlineCallbackList = append(ns.offlineCallbackList, offlineCallback{node.node, node.state, fields}) + log.Debug("Loaded node state", "id", id, "state", Flags{mask: enc.State, setup: ns.setup}) +} + +// saveNode saves the given node info to the database +func (ns *NodeStateMachine) saveNode(id enode.ID, node *nodeInfo) error { + if ns.db == nil { + return nil + } + + storedState := node.state & ns.saveFlags + for _, t := range node.timeouts { + storedState &= ^t.mask + } + if storedState == 0 { + if node.db { + node.db = false + ns.deleteNode(id) + } + node.dirty = false + return nil + } + + enc := nodeInfoEnc{ + Enr: *node.node.Record(), + Version: ns.setup.Version, + State: storedState, + Fields: make([][]byte, len(ns.fields)), + } + log.Debug("Saved node state", "id", id, "state", Flags{mask: enc.State, setup: ns.setup}) + lastIndex := -1 + for i, f := range node.fields { + if f == nil { + continue + } + encode := ns.fields[i].encode + if encode == nil { + continue + } + blob, err := encode(f) + if err != nil { + return err + } + enc.Fields[i] = blob + lastIndex = i + } + enc.Fields = enc.Fields[:lastIndex+1] + data, err := rlp.EncodeToBytes(&enc) + if err != nil { + return err + } + if err := ns.db.Put(append(ns.dbNodeKey, id[:]...), data); err != nil { + return err + } + node.dirty, node.db = false, true + + if ns.saveNodeHook != nil { + ns.saveNodeHook(node) + } + return nil +} + +// deleteNode removes a node info from the database +func (ns *NodeStateMachine) deleteNode(id enode.ID) { + ns.db.Delete(append(ns.dbNodeKey, id[:]...)) +} + +// saveToDb saves the persistent flags and fields of all nodes that have been changed +func (ns *NodeStateMachine) saveToDb() { + for id, node := range ns.nodes { + if node.dirty { + err := ns.saveNode(id, node) + if err != nil { + log.Error("Failed to save node", "id", id, "error", err) + } + } + } +} + +// updateEnode updates the enode entry belonging to the given node if it already exists +func (ns *NodeStateMachine) updateEnode(n *enode.Node) (enode.ID, *nodeInfo) { + id := n.ID() + node := ns.nodes[id] + if node != nil && n.Seq() > node.node.Seq() { + node.node = n + } + return id, node +} + +// Persist saves the persistent state and fields of the given node immediately +func (ns *NodeStateMachine) Persist(n *enode.Node) error { + ns.lock.Lock() + defer ns.lock.Unlock() + + ns.checkStarted() + if id, node := ns.updateEnode(n); node != nil && node.dirty { + err := ns.saveNode(id, node) + if err != nil { + log.Error("Failed to save node", "id", id, "error", err) + } + return err + } + return nil +} + +// SetState updates the given node state flags and processes all resulting callbacks. +// It only returns after all subsequent immediate changes (including those changed by the +// callbacks) have been processed. If a flag with a timeout is set again, the operation +// removes or replaces the existing timeout. +func (ns *NodeStateMachine) SetState(n *enode.Node, setFlags, resetFlags Flags, timeout time.Duration) { + ns.lock.Lock() + ns.checkStarted() + if ns.stopped { + ns.lock.Unlock() + return + } + + set, reset := ns.stateMask(setFlags), ns.stateMask(resetFlags) + id, node := ns.updateEnode(n) + if node == nil { + if set == 0 { + ns.lock.Unlock() + return + } + node = ns.newNode(n) + ns.nodes[id] = node + } + oldState := node.state + newState := (node.state & (^reset)) | set + changed := oldState ^ newState + node.state = newState + + // Remove the timeout callbacks for all reset and set flags, + // even they are not existent(it's noop). + ns.removeTimeouts(node, set|reset) + + // Register the timeout callback if the new state is not empty + // and timeout itself is required. + if timeout != 0 && newState != 0 { + ns.addTimeout(n, set, timeout) + } + if newState == oldState { + ns.lock.Unlock() + return + } + if newState == 0 { + delete(ns.nodes, id) + if node.db { + ns.deleteNode(id) + } + } else { + if changed&ns.saveFlags != 0 { + node.dirty = true + } + } + ns.lock.Unlock() + // call state update subscription callbacks without holding the mutex + for _, sub := range ns.stateSubs { + if changed&sub.mask != 0 { + sub.callback(n, Flags{mask: oldState & sub.mask, setup: ns.setup}, Flags{mask: newState & sub.mask, setup: ns.setup}) + } + } + if newState == 0 { + // call field subscriptions for discarded fields + for i, v := range node.fields { + if v != nil { + f := ns.fields[i] + if len(f.subs) > 0 { + for _, cb := range f.subs { + cb(n, Flags{setup: ns.setup}, v, nil) + } + } + } + } + } +} + +// offlineCallbacks calls state update callbacks at startup or shutdown +func (ns *NodeStateMachine) offlineCallbacks(start bool) { + for _, cb := range ns.offlineCallbackList { + for _, sub := range ns.stateSubs { + offState := offlineState & sub.mask + onState := cb.state & sub.mask + if offState != onState { + if start { + sub.callback(cb.node, Flags{mask: offState, setup: ns.setup}, Flags{mask: onState, setup: ns.setup}) + } else { + sub.callback(cb.node, Flags{mask: onState, setup: ns.setup}, Flags{mask: offState, setup: ns.setup}) + } + } + } + for i, f := range cb.fields { + if f != nil && ns.fields[i].subs != nil { + for _, fsub := range ns.fields[i].subs { + if start { + fsub(cb.node, Flags{mask: offlineState, setup: ns.setup}, nil, f) + } else { + fsub(cb.node, Flags{mask: offlineState, setup: ns.setup}, f, nil) + } + } + } + } + } + ns.offlineCallbackList = nil +} + +// AddTimeout adds a node state timeout associated to the given state flag(s). +// After the specified time interval, the relevant states will be reset. +func (ns *NodeStateMachine) AddTimeout(n *enode.Node, flags Flags, timeout time.Duration) { + ns.lock.Lock() + defer ns.lock.Unlock() + + ns.checkStarted() + if ns.stopped { + return + } + ns.addTimeout(n, ns.stateMask(flags), timeout) +} + +// addTimeout adds a node state timeout associated to the given state flag(s). +func (ns *NodeStateMachine) addTimeout(n *enode.Node, mask bitMask, timeout time.Duration) { + _, node := ns.updateEnode(n) + if node == nil { + return + } + mask &= node.state + if mask == 0 { + return + } + ns.removeTimeouts(node, mask) + t := &nodeStateTimeout{mask: mask} + t.timer = ns.clock.AfterFunc(timeout, func() { + ns.SetState(n, Flags{}, Flags{mask: t.mask, setup: ns.setup}, 0) + }) + node.timeouts = append(node.timeouts, t) + if mask&ns.saveFlags != 0 { + node.dirty = true + } +} + +// removeTimeout removes node state timeouts associated to the given state flag(s). +// If a timeout was associated to multiple flags which are not all included in the +// specified remove mask then only the included flags are de-associated and the timer +// stays active. +func (ns *NodeStateMachine) removeTimeouts(node *nodeInfo, mask bitMask) { + for i := 0; i < len(node.timeouts); i++ { + t := node.timeouts[i] + match := t.mask & mask + if match == 0 { + continue + } + t.mask -= match + if t.mask != 0 { + continue + } + t.timer.Stop() + node.timeouts[i] = node.timeouts[len(node.timeouts)-1] + node.timeouts = node.timeouts[:len(node.timeouts)-1] + i-- + if match&ns.saveFlags != 0 { + node.dirty = true + } + } +} + +// GetField retrieves the given field of the given node +func (ns *NodeStateMachine) GetField(n *enode.Node, field Field) interface{} { + ns.lock.Lock() + defer ns.lock.Unlock() + + ns.checkStarted() + if ns.stopped { + return nil + } + if _, node := ns.updateEnode(n); node != nil { + return node.fields[ns.fieldIndex(field)] + } + return nil +} + +// SetField sets the given field of the given node +func (ns *NodeStateMachine) SetField(n *enode.Node, field Field, value interface{}) error { + ns.lock.Lock() + ns.checkStarted() + if ns.stopped { + ns.lock.Unlock() + return nil + } + _, node := ns.updateEnode(n) + if node == nil { + ns.lock.Unlock() + return nil + } + fieldIndex := ns.fieldIndex(field) + f := ns.fields[fieldIndex] + if value != nil && reflect.TypeOf(value) != f.ftype { + log.Error("Invalid field type", "type", reflect.TypeOf(value), "required", f.ftype) + ns.lock.Unlock() + return errors.New("invalid field type") + } + oldValue := node.fields[fieldIndex] + if value == oldValue { + ns.lock.Unlock() + return nil + } + node.fields[fieldIndex] = value + if f.encode != nil { + node.dirty = true + } + + state := node.state + ns.lock.Unlock() + if len(f.subs) > 0 { + for _, cb := range f.subs { + cb(n, Flags{mask: state, setup: ns.setup}, oldValue, value) + } + } + return nil +} + +// ForEach calls the callback for each node having all of the required and none of the +// disabled flags set +func (ns *NodeStateMachine) ForEach(requireFlags, disableFlags Flags, cb func(n *enode.Node, state Flags)) { + ns.lock.Lock() + ns.checkStarted() + type callback struct { + node *enode.Node + state bitMask + } + require, disable := ns.stateMask(requireFlags), ns.stateMask(disableFlags) + var callbacks []callback + for _, node := range ns.nodes { + if node.state&require == require && node.state&disable == 0 { + callbacks = append(callbacks, callback{node.node, node.state & (require | disable)}) + } + } + ns.lock.Unlock() + for _, c := range callbacks { + cb(c.node, Flags{mask: c.state, setup: ns.setup}) + } +} + +// GetNode returns the enode currently associated with the given ID +func (ns *NodeStateMachine) GetNode(id enode.ID) *enode.Node { + ns.lock.Lock() + defer ns.lock.Unlock() + + ns.checkStarted() + if node := ns.nodes[id]; node != nil { + return node.node + } + return nil +} + +// AddLogMetrics adds logging and/or metrics for nodes entering, exiting and currently +// being in a given set specified by required and disabled state flags +func (ns *NodeStateMachine) AddLogMetrics(requireFlags, disableFlags Flags, name string, inMeter, outMeter metrics.Meter, gauge metrics.Gauge) { + var count int64 + ns.SubscribeState(requireFlags.Or(disableFlags), func(n *enode.Node, oldState, newState Flags) { + oldMatch := oldState.HasAll(requireFlags) && oldState.HasNone(disableFlags) + newMatch := newState.HasAll(requireFlags) && newState.HasNone(disableFlags) + if newMatch == oldMatch { + return + } + + if newMatch { + count++ + if name != "" { + log.Debug("Node entered", "set", name, "id", n.ID(), "count", count) + } + if inMeter != nil { + inMeter.Mark(1) + } + } else { + count-- + if name != "" { + log.Debug("Node left", "set", name, "id", n.ID(), "count", count) + } + if outMeter != nil { + outMeter.Mark(1) + } + } + if gauge != nil { + gauge.Update(count) + } + }) +} diff --git a/p2p/nodestate/nodestate_test.go b/p2p/nodestate/nodestate_test.go new file mode 100644 index 0000000000..5f60176f02 --- /dev/null +++ b/p2p/nodestate/nodestate_test.go @@ -0,0 +1,389 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package nodestate + +import ( + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/celo-org/celo-blockchain/common/mclock" + "github.com/celo-org/celo-blockchain/core/rawdb" + "github.com/celo-org/celo-blockchain/p2p/enode" + "github.com/celo-org/celo-blockchain/p2p/enr" + "github.com/celo-org/celo-blockchain/rlp" +) + +func testSetup(flagPersist []bool, fieldType []reflect.Type) (*Setup, []Flags, []Field) { + setup := &Setup{} + flags := make([]Flags, len(flagPersist)) + for i, persist := range flagPersist { + if persist { + flags[i] = setup.NewPersistentFlag(fmt.Sprintf("flag-%d", i)) + } else { + flags[i] = setup.NewFlag(fmt.Sprintf("flag-%d", i)) + } + } + fields := make([]Field, len(fieldType)) + for i, ftype := range fieldType { + switch ftype { + case reflect.TypeOf(uint64(0)): + fields[i] = setup.NewPersistentField(fmt.Sprintf("field-%d", i), ftype, uint64FieldEnc, uint64FieldDec) + case reflect.TypeOf(""): + fields[i] = setup.NewPersistentField(fmt.Sprintf("field-%d", i), ftype, stringFieldEnc, stringFieldDec) + default: + fields[i] = setup.NewField(fmt.Sprintf("field-%d", i), ftype) + } + } + return setup, flags, fields +} + +func testNode(b byte) *enode.Node { + r := &enr.Record{} + r.SetSig(dummyIdentity{b}, []byte{42}) + n, _ := enode.New(dummyIdentity{b}, r) + return n +} + +func TestCallback(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, _ := testSetup([]bool{false, false, false}, nil) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + set0 := make(chan struct{}, 1) + set1 := make(chan struct{}, 1) + set2 := make(chan struct{}, 1) + ns.SubscribeState(flags[0], func(n *enode.Node, oldState, newState Flags) { set0 <- struct{}{} }) + ns.SubscribeState(flags[1], func(n *enode.Node, oldState, newState Flags) { set1 <- struct{}{} }) + ns.SubscribeState(flags[2], func(n *enode.Node, oldState, newState Flags) { set2 <- struct{}{} }) + + ns.Start() + + ns.SetState(testNode(1), flags[0], Flags{}, 0) + ns.SetState(testNode(1), flags[1], Flags{}, time.Second) + ns.SetState(testNode(1), flags[2], Flags{}, 2*time.Second) + + for i := 0; i < 3; i++ { + select { + case <-set0: + case <-set1: + case <-set2: + case <-time.After(time.Second): + t.Fatalf("failed to invoke callback") + } + } +} + +func TestPersistentFlags(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, _ := testSetup([]bool{true, true, true, false}, nil) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + saveNode := make(chan *nodeInfo, 5) + ns.saveNodeHook = func(node *nodeInfo) { + saveNode <- node + } + + ns.Start() + + ns.SetState(testNode(1), flags[0], Flags{}, time.Second) // state with timeout should not be saved + ns.SetState(testNode(2), flags[1], Flags{}, 0) + ns.SetState(testNode(3), flags[2], Flags{}, 0) + ns.SetState(testNode(4), flags[3], Flags{}, 0) + ns.SetState(testNode(5), flags[0], Flags{}, 0) + ns.Persist(testNode(5)) + select { + case <-saveNode: + case <-time.After(time.Second): + t.Fatalf("Timeout") + } + ns.Stop() + + for i := 0; i < 2; i++ { + select { + case <-saveNode: + case <-time.After(time.Second): + t.Fatalf("Timeout") + } + } + select { + case <-saveNode: + t.Fatalf("Unexpected saveNode") + case <-time.After(time.Millisecond * 100): + } +} + +func TestSetField(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, fields := testSetup([]bool{true}, []reflect.Type{reflect.TypeOf("")}) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + saveNode := make(chan *nodeInfo, 1) + ns.saveNodeHook = func(node *nodeInfo) { + saveNode <- node + } + + ns.Start() + + // Set field before setting state + ns.SetField(testNode(1), fields[0], "hello world") + field := ns.GetField(testNode(1), fields[0]) + if field != nil { + t.Fatalf("Field shouldn't be set before setting states") + } + // Set field after setting state + ns.SetState(testNode(1), flags[0], Flags{}, 0) + ns.SetField(testNode(1), fields[0], "hello world") + field = ns.GetField(testNode(1), fields[0]) + if field == nil { + t.Fatalf("Field should be set after setting states") + } + if err := ns.SetField(testNode(1), fields[0], 123); err == nil { + t.Fatalf("Invalid field should be rejected") + } + // Dirty node should be written back + ns.Stop() + select { + case <-saveNode: + case <-time.After(time.Second): + t.Fatalf("Timeout") + } +} + +func TestUnsetField(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, fields := testSetup([]bool{false}, []reflect.Type{reflect.TypeOf("")}) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + ns.Start() + + ns.SetState(testNode(1), flags[0], Flags{}, time.Second) + ns.SetField(testNode(1), fields[0], "hello world") + + ns.SetState(testNode(1), Flags{}, flags[0], 0) + if field := ns.GetField(testNode(1), fields[0]); field != nil { + t.Fatalf("Field should be unset") + } +} + +func TestSetState(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, _ := testSetup([]bool{false, false, false}, nil) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + type change struct{ old, new Flags } + set := make(chan change, 1) + ns.SubscribeState(flags[0].Or(flags[1]), func(n *enode.Node, oldState, newState Flags) { + set <- change{ + old: oldState, + new: newState, + } + }) + + ns.Start() + + check := func(expectOld, expectNew Flags, expectChange bool) { + if expectChange { + select { + case c := <-set: + if !c.old.Equals(expectOld) { + t.Fatalf("Old state mismatch") + } + if !c.new.Equals(expectNew) { + t.Fatalf("New state mismatch") + } + case <-time.After(time.Second): + } + return + } + select { + case <-set: + t.Fatalf("Unexpected change") + case <-time.After(time.Millisecond * 100): + return + } + } + ns.SetState(testNode(1), flags[0], Flags{}, 0) + check(Flags{}, flags[0], true) + + ns.SetState(testNode(1), flags[1], Flags{}, 0) + check(flags[0], flags[0].Or(flags[1]), true) + + ns.SetState(testNode(1), flags[2], Flags{}, 0) + check(Flags{}, Flags{}, false) + + ns.SetState(testNode(1), Flags{}, flags[0], 0) + check(flags[0].Or(flags[1]), flags[1], true) + + ns.SetState(testNode(1), Flags{}, flags[1], 0) + check(flags[1], Flags{}, true) + + ns.SetState(testNode(1), Flags{}, flags[2], 0) + check(Flags{}, Flags{}, false) + + ns.SetState(testNode(1), flags[0].Or(flags[1]), Flags{}, time.Second) + check(Flags{}, flags[0].Or(flags[1]), true) + clock.Run(time.Second) + check(flags[0].Or(flags[1]), Flags{}, true) +} + +func uint64FieldEnc(field interface{}) ([]byte, error) { + if u, ok := field.(uint64); ok { + enc, err := rlp.EncodeToBytes(&u) + return enc, err + } else { + return nil, errors.New("invalid field type") + } +} + +func uint64FieldDec(enc []byte) (interface{}, error) { + var u uint64 + err := rlp.DecodeBytes(enc, &u) + return u, err +} + +func stringFieldEnc(field interface{}) ([]byte, error) { + if s, ok := field.(string); ok { + return []byte(s), nil + } else { + return nil, errors.New("invalid field type") + } +} + +func stringFieldDec(enc []byte) (interface{}, error) { + return string(enc), nil +} + +func TestPersistentFields(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, fields := testSetup([]bool{true}, []reflect.Type{reflect.TypeOf(uint64(0)), reflect.TypeOf("")}) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + ns.Start() + ns.SetState(testNode(1), flags[0], Flags{}, 0) + ns.SetField(testNode(1), fields[0], uint64(100)) + ns.SetField(testNode(1), fields[1], "hello world") + ns.Stop() + + ns2 := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + ns2.Start() + field0 := ns2.GetField(testNode(1), fields[0]) + if !reflect.DeepEqual(field0, uint64(100)) { + t.Fatalf("Field changed") + } + field1 := ns2.GetField(testNode(1), fields[1]) + if !reflect.DeepEqual(field1, "hello world") { + t.Fatalf("Field changed") + } + + s.Version++ + ns3 := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + ns3.Start() + if ns3.GetField(testNode(1), fields[0]) != nil { + t.Fatalf("Old field version should have been discarded") + } +} + +func TestFieldSub(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, fields := testSetup([]bool{true}, []reflect.Type{reflect.TypeOf(uint64(0))}) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + var ( + lastState Flags + lastOldValue, lastNewValue interface{} + ) + ns.SubscribeField(fields[0], func(n *enode.Node, state Flags, oldValue, newValue interface{}) { + lastState, lastOldValue, lastNewValue = state, oldValue, newValue + }) + check := func(state Flags, oldValue, newValue interface{}) { + if !lastState.Equals(state) || lastOldValue != oldValue || lastNewValue != newValue { + t.Fatalf("Incorrect field sub callback (expected [%v %v %v], got [%v %v %v])", state, oldValue, newValue, lastState, lastOldValue, lastNewValue) + } + } + ns.Start() + ns.SetState(testNode(1), flags[0], Flags{}, 0) + ns.SetField(testNode(1), fields[0], uint64(100)) + check(flags[0], nil, uint64(100)) + ns.Stop() + check(s.OfflineFlag(), uint64(100), nil) + + ns2 := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + ns2.SubscribeField(fields[0], func(n *enode.Node, state Flags, oldValue, newValue interface{}) { + lastState, lastOldValue, lastNewValue = state, oldValue, newValue + }) + ns2.Start() + check(s.OfflineFlag(), nil, uint64(100)) + ns2.SetState(testNode(1), Flags{}, flags[0], 0) + check(Flags{}, uint64(100), nil) + ns2.Stop() +} + +func TestDuplicatedFlags(t *testing.T) { + mdb, clock := rawdb.NewMemoryDatabase(), &mclock.Simulated{} + + s, flags, _ := testSetup([]bool{true}, nil) + ns := NewNodeStateMachine(mdb, []byte("-ns"), clock, s) + + type change struct{ old, new Flags } + set := make(chan change, 1) + ns.SubscribeState(flags[0], func(n *enode.Node, oldState, newState Flags) { + set <- change{oldState, newState} + }) + + ns.Start() + defer ns.Stop() + + check := func(expectOld, expectNew Flags, expectChange bool) { + if expectChange { + select { + case c := <-set: + if !c.old.Equals(expectOld) { + t.Fatalf("Old state mismatch") + } + if !c.new.Equals(expectNew) { + t.Fatalf("New state mismatch") + } + case <-time.After(time.Second): + } + return + } + select { + case <-set: + t.Fatalf("Unexpected change") + case <-time.After(time.Millisecond * 100): + return + } + } + ns.SetState(testNode(1), flags[0], Flags{}, time.Second) + check(Flags{}, flags[0], true) + ns.SetState(testNode(1), flags[0], Flags{}, 2*time.Second) // extend the timeout to 2s + check(Flags{}, flags[0], false) + + clock.Run(2 * time.Second) + check(flags[0], Flags{}, true) +} diff --git a/p2p/rlpx_test.go b/p2p/rlpx_test.go index cf87df4951..0ff5b76f9a 100644 --- a/p2p/rlpx_test.go +++ b/p2p/rlpx_test.go @@ -55,7 +55,7 @@ func TestSharedSecret(t *testing.T) { } t.Logf("Secret:\n%v %x\n%v %x", len(ss0), ss0, len(ss0), ss1) if !bytes.Equal(ss0, ss1) { - t.Errorf("dont match :(") + t.Errorf("don't match :(") } } diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 5064425ed8..a9c5e50e77 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -698,7 +698,7 @@ func (s *Server) JSON(w http.ResponseWriter, status int, data interface{}) { json.NewEncoder(w).Encode(data) } -// wrapHandler returns a httprouter.Handle which wraps a http.HandlerFunc by +// wrapHandler returns an httprouter.Handle which wraps an http.HandlerFunc by // populating request.Context with any objects from the URL params func (s *Server) wrapHandler(handler http.HandlerFunc) httprouter.Handle { return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { diff --git a/params/bootnodes.go b/params/bootnodes.go index c68fb208a8..49a52d76bd 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -41,11 +41,10 @@ var AlfajoresBootnodes = []string{ "enode://703cf979becdc501c4221090296fe75299cb9520f19a344098154c14c7133ebf6b649dad7f3f42947ad96312930bea5380a8ff86faa5a3795b0b6cc483adcfc8@35.230.23.131:30303", } -// These DNS names provide bootstrap connectivity for public testnets and the mainnet. -// See https://github.com/ethereum/discv4-dns-lists for more information. -// For now, Celo doesn't use DNS discovery, so urls are blank -var KnownDNSNetworks = map[common.Hash]string{ - MainnetGenesisHash: "", - AlfajoresGenesisHash: "", - BaklavaGenesisHash: "", +// KnownDNSNetwork returns the address of a public DNS-based node list for the given +// genesis hash and protocol. See https://github.com/ethereum/discv4-dns-lists for more +// information. +func KnownDNSNetwork(genesis common.Hash, protocol string) string { + // For now, Celo doesn't use DNS discovery, so urls are blank + return "" } diff --git a/params/config.go b/params/config.go index 9e8c76d5a4..940dd6f5e6 100644 --- a/params/config.go +++ b/params/config.go @@ -352,21 +352,22 @@ func (c *ChainConfig) CheckCompatible(newcfg *ChainConfig, height uint64) *Confi // to guarantee that forks can be implemented in a different order than on official networks func (c *ChainConfig) CheckConfigForkOrder() error { type fork struct { - name string - block *big.Int + name string + block *big.Int + optional bool // if true, the fork may be nil and next fork is still allowed } var lastFork fork for _, cur := range []fork{ - {"homesteadBlock", c.HomesteadBlock}, - {"eip150Block", c.EIP150Block}, - {"eip155Block", c.EIP155Block}, - {"eip158Block", c.EIP158Block}, - {"byzantiumBlock", c.ByzantiumBlock}, - {"constantinopleBlock", c.ConstantinopleBlock}, - {"petersburgBlock", c.PetersburgBlock}, - {"istanbulBlock", c.IstanbulBlock}, - {"churritoBlock", c.ChurritoBlock}, - {"donutBlock", c.DonutBlock}, + {name: "homesteadBlock", block: c.HomesteadBlock}, + {name: "eip150Block", block: c.EIP150Block}, + {name: "eip155Block", block: c.EIP155Block}, + {name: "eip158Block", block: c.EIP158Block}, + {name: "byzantiumBlock", block: c.ByzantiumBlock}, + {name: "constantinopleBlock", block: c.ConstantinopleBlock}, + {name: "petersburgBlock", block: c.PetersburgBlock}, + {name: "istanbulBlock", block: c.IstanbulBlock}, + {name: "churritoBlock", block: c.ChurritoBlock}, + {name: "donutBlock", block: c.DonutBlock}, } { if lastFork.name != "" { // Next one must be higher number @@ -381,7 +382,10 @@ func (c *ChainConfig) CheckConfigForkOrder() error { } } } - lastFork = cur + // If it was optional and not set, then ignore it + if !cur.optional || cur.block != nil { + lastFork = cur + } } return nil } diff --git a/params/network_params.go b/params/network_params.go index 68255b70c0..b9046c3e4d 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -56,6 +56,6 @@ const ( // ImmutabilityThreshold is the number of blocks after which a chain segment is // considered immutable (i.e. soft finality). It is used by the downloader as a // hard limit against deep ancestors, by the blockchain against deep reorgs, by - // the freezer as the cutoff treshold and by clique as the snapshot trust limit. + // the freezer as the cutoff threshold and by clique as the snapshot trust limit. FullImmutabilityThreshold = 90000 ) diff --git a/rpc/server.go b/rpc/server.go index 8fd3ce86f9..9f60fc4899 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -36,7 +36,7 @@ const ( // OptionMethodInvocation is an indication that the codec supports RPC method calls OptionMethodInvocation CodecOption = 1 << iota - // OptionSubscriptions is an indication that the codec suports RPC notifications + // OptionSubscriptions is an indication that the codec supports RPC notifications OptionSubscriptions = 1 << iota // support pub sub ) diff --git a/rpc/websocket.go b/rpc/websocket.go index 5c7507362f..130b9fc1a7 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -25,6 +25,7 @@ import ( "os" "strings" "sync" + "time" "github.com/celo-org/celo-blockchain/log" mapset "github.com/deckarep/golang-set" @@ -32,8 +33,10 @@ import ( ) const ( - wsReadBuffer = 1024 - wsWriteBuffer = 1024 + wsReadBuffer = 1024 + wsWriteBuffer = 1024 + wsPingInterval = 60 * time.Second + wsPingWriteTimeout = 5 * time.Second ) var wsBufferPool = new(sync.Pool) @@ -168,7 +171,64 @@ func wsClientHeaders(endpoint, origin string) (string, http.Header, error) { return endpointURL.String(), header, nil } +type websocketCodec struct { + *jsonCodec + conn *websocket.Conn + + wg sync.WaitGroup + pingReset chan struct{} +} + func newWebsocketCodec(conn *websocket.Conn) ServerCodec { conn.SetReadLimit(maxRequestContentLength) - return NewFuncCodec(conn, conn.WriteJSON, conn.ReadJSON) + wc := &websocketCodec{ + jsonCodec: NewFuncCodec(conn, conn.WriteJSON, conn.ReadJSON).(*jsonCodec), + conn: conn, + pingReset: make(chan struct{}, 1), + } + wc.wg.Add(1) + go wc.pingLoop() + return wc +} + +func (wc *websocketCodec) close() { + wc.jsonCodec.close() + wc.wg.Wait() +} + +func (wc *websocketCodec) writeJSON(ctx context.Context, v interface{}) error { + err := wc.jsonCodec.writeJSON(ctx, v) + if err == nil { + // Notify pingLoop to delay the next idle ping. + select { + case wc.pingReset <- struct{}{}: + default: + } + } + return err +} + +// pingLoop sends periodic ping frames when the connection is idle. +func (wc *websocketCodec) pingLoop() { + var timer = time.NewTimer(wsPingInterval) + defer wc.wg.Done() + defer timer.Stop() + + for { + select { + case <-wc.closed(): + return + case <-wc.pingReset: + if !timer.Stop() { + <-timer.C + } + timer.Reset(wsPingInterval) + case <-timer.C: + wc.jsonCodec.encMu.Lock() + wc.conn.SetWriteDeadline(time.Now().Add(wsPingWriteTimeout)) + wc.conn.WriteMessage(websocket.PingMessage, nil) + wc.jsonCodec.encMu.Unlock() + timer.Reset(wsPingInterval) + } + } } diff --git a/shared/signer/signed_data.go b/shared/signer/signed_data.go index ae8eaed37a..46ab4c37d1 100644 --- a/shared/signer/signed_data.go +++ b/shared/signer/signed_data.go @@ -68,7 +68,7 @@ func (t *Type) isReferenceType() bool { if len(t.Type) == 0 { return false } - // Reference types must have a leading uppercase characer + // Reference types must have a leading uppercase character return unicode.IsUpper([]rune(t.Type)[0]) } diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 2369ec9345..c5191efd48 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -25,9 +25,9 @@ import ( "sync" "github.com/celo-org/celo-blockchain/common/hexutil" + "github.com/celo-org/celo-blockchain/console/prompt" "github.com/celo-org/celo-blockchain/internal/ethapi" "github.com/celo-org/celo-blockchain/log" - "golang.org/x/crypto/ssh/terminal" ) type CommandlineUI struct { @@ -61,17 +61,16 @@ func (ui *CommandlineUI) readString() string { func (ui *CommandlineUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { fmt.Printf("## %s\n\n%s\n", info.Title, info.Prompt) + defer fmt.Println("-----------------------") if info.IsPassword { - fmt.Printf("> ") - text, err := terminal.ReadPassword(int(os.Stdin.Fd())) + text, err := prompt.Stdin.PromptPassword("> ") if err != nil { - log.Error("Failed to read password", "err", err) + log.Error("Failed to read password", "error", err) + return UserInputResponse{}, err } - fmt.Println("-----------------------") - return UserInputResponse{string(text)}, err + return UserInputResponse{text}, nil } text := ui.readString() - fmt.Println("-----------------------") return UserInputResponse{text}, nil } diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go index 95ac63f761..3d7ffb52e9 100644 --- a/signer/core/uiapi.go +++ b/signer/core/uiapi.go @@ -109,7 +109,7 @@ func (s *UIServerAPI) DeriveAccount(url string, path string, pin *bool) (account return wallet.Derive(derivPath, *pin) } -// fetchKeystore retrives the encrypted keystore from the account manager. +// fetchKeystore retrieves the encrypted keystore from the account manager. func fetchKeystore(am *accounts.Manager) *keystore.KeyStore { return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) } diff --git a/tests/fuzzers/README.md b/tests/fuzzers/README.md index fd8c4ec57f..7611c53698 100644 --- a/tests/fuzzers/README.md +++ b/tests/fuzzers/README.md @@ -34,7 +34,7 @@ go-fuzz -bin ./rlp/rlp-fuzz.zip Once a 'crasher' is found, the fuzzer tries to avoid reporting the same vector twice, so stores the fault in the `suppressions` folder. Thus, if you e.g. make changes to fix a bug, you should _remove_ all data from the `suppressions`-folder, to verify that the issue is indeed resolved. -Also, if you have only one and the same exit-point for multiple different types of test, the suppression can make the fuzzer hide differnent types of errors. So make +Also, if you have only one and the same exit-point for multiple different types of test, the suppression can make the fuzzer hide different types of errors. So make sure that each type of failure is unique (for an example, see the rlp fuzzer, where a counter `i` is used to differentiate between failures: ```golang diff --git a/trie/committer.go b/trie/committer.go index 8cabd17c5b..cb0e883e72 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -27,7 +27,7 @@ import ( ) // leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow -// some paralellism but not incur too much memory overhead. +// some parallelism but not incur too much memory overhead. const leafChanSize = 200 // leaf represents a trie leaf value @@ -41,7 +41,7 @@ type leaf struct { // committer is a type used for the trie Commit operation. A committer has some // internal preallocated temp space, and also a callback that is invoked when // leaves are committed. The leafs are passed through the `leafCh`, to allow -// some level of paralellism. +// some level of parallelism. // By 'some level' of parallelism, it's still the case that all leaves will be // processed sequentially - onleaf will never be called in parallel or out of order. type committer struct { diff --git a/trie/database.go b/trie/database.go index fae5cf491e..0ddd5dc205 100644 --- a/trie/database.go +++ b/trie/database.go @@ -399,7 +399,7 @@ func (db *Database) node(hash common.Hash) node { // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. func (db *Database) Node(hash common.Hash) ([]byte, error) { - // It doens't make sense to retrieve the metaroot + // It doesn't make sense to retrieve the metaroot if hash == (common.Hash{}) { return nil, errors.New("not found") } diff --git a/trie/proof.go b/trie/proof.go index 133cfd8931..2a2738aa16 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -133,7 +133,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) // The main purpose of this function is recovering a node // path from the merkle proof stream. All necessary nodes // will be resolved and leave the remaining as hashnode. -func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyValueReader) (node, error) { +func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyValueReader, allowNonExistent bool) (node, []byte, error) { // resolveNode retrieves and resolves trie node from merkle proof stream resolveNode := func(hash common.Hash) (node, error) { buf, _ := proofDb.Get(hash[:]) @@ -146,11 +146,12 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV } return n, err } - // If the root node is empty, resolve it first + // If the root node is empty, resolve it first. + // Root node must be included in the proof. if root == nil { n, err := resolveNode(rootHash) if err != nil { - return nil, err + return nil, nil, err } root = n } @@ -158,15 +159,21 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV err error child, parent node keyrest []byte - terminate bool + valnode []byte ) key, parent = keybytesToHex(key), root for { keyrest, child = get(parent, key, false) switch cld := child.(type) { case nil: - // The trie doesn't contain the key. - return nil, errors.New("the node is not contained in trie") + // The trie doesn't contain the key. It's possible + // the proof is a non-existing proof, but at least + // we can prove all resolved nodes are correct, it's + // enough for us to prove range. + if allowNonExistent { + return root, nil, nil + } + return nil, nil, errors.New("the node is not contained in trie") case *shortNode: key, parent = keyrest, child // Already resolved continue @@ -176,10 +183,10 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV case hashNode: child, err = resolveNode(common.BytesToHash(cld)) if err != nil { - return nil, err + return nil, nil, err } case valueNode: - terminate = true + valnode = cld } // Link the parent and child. switch pnode := parent.(type) { @@ -190,8 +197,8 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV default: panic(fmt.Sprintf("%T: invalid node: %v", pnode, pnode)) } - if terminate { - return root, nil // The whole path is resolved + if len(valnode) > 0 { + return root, valnode, nil // The whole path is resolved } key, parent = keyrest, child } @@ -205,111 +212,255 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV // since the node content might be modified. Besides it can happen that some // fullnodes only have one child which is disallowed. But if the proof is valid, // the missing children will be filled, otherwise it will be thrown anyway. -func unsetInternal(node node, left []byte, right []byte) error { +func unsetInternal(n node, left []byte, right []byte) error { left, right = keybytesToHex(left), keybytesToHex(right) // todo(rjl493456442) different length edge keys should be supported if len(left) != len(right) { return errors.New("inconsistent edge path") } - // Step down to the fork point - prefix, pos := prefixLen(left, right), 0 + // Step down to the fork point. There are two scenarios can happen: + // - the fork point is a shortnode: the left proof MUST point to a + // non-existent key and the key doesn't match with the shortnode + // - the fork point is a fullnode: the left proof can point to an + // existent key or not. + var ( + pos = 0 + parent node + ) +findFork: for { - if pos >= prefix { - break - } - switch n := (node).(type) { + switch rn := (n).(type) { case *shortNode: - if len(left)-pos < len(n.Key) || !bytes.Equal(n.Key, left[pos:pos+len(n.Key)]) { + // The right proof must point to an existent key. + if len(right)-pos < len(rn.Key) || !bytes.Equal(rn.Key, right[pos:pos+len(rn.Key)]) { return errors.New("invalid edge path") } - n.flags = nodeFlag{dirty: true} - node, pos = n.Val, pos+len(n.Key) + rn.flags = nodeFlag{dirty: true} + // Special case, the non-existent proof points to the same path + // as the existent proof, but the path of existent proof is longer. + // In this case, the fork point is this shortnode. + if len(left)-pos < len(rn.Key) || !bytes.Equal(rn.Key, left[pos:pos+len(rn.Key)]) { + break findFork + } + parent = n + n, pos = rn.Val, pos+len(rn.Key) case *fullNode: - n.flags = nodeFlag{dirty: true} - node, pos = n.Children[left[pos]], pos+1 + leftnode, rightnode := rn.Children[left[pos]], rn.Children[right[pos]] + // The right proof must point to an existent key. + if rightnode == nil { + return errors.New("invalid edge path") + } + rn.flags = nodeFlag{dirty: true} + if leftnode != rightnode { + break findFork + } + parent = n + n, pos = rn.Children[left[pos]], pos+1 default: - panic(fmt.Sprintf("%T: invalid node: %v", node, node)) + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } } - fn, ok := node.(*fullNode) - if !ok { - return errors.New("the fork point must be a fullnode") - } - // Find the fork point! Unset all intermediate references - for i := left[prefix] + 1; i < right[prefix]; i++ { - fn.Children[i] = nil + switch rn := n.(type) { + case *shortNode: + if _, ok := rn.Val.(valueNode); ok { + parent.(*fullNode).Children[right[pos-1]] = nil + return nil + } + return unset(rn, rn.Val, right[pos:], len(rn.Key), true) + case *fullNode: + for i := left[pos] + 1; i < right[pos]; i++ { + rn.Children[i] = nil + } + if err := unset(rn, rn.Children[left[pos]], left[pos:], 1, false); err != nil { + return err + } + if err := unset(rn, rn.Children[right[pos]], right[pos:], 1, true); err != nil { + return err + } + return nil + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } - fn.flags = nodeFlag{dirty: true} - unset(fn.Children[left[prefix]], left[prefix+1:], false) - unset(fn.Children[right[prefix]], right[prefix+1:], true) - return nil } // unset removes all internal node references either the left most or right most. -func unset(root node, rest []byte, removeLeft bool) { - switch rn := root.(type) { +// If we try to unset all right most references, it can meet these scenarios: +// +// - The given path is existent in the trie, unset the associated shortnode +// - The given path is non-existent in the trie +// - the fork point is a fullnode, the corresponding child pointed by path +// is nil, return +// - the fork point is a shortnode, the key of shortnode is less than path, +// keep the entire branch and return. +// - the fork point is a shortnode, the key of shortnode is greater than path, +// unset the entire branch. +// +// If we try to unset all left most references, then the given path should +// be existent. +func unset(parent node, child node, key []byte, pos int, removeLeft bool) error { + switch cld := child.(type) { case *fullNode: if removeLeft { - for i := 0; i < int(rest[0]); i++ { - rn.Children[i] = nil + for i := 0; i < int(key[pos]); i++ { + cld.Children[i] = nil } - rn.flags = nodeFlag{dirty: true} + cld.flags = nodeFlag{dirty: true} } else { - for i := rest[0] + 1; i < 16; i++ { - rn.Children[i] = nil + for i := key[pos] + 1; i < 16; i++ { + cld.Children[i] = nil } - rn.flags = nodeFlag{dirty: true} + cld.flags = nodeFlag{dirty: true} } - unset(rn.Children[rest[0]], rest[1:], removeLeft) + return unset(cld, cld.Children[key[pos]], key, pos+1, removeLeft) case *shortNode: - rn.flags = nodeFlag{dirty: true} - if _, ok := rn.Val.(valueNode); ok { - rn.Val = nilValueNode - return + if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { + // Find the fork point, it's an non-existent branch. + if removeLeft { + return errors.New("invalid right edge proof") + } + if bytes.Compare(cld.Key, key[pos:]) > 0 { + // The key of fork shortnode is greater than the + // path(it belongs to the range), unset the entrie + // branch. The parent must be a fullnode. + fn := parent.(*fullNode) + fn.Children[key[pos-1]] = nil + } // else { + // The key of fork shortnode is less than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + // } + return nil + } + if _, ok := cld.Val.(valueNode); ok { + fn := parent.(*fullNode) + fn.Children[key[pos-1]] = nil + return nil } - unset(rn.Val, rest[len(rn.Key):], removeLeft) - case hashNode, nil, valueNode: - panic("it shouldn't happen") + cld.flags = nodeFlag{dirty: true} + return unset(cld, cld.Val, key, pos+len(cld.Key), removeLeft) + case nil: + // If the node is nil, it's a child of the fork point + // fullnode(it's an non-existent branch). + if removeLeft { + return errors.New("invalid right edge proof") + } + return nil + default: + panic("it shouldn't happen") // hashNode, valueNode } } -// VerifyRangeProof checks whether the given leave nodes and edge proofs +// hasRightElement returns the indicator whether there exists more elements +// in the right side of the given path. The given path can point to an existent +// key or a non-existent one. This function has the assumption that the whole +// path should already be resolved. +func hasRightElement(node node, key []byte) bool { + pos, key := 0, keybytesToHex(key) + for node != nil { + switch rn := node.(type) { + case *fullNode: + for i := key[pos] + 1; i < 16; i++ { + if rn.Children[i] != nil { + return true + } + } + node, pos = rn.Children[key[pos]], pos+1 + case *shortNode: + if len(key)-pos < len(rn.Key) || !bytes.Equal(rn.Key, key[pos:pos+len(rn.Key)]) { + return bytes.Compare(rn.Key, key[pos:]) > 0 + } + node, pos = rn.Val, pos+len(rn.Key) + case valueNode: + return false // We have resolved the whole path + default: + panic(fmt.Sprintf("%T: invalid node: %v", node, node)) // hashnode + } + } + return false +} + +// VerifyRangeProof checks whether the given leaf nodes and edge proofs // can prove the given trie leaves range is matched with given root hash -// and the range is consecutive(no gap inside). -func VerifyRangeProof(rootHash common.Hash, keys [][]byte, values [][]byte, firstProof ethdb.KeyValueReader, lastProof ethdb.KeyValueReader) error { +// and the range is consecutive(no gap inside) and monotonic increasing. +// +// Note the given first edge proof can be non-existing proof. For example +// the first proof is for an non-existent values 0x03. The given batch +// leaves are [0x04, 0x05, .. 0x09]. It's still feasible to prove. But the +// last edge proof should always be an existent proof. +// +// The firstKey is paired with firstProof, not necessarily the same as keys[0] +// (unless firstProof is an existent proof). +// +// Expect the normal case, this function can also be used to verify the following +// range proofs(note this function doesn't accept zero element proof): +// +// - All elements proof. In this case the left and right proof can be nil, but the +// range should be all the leaves in the trie. +// +// - One element proof. In this case no matter the left edge proof is a non-existent +// proof or not, we can always verify the correctness of the proof. +// +// Except returning the error to indicate the proof is valid or not, the function will +// also return a flag to indicate whether there exists more accounts/slots in the trie. +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, values [][]byte, firstProof ethdb.KeyValueReader, lastProof ethdb.KeyValueReader) (error, bool) { if len(keys) != len(values) { - return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) + return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false } if len(keys) == 0 { - return fmt.Errorf("nothing to verify") + return errors.New("empty proof"), false } - if len(keys) == 1 { - value, err := VerifyProof(rootHash, keys[0], firstProof) + // Ensure the received batch is monotonic increasing. + for i := 0; i < len(keys)-1; i++ { + if bytes.Compare(keys[i], keys[i+1]) >= 0 { + return errors.New("range is not monotonically increasing"), false + } + } + // Special case, there is no edge proof at all. The given range is expected + // to be the whole leaf-set in the trie. + if firstProof == nil && lastProof == nil { + emptytrie, err := New(common.Hash{}, NewDatabase(memorydb.New())) if err != nil { - return err + return err, false } - if !bytes.Equal(value, values[0]) { - return fmt.Errorf("correct proof but invalid data") + for index, key := range keys { + emptytrie.TryUpdate(key, values[index]) } - return nil + if emptytrie.Hash() != rootHash { + return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, emptytrie.Hash()), false + } + return nil, false // no more element. + } + // Special case, there is only one element and left edge + // proof is an existent one. + if len(keys) == 1 && bytes.Equal(keys[0], firstKey) { + root, val, err := proofToPath(rootHash, nil, firstKey, firstProof, false) + if err != nil { + return err, false + } + if !bytes.Equal(val, values[0]) { + return fmt.Errorf("correct proof but invalid data"), false + } + return nil, hasRightElement(root, keys[0]) } // Convert the edge proofs to edge trie paths. Then we can // have the same tree architecture with the original one. - root, err := proofToPath(rootHash, nil, keys[0], firstProof) + // For the first edge proof, non-existent proof is allowed. + root, _, err := proofToPath(rootHash, nil, firstKey, firstProof, true) if err != nil { - return err + return err, false } // Pass the root node here, the second path will be merged - // with the first one. - root, err = proofToPath(rootHash, root, keys[len(keys)-1], lastProof) + // with the first one. For the last edge proof, non-existent + // proof is not allowed. + root, _, err = proofToPath(rootHash, root, keys[len(keys)-1], lastProof, false) if err != nil { - return err + return err, false } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. - if err := unsetInternal(root, keys[0], keys[len(keys)-1]); err != nil { - return err + if err := unsetInternal(root, firstKey, keys[len(keys)-1]); err != nil { + return err, false } // Rebuild the trie with the leave stream, the shape of trie // should be same with the original one. @@ -318,9 +469,9 @@ func VerifyRangeProof(rootHash common.Hash, keys [][]byte, values [][]byte, firs newtrie.TryUpdate(key, values[index]) } if newtrie.Hash() != rootHash { - return fmt.Errorf("invalid proof, wanthash %x, got %x", rootHash, newtrie.Hash()) + return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, newtrie.Hash()), false } - return nil + return nil, hasRightElement(root, keys[len(keys)-1]) } // get returns the child of the given node. Return nil if the diff --git a/trie/proof_test.go b/trie/proof_test.go index 9917e956b3..ec98df9c51 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -98,12 +98,65 @@ func TestOneElementProof(t *testing.T) { } } +func TestBadProof(t *testing.T) { + trie, vals := randomTrie(800) + root := trie.Hash() + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + it := proof.NewIterator(nil, nil) + for i, d := 0, mrand.Intn(proof.Len()); i <= d; i++ { + it.Next() + } + key := it.Key() + val, _ := proof.Get(key) + proof.Delete(key) + it.Release() + + mutateByte(val) + proof.Put(crypto.Keccak256(val), val) + + if _, err := VerifyProof(root, kv.k, proof); err == nil { + t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k) + } + } + } +} + +// Tests that missing keys can also be proven. The test explicitly uses a single +// entry trie and checks for missing keys both before and after the single entry. +func TestMissingKeyProof(t *testing.T) { + trie := new(Trie) + updateString(trie, "k", "v") + + for i, key := range []string{"a", "j", "l", "z"} { + proof := memorydb.New() + trie.Prove([]byte(key), 0, proof) + + if proof.Len() != 1 { + t.Errorf("test %d: proof should have one element", i) + } + val, err := VerifyProof(trie.Hash(), []byte(key), proof) + if err != nil { + t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if val != nil { + t.Fatalf("test %d: verified value mismatch: have %x, want nil", i, val) + } + } +} + type entrySlice []*kv func (p entrySlice) Len() int { return len(p) } func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +// TestRangeProof tests normal range proof with both edge proofs +// as the existent proof. The test cases are generated randomly. func TestRangeProof(t *testing.T) { trie, vals := randomTrie(4096) var entries entrySlice @@ -130,13 +183,222 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err := VerifyRangeProof(trie.Hash(), keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } } } +// TestRangeProof tests normal range proof with the first edge proof +// as the non-existent proof. The test cases are generated randomly. +func TestRangeProofWithNonExistentProof(t *testing.T) { + trie, vals := randomTrie(4096) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + for i := 0; i < 500; i++ { + start := mrand.Intn(len(entries)) + end := mrand.Intn(len(entries)-start) + start + if start == end { + continue + } + firstProof, lastProof := memorydb.New(), memorydb.New() + + first := decreseKey(common.CopyBytes(entries[start].k)) + if start != 0 && bytes.Equal(first, entries[start-1].k) { + continue + } + if err := trie.Prove(first, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + var keys [][]byte + var vals [][]byte + for i := start; i < end; i++ { + keys = append(keys, entries[i].k) + vals = append(vals, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), first, keys, vals, firstProof, lastProof) + if err != nil { + t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) + } + } +} + +// TestRangeProofWithInvalidNonExistentProof tests such scenarios: +// - The last edge proof is an non-existent proof +// - There exists a gap between the first element and the left edge proof +func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { + trie, vals := randomTrie(4096) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + + // Case 1 + start, end := 100, 200 + first, last := decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[end].k)) + firstProof, lastProof := memorydb.New(), memorydb.New() + if err := trie.Prove(first, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + var k [][]byte + var v [][]byte + for i := start; i < end; i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) + if err == nil { + t.Fatalf("Expected to detect the error, got nil") + } + + // Case 2 + start, end = 100, 200 + first = decreseKey(common.CopyBytes(entries[start].k)) + + firstProof, lastProof = memorydb.New(), memorydb.New() + if err := trie.Prove(first, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + start = 105 // Gap created + k = make([][]byte, 0) + v = make([][]byte, 0) + for i := start; i < end; i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ = VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) + if err == nil { + t.Fatalf("Expected to detect the error, got nil") + } +} + +// TestOneElementRangeProof tests the proof with only one +// element. The first edge proof can be existent one or +// non-existent one. +func TestOneElementRangeProof(t *testing.T) { + trie, vals := randomTrie(4096) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + + // One element with existent edge proof + start := 1000 + firstProof, lastProof := memorydb.New(), memorydb.New() + if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + // One element with non-existent edge proof + start = 1000 + first := decreseKey(common.CopyBytes(entries[start].k)) + firstProof, lastProof = memorydb.New(), memorydb.New() + if err := trie.Prove(first, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} + +// TestAllElementsProof tests the range proof with all elements. +// The edge proofs can be nil. +func TestAllElementsProof(t *testing.T) { + trie, vals := randomTrie(4096) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + + var k [][]byte + var v [][]byte + for i := 0; i < len(entries); i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), k[0], k, v, nil, nil) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + // Even with edge proofs, it should still work. + firstProof, lastProof := memorydb.New(), memorydb.New() + if err := trie.Prove(entries[0].k, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[len(entries)-1].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), k[0], k, v, firstProof, lastProof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} + +// TestSingleSideRangeProof tests the range starts from zero. +func TestSingleSideRangeProof(t *testing.T) { + for i := 0; i < 64; i++ { + trie := new(Trie) + var entries entrySlice + for i := 0; i < 4096; i++ { + value := &kv{randBytes(32), randBytes(20), false} + trie.Update(value.k, value.v) + entries = append(entries, value) + } + sort.Sort(entries) + + var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} + for _, pos := range cases { + firstProof, lastProof := memorydb.New(), memorydb.New() + if err := trie.Prove(common.Hash{}.Bytes(), 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(entries[pos].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + k := make([][]byte, 0) + v := make([][]byte, 0) + for i := 0; i <= pos; i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k, v, firstProof, lastProof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + } + } +} + +// TestBadRangeProof tests a few cases which the proof is wrong. +// The prover is expected to detect the error. func TestBadRangeProof(t *testing.T) { trie, vals := randomTrie(4096) var entries entrySlice @@ -208,7 +470,7 @@ func TestBadRangeProof(t *testing.T) { index = mrand.Intn(end - start) vals[index] = nil } - err := VerifyRangeProof(trie.Hash(), keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -242,59 +504,69 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err := VerifyRangeProof(trie.Hash(), keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) if err == nil { t.Fatal("expect error, got nil") } } -func TestBadProof(t *testing.T) { - trie, vals := randomTrie(800) - root := trie.Hash() - for i, prover := range makeProvers(trie) { - for _, kv := range vals { - proof := prover(kv.k) - if proof == nil { - t.Fatalf("prover %d: nil proof", i) - } - it := proof.NewIterator(nil, nil) - for i, d := 0, mrand.Intn(proof.Len()); i <= d; i++ { - it.Next() - } - key := it.Key() - val, _ := proof.Get(key) - proof.Delete(key) - it.Release() - - mutateByte(val) - proof.Put(crypto.Keccak256(val), val) +func TestHasRightElement(t *testing.T) { + trie := new(Trie) + var entries entrySlice + for i := 0; i < 4096; i++ { + value := &kv{randBytes(32), randBytes(20), false} + trie.Update(value.k, value.v) + entries = append(entries, value) + } + sort.Sort(entries) - if _, err := VerifyProof(root, kv.k, proof); err == nil { - t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k) + var cases = []struct { + start int + end int + hasMore bool + }{ + {-1, 1, true}, // single element with non-existent left proof + {0, 1, true}, // single element with existent left proof + {0, 10, true}, + {50, 100, true}, + {50, len(entries), false}, // No more element expected + {len(entries) - 1, len(entries), false}, // Single last element + {0, len(entries), false}, // The whole set with existent left proof + {-1, len(entries), false}, // The whole set with non-existent left proof + } + for _, c := range cases { + var ( + firstKey []byte + start = c.start + firstProof = memorydb.New() + lastProof = memorydb.New() + ) + if c.start == -1 { + firstKey, start = common.Hash{}.Bytes(), 0 + if err := trie.Prove(firstKey, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + } else { + firstKey = entries[c.start].k + if err := trie.Prove(entries[c.start].k, 0, firstProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) } } - } -} - -// Tests that missing keys can also be proven. The test explicitly uses a single -// entry trie and checks for missing keys both before and after the single entry. -func TestMissingKeyProof(t *testing.T) { - trie := new(Trie) - updateString(trie, "k", "v") - - for i, key := range []string{"a", "j", "l", "z"} { - proof := memorydb.New() - trie.Prove([]byte(key), 0, proof) - - if proof.Len() != 1 { - t.Errorf("test %d: proof should have one element", i) + if err := trie.Prove(entries[c.end-1].k, 0, lastProof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) } - val, err := VerifyProof(trie.Hash(), []byte(key), proof) + k := make([][]byte, 0) + v := make([][]byte, 0) + for i := start; i < c.end; i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, k, v, firstProof, lastProof) if err != nil { - t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + t.Fatalf("Expected no error, got %v", err) } - if val != nil { - t.Fatalf("test %d: verified value mismatch: have %x, want nil", i, val) + if hasMore != c.hasMore { + t.Fatalf("Wrong hasMore indicator, want %t, got %t", c.hasMore, hasMore) } } } @@ -310,6 +582,26 @@ func mutateByte(b []byte) { } } +func increseKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]++ + if key[i] != 0x0 { + break + } + } + return key +} + +func decreseKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]-- + if key[i] != 0xff { + break + } + } + return key +} + func BenchmarkProve(b *testing.B) { trie, vals := randomTrie(100) var keys []string @@ -379,7 +671,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - err := VerifyRangeProof(trie.Hash(), keys, values, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, values, firstProof, lastProof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) }