diff --git a/VERSION b/VERSION index 267637b120..7d2424c90b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.11 +1.8.12 diff --git a/build/ci.go b/build/ci.go index 9f3ed829c6..5939d91e96 100644 --- a/build/ci.go +++ b/build/ci.go @@ -330,7 +330,7 @@ func doLint(cmdline []string) { configs := []string{ "--vendor", "--tests", - "--deadline=10m", + "--deadline=2m", "--disable-all", "--enable=goimports", "--enable=varcheck", diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 3a1ae6f4c3..3ac37ba7ad 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -29,7 +29,7 @@ import ( ) var ( - abiFlag = flag.String("abi", "", "Path to the Ethereum contract ABI json to bind") + abiFlag = flag.String("abi", "", "Path to the Ethereum contract ABI json to bind, - for STDIN") binFlag = flag.String("bin", "", "Path to the Ethereum contract bytecode (generate deploy method)") typFlag = flag.String("type", "", "Struct name for the binding (default = package name)") @@ -75,16 +75,27 @@ func main() { bins []string types []string ) - if *solFlag != "" { + if *solFlag != "" || *abiFlag == "-" { // Generate the list of types to exclude from binding exclude := make(map[string]bool) for _, kind := range strings.Split(*excFlag, ",") { exclude[strings.ToLower(kind)] = true } - contracts, err := compiler.CompileSolidity(*solcFlag, *solFlag) - if err != nil { - fmt.Printf("Failed to build Solidity contract: %v\n", err) - os.Exit(-1) + + var contracts map[string]*compiler.Contract + var err error + if *solFlag != "" { + contracts, err = compiler.CompileSolidity(*solcFlag, *solFlag) + if err != nil { + fmt.Printf("Failed to build Solidity contract: %v\n", err) + os.Exit(-1) + } + } else { + contracts, err = contractsFromStdin() + if err != nil { + fmt.Printf("Failed to read input ABIs from STDIN: %v\n", err) + os.Exit(-1) + } } // Gather all non-excluded contract for binding for name, contract := range contracts { @@ -138,3 +149,12 @@ func main() { os.Exit(-1) } } + +func contractsFromStdin() (map[string]*compiler.Contract, error) { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + + return compiler.ParseCombinedJSON(bytes, "", "", "", "") +} diff --git a/cmd/ethkey/changepassphrase.go b/cmd/ethkey/changepassphrase.go new file mode 100644 index 0000000000..d1ae2ae0d8 --- /dev/null +++ b/cmd/ethkey/changepassphrase.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "io/ioutil" + "strings" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/cmd/utils" + "gopkg.in/urfave/cli.v1" +) + +var newPassphraseFlag = cli.StringFlag{ + Name: "newpasswordfile", + Usage: "the file that contains the new passphrase for the keyfile", +} + +var commandChangePassphrase = cli.Command{ + Name: "changepassphrase", + Usage: "change the passphrase on a keyfile", + ArgsUsage: "", + Description: ` +Change the passphrase of a keyfile.`, + Flags: []cli.Flag{ + passphraseFlag, + newPassphraseFlag, + }, + Action: func(ctx *cli.Context) error { + keyfilepath := ctx.Args().First() + + // Read key from file. + keyjson, err := ioutil.ReadFile(keyfilepath) + if err != nil { + utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err) + } + + // Decrypt key with passphrase. + passphrase := getPassphrase(ctx) + key, err := keystore.DecryptKey(keyjson, passphrase) + if err != nil { + utils.Fatalf("Error decrypting key: %v", err) + } + + // Get a new passphrase. + fmt.Println("Please provide a new passphrase") + var newPhrase string + if passFile := ctx.String(newPassphraseFlag.Name); passFile != "" { + content, err := ioutil.ReadFile(passFile) + if err != nil { + utils.Fatalf("Failed to read new passphrase file '%s': %v", passFile, err) + } + newPhrase = strings.TrimRight(string(content), "\r\n") + } else { + newPhrase = promptPassphrase(true) + } + + // Encrypt the key with the new passphrase. + newJson, err := keystore.EncryptKey(key, newPhrase, keystore.StandardScryptN, keystore.StandardScryptP) + if err != nil { + utils.Fatalf("Error encrypting with new passphrase: %v", err) + } + + // Then write the new keyfile in place of the old one. + if err := ioutil.WriteFile(keyfilepath, newJson, 600); err != nil { + utils.Fatalf("Error writing new keyfile to disk: %v", err) + } + + // Don't print anything. Just return successfully, + // producing a positive exit code. + return nil + }, +} diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index 6d57d17fb4..fe9a0c1519 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -90,7 +90,7 @@ If you want to encrypt an existing private key, it can be specified by setting } // Encrypt key with passphrase. - passphrase := getPassPhrase(ctx, true) + passphrase := promptPassphrase(true) keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP) if err != nil { utils.Fatalf("Error encrypting key: %v", err) diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index dbf5afc0ce..ba03d4d936 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -60,7 +60,7 @@ make sure to use this feature with great caution!`, } // Decrypt key with passphrase. - passphrase := getPassPhrase(ctx, false) + passphrase := getPassphrase(ctx) key, err := keystore.DecryptKey(keyjson, passphrase) if err != nil { utils.Fatalf("Error decrypting key: %v", err) diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index 4127f5566f..c434da0c05 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -38,6 +38,7 @@ func init() { app.Commands = []cli.Command{ commandGenerate, commandInspect, + commandChangePassphrase, commandSignMessage, commandVerifyMessage, } diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index 531a931c82..5caea69ff6 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -62,7 +62,7 @@ To sign a message contained in a file, use the --msgfile flag. } // Decrypt key with passphrase. - passphrase := getPassPhrase(ctx, false) + passphrase := getPassphrase(ctx) key, err := keystore.DecryptKey(keyjson, passphrase) if err != nil { utils.Fatalf("Error decrypting key: %v", err) diff --git a/cmd/ethkey/utils.go b/cmd/ethkey/utils.go index 0e563bf922..6f60ebaf1b 100644 --- a/cmd/ethkey/utils.go +++ b/cmd/ethkey/utils.go @@ -28,26 +28,14 @@ import ( "gopkg.in/urfave/cli.v1" ) -// getPassPhrase obtains a passphrase given by the user. It first checks the -// --passphrase command line flag and ultimately prompts the user for a -// passphrase. -func getPassPhrase(ctx *cli.Context, confirmation bool) string { - // Look for the --passphrase flag. - passphraseFile := ctx.String(passphraseFlag.Name) - if passphraseFile != "" { - content, err := ioutil.ReadFile(passphraseFile) - if err != nil { - utils.Fatalf("Failed to read passphrase file '%s': %v", - passphraseFile, err) - } - return strings.TrimRight(string(content), "\r\n") - } - - // Otherwise prompt the user for the passphrase. +// promptPassphrase prompts the user for a passphrase. Set confirmation to true +// to require the user to confirm the passphrase. +func promptPassphrase(confirmation bool) string { passphrase, err := console.Stdin.PromptPassword("Passphrase: ") if err != nil { utils.Fatalf("Failed to read passphrase: %v", err) } + if confirmation { confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ") if err != nil { @@ -57,9 +45,29 @@ func getPassPhrase(ctx *cli.Context, confirmation bool) string { utils.Fatalf("Passphrases do not match") } } + return passphrase } +// getPassphrase obtains a passphrase given by the user. It first checks the +// --passfile command line flag and ultimately prompts the user for a +// passphrase. +func getPassphrase(ctx *cli.Context) string { + // Look for the --passwordfile flag. + passphraseFile := ctx.String(passphraseFlag.Name) + if passphraseFile != "" { + content, err := ioutil.ReadFile(passphraseFile) + if err != nil { + utils.Fatalf("Failed to read passphrase file '%s': %v", + passphraseFile, err) + } + return strings.TrimRight(string(content), "\r\n") + } + + // Otherwise prompt the user for the passphrase. + return promptPassphrase(false) +} + // signHash is a helper function that calculates a hash for the given message // that can be safely used to calculate a signature from. // diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 5bad09bbd5..5f00dde741 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -474,7 +474,7 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { amount = new(big.Int).Div(amount, new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(msg.Tier)), nil)) tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, 21000, f.price, nil) - signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId) + signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainID) if err != nil { f.lock.Unlock() if err = sendError(conn, err); err != nil { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 09d9c493d1..edf2a557a7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -19,12 +19,16 @@ package main import ( "fmt" + "math" "os" "runtime" + godebug "runtime/debug" "sort" + "strconv" "strings" "time" + "github.com/elastic/gosigar" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" @@ -188,6 +192,22 @@ func init() { if err := debug.Setup(ctx); err != nil { return err } + // Cap the cache allowance and tune the garbage colelctor + var mem gosigar.Mem + if err := mem.Get(); err == nil { + allowance := int(mem.Total / 1024 / 1024 / 3) + if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance { + log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) + ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance)) + } + } + // Ensure Go's GC ignores the database cache for trigger percentage + cache := ctx.GlobalInt(utils.CacheFlag.Name) + gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) + + log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) + godebug.SetGCPercent(int(gogc)) + // Start system runtime metrics collection go metrics.CollectProcessMetrics(3 * time.Second) diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 1974a94aa2..5f39a889d1 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -103,8 +103,8 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()) spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64) - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) + spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) + spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) @@ -284,7 +284,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) + spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() spec.Params.EIP98Transition = math.MaxUint64 diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 4f9e88899a..a517623381 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -609,7 +609,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da } template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{ "Network": network, - "NetworkID": conf.Genesis.Config.ChainId, + "NetworkID": conf.Genesis.Config.ChainID, "NetworkTitle": strings.Title(network), "EthstatsPage": config.ethstats, "ExplorerPage": config.explorer, diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 9a429bc96d..6f08408947 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -49,7 +49,7 @@ func (w *wizard) deployFaucet() { existed := err == nil infos.node.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", " ") - infos.node.network = w.conf.Genesis.Config.ChainId.Int64() + infos.node.network = w.conf.Genesis.Config.ChainID.Int64() // Figure out which port to listen on fmt.Println() diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index f255ef6e65..6c4cd571fb 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -121,7 +121,7 @@ func (w *wizard) makeGenesis() { // Query the user for some custom extras fmt.Println() fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") - genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) + genesis.Config.ChainID = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) // All done, store the genesis and flush to disk log.Info("Configured new genesis block") diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index a60948bc67..9b22da4460 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -56,7 +56,7 @@ func (w *wizard) deployNode(boot bool) { existed := err == nil infos.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", " ") - infos.network = w.conf.Genesis.Config.ChainId.Int64() + infos.network = w.conf.Genesis.Config.ChainID.Int64() // Figure out where the user wants to store the persistent data fmt.Println() @@ -107,7 +107,7 @@ func (w *wizard) deployNode(boot bool) { // Ethash based miners only need an etherbase to mine against fmt.Println() if infos.etherbase == "" { - fmt.Printf("What address should the miner user?\n") + fmt.Printf("What address should the miner use?\n") for { if address := w.readAddress(); address != nil { infos.etherbase = address.Hex() @@ -115,7 +115,7 @@ func (w *wizard) deployNode(boot bool) { } } } else { - fmt.Printf("What address should the miner user? (default = %s)\n", infos.etherbase) + fmt.Printf("What address should the miner use? (default = %s)\n", infos.etherbase) infos.etherbase = w.readDefaultAddress(common.HexToAddress(infos.etherbase)).Hex() } } else if w.conf.Genesis.Config.Clique != nil { diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go index 933cd9ae59..7624d11e20 100644 --- a/cmd/puppeth/wizard_wallet.go +++ b/cmd/puppeth/wizard_wallet.go @@ -52,7 +52,7 @@ func (w *wizard) deployWallet() { existed := err == nil infos.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", " ") - infos.network = w.conf.Genesis.Config.ChainId.Int64() + infos.network = w.conf.Genesis.Config.ChainID.Int64() // Figure out which port to listen on fmt.Println() diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ef5f6a9f08..41a1ac35fe 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1084,6 +1084,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.Genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 1337 + } // Create new developer account or reuse existing one var ( developer accounts.Account diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 5031a088cb..31b65c1da9 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -140,8 +140,8 @@ func processArgs() { } if *asymmetricMode && len(*argPub) > 0 { - pub = crypto.ToECDSAPub(common.FromHex(*argPub)) - if !isKeyValid(pub) { + var err error + if pub, err = crypto.UnmarshalPubkey(common.FromHex(*argPub)); err != nil { utils.Fatalf("invalid public key") } } @@ -321,10 +321,6 @@ func startServer() error { return nil } -func isKeyValid(k *ecdsa.PublicKey) bool { - return k.X != nil && k.Y != nil -} - func configureNode() { var err error var p2pAccept bool @@ -340,9 +336,8 @@ func configureNode() { if b == nil { utils.Fatalf("Error: can not convert hexadecimal string") } - pub = crypto.ToECDSAPub(b) - if !isKeyValid(pub) { - utils.Fatalf("Error: invalid public key") + if pub, err = crypto.UnmarshalPubkey(b); err != nil { + utils.Fatalf("Error: invalid peer public key") } } } diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index 234714a2b9..f6e8d2e42a 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -31,11 +31,17 @@ import ( var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) +// Contract contains information about a compiled contract, alongside its code. type Contract struct { Code string `json:"code"` Info ContractInfo `json:"info"` } +// ContractInfo contains information about a compiled contract, including access +// to the ABI definition, user and developer docs, and metadata. +// +// Depending on the source, language version, compiler version, and compiler +// options will provide information about how the contract was compiled. type ContractInfo struct { Source string `json:"source"` Language string `json:"language"` @@ -142,8 +148,22 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro if err := cmd.Run(); err != nil { return nil, fmt.Errorf("solc: %v\n%s", err, stderr.Bytes()) } + + return ParseCombinedJSON(stdout.Bytes(), source, s.Version, s.Version, strings.Join(s.makeArgs(), " ")) +} + +// ParseCombinedJSON takes the direct output of a solc --combined-output run and +// parses it into a map of string contract name to Contract structs. The +// provided source, language and compiler version, and compiler options are all +// passed through into the Contract structs. +// +// The solc output is expected to contain ABI, user docs, and dev docs. +// +// Returns an error if the JSON is malformed or missing data, or if the JSON +// embedded within the JSON is malformed. +func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion string, compilerVersion string, compilerOptions string) (map[string]*Contract, error) { var output solcOutput - if err := json.Unmarshal(stdout.Bytes(), &output); err != nil { + if err := json.Unmarshal(combinedJSON, &output); err != nil { return nil, err } @@ -168,9 +188,9 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro Info: ContractInfo{ Source: source, Language: "Solidity", - LanguageVersion: s.Version, - CompilerVersion: s.Version, - CompilerOptions: strings.Join(s.makeArgs(), " "), + LanguageVersion: languageVersion, + CompilerVersion: compilerVersion, + CompilerOptions: compilerOptions, AbiDefinition: abi, UserDoc: userdoc, DeveloperDoc: devdoc, diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 905a7b1ea7..fa1c2c8246 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -94,14 +94,25 @@ func calcDatasetSize(epoch int) uint64 { // reused between hash runs instead of requiring new ones to be created. type hasher func(dest []byte, data []byte) -// makeHasher creates a repetitive hasher, allowing the same hash data structures -// to be reused between hash runs instead of requiring new ones to be created. -// The returned function is not thread safe! +// makeHasher creates a repetitive hasher, allowing the same hash data structures to +// be reused between hash runs instead of requiring new ones to be created. The returned +// function is not thread safe! func makeHasher(h hash.Hash) hasher { + // sha3.state supports Read to get the sum, use it to avoid the overhead of Sum. + // Read alters the state but we reset the hash before every operation. + type readerHash interface { + hash.Hash + Read([]byte) (int, error) + } + rh, ok := h.(readerHash) + if !ok { + panic("can't find Read method on hash") + } + outputLen := rh.Size() return func(dest []byte, data []byte) { - h.Write(data) - h.Sum(dest[:0]) - h.Reset() + rh.Reset() + rh.Write(data) + rh.Read(dest[:outputLen]) } } diff --git a/console/bridge.go b/console/bridge.go index f2120351c0..b0b4d37985 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -271,7 +271,7 @@ func (b *bridge) SleepBlocks(call otto.FunctionCall) (response otto.Value) { } type jsonrpcCall struct { - Id int64 + ID int64 Method string Params []interface{} } @@ -304,7 +304,7 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { resps, _ := call.Otto.Object("new Array()") for _, req := range reqs { resp, _ := call.Otto.Object(`({"jsonrpc":"2.0"})`) - resp.Set("id", req.Id) + resp.Set("id", req.ID) var result json.RawMessage err = b.client.Call(&result, req.Method, req.Params...) switch err := err.(type) { diff --git a/console/console.go b/console/console.go index b280d4e65d..56e03837ac 100644 --- a/console/console.go +++ b/console/console.go @@ -60,7 +60,7 @@ type Config struct { Preload []string // Absolute paths to JavaScript files to preload } -// Console is a JavaScript interpreted runtime environment. It is a fully fleged +// Console is a JavaScript interpreted runtime environment. It is a fully fledged // JavaScript console attached to a running node via an external or in-process RPC // client. type Console struct { @@ -73,6 +73,8 @@ type Console struct { printer io.Writer // Output writer to serialize any display strings to } +// New initializes a JavaScript interpreted runtime environment and sets defaults +// with the config struct. func New(config Config) (*Console, error) { // Handle unset config values gracefully if config.Prompter == nil { diff --git a/contracts/ens/ens.go b/contracts/ens/ens.go index 329109b943..75d9d0e4b1 100644 --- a/contracts/ens/ens.go +++ b/contracts/ens/ens.go @@ -95,16 +95,11 @@ func ensParentNode(name string) (common.Hash, common.Hash) { } } -func ensNode(name string) common.Hash { +func EnsNode(name string) common.Hash { parentNode, parentLabel := ensParentNode(name) return crypto.Keccak256Hash(parentNode[:], parentLabel[:]) } -// Suggest exporting ensNode so external code can use it for generating ens namehashes -func EnsNode(name string) common.Hash { - return ensNode(name) -} - func (self *ENS) getResolver(node [32]byte) (*contract.PublicResolverSession, error) { resolverAddr, err := self.Resolver(node) if err != nil { @@ -141,7 +136,7 @@ func (self *ENS) getRegistrar(node [32]byte) (*contract.FIFSRegistrarSession, er // Resolve is a non-transactional call that returns the content hash associated with a name. func (self *ENS) Resolve(name string) (common.Hash, error) { - node := ensNode(name) + node := EnsNode(name) resolver, err := self.getResolver(node) if err != nil { @@ -170,7 +165,7 @@ func (self *ENS) Register(name string) (*types.Transaction, error) { // SetContentHash sets the content hash associated with a name. Only works if the caller // owns the name, and the associated resolver implements a `setContent` function. func (self *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) { - node := ensNode(name) + node := EnsNode(name) resolver, err := self.getResolver(node) if err != nil { diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index 0016f47dbf..6ad8447082 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -55,7 +55,7 @@ func TestENS(t *testing.T) { if err != nil { t.Fatalf("can't deploy resolver: %v", err) } - if _, err := ens.SetResolver(ensNode(name), resolverAddr); err != nil { + if _, err := ens.SetResolver(EnsNode(name), resolverAddr); err != nil { t.Fatalf("can't set resolver: %v", err) } contractBackend.Commit() diff --git a/core/asm/lexer.go b/core/asm/lexer.go index 4d62159e55..91caeb27bc 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -242,7 +242,7 @@ func lexLabel(l *lexer) stateFn { } // lexInsideString lexes the inside of a string until -// until the state function finds the closing quote. +// the state function finds the closing quote. // It returns the lex text state function. func lexInsideString(l *lexer) stateFn { if l.acceptRunUntil('"') { diff --git a/core/blockchain.go b/core/blockchain.go index f74a0f5b27..34832252a7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -674,7 +674,7 @@ func (bc *BlockChain) Stop() { for !bc.triegc.Empty() { triedb.Dereference(bc.triegc.PopItem().(common.Hash), common.Hash{}) } - if size := triedb.Size(); size != 0 { + if size, _ := triedb.Size(); size != 0 { log.Error("Dangling trie nodes after full cleanup") } } @@ -916,33 +916,29 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, -float32(block.NumberU64())) if current := block.NumberU64(); current > triesInMemory { + // If we exceeded our memory allowance, flush matured singleton nodes to disk + var ( + nodes, imgs = triedb.Size() + limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 + ) + if nodes > limit || imgs > 4*1024*1024 { + triedb.Cap(limit - ethdb.IdealBatchSize) + } // Find the next state trie we need to commit header := bc.GetHeaderByNumber(current - triesInMemory) chosen := header.Number.Uint64() - // Only write to disk if we exceeded our memory allowance *and* also have at - // least a given number of tries gapped. - var ( - size = triedb.Size() - limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 - ) - if size > limit || bc.gcproc > bc.cacheConfig.TrieTimeLimit { + // If we exceeded out time allowance, flush an entire trie to disk + if bc.gcproc > bc.cacheConfig.TrieTimeLimit { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory { - switch { - case size >= 2*limit: - log.Warn("State memory usage too high, committing", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory) - case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit: - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory) - } - } - // If optimum or critical limits reached, write to disk - if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - triedb.Commit(header.Root, true) - lastWrite = chosen - bc.gcproc = 0 + if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory) } + // Flush an entire trie and restart the counters + triedb.Commit(header.Root, true) + lastWrite = chosen + bc.gcproc = 0 } // Garbage collect anything below our required write retention for !bc.triegc.Empty() { @@ -1009,6 +1005,10 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // only reason this method exists as a separate one is to make locking cleaner // with deferred statements. func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { + // Sanity check that we have something meaningful to import + if len(chain) == 0 { + return 0, nil, nil, nil + } // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() { @@ -1047,6 +1047,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty abort, results := bc.engine.VerifyHeaders(bc, headers, seals) defer close(abort) + // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) + senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain) + // Iterate over the blocks and insert when the verifier permits for i, block := range chain { // If the chain is terminating, stop processing blocks @@ -1181,7 +1184,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty } stats.processed++ stats.usedGas += usedGas - stats.report(chain, i, bc.stateCache.TrieDB().Size()) + + cache, _ := bc.stateCache.TrieDB().Size() + stats.report(chain, i, cache) } // Append a single chain head event if we've progressed the chain if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { @@ -1387,27 +1392,21 @@ func (bc *BlockChain) update() { } } -// BadBlockArgs represents the entries in the list returned when bad blocks are queried. -type BadBlockArgs struct { - Hash common.Hash `json:"hash"` - Header *types.Header `json:"header"` -} - // BadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -func (bc *BlockChain) BadBlocks() ([]BadBlockArgs, error) { - headers := make([]BadBlockArgs, 0, bc.badBlocks.Len()) +func (bc *BlockChain) BadBlocks() []*types.Block { + blocks := make([]*types.Block, 0, bc.badBlocks.Len()) for _, hash := range bc.badBlocks.Keys() { - if hdr, exist := bc.badBlocks.Peek(hash); exist { - header := hdr.(*types.Header) - headers = append(headers, BadBlockArgs{header.Hash(), header}) + if blk, exist := bc.badBlocks.Peek(hash); exist { + block := blk.(*types.Block) + blocks = append(blocks, block) } } - return headers, nil + return blocks } // addBadBlock adds a bad block to the bad-block LRU cache func (bc *BlockChain) addBadBlock(block *types.Block) { - bc.badBlocks.Add(block.Header().Hash(), block.Header()) + bc.badBlocks.Add(block.Hash(), block) } // reportBlock logs a bad block error. @@ -1525,6 +1524,18 @@ func (bc *BlockChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []com return bc.hc.GetBlockHashesFromHash(hash, max) } +// GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or +// a close ancestor of it is canonical. maxNonCanonical points to a downwards counter limiting the +// number of blocks to be individually checked before we reach the canonical chain. +// +// Note: ancestor == 0 returns the same block, 1 returns its parent and so on. +func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { + bc.chainmu.Lock() + defer bc.chainmu.Unlock() + + return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) +} + // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 89c071174f..5dbf63d1d6 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -578,7 +578,7 @@ func TestFastVsFullChains(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, } genesis = gspec.MustCommit(gendb) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.NewEIP155Signer(gspec.Config.ChainID) ) blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1024, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) @@ -753,7 +753,7 @@ func TestChainTxReorgs(t *testing.T) { }, } genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.NewEIP155Signer(gspec.Config.ChainID) ) // Create two transactions shared between the chains: @@ -859,7 +859,7 @@ func TestLogReorgs(t *testing.T) { code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.NewEIP155Signer(gspec.Config.ChainID) ) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -906,7 +906,7 @@ func TestReorgSideEvent(t *testing.T) { Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}, } genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.NewEIP155Signer(gspec.Config.ChainID) ) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}) @@ -1032,7 +1032,7 @@ func TestEIP155Transition(t *testing.T) { funds = big.NewInt(1000000000) deleteAddr = common.Address{1} gspec = &Genesis{ - Config: ¶ms.ChainConfig{ChainId: big.NewInt(1), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)}, + Config: ¶ms.ChainConfig{ChainID: big.NewInt(1), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)}, Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } genesis = gspec.MustCommit(db) @@ -1063,7 +1063,7 @@ func TestEIP155Transition(t *testing.T) { } block.AddTx(tx) - tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainId)) + tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainID)) if err != nil { t.Fatal(err) } @@ -1075,7 +1075,7 @@ func TestEIP155Transition(t *testing.T) { } block.AddTx(tx) - tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainId)) + tx, err = basicTx(types.NewEIP155Signer(gspec.Config.ChainID)) if err != nil { t.Fatal(err) } @@ -1103,7 +1103,7 @@ func TestEIP155Transition(t *testing.T) { } // generate an invalid chain id transaction - config := ¶ms.ChainConfig{ChainId: big.NewInt(2), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)} + config := ¶ms.ChainConfig{ChainID: big.NewInt(2), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)} blocks, _ = GenerateChain(config, blocks[len(blocks)-1], ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { var ( tx *types.Transaction @@ -1137,7 +1137,7 @@ func TestEIP161AccountRemoval(t *testing.T) { theAddr = common.Address{1} gspec = &Genesis{ Config: ¶ms.ChainConfig{ - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: new(big.Int), EIP155Block: new(big.Int), EIP158Block: big.NewInt(2), @@ -1153,7 +1153,7 @@ func TestEIP161AccountRemoval(t *testing.T) { var ( tx *types.Transaction err error - signer = types.NewEIP155Signer(gspec.Config.ChainId) + signer = types.NewEIP155Signer(gspec.Config.ChainID) ) switch i { case 0: diff --git a/core/headerchain.go b/core/headerchain.go index 2ac0cccc72..6e759ed1c1 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -307,6 +307,43 @@ func (hc *HeaderChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []co return chain } +// GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or +// a close ancestor of it is canonical. maxNonCanonical points to a downwards counter limiting the +// number of blocks to be individually checked before we reach the canonical chain. +// +// Note: ancestor == 0 returns the same block, 1 returns its parent and so on. +func (hc *HeaderChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { + if ancestor > number { + return common.Hash{}, 0 + } + if ancestor == 1 { + // in this case it is cheaper to just read the header + if header := hc.GetHeader(hash, number); header != nil { + return header.ParentHash, number - 1 + } else { + return common.Hash{}, 0 + } + } + for ancestor != 0 { + if rawdb.ReadCanonicalHash(hc.chainDb, number) == hash { + number -= ancestor + return rawdb.ReadCanonicalHash(hc.chainDb, number), number + } + if *maxNonCanonical == 0 { + return common.Hash{}, 0 + } + *maxNonCanonical-- + ancestor-- + header := hc.GetHeader(hash, number) + if header == nil { + return common.Hash{}, 0 + } + hash = header.ParentHash + number-- + } + return hash, number +} + // GetTd retrieves a block's total difficulty in the canonical chain from the // database by hash and number, caching it if found. func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index a26a42ba7c..da54328326 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -29,7 +29,7 @@ import ( // ReadCanonicalHash retrieves the hash assigned to a canonical block number. func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)) + data, _ := db.Get(headerHashKey(number)) if len(data) == 0 { return common.Hash{} } @@ -38,22 +38,21 @@ func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { // WriteCanonicalHash stores the hash assigned to a canonical block number. func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) { - key := append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) - if err := db.Put(key, hash.Bytes()); err != nil { + if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) } } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { - if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)); err != nil { + if err := db.Delete(headerHashKey(number)); err != nil { log.Crit("Failed to delete number to hash mapping", "err", err) } } // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { - data, _ := db.Get(append(headerNumberPrefix, hash.Bytes()...)) + data, _ := db.Get(headerNumberKey(hash)) if len(data) != 8 { return nil } @@ -129,14 +128,13 @@ func WriteFastTrieProgress(db DatabaseWriter, count uint64) { // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(headerKey(number, hash)) return data } // HasHeader verifies the existence of a block header corresponding to the hash. func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { - key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) - if has, err := db.Has(key); !has || err != nil { + if has, err := db.Has(headerKey(number, hash)); !has || err != nil { return false } return true @@ -161,11 +159,11 @@ func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Heade func WriteHeader(db DatabaseWriter, header *types.Header) { // Write the hash -> number mapping var ( - hash = header.Hash().Bytes() + hash = header.Hash() number = header.Number.Uint64() encoded = encodeBlockNumber(number) ) - key := append(headerNumberPrefix, hash...) + key := headerNumberKey(hash) if err := db.Put(key, encoded); err != nil { log.Crit("Failed to store hash to number mapping", "err", err) } @@ -174,7 +172,7 @@ func WriteHeader(db DatabaseWriter, header *types.Header) { if err != nil { log.Crit("Failed to RLP encode header", "err", err) } - key = append(append(headerPrefix, encoded...), hash...) + key = headerKey(number, hash) if err := db.Put(key, data); err != nil { log.Crit("Failed to store header", "err", err) } @@ -182,32 +180,30 @@ func WriteHeader(db DatabaseWriter, header *types.Header) { // DeleteHeader removes all block header data associated with a hash. func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(headerKey(number, hash)); err != nil { log.Crit("Failed to delete header", "err", err) } - if err := db.Delete(append(headerNumberPrefix, hash.Bytes()...)); err != nil { + if err := db.Delete(headerNumberKey(hash)); err != nil { log.Crit("Failed to delete hash to number mapping", "err", err) } } // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(blockBodyKey(number, hash)) return data } // WriteBodyRLP stores an RLP encoded block body into the database. func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { - key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, rlp); err != nil { + if err := db.Put(blockBodyKey(number, hash), rlp); err != nil { log.Crit("Failed to store block body", "err", err) } } // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { - key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if has, err := db.Has(key); !has || err != nil { + if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false } return true @@ -238,14 +234,14 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B // DeleteBody removes all block body data associated with a hash. func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(blockBodyKey(number, hash)); err != nil { log.Crit("Failed to delete block body", "err", err) } } // ReadTd retrieves a block's total difficulty corresponding to the hash. func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { - data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), headerTDSuffix...)) + data, _ := db.Get(headerTDKey(number, hash)) if len(data) == 0 { return nil } @@ -263,15 +259,14 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { if err != nil { log.Crit("Failed to RLP encode block total difficulty", "err", err) } - key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...) - if err := db.Put(key, data); err != nil { + if err := db.Put(headerTDKey(number, hash), data); err != nil { log.Crit("Failed to store block total difficulty", "err", err) } } // DeleteTd removes all block total difficulty data associated with a hash. func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...)); err != nil { + if err := db.Delete(headerTDKey(number, hash)); err != nil { log.Crit("Failed to delete block total difficulty", "err", err) } } @@ -279,7 +274,7 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { // ReadReceipts retrieves all the transaction receipts belonging to a block. func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice - data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + data, _ := db.Get(blockReceiptsKey(number, hash)) if len(data) == 0 { return nil } @@ -308,15 +303,14 @@ func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts log.Crit("Failed to encode block receipts", "err", err) } // Store the flattened receipt slice - key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, bytes); err != nil { + if err := db.Put(blockReceiptsKey(number, hash), bytes); err != nil { log.Crit("Failed to store block receipts", "err", err) } } // DeleteReceipts removes all receipt data associated with a block hash. func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { log.Crit("Failed to delete block receipts", "err", err) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 9abad14e0c..4ff7e5bd35 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -17,8 +17,6 @@ package rawdb import ( - "encoding/binary" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -28,7 +26,7 @@ import ( // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { - data, _ := db.Get(append(txLookupPrefix, hash.Bytes()...)) + data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { return common.Hash{}, 0, 0 } @@ -53,7 +51,7 @@ func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { if err != nil { log.Crit("Failed to encode transaction lookup entry", "err", err) } - if err := db.Put(append(txLookupPrefix, tx.Hash().Bytes()...), data); err != nil { + if err := db.Put(txLookupKey(tx.Hash()), data); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) } } @@ -61,7 +59,7 @@ func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { // DeleteTxLookupEntry removes all transaction data associated with a hash. func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { - db.Delete(append(txLookupPrefix, hash.Bytes()...)) + db.Delete(txLookupKey(hash)) } // ReadTransaction retrieves a specific transaction from the database, along with @@ -97,23 +95,13 @@ func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Ha // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given // section and bit index from the. func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - return db.Get(key) + return db.Get(bloomBitsKey(bit, section, head)) } // WriteBloomBits stores the compressed bloom bits vector belonging to the given // section and bit index. func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - if err := db.Put(key, bits); err != nil { + if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil { log.Crit("Failed to store bloom bits", "err", err) } } diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 73ab983f2b..514328e87b 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -45,7 +45,7 @@ func WriteDatabaseVersion(db DatabaseWriter, version int) { // ReadChainConfig retrieves the consensus settings based on the given genesis hash. func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { - data, _ := db.Get(append(configPrefix, hash[:]...)) + data, _ := db.Get(configKey(hash)) if len(data) == 0 { return nil } @@ -66,14 +66,14 @@ func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConf if err != nil { log.Crit("Failed to JSON encode chain config", "err", err) } - if err := db.Put(append(configPrefix, hash[:]...), data); err != nil { + if err := db.Put(configKey(hash), data); err != nil { log.Crit("Failed to store chain config", "err", err) } } // ReadPreimage retrieves a single preimage of the provided hash. func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { - data, _ := db.Get(append(preimagePrefix, hash.Bytes()...)) + data, _ := db.Get(preimageKey(hash)) return data } @@ -81,7 +81,7 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { // current block number, and is used for debug messages only. func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { - if err := db.Put(append(preimagePrefix, hash.Bytes()...), preimage); err != nil { + if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) } } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index a4b1596fde..ef597ef309 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -77,3 +77,58 @@ func encodeBlockNumber(number uint64) []byte { binary.BigEndian.PutUint64(enc, number) return enc } + +// headerKey = headerPrefix + num (uint64 big endian) + hash +func headerKey(number uint64, hash common.Hash) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix +func headerTDKey(number uint64, hash common.Hash) []byte { + return append(headerKey(number, hash), headerTDSuffix...) +} + +// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix +func headerHashKey(number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) +} + +// headerNumberKey = headerNumberPrefix + hash +func headerNumberKey(hash common.Hash) []byte { + return append(headerNumberPrefix, hash.Bytes()...) +} + +// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash +func blockBodyKey(number uint64, hash common.Hash) []byte { + return append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash +func blockReceiptsKey(number uint64, hash common.Hash) []byte { + return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// txLookupKey = txLookupPrefix + hash +func txLookupKey(hash common.Hash) []byte { + return append(txLookupPrefix, hash.Bytes()...) +} + +// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash +func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + return key +} + +// preimageKey = preimagePrefix + hash +func preimageKey(hash common.Hash) []byte { + return append(preimagePrefix, hash.Bytes()...) +} + +// configKey = configPrefix + hash +func configKey(hash common.Hash) []byte { + return append(configPrefix, hash.Bytes()...) +} diff --git a/core/state/state_object.go b/core/state/state_object.go index 5d203fddd8..091d24184a 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -219,7 +219,7 @@ func (self *stateObject) updateRoot(db Database) { self.data.Root = self.trie.Hash() } -// CommitTrie the storage trie of the object to dwb. +// CommitTrie the storage trie of the object to db. // This updates the trie root. func (self *stateObject) CommitTrie(db Database) error { self.updateTrie(db) diff --git a/core/state_processor.go b/core/state_processor.go index 4dc58b9dea..8e238ce1f0 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -85,7 +85,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, uint64, error) { +func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, uint64, error) { msg, err := tx.AsMessage(types.MakeSigner(config, header.Number)) if err != nil { return nil, 0, err diff --git a/core/tx_cacher.go b/core/tx_cacher.go new file mode 100644 index 0000000000..6d989c83d9 --- /dev/null +++ b/core/tx_cacher.go @@ -0,0 +1,105 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "runtime" + + "github.com/ethereum/go-ethereum/core/types" +) + +// senderCacher is a concurrent tranaction sender recoverer anc cacher. +var senderCacher = newTxSenderCacher(runtime.NumCPU()) + +// txSenderCacherRequest is a request for recovering transaction senders with a +// specific signature scheme and caching it into the transactions themselves. +// +// The inc field defines the number of transactions to skip after each recovery, +// which is used to feed the same underlying input array to different threads but +// ensure they process the early transactions fast. +type txSenderCacherRequest struct { + signer types.Signer + txs []*types.Transaction + inc int +} + +// txSenderCacher is a helper structure to concurrently ecrecover transaction +// senders from digital signatures on background threads. +type txSenderCacher struct { + threads int + tasks chan *txSenderCacherRequest +} + +// newTxSenderCacher creates a new transaction sender background cacher and starts +// as many procesing goroutines as allowed by the GOMAXPROCS on construction. +func newTxSenderCacher(threads int) *txSenderCacher { + cacher := &txSenderCacher{ + tasks: make(chan *txSenderCacherRequest, threads), + threads: threads, + } + for i := 0; i < threads; i++ { + go cacher.cache() + } + return cacher +} + +// cache is an infinite loop, caching transaction senders from various forms of +// data structures. +func (cacher *txSenderCacher) cache() { + for task := range cacher.tasks { + for i := 0; i < len(task.txs); i += task.inc { + types.Sender(task.signer, task.txs[i]) + } + } +} + +// recover recovers the senders from a batch of transactions and caches them +// back into the same data structures. There is no validation being done, nor +// any reaction to invalid signatures. That is up to calling code later. +func (cacher *txSenderCacher) recover(signer types.Signer, txs []*types.Transaction) { + // If there's nothing to recover, abort + if len(txs) == 0 { + return + } + // Ensure we have meaningful task sizes and schedule the recoveries + tasks := cacher.threads + if len(txs) < tasks*4 { + tasks = (len(txs) + 3) / 4 + } + for i := 0; i < tasks; i++ { + cacher.tasks <- &txSenderCacherRequest{ + signer: signer, + txs: txs[i:], + inc: tasks, + } + } +} + +// recoverFromBlocks recovers the senders from a batch of blocks and caches them +// back into the same data structures. There is no validation being done, nor +// any reaction to invalid signatures. That is up to calling code later. +func (cacher *txSenderCacher) recoverFromBlocks(signer types.Signer, blocks []*types.Block) { + count := 0 + for _, block := range blocks { + count += len(block.Transactions()) + } + txs := make([]*types.Transaction, 0, count) + for _, block := range blocks { + txs = append(txs, block.Transactions()...) + } + cacher.recover(signer, txs) +} diff --git a/core/tx_pool.go b/core/tx_pool.go index 7393c8286f..b0b55fcc25 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -222,7 +222,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block config: config, chainconfig: chainconfig, chain: chain, - signer: types.NewEIP155Signer(chainconfig.ChainId), + signer: types.NewEIP155Signer(chainconfig.ChainID), pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), @@ -411,6 +411,7 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { // Inject any transactions discarded due to reorgs log.Debug("Reinjecting stale transactions", "count", len(reinject)) + senderCacher.recover(pool.signer, reinject) pool.addTxsLocked(reinject, false) // validate the pool of pending transactions, this will remove @@ -1106,7 +1107,7 @@ func (pool *TxPool) demoteUnexecutables() { log.Trace("Demoting pending transaction", "hash", hash) pool.enqueueTx(hash, tx) } - // If there's a gap in front, warn (should never happen) and postpone all transactions + // If there's a gap in front, alert (should never happen) and postpone all transactions if list.Len() > 0 && list.txs.Get(nonce) == nil { for _, tx := range list.Cap(0) { hash := tx.Hash() diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index dfc84fdac7..c195d23a32 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -43,7 +43,7 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int) Signer { var signer Signer switch { case config.IsEIP155(blockNumber): - signer = NewEIP155Signer(config.ChainId) + signer = NewEIP155Signer(config.ChainID) case config.IsHomestead(blockNumber): signer = HomesteadSigner{} default: diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 3a67e1865f..9475322cdf 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -556,7 +556,7 @@ func opMload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *St func opMstore(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { // pop value of the stack mStart, val := stack.pop(), stack.pop() - memory.Set(mStart.Uint64(), 32, math.PaddedBigBytes(val, 32)) + memory.Set32(mStart.Uint64(), val) evm.interpreter.intPool.put(mStart, val) return nil, nil @@ -570,9 +570,9 @@ func opMstore8(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack * } func opSload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - loc := common.BigToHash(stack.pop()) - val := evm.StateDB.GetState(contract.Address(), loc).Big() - stack.push(val) + loc := stack.peek() + val := evm.StateDB.GetState(contract.Address(), common.BigToHash(loc)) + loc.SetBytes(val.Bytes()) return nil, nil } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 0de558612c..f51e6363f6 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -425,3 +425,42 @@ func BenchmarkOpIsZero(b *testing.B) { x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" opBenchmark(b, opIszero, x) } + +func TestOpMstore(t *testing.T) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() + ) + mem.Resize(64) + pc := uint64(0) + v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" + stack.pushN(new(big.Int).SetBytes(common.Hex2Bytes(v)), big.NewInt(0)) + opMstore(&pc, env, nil, mem, stack) + if got := common.Bytes2Hex(mem.Get(0, 32)); got != v { + t.Fatalf("Mstore fail, got %v, expected %v", got, v) + } + stack.pushN(big.NewInt(0x1), big.NewInt(0)) + opMstore(&pc, env, nil, mem, stack) + if common.Bytes2Hex(mem.Get(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { + t.Fatalf("Mstore failed to overwrite previous value") + } +} + +func BenchmarkOpMstore(bench *testing.B) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() + ) + mem.Resize(64) + pc := uint64(0) + memStart := big.NewInt(0) + value := big.NewInt(0x1337) + + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + stack.pushN(value, memStart) + opMstore(&pc, env, nil, mem, stack) + } +} diff --git a/core/vm/memory.go b/core/vm/memory.go index d5cc7870bc..43f6ff5bff 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -16,7 +16,12 @@ package vm -import "fmt" +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common/math" +) // Memory implements a simple memory model for the ethereum virtual machine. type Memory struct { @@ -30,19 +35,32 @@ func NewMemory() *Memory { // Set sets offset + size to value func (m *Memory) Set(offset, size uint64, value []byte) { - // length of store may never be less than offset + size. - // The store should be resized PRIOR to setting the memory - if size > uint64(len(m.store)) { - panic("INVALID memory: store empty") - } - // It's possible the offset is greater than 0 and size equals 0. This is because // the calcMemSize (common.go) could potentially return 0 when size is zero (NO-OP) if size > 0 { + // length of store may never be less than offset + size. + // The store should be resized PRIOR to setting the memory + if offset+size > uint64(len(m.store)) { + panic("invalid memory: store empty") + } copy(m.store[offset:offset+size], value) } } +// Set32 sets the 32 bytes starting at offset to the value of val, left-padded with zeroes to +// 32 bytes. +func (m *Memory) Set32(offset uint64, val *big.Int) { + // length of store may never be less than offset + size. + // The store should be resized PRIOR to setting the memory + if offset+32 > uint64(len(m.store)) { + panic("invalid memory: store empty") + } + // Zero the memory area + copy(m.store[offset:offset+32], []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}) + // Fill in relevant bits + math.ReadBits(val, m.store[offset:offset+32]) +} + // Resize resizes the memory to size func (m *Memory) Resize(size uint64) { if uint64(m.Len()) < size { diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index c8c5d34d9d..cda49a34b4 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -52,7 +52,7 @@ type Config struct { func setDefaults(cfg *Config) { if cfg.ChainConfig == nil { cfg.ChainConfig = ¶ms.ChainConfig{ - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: new(big.Int), DAOForkBlock: new(big.Int), DAOForkSupport: false, diff --git a/crypto/crypto.go b/crypto/crypto.go index 76d1ffaf64..619440e817 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -39,6 +39,8 @@ var ( secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) ) +var errInvalidPubkey = errors.New("invalid secp256k1 public key") + // Keccak256 calculates and returns the Keccak256 hash of the input data. func Keccak256(data ...[]byte) []byte { d := sha3.NewKeccak256() @@ -122,12 +124,13 @@ func FromECDSA(priv *ecdsa.PrivateKey) []byte { return math.PaddedBigBytes(priv.D, priv.Params().BitSize/8) } -func ToECDSAPub(pub []byte) *ecdsa.PublicKey { - if len(pub) == 0 { - return nil - } +// UnmarshalPubkey converts bytes to a secp256k1 public key. +func UnmarshalPubkey(pub []byte) (*ecdsa.PublicKey, error) { x, y := elliptic.Unmarshal(S256(), pub) - return &ecdsa.PublicKey{Curve: S256(), X: x, Y: y} + if x == nil { + return nil, errInvalidPubkey + } + return &ecdsa.PublicKey{Curve: S256(), X: x, Y: y}, nil } func FromECDSAPub(pub *ecdsa.PublicKey) []byte { diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 804de3fe22..177c19c0cf 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -23,9 +23,11 @@ import ( "io/ioutil" "math/big" "os" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" ) var testAddrHex = "970e8128ab834e8eac17ab8e3812f010678cf791" @@ -56,6 +58,33 @@ func BenchmarkSha3(b *testing.B) { } } +func TestUnmarshalPubkey(t *testing.T) { + key, err := UnmarshalPubkey(nil) + if err != errInvalidPubkey || key != nil { + t.Fatalf("expected error, got %v, %v", err, key) + } + key, err = UnmarshalPubkey([]byte{1, 2, 3}) + if err != errInvalidPubkey || key != nil { + t.Fatalf("expected error, got %v, %v", err, key) + } + + var ( + enc, _ = hex.DecodeString("04760c4460e5336ac9bbd87952a3c7ec4363fc0a97bd31c86430806e287b437fd1b01abc6e1db640cf3106b520344af1d58b00b57823db3e1407cbc433e1b6d04d") + dec = &ecdsa.PublicKey{ + Curve: S256(), + X: hexutil.MustDecodeBig("0x760c4460e5336ac9bbd87952a3c7ec4363fc0a97bd31c86430806e287b437fd1"), + Y: hexutil.MustDecodeBig("0xb01abc6e1db640cf3106b520344af1d58b00b57823db3e1407cbc433e1b6d04d"), + } + ) + key, err = UnmarshalPubkey(enc) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !reflect.DeepEqual(key, dec) { + t.Fatal("wrong result") + } +} + func TestSign(t *testing.T) { key, _ := HexToECDSA(testPrivHex) addr := common.HexToAddress(testAddrHex) @@ -69,7 +98,7 @@ func TestSign(t *testing.T) { if err != nil { t.Errorf("ECRecover error: %s", err) } - pubKey := ToECDSAPub(recoveredPub) + pubKey, _ := UnmarshalPubkey(recoveredPub) recoveredAddr := PubkeyToAddress(*pubKey) if addr != recoveredAddr { t.Errorf("Address mismatch: want: %x have: %x", addr, recoveredAddr) diff --git a/eth/api.go b/eth/api.go index 247ca7485c..446161cc4a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" @@ -351,10 +352,34 @@ func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hex return nil, errors.New("unknown preimage") } +// BadBlockArgs represents the entries in the list returned when bad blocks are queried. +type BadBlockArgs struct { + Hash common.Hash `json:"hash"` + Block map[string]interface{} `json:"block"` + RLP string `json:"rlp"` +} + // GetBadBLocks returns a list of the last 'bad blocks' that the client has seen on the network // and returns them as a JSON list of block-hashes -func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]core.BadBlockArgs, error) { - return api.eth.BlockChain().BadBlocks() +func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { + blocks := api.eth.BlockChain().BadBlocks() + results := make([]*BadBlockArgs, len(blocks)) + + var err error + for i, block := range blocks { + results[i] = &BadBlockArgs{ + Hash: block.Hash(), + } + if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { + results[i].RLP = err.Error() // Hacky, but hey, it works + } else { + results[i].RLP = fmt.Sprintf("0x%x", rlpBytes) + } + if results[i].Block, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { + results[i].Block = map[string]interface{}{"error": err.Error()} + } + } + return results, nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 45a819022a..61f5c71d64 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -251,7 +251,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { if number > origin { - log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin), "memory", database.TrieDB().Size()) + nodes, imgs := database.TrieDB().Size() + log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin), "memory", nodes+imgs) } else { log.Info("Preparing state for chain trace", "block", number, "start", origin, "elapsed", time.Since(begin)) } @@ -298,6 +299,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Dereference all past tries we ourselves are done working with database.TrieDB().Dereference(proot, common.Hash{}) proot = root + + // TODO(karalabe): Do we need the preimages? Won't they accumulate too much? } }() @@ -526,7 +529,8 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* database.TrieDB().Dereference(proot, common.Hash{}) proot = root } - log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", database.TrieDB().Size()) + nodes, imgs := database.TrieDB().Size() + log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) return statedb, nil } diff --git a/eth/config.go b/eth/config.go index dd7f42c7d9..426d2bf1ef 100644 --- a/eth/config.go +++ b/eth/config.go @@ -47,7 +47,7 @@ var DefaultConfig = Config{ LightPeers: 100, DatabaseCache: 768, TrieCache: 256, - TrieTimeout: 5 * time.Minute, + TrieTimeout: 60 * time.Minute, GasPrice: big.NewInt(18 * params.Shannon), TxPool: core.DefaultTxPoolConfig, diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index a2e7cdecf9..6383596dce 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -88,7 +88,7 @@ type headerFilterTask struct { time time.Time // Arrival time of the headers } -// headerFilterTask represents a batch of block bodies (transactions and uncles) +// bodyFilterTask represents a batch of block bodies (transactions and uncles) // needing fetcher filtering. type bodyFilterTask struct { peer string // The source peer of block bodies diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 67b4612ae4..7000d74fa4 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -258,9 +258,9 @@ Logs: if len(topics) > len(log.Topics) { continue Logs } - for i, topics := range topics { - match := len(topics) == 0 // empty rule set == wildcard - for _, topic := range topics { + for i, sub := range topics { + match := len(sub) == 0 // empty rule set == wildcard + for _, topic := range sub { if log.Topics[i] == topic { match = true break diff --git a/eth/handler.go b/eth/handler.go index 918d71088d..a46e7f13cb 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -340,6 +340,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "%v: %v", msg, err) } hashMode := query.Origin.Hash != (common.Hash{}) + first := true + maxNonCanonical := uint64(100) // Gather headers until the fetch or network limits is reached var ( @@ -351,31 +353,36 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Retrieve the next header satisfying the query var origin *types.Header if hashMode { - origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if first { + first = false + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) + } } else { origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } if origin == nil { break } - number := origin.Number.Uint64() headers = append(headers, origin) bytes += estHeaderRlpSize // Advance to the next header of the query switch { - case query.Origin.Hash != (common.Hash{}) && query.Reverse: + case hashMode && query.Reverse: // Hash based traversal towards the genesis block - for i := 0; i < int(query.Skip)+1; i++ { - if header := pm.blockchain.GetHeader(query.Origin.Hash, number); header != nil { - query.Origin.Hash = header.ParentHash - number-- - } else { - unknown = true - break - } + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) } - case query.Origin.Hash != (common.Hash{}) && !query.Reverse: + case hashMode && !query.Reverse: // Hash based traversal towards the leaf block var ( current = origin.Number.Uint64() @@ -387,8 +394,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { unknown = true } else { if header := pm.blockchain.GetHeaderByNumber(next); header != nil { - if pm.blockchain.GetBlockHashesFromHash(header.Hash(), query.Skip+1)[query.Skip] == query.Origin.Hash { - query.Origin.Hash = header.Hash() + nextHash := header.Hash() + expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next } else { unknown = true } diff --git a/eth/tracers/internal/tracers/4byte_tracer.js b/eth/tracers/internal/tracers/4byte_tracer.js index 0a6b088cca..2629aba3cf 100644 --- a/eth/tracers/internal/tracers/4byte_tracer.js +++ b/eth/tracers/internal/tracers/4byte_tracer.js @@ -78,7 +78,7 @@ // the final result of the tracing. result: function(ctx) { // Save the outer calldata also - if (ctx.input.length > 4) { + if (ctx.input.length >= 4) { this.store(slice(ctx.input, 0, 4), ctx.input.length-4) } return this.ids; diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 1912f74edd..2fdf5a646d 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -12,6 +12,7 @@ package tracers import ( "bytes" "compress/gzip" + "crypto/sha256" "fmt" "io" "io/ioutil" @@ -42,8 +43,9 @@ func bindataRead(data []byte, name string) ([]byte, error) { } type asset struct { - bytes []byte - info os.FileInfo + bytes []byte + info os.FileInfo + digest [sha256.Size]byte } type bindataFileInfo struct { @@ -72,7 +74,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var __4byte_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x56\x5b\x6f\xdb\x4a\x0e\x7e\xb6\x7f\x05\xd7\x2f\xb5\x51\x59\x8e\x2f\x89\x2f\xd9\x16\xf0\xe6\xa4\x6d\x80\x9c\x24\x88\xdd\x3d\x28\x16\xfb\x30\x9e\xa1\xac\xd9\xc8\x33\xc2\x0c\xe5\x4b\x73\xf2\xdf\x17\x1c\x49\x89\x93\xd3\x62\xbb\x4f\x96\x47\xc3\x8f\x1f\xc9\x8f\xa4\x7a\x3d\xb8\xb0\xf9\xc1\xe9\x75\x4a\x30\x38\xe9\x8f\x61\x99\x22\xac\x6d\x17\x29\x45\x87\xc5\x06\xe6\x05\xa5\xd6\xf9\x66\xaf\x07\xcb\x54\x7b\x48\x74\x86\xa0\x3d\xe4\xc2\x11\xd8\x04\xe8\xcd\xfd\x4c\xaf\x9c\x70\x87\xb8\xd9\xeb\x95\x36\x3f\x7c\xcd\x08\x89\x43\x04\x6f\x13\xda\x09\x87\x33\x38\xd8\x02\xa4\x30\xe0\x50\x69\x4f\x4e\xaf\x0a\x42\xd0\x04\xc2\xa8\x9e\x75\xb0\xb1\x4a\x27\x07\x86\xd4\x04\x85\x51\xe8\x82\x6b\x42\xb7\xf1\x35\x8f\xcf\x37\x5f\xe1\x1a\xbd\x47\x07\x9f\xd1\xa0\x13\x19\xdc\x15\xab\x4c\x4b\xb8\xd6\x12\x8d\x47\x10\x1e\x72\x3e\xf1\x29\x2a\x58\x05\x38\x36\xfc\xc4\x54\x16\x15\x15\xf8\x64\x0b\xa3\x04\x69\x6b\x22\x40\xcd\xcc\x61\x8b\xce\x6b\x6b\x60\x58\xbb\xaa\x00\x23\xb0\x8e\x41\xda\x82\x38\x00\x07\x36\x67\xbb\x0e\x08\x73\x80\x4c\xd0\x8b\xe9\x2f\x24\xe4\x25\x6e\x05\xda\x04\x37\xa9\xcd\x11\x28\x15\xc4\x51\xef\x74\x96\xc1\x0a\xa1\xf0\x98\x14\x59\xc4\x68\xab\x82\xe0\x8f\xab\xe5\x97\xdb\xaf\x4b\x98\xdf\x7c\x83\x3f\xe6\xf7\xf7\xf3\x9b\xe5\xb7\x73\xd8\x69\x4a\x6d\x41\x80\x5b\x2c\xa1\xf4\x26\xcf\x34\x2a\xd8\x09\xe7\x84\xa1\x03\xd8\x84\x11\x7e\xbf\xbc\xbf\xf8\x32\xbf\x59\xce\xff\x71\x75\x7d\xb5\xfc\x06\xd6\xc1\xa7\xab\xe5\xcd\xe5\x62\x01\x9f\x6e\xef\x61\x0e\x77\xf3\xfb\xe5\xd5\xc5\xd7\xeb\xf9\x3d\xdc\x7d\xbd\xbf\xbb\x5d\x5c\xc6\xb0\x40\x66\x85\x6c\xff\xbf\x73\x9e\x84\xea\x39\x04\x85\x24\x74\xe6\xeb\x4c\x7c\xb3\x05\xf8\xd4\x16\x99\x82\x54\x6c\x11\x1c\x4a\xd4\x5b\x54\x20\x40\xda\xfc\xf0\xcb\x45\x65\x2c\x91\x59\xb3\x0e\x31\xff\x54\x90\x70\x95\x80\xb1\x14\x81\x47\x84\xbf\xa7\x44\xf9\xac\xd7\xdb\xed\x76\xf1\xda\x14\xb1\x75\xeb\x5e\x56\xc2\xf9\xde\xc7\xb8\xc9\x98\xa3\xd5\x81\x70\xe9\x84\x44\x07\x1e\x85\x93\x29\xfa\x10\x4c\x78\xd1\xd5\x0a\x0d\xe9\x44\xa3\xf3\x11\x8b\x14\xa4\xcd\x32\x94\xe4\x99\xc1\x26\x5c\xcc\xad\xa7\x6e\xee\xac\x44\xef\xb5\x59\x73\xe0\x70\x45\xaf\x2e\xc2\x06\x29\xb5\xca\xc3\x11\xdc\xdb\x68\xbc\xfe\x8e\x75\x36\x7c\x91\x97\x65\x54\x82\x44\x04\xde\x86\xe8\xc1\x21\xcb\x0c\x15\x78\xbd\x36\x82\x0a\x87\xa1\x97\x56\x08\x1b\x41\x92\xc5\x2e\xd6\x42\x1b\x4f\x7f\x01\x64\x9c\xba\x22\x97\x7b\xb1\xc9\x33\x9c\xf1\x33\xc0\x47\x50\xb8\x2a\xd6\x31\x71\x0a\x96\x4e\x18\x2f\x24\x8b\xbb\x0d\xad\x93\xfd\xa0\x3f\xc2\xd3\xe9\x18\x87\xa7\x4a\x9c\x4c\x86\x67\xd3\x41\x72\x3a\x9c\x9c\xf5\x47\x7d\x3c\x9b\x26\xa3\x31\x4e\xc7\xc3\xd5\x40\x9e\x9e\xe1\x58\x4c\x4e\xc6\xc3\x55\x1f\xc5\xc9\x24\x51\xe3\xd3\x71\x1f\xa7\x0a\x5b\x11\x3c\x06\x60\x37\x83\xd6\x51\xa6\x5b\x4f\x9d\xd2\xfb\x63\xf9\x03\x70\xb2\x1f\x8c\x95\x1c\x4c\xc7\xd8\xed\x0f\x26\x33\xe8\x47\x2f\x6f\x86\x13\x29\x47\x93\x61\xbf\x7b\x32\x83\xc1\xd1\xf9\xe9\x60\x94\x0c\x27\x93\x69\x77\x7a\xf6\xda\x40\xa8\xe4\x74\x9a\x4c\xa7\xdd\xc1\xe4\x0d\x94\x1c\x4c\xfa\xaa\x3f\x45\x86\xea\x97\xc7\x4f\xcd\xc7\x66\x83\x07\x8e\xf2\x20\xd6\x6b\x87\x6b\x41\x58\x56\x2d\x30\x0e\x2f\x12\x1e\x16\x71\xb3\xc1\xcf\x33\x78\x7c\x8a\x9a\xc1\x46\x8a\x2c\x5b\x1e\x72\x56\x35\x15\xce\x78\x78\x97\x88\xcc\xe3\xbb\xa0\x0b\x63\x4d\x97\x2f\x78\x1e\x1f\x01\x2f\x47\x7c\xe8\x6a\xa3\x70\x1f\x2e\xf0\x51\xa2\x9d\x27\x1e\xb3\x62\x13\x10\x45\xc2\xd3\xe4\xdd\x56\x64\x05\xbe\x8b\x40\xc7\x18\xc3\x06\x37\x5c\x54\xe1\x28\x6e\x36\x6a\x97\x33\x48\x0a\x53\x56\xca\xe6\x9e\x5c\xe7\xb1\xd9\x68\xf8\x9d\x26\x99\x1e\x1d\x48\xe1\x11\x5a\x17\xf3\xeb\xeb\xd6\x0c\x5e\xfe\x5c\xdc\xfe\x76\xd9\x9a\x35\x1b\x0d\x76\xb9\x16\x2c\x6d\xa5\x5c\x04\x5b\x91\x45\xa5\xbb\xea\xc7\x7f\x0f\x0f\xb6\xa0\xfa\xd7\x7f\x67\xb3\x32\x5e\x18\x9e\x43\xaf\x07\x9e\x84\x7c\x80\x9c\x1c\x90\x2d\xcd\x9a\xcf\xae\x7f\xbb\xbc\xbe\xfc\x3c\x5f\x5e\xbe\xa2\xb0\x58\xce\x97\x57\x17\xe5\xd1\x5f\x49\xfc\x1f\xfe\x07\x3f\xf3\xdf\x68\x3c\x35\x9f\x6f\x85\x9a\x9c\x37\x1b\x75\xd5\x3c\xf1\x9c\xf2\x3c\x8d\xc2\x18\xd1\x3c\x3c\xb9\x2c\x55\x6b\x86\x3e\xe7\x8e\xe1\x0e\x8a\x9b\x8d\x70\xff\x28\xdf\x5a\x45\xa1\xb9\x42\x86\xb7\xc2\xc1\x03\x1e\xe0\x03\xb4\x5a\xf0\x1e\xc8\x7e\xc1\x7d\x5b\xab\x0e\xbc\x87\x56\x97\x4f\xf8\xe6\x79\xb3\xd1\xa0\x54\xfb\x58\x2b\xff\xaf\x07\x3c\xfc\x1b\x3e\xc0\xeb\xff\xef\xa1\x0f\x7f\xfe\x09\xfd\x57\x34\x31\xe7\x85\xa1\xcd\xd6\x3e\xa0\x0a\x92\xe1\x01\x70\x00\x9b\x4b\xab\xaa\x8d\xc1\x11\xfc\xf3\x77\xc0\x3d\xca\x82\xd0\x07\xba\x98\x1f\xb1\xcd\xec\x3a\x02\xb5\xea\x00\xb3\xed\xf5\x60\xf1\xa0\xf3\xb0\xb8\x4a\x14\x5f\xc2\xf0\x46\x34\x96\x40\x1b\x42\x67\x44\x16\xa4\xed\xab\xf8\x24\xd5\x7c\x6b\xf5\x31\x6a\x6c\xf3\x98\xec\x82\x9c\x36\xeb\x76\xa7\xc3\x31\xea\x04\xda\x7f\x93\x54\xfa\xaa\xd2\x7f\x5e\x15\xe3\xd8\x75\xee\xb0\x2b\xed\x26\x0f\x5f\x19\x66\x6b\x65\xd8\xc3\x3e\x02\x4a\x2d\xef\x6f\x87\xf0\x9f\xc2\x13\x24\xc2\xc8\x67\xa2\x15\xbe\xf6\x77\x0e\x2b\x63\xd5\x26\x3b\x57\xca\xa1\xf7\x81\x51\x50\x42\xcc\x6d\xd6\xee\x77\x3a\x9d\x9f\xf1\xf8\x2c\xc2\xba\x7f\x15\x6b\xbd\xb7\xaa\x90\xb5\x59\x7c\x87\x0f\xf0\x06\x54\x12\x17\xaa\x13\x87\xf6\xbc\x4d\xda\xcf\x41\x87\xeb\x1f\x3f\xc0\xa8\x72\x59\x42\xdc\x26\xc9\x8f\x30\xde\xd8\x97\xca\x08\x22\x0b\x41\xb0\xce\xdd\x21\xf6\xbc\xa9\xda\x01\x24\xaa\xb0\xde\xc3\xa8\x13\x05\x6a\xdd\x51\xa7\x8a\xa7\x56\x4b\x22\x8a\x8c\x8e\xe5\xb2\x4b\xab\x4f\x02\x21\xa9\x10\x59\xa5\x10\xfe\xbc\xb1\x09\x08\x53\x8b\x28\x29\x97\x75\x23\xd8\xff\x50\x36\x50\xbb\x70\xe8\x7f\xe4\x83\x93\xc7\x7e\x6a\x3d\x85\x35\xbf\x42\xee\x29\x42\x27\xf8\x3b\xc7\x6e\xab\xae\xaa\xe6\x64\x80\x2b\xc7\x1f\xe7\xbf\x02\xae\x76\x15\x2f\x8c\xb0\x47\x1b\xe5\xf9\x11\x29\x49\xfb\x17\x1d\xd7\xfd\x6b\x0b\x1e\x99\x5c\x43\xee\x59\x10\x99\xb7\x55\x55\x24\xed\x63\x6d\xf2\x82\xe2\x0c\xcd\x9a\x52\xf8\xf8\x5c\xa0\xa3\x9c\x97\x89\x7e\xbe\x1b\xc1\x49\x14\xf2\xfc\xd6\xba\x3b\xea\xbc\x9e\x2b\x75\x07\x97\x3d\xfb\xd4\xfc\x6f\x00\x00\x00\xff\xff\x08\x1e\xd9\xac\x67\x0b\x00\x00") +var __4byte_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x56\x5b\x6f\xdb\x4a\x0e\x7e\xb6\x7f\x05\xd7\x2f\xb5\x51\x59\x8e\x2f\x89\x2f\xd9\x16\xf0\xe6\xa4\x6d\x80\x9c\x24\x88\xdd\x3d\x28\x16\xfb\x30\x9e\xa1\xac\xd9\xc8\x33\xc2\x0c\xe5\x4b\x73\xf2\xdf\x17\x1c\x49\x89\x93\xd3\x62\xbb\x4f\x96\x47\xc3\x8f\x1f\xc9\x8f\xa4\x7a\x3d\xb8\xb0\xf9\xc1\xe9\x75\x4a\x30\x38\xe9\x8f\x61\x99\x22\xac\x6d\x17\x29\x45\x87\xc5\x06\xe6\x05\xa5\xd6\xf9\x66\xaf\x07\xcb\x54\x7b\x48\x74\x86\xa0\x3d\xe4\xc2\x11\xd8\x04\xe8\xcd\xfd\x4c\xaf\x9c\x70\x87\xb8\xd9\xeb\x95\x36\x3f\x7c\xcd\x08\x89\x43\x04\x6f\x13\xda\x09\x87\x33\x38\xd8\x02\xa4\x30\xe0\x50\x69\x4f\x4e\xaf\x0a\x42\xd0\x04\xc2\xa8\x9e\x75\xb0\xb1\x4a\x27\x07\x86\xd4\x04\x85\x51\xe8\x82\x6b\x42\xb7\xf1\x35\x8f\xcf\x37\x5f\xe1\x1a\xbd\x47\x07\x9f\xd1\xa0\x13\x19\xdc\x15\xab\x4c\x4b\xb8\xd6\x12\x8d\x47\x10\x1e\x72\x3e\xf1\x29\x2a\x58\x05\x38\x36\xfc\xc4\x54\x16\x15\x15\xf8\x64\x0b\xa3\x04\x69\x6b\x22\x40\xcd\xcc\x61\x8b\xce\x6b\x6b\x60\x58\xbb\xaa\x00\x23\xb0\x8e\x41\xda\x82\x38\x00\x07\x36\x67\xbb\x0e\x08\x73\x80\x4c\xd0\x8b\xe9\x2f\x24\xe4\x25\x6e\x05\xda\x04\x37\xa9\xcd\x11\x28\x15\xc4\x51\xef\x74\x96\xc1\x0a\xa1\xf0\x98\x14\x59\xc4\x68\xab\x82\xe0\x8f\xab\xe5\x97\xdb\xaf\x4b\x98\xdf\x7c\x83\x3f\xe6\xf7\xf7\xf3\x9b\xe5\xb7\x73\xd8\x69\x4a\x6d\x41\x80\x5b\x2c\xa1\xf4\x26\xcf\x34\x2a\xd8\x09\xe7\x84\xa1\x03\xd8\x84\x11\x7e\xbf\xbc\xbf\xf8\x32\xbf\x59\xce\xff\x71\x75\x7d\xb5\xfc\x06\xd6\xc1\xa7\xab\xe5\xcd\xe5\x62\x01\x9f\x6e\xef\x61\x0e\x77\xf3\xfb\xe5\xd5\xc5\xd7\xeb\xf9\x3d\xdc\x7d\xbd\xbf\xbb\x5d\x5c\xc6\xb0\x40\x66\x85\x6c\xff\xbf\x73\x9e\x84\xea\x39\x04\x85\x24\x74\xe6\xeb\x4c\x7c\xb3\x05\xf8\xd4\x16\x99\x82\x54\x6c\x11\x1c\x4a\xd4\x5b\x54\x20\x40\xda\xfc\xf0\xcb\x45\x65\x2c\x91\x59\xb3\x0e\x31\xff\x54\x90\x70\x95\x80\xb1\x14\x81\x47\x84\xbf\xa7\x44\xf9\xac\xd7\xdb\xed\x76\xf1\xda\x14\xb1\x75\xeb\x5e\x56\xc2\xf9\xde\xc7\xb8\xc9\x98\xa3\xd5\x81\x70\xe9\x84\x44\x07\x1e\x85\x93\x29\xfa\x10\x4c\x78\xd1\xd5\x0a\x0d\xe9\x44\xa3\xf3\x11\x8b\x14\xa4\xcd\x32\x94\xe4\x99\xc1\x26\x5c\xcc\xad\xa7\x6e\xee\xac\x44\xef\xb5\x59\x73\xe0\x70\x45\xaf\x2e\xc2\x06\x29\xb5\xca\xc3\x11\xdc\xdb\x68\xbc\xfe\x8e\x75\x36\x7c\x91\x97\x65\x54\x82\x44\x04\xde\x86\xe8\xc1\x21\xcb\x0c\x15\x78\xbd\x36\x82\x0a\x87\xa1\x97\x56\x08\x1b\x41\x92\xc5\x2e\xd6\x42\x1b\x4f\x7f\x01\x64\x9c\xba\x22\x97\x7b\xb1\xc9\x33\x9c\xf1\x33\xc0\x47\x50\xb8\x2a\xd6\x31\x71\x0a\x96\x4e\x18\x2f\x24\x8b\xbb\x0d\xad\x93\xfd\xa0\x3f\xc2\xd3\xe9\x18\x87\xa7\x4a\x9c\x4c\x86\x67\xd3\x41\x72\x3a\x9c\x9c\xf5\x47\x7d\x3c\x9b\x26\xa3\x31\x4e\xc7\xc3\xd5\x40\x9e\x9e\xe1\x58\x4c\x4e\xc6\xc3\x55\x1f\xc5\xc9\x24\x51\xe3\xd3\x71\x1f\xa7\x0a\x5b\x11\x3c\x06\x60\x37\x83\xd6\x51\xa6\x5b\x4f\x9d\xd2\xfb\x63\xf9\x03\x70\xb2\x1f\x8c\x95\x1c\x4c\xc7\xd8\xed\x0f\x26\x33\xe8\x47\x2f\x6f\x86\x13\x29\x47\x93\x61\xbf\x7b\x32\x83\xc1\xd1\xf9\xe9\x60\x94\x0c\x27\x93\x69\x77\x7a\xf6\xda\x40\xa8\xe4\x74\x9a\x4c\xa7\xdd\xc1\xe4\x0d\x94\x1c\x4c\xfa\xaa\x3f\x45\x86\xea\x97\xc7\x4f\xcd\xc7\x66\x83\x07\x8e\xf2\x20\xd6\x6b\x87\x6b\x41\x58\x56\x2d\x30\x0e\x2f\x12\x1e\x16\x71\xb3\xc1\xcf\x33\x78\x7c\x8a\x9a\xc1\x46\x8a\x2c\x5b\x1e\x72\x56\x35\x15\xce\x78\x78\x97\x88\xcc\xe3\xbb\xa0\x0b\x63\x4d\x97\x2f\x78\x1e\x1f\x01\x2f\x47\x7c\xe8\x6a\xa3\x70\x1f\x2e\xf0\x51\xa2\x9d\x27\x1e\xb3\x62\x13\x10\x45\xc2\xd3\xe4\xdd\x56\x64\x05\xbe\x8b\x40\xc7\x18\xc3\x06\x37\x5c\x54\xe1\x28\x6e\x36\x6a\x97\x33\x48\x0a\x53\x56\xca\xe6\x9e\x5c\xe7\xb1\xd9\x68\xf8\x9d\x26\x99\x1e\x1d\x48\xe1\x11\x5a\x17\xf3\xeb\xeb\xd6\x0c\x5e\xfe\x5c\xdc\xfe\x76\xd9\x9a\x35\x1b\x0d\x76\xb9\x16\x2c\x6d\xa5\x5c\x04\x5b\x91\x45\xa5\xbb\xea\xc7\x7f\x0f\x0f\xb6\xa0\xfa\xd7\x7f\x67\xb3\x32\x5e\x18\x9e\x43\xaf\x07\x9e\x84\x7c\x80\x9c\x1c\x90\x2d\xcd\x9a\xcf\xae\x7f\xbb\xbc\xbe\xfc\x3c\x5f\x5e\xbe\xa2\xb0\x58\xce\x97\x57\x17\xe5\xd1\x5f\x49\xfc\x1f\xfe\x07\x3f\xf3\xdf\x68\x3c\x35\x9f\x6f\x85\x9a\x9c\x37\x1b\x75\xd5\x3c\xf1\x9c\xf2\x3c\x8d\xc2\x18\xd1\x3c\x3c\xb9\x2c\x55\x6b\x86\x3e\xe7\x8e\xe1\x0e\x8a\x9b\x8d\x70\xff\x28\xdf\x5a\x45\xa1\xb9\x42\x86\xb7\xc2\xc1\x03\x1e\xe0\x03\xb4\x5a\xf0\x1e\xc8\x7e\xc1\x7d\x5b\xab\x0e\xbc\x87\x56\x97\x4f\xf8\xe6\x79\xb3\xd1\xa0\x54\xfb\x58\x2b\xff\xaf\x07\x3c\xfc\x1b\x3e\xc0\xeb\xff\xef\xa1\x0f\x7f\xfe\x09\xfd\x57\x34\x31\xe7\x85\xa1\xcd\xd6\x3e\xa0\x0a\x92\xe1\x01\x70\x00\x9b\x4b\xab\xaa\x8d\xc1\x11\xfc\xf3\x77\xc0\x3d\xca\x82\xd0\x07\xba\x98\x1f\xb1\xcd\xec\x3a\x02\xb5\xea\x00\xb3\xed\xf5\x60\xf1\xa0\xf3\xb0\xb8\x4a\x14\x5f\xc2\xf0\x46\x34\x96\x40\x1b\x42\x67\x44\x16\xa4\xed\xab\xf8\x24\xd5\x7c\x6b\xf5\x31\x6a\x6c\xf3\x98\xec\x82\x9c\x36\xeb\x76\xa7\xc3\x31\xea\x04\xda\x7f\x93\x54\xfa\xaa\xd2\x7f\x5e\x15\xe3\xd8\x75\xee\xb0\x2b\xed\x26\x0f\x5f\x19\x66\x6b\x65\xd8\xc3\x3e\x02\x4a\x2d\xef\x6f\x87\xf0\x9f\xc2\x13\x24\xc2\xc8\x67\xa2\x15\xbe\xf6\x77\x0e\x2b\x63\xd5\x26\x3b\x57\xca\xa1\xf7\x81\x51\x50\x42\xcc\x6d\xd6\xee\x77\x3a\x9d\x9f\xf1\xf8\x2c\xc2\xba\x7f\x15\x6b\xbd\xb7\xaa\x90\xb5\x59\x7c\x87\x0f\xf0\x06\x54\x12\x17\xaa\x13\x87\xf6\xbc\x4d\xda\xcf\x41\x87\xeb\x1f\x3f\xc0\xa8\x72\x59\x42\xdc\x26\xc9\x8f\x30\xde\xd8\x97\xca\x08\x22\x0b\x41\xb0\xce\xdd\x21\xf6\xbc\xa9\xda\x01\x24\xaa\xb0\xde\xc3\xa8\x13\x05\x6a\xdd\x51\xa7\x8a\xa7\x56\x4b\x22\x8a\x8c\x8e\xe5\xb2\x4b\xab\x4f\x02\x21\xa9\x10\x59\xa5\x10\xfe\xbc\xb1\x09\x08\x53\x8b\x28\x29\x97\x75\x23\xd8\xff\x50\x36\x50\xbb\x70\xe8\x7f\xe4\x83\x93\xc7\x7e\x6a\x3d\x85\x35\xbf\x42\xee\x29\x42\x27\xf8\x3b\xc7\x6e\xab\xae\xaa\xe6\x64\x80\x2b\xc7\x1f\xe7\xbf\x02\xae\x76\x15\x2f\x8c\xb0\x47\x1b\xe5\xf9\x11\x29\x49\xfb\x17\x1d\xd7\xfd\x6b\x0b\x1e\x99\x5c\x43\xee\x59\x10\x99\xb7\x55\x55\x24\xed\x63\x6d\xf2\x82\xe2\x0c\xcd\x9a\xd2\xe3\x0a\x1d\x25\xbd\xcc\xf4\xf3\xe5\x08\x4e\xa2\x90\xe8\xb7\xe6\xdd\x51\xe7\xf5\x60\xa9\x5b\xb8\x6c\xda\xa7\xe6\x7f\x03\x00\x00\xff\xff\x8b\x90\x53\x6e\x68\x0b\x00\x00") func _4byte_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -88,7 +90,7 @@ func _4byte_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "4byte_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xdc, 0x44, 0x40, 0x64, 0xa7, 0xa2, 0x19, 0xea, 0x36, 0x7, 0xf8, 0x62, 0x5, 0x90, 0xda, 0x9c, 0xc1, 0x71, 0xab, 0xc6, 0x14, 0x63, 0xe5, 0x52, 0x34, 0xb9, 0x53, 0x9b, 0x89, 0x2, 0x5b, 0xa4}} return a, nil } @@ -108,7 +110,7 @@ func call_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "call_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xf5, 0xb3, 0xb6, 0xe8, 0x19, 0xc3, 0xa, 0xce, 0xfd, 0x50, 0x84, 0xf7, 0x8a, 0xc5, 0x99, 0x10, 0x58, 0xc4, 0x69, 0xfb, 0x8, 0xad, 0x67, 0xea, 0x12, 0x38, 0xcb, 0xd, 0x2a, 0x94, 0xa1, 0x70}} return a, nil } @@ -128,7 +130,7 @@ func evmdis_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "evmdis_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd5, 0xe8, 0x96, 0xa1, 0x8b, 0xc, 0x68, 0x3c, 0xe8, 0x5d, 0x7e, 0xf0, 0xab, 0xfe, 0xec, 0xd1, 0xb, 0x3d, 0xfc, 0xc7, 0xac, 0xb5, 0xa, 0x41, 0x55, 0x0, 0x3a, 0x60, 0xa7, 0x8e, 0x46, 0x93}} return a, nil } @@ -148,7 +150,7 @@ func noop_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "noop_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4d, 0xcc, 0x83, 0xe9, 0x9e, 0xc1, 0x56, 0x41, 0x6c, 0x6a, 0x3b, 0x46, 0xc9, 0x5f, 0xe1, 0x5b, 0xcd, 0x6b, 0x53, 0x45, 0xcd, 0xfe, 0x1e, 0x86, 0x8c, 0x6b, 0xb, 0x70, 0x73, 0x9f, 0x4c, 0xb1}} return a, nil } @@ -168,7 +170,7 @@ func opcount_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "opcount_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x27, 0xe, 0x97, 0x88, 0x9b, 0x53, 0xbb, 0x20, 0x44, 0xd8, 0xf5, 0xeb, 0x41, 0xd2, 0x7e, 0xd6, 0xda, 0x6b, 0xf5, 0xaf, 0x0, 0x75, 0x9f, 0xd9, 0x22, 0xc, 0x6e, 0x74, 0xac, 0x2a, 0xa9, 0xa7}} return a, nil } @@ -188,7 +190,7 @@ func prestate_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "prestate_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd0, 0xd5, 0x5, 0x92, 0xed, 0xf4, 0x69, 0x2e, 0x14, 0x48, 0x35, 0x67, 0xcc, 0xf2, 0x3e, 0xc7, 0xf, 0x18, 0x22, 0x7a, 0x4d, 0x6f, 0x31, 0xad, 0x3c, 0x92, 0x77, 0xb4, 0x1, 0x2a, 0xd3, 0x7c}} return a, nil } @@ -207,6 +209,12 @@ func Asset(name string) ([]byte, error) { return nil, fmt.Errorf("Asset %s not found", name) } +// AssetString returns the asset contents as a string (instead of a []byte). +func AssetString(name string) (string, error) { + data, err := Asset(name) + return string(data), err +} + // MustAsset is like Asset but panics when Asset would return an error. // It simplifies safe initialization of global variables. func MustAsset(name string) []byte { @@ -218,6 +226,12 @@ func MustAsset(name string) []byte { return a } +// MustAssetString is like AssetString but panics when Asset would return an +// error. It simplifies safe initialization of global variables. +func MustAssetString(name string) string { + return string(MustAsset(name)) +} + // AssetInfo loads and returns the asset info for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -233,6 +247,33 @@ func AssetInfo(name string) (os.FileInfo, error) { return nil, fmt.Errorf("AssetInfo %s not found", name) } +// AssetDigest returns the digest of the file with the given name. It returns an +// error if the asset could not be found or the digest could not be loaded. +func AssetDigest(name string) ([sha256.Size]byte, error) { + canonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[canonicalName]; ok { + a, err := f() + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s can't read by error: %v", name, err) + } + return a.digest, nil + } + return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s not found", name) +} + +// Digests returns a map of all known files and their checksums. +func Digests() (map[string][sha256.Size]byte, error) { + mp := make(map[string][sha256.Size]byte, len(_bindata)) + for name := range _bindata { + a, err := _bindata[name]() + if err != nil { + return nil, err + } + mp[name] = a.digest + } + return mp, nil +} + // AssetNames returns the names of the assets. func AssetNames() []string { names := make([]string, 0, len(_bindata)) @@ -266,9 +307,9 @@ var _bindata = map[string]func() (*asset, error){ // img/ // a.png // b.png -// then AssetDir("data") would return []string{"foo.txt", "img"} -// AssetDir("data/img") would return []string{"a.png", "b.png"} -// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// then AssetDir("data") would return []string{"foo.txt", "img"}, +// AssetDir("data/img") would return []string{"a.png", "b.png"}, +// AssetDir("foo.txt") and AssetDir("notexist") would return an error, and // AssetDir("") will return []string{"data"}. func AssetDir(name string) ([]string, error) { node := _bintree @@ -306,7 +347,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "prestate_tracer.js": {prestate_tracerJs, map[string]*bintree{}}, }} -// RestoreAsset restores an asset under the given directory +// RestoreAsset restores an asset under the given directory. func RestoreAsset(dir, name string) error { data, err := Asset(name) if err != nil { @@ -327,7 +368,7 @@ func RestoreAsset(dir, name string) error { return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) } -// RestoreAssets restores an asset under the given directory recursively +// RestoreAssets restores an asset under the given directory recursively. func RestoreAssets(dir, name string) error { children, err := AssetDir(name) // File diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index b482245878..b40837c8cc 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -141,7 +141,9 @@ func (ec *Client) getBlock(ctx context.Context, method string, args ...interface // Fill the sender cache of transactions in the block. txs := make([]*types.Transaction, len(body.Transactions)) for i, tx := range body.Transactions { - setSenderFromServer(tx.tx, tx.From, body.Hash) + if tx.From != nil { + setSenderFromServer(tx.tx, *tx.From, body.Hash) + } txs[i] = tx.tx } return types.NewBlockWithHeader(head).WithBody(txs, uncles), nil @@ -174,9 +176,9 @@ type rpcTransaction struct { } type txExtraInfo struct { - BlockNumber *string - BlockHash common.Hash - From common.Address + BlockNumber *string `json:"blockNumber,omitempty"` + BlockHash *common.Hash `json:"blockHash,omitempty"` + From *common.Address `json:"from,omitempty"` } func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error { @@ -197,7 +199,9 @@ func (ec *Client) TransactionByHash(ctx context.Context, hash common.Hash) (tx * } else if _, r, _ := json.tx.RawSignatureValues(); r == nil { return nil, false, fmt.Errorf("server returned transaction without signature") } - setSenderFromServer(json.tx, json.From, json.BlockHash) + if json.From != nil && json.BlockHash != nil { + setSenderFromServer(json.tx, *json.From, *json.BlockHash) + } return json.tx, json.BlockNumber == nil, nil } @@ -244,7 +248,9 @@ func (ec *Client) TransactionInBlock(ctx context.Context, blockHash common.Hash, return nil, fmt.Errorf("server returned transaction without signature") } } - setSenderFromServer(json.tx, json.From, json.BlockHash) + if json.From != nil && json.BlockHash != nil { + setSenderFromServer(json.tx, *json.From, *json.BlockHash) + } return json.tx, err } diff --git a/ethdb/database.go b/ethdb/database.go index 86dd210762..f4a5ce2c8d 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -141,6 +141,7 @@ func (db *LDBDatabase) Close() { if err := <-errc; err != nil { db.log.Error("Metrics collection failed", "err", err) } + db.quitChan = nil } err := db.db.Close() if err == nil { @@ -189,7 +190,7 @@ func (db *LDBDatabase) Meter(prefix string) { // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 // // This is how the write delay look like (currently): -// DelayN:5 Delay:406.604657ms +// DelayN:5 Delay:406.604657ms Paused: false // // This is how the iostats look like (currently): // Read(MB):3895.04860 Write(MB):3654.64712 @@ -210,13 +211,19 @@ func (db *LDBDatabase) meter(refresh time.Duration) { lastWritePaused time.Time ) + var ( + errc chan error + merr error + ) + // Iterate ad infinitum and collect the stats - for i := 1; ; i++ { + for i := 1; errc == nil && merr == nil; i++ { // Retrieve the database stats stats, err := db.db.GetProperty("leveldb.stats") if err != nil { db.log.Error("Failed to read database stats", "err", err) - return + merr = err + continue } // Find the compaction table, skip the header lines := strings.Split(stats, "\n") @@ -225,7 +232,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { } if len(lines) <= 3 { db.log.Error("Compaction table not found") - return + merr = errors.New("compaction table not found") + continue } lines = lines[3:] @@ -242,7 +250,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) if err != nil { db.log.Error("Compaction entry parsing failed", "err", err) - return + merr = err + continue } compactions[i%2][idx] += value } @@ -262,7 +271,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { writedelay, err := db.db.GetProperty("leveldb.writedelay") if err != nil { db.log.Error("Failed to read database write delay statistic", "err", err) - return + merr = err + continue } var ( delayN int64 @@ -272,12 +282,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) { ) if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { db.log.Error("Write delay statistic not found") - return + merr = err + continue } duration, err = time.ParseDuration(delayDuration) if err != nil { db.log.Error("Failed to parse delay duration", "err", err) - return + merr = err + continue } if db.writeDelayNMeter != nil { db.writeDelayNMeter.Mark(delayN - delaystats[0]) @@ -317,53 +329,47 @@ func (db *LDBDatabase) meter(refresh time.Duration) { ioStats, err := db.db.GetProperty("leveldb.iostats") if err != nil { db.log.Error("Failed to read database iostats", "err", err) - return + merr = err + continue } + var nRead, nWrite float64 parts := strings.Split(ioStats, " ") if len(parts) < 2 { db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - return + merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) + continue } - r := strings.Split(parts[0], ":") - if len(r) < 2 { + if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { db.log.Error("Bad syntax of read entry", "entry", parts[0]) - return - } - read, err := strconv.ParseFloat(r[1], 64) - if err != nil { - db.log.Error("Read entry parsing failed", "err", err) - return + merr = err + continue } - w := strings.Split(parts[1], ":") - if len(w) < 2 { + if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { db.log.Error("Bad syntax of write entry", "entry", parts[1]) - return - } - write, err := strconv.ParseFloat(w[1], 64) - if err != nil { - db.log.Error("Write entry parsing failed", "err", err) - return + merr = err + continue } if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((read - iostats[0]) * 1024 * 1024)) + db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) } if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((write - iostats[1]) * 1024 * 1024)) + db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) } - iostats[0] = read - iostats[1] = write + iostats[0], iostats[1] = nRead, nWrite // Sleep a bit, then repeat the stats collection select { - case errc := <-db.quitChan: + case errc = <-db.quitChan: // Quit requesting, stop hammering the database - errc <- nil - return - case <-time.After(refresh): // Timeout, gather a new set of stats } } + + if errc == nil { + errc = <-db.quitChan + } + errc <- merr } func (db *LDBDatabase) NewBatch() Batch { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a524dbadd8..a465a59046 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -62,8 +62,9 @@ func NewPublicEthereumAPI(b Backend) *PublicEthereumAPI { } // GasPrice returns a suggestion for a gas price. -func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*big.Int, error) { - return s.b.SuggestPrice(ctx) +func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { + price, err := s.b.SuggestPrice(ctx) + return (*hexutil.Big)(price), err } // ProtocolVersion returns the current Ethereum protocol version this node supports @@ -354,7 +355,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs var chainID *big.Int if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainId + chainID = config.ChainID } return wallet.SignTxWithPassphrase(account, passwd, tx, chainID) } @@ -460,13 +461,11 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt } sig[64] -= 27 // Transform yellow paper V from 27/28 to 0/1 - rpk, err := crypto.Ecrecover(signHash(data), sig) + rpk, err := crypto.SigToPub(signHash(data), sig) if err != nil { return common.Address{}, err } - pubKey := crypto.ToECDSAPub(rpk) - recoveredAddr := crypto.PubkeyToAddress(*pubKey) - return recoveredAddr, nil + return crypto.PubkeyToAddress(*rpk), nil } // SignAndSendTransaction was renamed to SendTransaction. This method is deprecated @@ -487,21 +486,20 @@ func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { } // BlockNumber returns the block number of the chain head. -func (s *PublicBlockChainAPI) BlockNumber() *big.Int { +func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available - return header.Number + return hexutil.Uint64(header.Number.Uint64()) } // GetBalance returns the amount of wei for the given address in the state of the // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. -func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*big.Int, error) { +func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Big, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { return nil, err } - b := state.GetBalance(address) - return b, state.Error() + return (*hexutil.Big)(state.GetBalance(address)), state.Error() } // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all @@ -792,10 +790,10 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { return formatted } -// rpcOutputBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are +// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { +func RPCMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { head := b.Header() // copies the header once fields := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), @@ -808,7 +806,6 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "stateRoot": head.Root, "miner": head.Coinbase, "difficulty": (*hexutil.Big)(head.Difficulty), - "totalDifficulty": (*hexutil.Big)(s.b.GetTd(b.Hash())), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(b.Size()), "gasLimit": hexutil.Uint64(head.GasLimit), @@ -822,17 +819,15 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx formatTx := func(tx *types.Transaction) (interface{}, error) { return tx.Hash(), nil } - if fullTx { formatTx = func(tx *types.Transaction) (interface{}, error) { return newRPCTransactionFromBlockHash(b, tx.Hash()), nil } } - txs := b.Transactions() transactions := make([]interface{}, len(txs)) var err error - for i, tx := range b.Transactions() { + for i, tx := range txs { if transactions[i], err = formatTx(tx); err != nil { return nil, err } @@ -850,6 +845,17 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx return fields, nil } +// rpcOutputBlock uses the generalized output filler, then adds the total difficulty field, which requires +// a `PublicBlockchainAPI`. +func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields, err := RPCMarshalBlock(b, inclTx, fullTx) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(b.Hash())) + return fields, err +} + // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { BlockHash common.Hash `json:"blockHash"` @@ -1096,7 +1102,7 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti // Request the wallet to sign the transaction var chainID *big.Int if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainId + chainID = config.ChainID } return wallet.SignTx(account, tx, chainID) } @@ -1216,7 +1222,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen var chainID *big.Int if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainId + chainID = config.ChainID } signed, err := wallet.SignTx(account, tx, chainID) if err != nil { @@ -1293,14 +1299,19 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Sen return &SignTransactionResult{data, tx}, nil } -// PendingTransactions returns the transactions that are in the transaction pool and have a from address that is one of -// the accounts this node manages. +// PendingTransactions returns the transactions that are in the transaction pool +// and have a from address that is one of the accounts this node manages. func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, error) { pending, err := s.b.GetPoolTransactions() if err != nil { return nil, err } - + accounts := make(map[common.Address]struct{}) + for _, wallet := range s.b.AccountManager().Wallets() { + for _, account := range wallet.Accounts() { + accounts[account.Address] = struct{}{} + } + } transactions := make([]*RPCTransaction, 0, len(pending)) for _, tx := range pending { var signer types.Signer = types.HomesteadSigner{} @@ -1308,7 +1319,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err signer = types.NewEIP155Signer(tx.ChainId()) } from, _ := types.Sender(signer, tx) - if _, err := s.b.AccountManager().Find(accounts.Account{Address: from}); err == nil { + if _, exists := accounts[from]; exists { transactions = append(transactions, newRPCPendingTransaction(tx)) } } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 9d6ce8c6c8..89ebceec7c 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -313,8 +313,8 @@ web3._extend({ params: 2 }), new web3._extend.Method({ - name: 'setMutexProfileRate', - call: 'debug_setMutexProfileRate', + name: 'setMutexProfileFraction', + call: 'debug_setMutexProfileFraction', params: 1 }), new web3._extend.Method({ diff --git a/les/backend.go b/les/backend.go index 2d8ada7b0e..35f67f29f8 100644 --- a/les/backend.go +++ b/les/backend.go @@ -127,7 +127,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, ClientProtocolVersions, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, quitSync, &leth.wg); err != nil { + if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, ClientProtocolVersions, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { return nil, err } leth.ApiBackend = &LesApiBackend{leth, nil} diff --git a/les/handler.go b/les/handler.go index 22899eb1bc..a1c16cb875 100644 --- a/les/handler.go +++ b/les/handler.go @@ -19,6 +19,7 @@ package les import ( "encoding/binary" + "encoding/json" "errors" "fmt" "math/big" @@ -82,7 +83,7 @@ type BlockChain interface { InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) Rollback(chain []common.Hash) GetHeaderByNumber(number uint64) *types.Header - GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash + GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) Genesis() *types.Block SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } @@ -128,7 +129,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protocolVersions []uint, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protocolVersions []uint, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, @@ -140,6 +141,7 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protoco networkId: networkId, txpool: txpool, txrelay: txrelay, + serverPool: serverPool, peers: peers, newPeerCh: make(chan *peer), quitSync: quitSync, @@ -417,6 +419,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } hashMode := query.Origin.Hash != (common.Hash{}) + first := true + maxNonCanonical := uint64(100) // Gather headers until the fetch or network limits is reached var ( @@ -428,40 +432,57 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Retrieve the next header satisfying the query var origin *types.Header if hashMode { - origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if first { + first = false + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) + } } else { origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } if origin == nil { break } - number := origin.Number.Uint64() headers = append(headers, origin) bytes += estHeaderRlpSize // Advance to the next header of the query switch { - case query.Origin.Hash != (common.Hash{}) && query.Reverse: + case hashMode && query.Reverse: // Hash based traversal towards the genesis block - for i := 0; i < int(query.Skip)+1; i++ { - if header := pm.blockchain.GetHeader(query.Origin.Hash, number); header != nil { - query.Origin.Hash = header.ParentHash - number-- - } else { - unknown = true - break - } + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) } - case query.Origin.Hash != (common.Hash{}) && !query.Reverse: + case hashMode && !query.Reverse: // Hash based traversal towards the leaf block - if header := pm.blockchain.GetHeaderByNumber(origin.Number.Uint64() + query.Skip + 1); header != nil { - if pm.blockchain.GetBlockHashesFromHash(header.Hash(), query.Skip+1)[query.Skip] == query.Origin.Hash { - query.Origin.Hash = header.Hash() + var ( + current = origin.Number.Uint64() + next = current + query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") + p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := pm.blockchain.GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next + } else { + unknown = true + } } else { unknown = true } - } else { - unknown = true } case query.Reverse: // Number based traversal towards the genesis block diff --git a/les/helper_test.go b/les/helper_test.go index 6d997a1a36..8fd01a39e0 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -178,7 +178,7 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor } else { protocolVersions = ServerProtocolVersions } - pm, err := NewProtocolManager(gspec.Config, lightSync, protocolVersions, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, make(chan struct{}), new(sync.WaitGroup)) + pm, err := NewProtocolManager(gspec.Config, lightSync, protocolVersions, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) if err != nil { return nil, err } diff --git a/les/retrieve.go b/les/retrieve.go index e262a3cb47..a9037a38ef 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -69,9 +69,9 @@ type sentReq struct { lock sync.RWMutex // protect access to sentTo map sentTo map[distPeer]sentReqToPeer - reqQueued bool // a request has been queued but not sent - reqSent bool // a request has been sent but not timed out - reqSrtoCount int // number of requests that reached soft (but not hard) timeout + lastReqQueued bool // last request has been queued but not sent + lastReqSentTo distPeer // if not nil then last request has been sent to given peer but not timed out + reqSrtoCount int // number of requests that reached soft (but not hard) timeout } // sentReqToPeer notifies the request-from-peer goroutine (tryRequest) about a response @@ -180,7 +180,7 @@ type reqStateFn func() reqStateFn // retrieveLoop is the retrieval state machine event loop func (r *sentReq) retrieveLoop() { go r.tryRequest() - r.reqQueued = true + r.lastReqQueued = true state := r.stateRequesting for state != nil { @@ -214,7 +214,7 @@ func (r *sentReq) stateRequesting() reqStateFn { case rpSoftTimeout: // last request timed out, try asking a new peer go r.tryRequest() - r.reqQueued = true + r.lastReqQueued = true return r.stateRequesting case rpDeliveredValid: r.stop(nil) @@ -233,7 +233,7 @@ func (r *sentReq) stateNoMorePeers() reqStateFn { select { case <-time.After(retryQueue): go r.tryRequest() - r.reqQueued = true + r.lastReqQueued = true return r.stateRequesting case ev := <-r.eventsCh: r.update(ev) @@ -260,22 +260,26 @@ func (r *sentReq) stateStopped() reqStateFn { func (r *sentReq) update(ev reqPeerEvent) { switch ev.event { case rpSent: - r.reqQueued = false - if ev.peer != nil { - r.reqSent = true - } + r.lastReqQueued = false + r.lastReqSentTo = ev.peer case rpSoftTimeout: - r.reqSent = false + r.lastReqSentTo = nil r.reqSrtoCount++ - case rpHardTimeout, rpDeliveredValid, rpDeliveredInvalid: + case rpHardTimeout: r.reqSrtoCount-- + case rpDeliveredValid, rpDeliveredInvalid: + if ev.peer == r.lastReqSentTo { + r.lastReqSentTo = nil + } else { + r.reqSrtoCount-- + } } } // waiting returns true if the retrieval mechanism is waiting for an answer from // any peer func (r *sentReq) waiting() bool { - return r.reqQueued || r.reqSent || r.reqSrtoCount > 0 + return r.lastReqQueued || r.lastReqSentTo != nil || r.reqSrtoCount > 0 } // tryRequest tries to send the request to a new peer and waits for it to either diff --git a/les/server.go b/les/server.go index 9eb8ee7f97..fca6124c9c 100644 --- a/les/server.go +++ b/les/server.go @@ -52,7 +52,7 @@ type LesServer struct { func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), false, ServerProtocolVersions, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager(eth.BlockChain().Config(), false, ServerProtocolVersions, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) if err != nil { return nil, err } diff --git a/light/lightchain.go b/light/lightchain.go index 9d0a4e4f73..30b9bd89a6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -433,6 +433,18 @@ func (self *LightChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []c return self.hc.GetBlockHashesFromHash(hash, max) } +// GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or +// a close ancestor of it is canonical. maxNonCanonical points to a downwards counter limiting the +// number of blocks to be individually checked before we reach the canonical chain. +// +// Note: ancestor == 0 returns the same block, 1 returns its parent and so on. +func (bc *LightChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { + bc.chainmu.Lock() + defer bc.chainmu.Unlock() + + return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) +} + // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. func (self *LightChain) GetHeaderByNumber(number uint64) *types.Header { diff --git a/light/postprocess.go b/light/postprocess.go index c06c18027e..121321ae66 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -59,18 +59,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 170, - sectionHead: common.HexToHash("3bb2c28bcce463d57968f14f56cdb3fbf35349ab7a701f44c1afb57349c9a356"), - chtRoot: common.HexToHash("d92b6d0853455f8439086292338e87f69781921680dd7aa072fb71547b87415e"), - bloomTrieRoot: common.HexToHash("e4e8250a2fefddead7ae42daecd848cbf9b66d748a8270f8bbd4370b764bb9e9"), + sectionIdx: 174, + sectionHead: common.HexToHash("a3ef48cd8f1c3a08419f0237fc7763491fe89497b3144b17adf87c1c43664613"), + chtRoot: common.HexToHash("dcbeed9f4dea1b3cb75601bb27c51b9960c28e5850275402ac49a150a667296e"), + bloomTrieRoot: common.HexToHash("6b7497a4a03e33870a2383cb6f5e70570f12b1bf5699063baf8c71d02ca90b02"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 97, - sectionHead: common.HexToHash("719448c67c01eb5b9f27833a36a4e34612f66801316d7ff37daf9e77fb4cd095"), - chtRoot: common.HexToHash("a7857afc15930ca6e583b6c3d563a025144011655843d52d28e2fdaadd417bea"), - bloomTrieRoot: common.HexToHash("9c71d4b50cbec86dfeaa8e08992de8a4667b81d13c54d6522b17ce2fc5d36416"), + sectionIdx: 102, + sectionHead: common.HexToHash("9017ab08465cb2b2dee035ee5b817bbd7fa28e2c8d2cd903e0aed1cccb249e89"), + chtRoot: common.HexToHash("f61c10a7a787a5ef15f0ae1ae6c13c64331e57e79d0466d2bd9b0c06833fe956"), + bloomTrieRoot: common.HexToHash("69f2ad19aa46d5213a90137b3d2c9bff8a7c9483f7170f0125096ff450c9a873"), } ) diff --git a/light/txpool.go b/light/txpool.go index 1fabc3dc5a..5b4d06d90d 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -89,7 +89,7 @@ type TxRelayBackend interface { func NewTxPool(config *params.ChainConfig, chain *LightChain, relay TxRelayBackend) *TxPool { pool := &TxPool{ config: config, - signer: types.NewEIP155Signer(config.ChainId), + signer: types.NewEIP155Signer(config.ChainID), nonce: make(map[common.Address]uint64), pending: make(map[common.Hash]*types.Transaction), mined: make(map[common.Hash][]*types.Transaction), diff --git a/log/logger.go b/log/logger.go index a8eef1a9a0..438aa548fa 100644 --- a/log/logger.go +++ b/log/logger.go @@ -12,6 +12,7 @@ const timeKey = "t" const lvlKey = "lvl" const msgKey = "msg" const errorKey = "LOG15_ERROR" +const skipLevel = 2 type Lvl int @@ -157,27 +158,27 @@ func newContext(prefix []interface{}, suffix []interface{}) []interface{} { } func (l *logger) Trace(msg string, ctx ...interface{}) { - l.write(msg, LvlTrace, ctx, 2) + l.write(msg, LvlTrace, ctx, skipLevel) } func (l *logger) Debug(msg string, ctx ...interface{}) { - l.write(msg, LvlDebug, ctx, 2) + l.write(msg, LvlDebug, ctx, skipLevel) } func (l *logger) Info(msg string, ctx ...interface{}) { - l.write(msg, LvlInfo, ctx, 2) + l.write(msg, LvlInfo, ctx, skipLevel) } func (l *logger) Warn(msg string, ctx ...interface{}) { - l.write(msg, LvlWarn, ctx, 2) + l.write(msg, LvlWarn, ctx, skipLevel) } func (l *logger) Error(msg string, ctx ...interface{}) { - l.write(msg, LvlError, ctx, 2) + l.write(msg, LvlError, ctx, skipLevel) } func (l *logger) Crit(msg string, ctx ...interface{}) { - l.write(msg, LvlCrit, ctx, 2) + l.write(msg, LvlCrit, ctx, skipLevel) os.Exit(1) } diff --git a/log/root.go b/log/root.go index dd24c05e32..9fb4c5ae0b 100644 --- a/log/root.go +++ b/log/root.go @@ -31,36 +31,40 @@ func Root() Logger { // Trace is a convenient alias for Root().Trace func Trace(msg string, ctx ...interface{}) { - root.write(msg, LvlTrace, ctx, 2) + root.write(msg, LvlTrace, ctx, skipLevel) } // Debug is a convenient alias for Root().Debug func Debug(msg string, ctx ...interface{}) { - root.write(msg, LvlDebug, ctx, 2) + root.write(msg, LvlDebug, ctx, skipLevel) } // Info is a convenient alias for Root().Info func Info(msg string, ctx ...interface{}) { - root.write(msg, LvlInfo, ctx, 2) + root.write(msg, LvlInfo, ctx, skipLevel) } // Warn is a convenient alias for Root().Warn func Warn(msg string, ctx ...interface{}) { - root.write(msg, LvlWarn, ctx, 2) + root.write(msg, LvlWarn, ctx, skipLevel) } // Error is a convenient alias for Root().Error func Error(msg string, ctx ...interface{}) { - root.write(msg, LvlError, ctx, 2) + root.write(msg, LvlError, ctx, skipLevel) } // Crit is a convenient alias for Root().Crit func Crit(msg string, ctx ...interface{}) { - root.write(msg, LvlCrit, ctx, 2) + root.write(msg, LvlCrit, ctx, skipLevel) os.Exit(1) } -// Output is a convenient alias for write -func Output(msg string, lvl Lvl, skip int, ctx ...interface{}) { - root.write(msg, lvl, ctx, skip) +// Output is a convenient alias for write, allowing for the modification of +// the calldepth (number of stack frames to skip). +// calldepth influences the reported line number of the log message. +// A calldepth of zero reports the immediate caller of Output. +// Non-zero calldepth skips as many stack frames. +func Output(msg string, lvl Lvl, calldepth int, ctx ...interface{}) { + root.write(msg, lvl, ctx, calldepth+skipLevel) } diff --git a/metrics/exp/exp.go b/metrics/exp/exp.go index c19d00a94d..625ffd4e8a 100644 --- a/metrics/exp/exp.go +++ b/metrics/exp/exp.go @@ -134,6 +134,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) { exp.getFloat(name + ".mean-rate").Set(t.RateMean()) } +func (exp *exp) publishResettingTimer(name string, metric metrics.ResettingTimer) { + t := metric.Snapshot() + ps := t.Percentiles([]float64{50, 75, 95, 99}) + exp.getInt(name + ".count").Set(int64(len(t.Values()))) + exp.getFloat(name + ".mean").Set(t.Mean()) + exp.getInt(name + ".50-percentile").Set(ps[0]) + exp.getInt(name + ".75-percentile").Set(ps[1]) + exp.getInt(name + ".95-percentile").Set(ps[2]) + exp.getInt(name + ".99-percentile").Set(ps[3]) +} + func (exp *exp) syncToExpvar() { exp.registry.Each(func(name string, i interface{}) { switch i.(type) { @@ -149,6 +160,8 @@ func (exp *exp) syncToExpvar() { exp.publishMeter(name, i.(metrics.Meter)) case metrics.Timer: exp.publishTimer(name, i.(metrics.Timer)) + case metrics.ResettingTimer: + exp.publishResettingTimer(name, i.(metrics.ResettingTimer)) default: panic(fmt.Sprintf("unsupported type for '%s': %T", name, i)) } diff --git a/metrics/metrics.go b/metrics/metrics.go index e24324814c..2356f2b148 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -68,17 +68,20 @@ func CollectProcessMetrics(refresh time.Duration) { } // Iterate loading the different stats and updating the meters for i := 1; ; i++ { - runtime.ReadMemStats(memstats[i%2]) - memAllocs.Mark(int64(memstats[i%2].Mallocs - memstats[(i-1)%2].Mallocs)) - memFrees.Mark(int64(memstats[i%2].Frees - memstats[(i-1)%2].Frees)) - memInuse.Mark(int64(memstats[i%2].Alloc - memstats[(i-1)%2].Alloc)) - memPauses.Mark(int64(memstats[i%2].PauseTotalNs - memstats[(i-1)%2].PauseTotalNs)) + location1 := i % 2 + location2 := (i - 1) % 2 - if ReadDiskStats(diskstats[i%2]) == nil { - diskReads.Mark(diskstats[i%2].ReadCount - diskstats[(i-1)%2].ReadCount) - diskReadBytes.Mark(diskstats[i%2].ReadBytes - diskstats[(i-1)%2].ReadBytes) - diskWrites.Mark(diskstats[i%2].WriteCount - diskstats[(i-1)%2].WriteCount) - diskWriteBytes.Mark(diskstats[i%2].WriteBytes - diskstats[(i-1)%2].WriteBytes) + runtime.ReadMemStats(memstats[location1]) + memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs)) + memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees)) + memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc)) + memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs)) + + if ReadDiskStats(diskstats[location1]) == nil { + diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount) + diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes) + diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount) + diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes) } time.Sleep(refresh) } diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 57bcb31343..e5327d3bd3 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -58,7 +58,11 @@ type NilResettingTimer struct { func (NilResettingTimer) Values() []int64 { return nil } // Snapshot is a no-op. -func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} } +func (NilResettingTimer) Snapshot() ResettingTimer { + return &ResettingTimerSnapshot{ + values: []int64{}, + } +} // Time is a no-op. func (NilResettingTimer) Time(func()) {} @@ -210,7 +214,7 @@ func (t *ResettingTimerSnapshot) calc(percentiles []float64) { // poor man's math.Round(x): // math.Floor(x + 0.5) indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5)) - if pct >= 0 { + if pct >= 0 && indexOfPerc > 0 { indexOfPerc -= 1 // index offset=0 } thresholdBoundary = t.values[indexOfPerc] diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go index 58fd47f352..77c49dc386 100644 --- a/metrics/resetting_timer_test.go +++ b/metrics/resetting_timer_test.go @@ -104,3 +104,113 @@ func TestResettingTimer(t *testing.T) { } } } + +func TestResettingTimerWithFivePercentiles(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP05 int64 + wantP20 int64 + wantP50 int64 + wantP95 int64 + wantP99 int64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP05: 1, wantP20: 2, wantP50: 5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP05: 5, wantP20: 20, wantP50: 50, wantP95: 95, wantP99: 99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for ind, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{5, 20, 50, 95, 99}) + + val := snap.Values() + + if len(val) > 0 { + if tt.wantMin != val[0] { + t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) + } + + if tt.wantMax != val[len(val)-1] { + t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) + } + } + + if tt.wantMean != snap.Mean() { + t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + } + + if tt.wantP05 != ps[0] { + t.Fatalf("%d: p05: got %d, want %d", ind, ps[0], tt.wantP05) + } + + if tt.wantP20 != ps[1] { + t.Fatalf("%d: p20: got %d, want %d", ind, ps[1], tt.wantP20) + } + + if tt.wantP50 != ps[2] { + t.Fatalf("%d: p50: got %d, want %d", ind, ps[2], tt.wantP50) + } + + if tt.wantP95 != ps[3] { + t.Fatalf("%d: p95: got %d, want %d", ind, ps[3], tt.wantP95) + } + + if tt.wantP99 != ps[4] { + t.Fatalf("%d: p99: got %d, want %d", ind, ps[4], tt.wantP99) + } + } +} diff --git a/miner/worker.go b/miner/worker.go index 640e9032e7..ff48ecabc3 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -297,7 +297,6 @@ func (self *worker) update() { func (self *worker) wait() { for { - mustCommitNewWork := true for result := range self.recv { atomic.AddInt32(&self.atWork, -1) @@ -322,11 +321,6 @@ func (self *worker) wait() { log.Error("Failed writing block to chain", "err", err) continue } - // check if canon block and write transactions - if stat == core.CanonStatTy { - // implicit by posting ChainHeadEvent - mustCommitNewWork = false - } // Broadcast the block and announce chain insertion event self.mux.Post(core.NewMinedBlockEvent{Block: block}) var ( @@ -341,10 +335,6 @@ func (self *worker) wait() { // Insert the block into the set of pending ones to wait for confirmations self.unconfirmed.Insert(block.NumberU64(), block.Hash()) - - if mustCommitNewWork { - self.commitNewWork() - } } } } @@ -370,7 +360,7 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error } work := &Work{ config: self.config, - signer: types.NewEIP155Signer(self.config.ChainId), + signer: types.NewEIP155Signer(self.config.ChainID), state: state, ancestors: set.New(), family: set.New(), diff --git a/node/api.go b/node/api.go index da9da5bd72..989d3884ac 100644 --- a/node/api.go +++ b/node/api.go @@ -338,6 +338,21 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { }, } + case metrics.ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{5, 20, 50, 80, 95}) + root[name] = map[string]interface{}{ + "Measurements": len(t.Values()), + "Mean": time.Duration(t.Mean()).String(), + "Percentiles": map[string]interface{}{ + "5": time.Duration(ps[0]).String(), + "20": time.Duration(ps[1]).String(), + "50": time.Duration(ps[2]).String(), + "80": time.Duration(ps[3]).String(), + "95": time.Duration(ps[4]).String(), + }, + } + default: root[name] = "Unknown metric type" } @@ -373,6 +388,21 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { }, } + case metrics.ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{5, 20, 50, 80, 95}) + root[name] = map[string]interface{}{ + "Measurements": len(t.Values()), + "Mean": time.Duration(t.Mean()).String(), + "Percentiles": map[string]interface{}{ + "5": time.Duration(ps[0]).String(), + "20": time.Duration(ps[1]).String(), + "50": time.Duration(ps[2]).String(), + "80": time.Duration(ps[3]).String(), + "95": time.Duration(ps[4]).String(), + }, + } + default: root[name] = "Unknown metric type" } diff --git a/p2p/rlpx.go b/p2p/rlpx.go index a320e81e7c..149eda689c 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -528,9 +528,9 @@ func importPublicKey(pubKey []byte) (*ecies.PublicKey, error) { return nil, fmt.Errorf("invalid public key length %v (expect 64/65)", len(pubKey)) } // TODO: fewer pointless conversions - pub := crypto.ToECDSAPub(pubKey65) - if pub.X == nil { - return nil, fmt.Errorf("invalid public key") + pub, err := crypto.UnmarshalPubkey(pubKey65) + if err != nil { + return nil, err } return ecies.ImportECDSAPublic(pub), nil } diff --git a/p2p/rlpx_test.go b/p2p/rlpx_test.go index bca4604021..7ae8007740 100644 --- a/p2p/rlpx_test.go +++ b/p2p/rlpx_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/pipes" "github.com/ethereum/go-ethereum/rlp" ) @@ -159,7 +160,7 @@ func TestProtocolHandshake(t *testing.T) { wg sync.WaitGroup ) - fd0, fd1, err := tcpPipe() + fd0, fd1, err := pipes.TCPPipe() if err != nil { t.Fatal(err) } @@ -601,31 +602,3 @@ func TestHandshakeForwardCompatibility(t *testing.T) { t.Errorf("ingress-mac('foo') mismatch:\ngot %x\nwant %x", fooIngressHash, wantFooIngressHash) } } - -// tcpPipe creates an in process full duplex pipe based on a localhost TCP socket -func tcpPipe() (net.Conn, net.Conn, error) { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return nil, nil, err - } - defer l.Close() - - var aconn net.Conn - aerr := make(chan error, 1) - go func() { - var err error - aconn, err = l.Accept() - aerr <- err - }() - - dconn, err := net.Dial("tcp", l.Addr().String()) - if err != nil { - <-aerr - return nil, nil, err - } - if err := <-aerr; err != nil { - dconn.Close() - return nil, nil, err - } - return aconn, dconn, nil -} diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index c81257e7df..b68d08f392 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -17,25 +17,23 @@ package adapters import ( - "crypto/rand" "errors" "fmt" "math" "net" - "os" "sync" - "syscall" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/pipes" "github.com/ethereum/go-ethereum/rpc" ) // SimAdapter is a NodeAdapter which creates in-memory simulation nodes and -// connects them using net.Pipe or OS socket connections +// connects them using net.Pipe type SimAdapter struct { pipe func() (net.Conn, net.Conn, error) mtx sync.RWMutex @@ -49,19 +47,7 @@ type SimAdapter struct { // the adapter uses a net.Pipe for in-memory simulated network connections func NewSimAdapter(services map[string]ServiceFunc) *SimAdapter { return &SimAdapter{ - pipe: netPipe, - nodes: make(map[discover.NodeID]*SimNode), - services: services, - } -} - -// NewSocketAdapter creates a SimAdapter which is capable of running in-memory -// simulation nodes running any of the given services (the services to run on a -// particular node are passed to the NewNode function in the NodeConfig) -// the adapter uses a OS socketpairs for in-memory simulated network connections -func NewSocketAdapter(services map[string]ServiceFunc) *SimAdapter { - return &SimAdapter{ - pipe: socketPipe, + pipe: pipes.NetPipe, nodes: make(map[discover.NodeID]*SimNode), services: services, } @@ -69,7 +55,7 @@ func NewSocketAdapter(services map[string]ServiceFunc) *SimAdapter { func NewTCPAdapter(services map[string]ServiceFunc) *SimAdapter { return &SimAdapter{ - pipe: tcpPipe, + pipe: pipes.TCPPipe, nodes: make(map[discover.NodeID]*SimNode), services: services, } @@ -128,7 +114,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { } // Dial implements the p2p.NodeDialer interface by connecting to the node using -// an in-memory net.Pipe or OS socket connection +// an in-memory net.Pipe func (s *SimAdapter) Dial(dest *discover.Node) (conn net.Conn, err error) { node, ok := s.GetNode(dest.ID) if !ok { @@ -138,7 +124,7 @@ func (s *SimAdapter) Dial(dest *discover.Node) (conn net.Conn, err error) { if srv == nil { return nil, fmt.Errorf("node not running: %s", dest.ID) } - // SimAdapter.pipe is either net.Pipe (NewSimAdapter) or socketPipe (NewSocketAdapter) + // SimAdapter.pipe is net.Pipe (NewSimAdapter) pipe1, pipe2, err := s.pipe() if err != nil { return nil, err @@ -173,8 +159,8 @@ func (s *SimAdapter) GetNode(id discover.NodeID) (*SimNode, bool) { } // SimNode is an in-memory simulation node which connects to other nodes using -// net.Pipe or OS socket connection (see SimAdapter.Dial), running devp2p -// protocols directly over that pipe +// net.Pipe (see SimAdapter.Dial), running devp2p protocols directly over that +// pipe type SimNode struct { lock sync.RWMutex ID discover.NodeID @@ -348,34 +334,6 @@ func (sn *SimNode) NodeInfo() *p2p.NodeInfo { return server.NodeInfo() } -// socketPipe creates an in process full duplex pipe based on OS sockets -// credit to @lmars & Flynn -// https://github.com/flynn/flynn/blob/master/host/containerinit/init.go#L743-L749 -// using this in large simulations requires raising OS's max open file limit -func socketPipe() (net.Conn, net.Conn, error) { - pair, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0) - if err != nil { - return nil, nil, err - } - nameb := make([]byte, 8) - _, err = rand.Read(nameb) - if err != nil { - return nil, nil, err - } - f1 := os.NewFile(uintptr(pair[0]), string(nameb)+".out") - f2 := os.NewFile(uintptr(pair[1]), string(nameb)+".in") - pipe1, err := net.FileConn(f1) - if err != nil { - return nil, nil, err - } - pipe2, err := net.FileConn(f2) - if err != nil { - return nil, nil, err - } - - return pipe1, pipe2, nil -} - func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) error { switch v := conn.(type) { case *net.UnixConn: @@ -390,64 +348,3 @@ func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) } return nil } - -// netPipe wraps net.Pipe in a signature returning an error -func netPipe() (net.Conn, net.Conn, error) { - p1, p2 := net.Pipe() - return p1, p2, nil -} - -// tcpPipe creates an in process full duplex pipe based on a localhost TCP socket -func tcpPipe() (net.Conn, net.Conn, error) { - type result struct { - conn net.Conn - err error - } - - cl := make(chan result) - cd := make(chan result) - - start := make(chan net.Addr) - - go func(res chan result, start chan net.Addr) { - // resolve - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - res <- result{err: err} - return - } - // listen - l, err := net.ListenTCP("tcp", addr) - if err != nil { - res <- result{err: err} - return - } - start <- l.Addr() - c, err := l.AcceptTCP() - if err != nil { - res <- result{err: err} - return - } - res <- result{conn: c} - }(cl, start) - - go func(res chan result, start chan net.Addr) { - addr := <-start - c, err := net.DialTCP("tcp", nil, addr.(*net.TCPAddr)) - if err != nil { - res <- result{err: err} - return - } - res <- result{conn: c} - }(cd, start) - - a := <-cl - if a.err != nil { - return nil, nil, a.err - } - b := <-cd - if b.err != nil { - return nil, nil, b.err - } - return a.conn, b.conn, nil -} diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index b1ef7add0b..e1e092f6e1 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -22,123 +22,12 @@ import ( "fmt" "testing" "time" -) - -func TestSocketPipe(t *testing.T) { - c1, c2, err := socketPipe() - if err != nil { - t.Fatal(err) - } - - done := make(chan struct{}) - - go func() { - msgs := 20 - size := 8 - - // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously - go func() { - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - _ = binary.PutUvarint(msg, uint64(i)) - - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) - } - } - }() - - for i := 0; i < msgs; i++ { - msg := make([]byte, size) - _ = binary.PutUvarint(msg, uint64(i)) - - out := make([]byte, size) - _, err := c2.Read(out) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(msg, out) { - t.Fatalf("expected %#v, got %#v", msg, out) - } - } - done <- struct{}{} - }() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("test timeout") - } -} - -func TestSocketPipeBidirections(t *testing.T) { - c1, c2, err := socketPipe() - if err != nil { - t.Fatal(err) - } - done := make(chan struct{}) - - go func() { - msgs := 100 - size := 4 - - // OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously - go func() { - for i := 0; i < msgs; i++ { - msg := []byte(`ping`) - - _, err := c1.Write(msg) - if err != nil { - t.Fatal(err) - } - } - }() - - for i := 0; i < msgs; i++ { - out := make([]byte, size) - _, err := c2.Read(out) - if err != nil { - t.Fatal(err) - } - - if bytes.Equal(out, []byte(`ping`)) { - msg := []byte(`pong`) - _, err := c2.Write(msg) - if err != nil { - t.Fatal(err) - } - } - } - - for i := 0; i < msgs; i++ { - expected := []byte(`pong`) - - out := make([]byte, size) - _, err := c1.Read(out) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(out, expected) { - t.Fatalf("expected %#v, got %#v", expected, out) - } - } - - done <- struct{}{} - }() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("test timeout") - } -} + "github.com/ethereum/go-ethereum/p2p/simulations/pipes" +) -func TestTcpPipe(t *testing.T) { - c1, c2, err := tcpPipe() +func TestTCPPipe(t *testing.T) { + c1, c2, err := pipes.TCPPipe() if err != nil { t.Fatal(err) } @@ -182,8 +71,8 @@ func TestTcpPipe(t *testing.T) { } } -func TestTcpPipeBidirections(t *testing.T) { - c1, c2, err := tcpPipe() +func TestTCPPipeBidirections(t *testing.T) { + c1, c2, err := pipes.TCPPipe() if err != nil { t.Fatal(err) } @@ -246,7 +135,7 @@ func TestTcpPipeBidirections(t *testing.T) { } func TestNetPipe(t *testing.T) { - c1, c2, err := netPipe() + c1, c2, err := pipes.NetPipe() if err != nil { t.Fatal(err) } @@ -295,7 +184,7 @@ func TestNetPipe(t *testing.T) { } func TestNetPipeBidirections(t *testing.T) { - c1, c2, err := netPipe() + c1, c2, err := pipes.NetPipe() if err != nil { t.Fatal(err) } diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index a104d4d576..0fb7485ad0 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -378,6 +378,15 @@ func (net *Network) GetNodeByName(name string) *Node { return net.getNodeByName(name) } +// GetNodes returns the existing nodes +func (net *Network) GetNodes() (nodes []*Node) { + net.lock.Lock() + defer net.lock.Unlock() + + nodes = append(nodes, net.Nodes...) + return nodes +} + func (net *Network) getNode(id discover.NodeID) *Node { i, found := net.nodeMap[id] if !found { @@ -395,27 +404,6 @@ func (net *Network) getNodeByName(name string) *Node { return nil } -// GetNodes returns the existing nodes -func (net *Network) GetNodes() (nodes []*Node) { - net.lock.Lock() - defer net.lock.Unlock() - - nodes = append(nodes, net.Nodes...) - return nodes -} - -// GetNodes returns the existing nodes that are up -func (net *Network) GetUpNodes() (nodes []*Node) { - net.lock.Lock() - defer net.lock.Unlock() - for _, n := range net.Nodes { - if n.Up { - nodes = append(nodes, n) - } - } - return nodes -} - // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection func (net *Network) GetConn(oneID, otherID discover.NodeID) *Conn { diff --git a/p2p/simulations/pipes/pipes.go b/p2p/simulations/pipes/pipes.go new file mode 100644 index 0000000000..8532c1bcf0 --- /dev/null +++ b/p2p/simulations/pipes/pipes.go @@ -0,0 +1,55 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pipes + +import ( + "net" +) + +// NetPipe wraps net.Pipe in a signature returning an error +func NetPipe() (net.Conn, net.Conn, error) { + p1, p2 := net.Pipe() + return p1, p2, nil +} + +// TCPPipe creates an in process full duplex pipe based on a localhost TCP socket +func TCPPipe() (net.Conn, net.Conn, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, nil, err + } + defer l.Close() + + var aconn net.Conn + aerr := make(chan error, 1) + go func() { + var err error + aconn, err = l.Accept() + aerr <- err + }() + + dconn, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + <-aerr + return nil, nil, err + } + if err := <-aerr; err != nil { + dconn.Close() + return nil, nil, err + } + return aconn, dconn, nil +} diff --git a/params/config.go b/params/config.go index dc02c7ca37..6e6a5cb8b2 100644 --- a/params/config.go +++ b/params/config.go @@ -23,15 +23,16 @@ import ( "github.com/ethereum/go-ethereum/common" ) +// Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") // Mainnet genesis hash to enforce below configs on - TestnetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") // Testnet genesis hash to enforce below configs on + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + TestnetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") ) var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(1150000), DAOForkBlock: big.NewInt(1920000), DAOForkSupport: true, @@ -46,7 +47,7 @@ var ( // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), + ChainID: big.NewInt(3), HomesteadBlock: big.NewInt(0), DAOForkBlock: nil, DAOForkSupport: true, @@ -61,7 +62,7 @@ var ( // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), + ChainID: big.NewInt(4), HomesteadBlock: big.NewInt(1), DAOForkBlock: nil, DAOForkSupport: true, @@ -101,7 +102,7 @@ var ( // that any network, identified by its genesis block, can have its own // set of configuration options. type ChainConfig struct { - ChainId *big.Int `json:"chainId"` // Chain id identifies the current chain and is used for replay protection + ChainID *big.Int `json:"chainId"` // chainId identifies the current chain and is used for replay protection HomesteadBlock *big.Int `json:"homesteadBlock,omitempty"` // Homestead switch block (nil = no fork, 0 = already homestead) @@ -154,7 +155,7 @@ func (c *ChainConfig) String() string { engine = "unknown" } return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", - c.ChainId, + c.ChainID, c.HomesteadBlock, c.DAOForkBlock, c.DAOForkSupport, @@ -172,27 +173,32 @@ func (c *ChainConfig) IsHomestead(num *big.Int) bool { return isForked(c.HomesteadBlock, num) } -// IsDAO returns whether num is either equal to the DAO fork block or greater. +// IsDAOFork returns whether num is either equal to the DAO fork block or greater. func (c *ChainConfig) IsDAOFork(num *big.Int) bool { return isForked(c.DAOForkBlock, num) } +// IsEIP150 returns whether num is either equal to the EIP150 fork block or greater. func (c *ChainConfig) IsEIP150(num *big.Int) bool { return isForked(c.EIP150Block, num) } +// IsEIP155 returns whether num is either equal to the EIP155 fork block or greater. func (c *ChainConfig) IsEIP155(num *big.Int) bool { return isForked(c.EIP155Block, num) } +// IsEIP158 returns whether num is either equal to the EIP158 fork block or greater. func (c *ChainConfig) IsEIP158(num *big.Int) bool { return isForked(c.EIP158Block, num) } +// IsByzantium returns whether num is either equal to the Byzantium fork block or greater. func (c *ChainConfig) IsByzantium(num *big.Int) bool { return isForked(c.ByzantiumBlock, num) } +// IsConstantinople returns whether num is either equal to the Constantinople fork block or greater. func (c *ChainConfig) IsConstantinople(num *big.Int) bool { return isForked(c.ConstantinopleBlock, num) } @@ -251,7 +257,7 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.EIP158Block, newcfg.EIP158Block, head) { return newCompatError("EIP158 fork block", c.EIP158Block, newcfg.EIP158Block) } - if c.IsEIP158(head) && !configNumEqual(c.ChainId, newcfg.ChainId) { + if c.IsEIP158(head) && !configNumEqual(c.ChainID, newcfg.ChainID) { return newCompatError("EIP158 chain ID", c.EIP158Block, newcfg.EIP158Block) } if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { @@ -324,15 +330,16 @@ func (err *ConfigCompatError) Error() string { // Rules is a one time interface meaning that it shouldn't be used in between transition // phases. type Rules struct { - ChainId *big.Int + ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium bool } +// Rules ensures c's ChainID is not nil. func (c *ChainConfig) Rules(num *big.Int) Rules { - chainId := c.ChainId - if chainId == nil { - chainId = new(big.Int) + chainID := c.ChainID + if chainID == nil { + chainID = new(big.Int) } - return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num)} + return Rules{ChainID: new(big.Int).Set(chainID), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num)} } diff --git a/params/denomination.go b/params/denomination.go index 9e1b52506f..1a309827da 100644 --- a/params/denomination.go +++ b/params/denomination.go @@ -16,12 +16,12 @@ package params +// These are the multipliers for ether denominations. +// Example: To get the wei value of an amount in 'douglas', use +// +// new(big.Int).Mul(value, big.NewInt(params.Douglas)) +// const ( - // These are the multipliers for ether denominations. - // Example: To get the wei value of an amount in 'douglas', use - // - // new(big.Int).Mul(value, big.NewInt(params.Douglas)) - // Wei = 1 Ada = 1e3 Babbage = 1e6 diff --git a/params/gas_table.go b/params/gas_table.go index 4969382b1b..d33bebbe53 100644 --- a/params/gas_table.go +++ b/params/gas_table.go @@ -16,6 +16,7 @@ package params +// GasTable organizes gas prices for different ethereum phases. type GasTable struct { ExtcodeSize uint64 ExtcodeCopy uint64 @@ -34,6 +35,7 @@ type GasTable struct { CreateBySuicide uint64 } +// Variables containing gas prices for different ethereum phases. var ( // GasTableHomestead contain the gas prices for // the homestead phase. @@ -47,8 +49,8 @@ var ( ExpByte: 10, } - // GasTableHomestead contain the gas re-prices for - // the homestead phase. + // GasTableEIP150 contain the gas re-prices for + // the EIP150 phase. GasTableEIP150 = GasTable{ ExtcodeSize: 700, ExtcodeCopy: 700, @@ -60,7 +62,8 @@ var ( CreateBySuicide: 25000, } - + // GasTableEIP158 contain the gas re-prices for + // the EIP15* phase. GasTableEIP158 = GasTable{ ExtcodeSize: 700, ExtcodeCopy: 700, diff --git a/params/protocol_params.go b/params/protocol_params.go index 5a0b14d61a..1ea9c58138 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -19,7 +19,7 @@ package params import "math/big" var ( - TargetGasLimit uint64 = GenesisGasLimit // The artificial target + TargetGasLimit = GenesisGasLimit // The artificial target ) const ( diff --git a/params/version.go b/params/version.go index 8689ccba7f..f679908ae4 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 11 // Patch version component of the current release + VersionPatch = 12 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rpc/http.go b/rpc/http.go index feaa7348c4..6388d68961 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -165,7 +165,12 @@ func NewHTTPServer(cors []string, vhosts []string, srv *Server) *http.Server { // Wrap the CORS-handler within a host-handler handler := newCorsHandler(srv, cors) handler = newVHostHandler(vhosts, handler) - return &http.Server{Handler: handler} + return &http.Server{ + Handler: handler, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 120 * time.Second, + } } // ServeHTTP serves JSON-RPC requests over HTTP. @@ -181,7 +186,7 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // All checks passed, create a codec that reads direct from the request body // untilEOF and writes the response to w and order the server to process a // single request. - ctx := context.Background() + ctx := r.Context() ctx = context.WithValue(ctx, "remote", r.RemoteAddr) ctx = context.WithValue(ctx, "scheme", r.Proto) ctx = context.WithValue(ctx, "local", r.Host) diff --git a/rpc/json.go b/rpc/json.go index 837011f51b..a523eeb8ef 100644 --- a/rpc/json.go +++ b/rpc/json.go @@ -320,9 +320,6 @@ func parsePositionalArguments(rawArgs json.RawMessage, types []reflect.Type) ([] // CreateResponse will create a JSON-RPC success response with the given id and reply as result. func (c *jsonCodec) CreateResponse(id interface{}, reply interface{}) interface{} { - if isHexNum(reflect.TypeOf(reply)) { - return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: fmt.Sprintf(`%#x`, reply)} - } return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: reply} } @@ -340,11 +337,6 @@ func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err Error, info // CreateNotification will create a JSON-RPC notification with the given subscription id and event as params. func (c *jsonCodec) CreateNotification(subid, namespace string, event interface{}) interface{} { - if isHexNum(reflect.TypeOf(event)) { - return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, - Params: jsonSubscription{Subscription: subid, Result: fmt.Sprintf(`%#x`, event)}} - } - return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, Params: jsonSubscription{Subscription: subid, Result: event}} } diff --git a/rpc/server.go b/rpc/server.go index 7f304d8d93..8925419fe0 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -145,7 +145,7 @@ func (s *Server) serveRequest(ctx context.Context, codec ServerCodec, singleShot defer cancel() // if the codec supports notification include a notifier that callbacks can use - // to send notification to clients. It is thight to the codec/connection. If the + // to send notification to clients. It is tied to the codec/connection. If the // connection is closed the notifier will stop and cancels all active subscriptions. if options&OptionSubscriptions == OptionSubscriptions { ctx = context.WithValue(ctx, notifierKey{}, newNotifier(codec)) @@ -313,7 +313,6 @@ func (s *Server) handle(ctx context.Context, codec ServerCodec, req *serverReque if len(reply) == 0 { return codec.CreateResponse(req.id, nil), nil } - if req.callb.errPos >= 0 { // test if method returned an error if !reply[req.callb.errPos].IsNil() { e := reply[req.callb.errPos].Interface().(error) diff --git a/rpc/utils.go b/rpc/utils.go index 9315cab591..7f7ac4520b 100644 --- a/rpc/utils.go +++ b/rpc/utils.go @@ -22,7 +22,6 @@ import ( crand "crypto/rand" "encoding/binary" "encoding/hex" - "math/big" "math/rand" "reflect" "strings" @@ -105,20 +104,6 @@ func formatName(name string) string { return string(ret) } -var bigIntType = reflect.TypeOf((*big.Int)(nil)).Elem() - -// Indication if this type should be serialized in hex -func isHexNum(t reflect.Type) bool { - if t == nil { - return false - } - for t.Kind() == reflect.Ptr { - t = t.Elem() - } - - return t == bigIntType -} - // suitableCallbacks iterates over the methods of the given type. It will determine if a method satisfies the criteria // for a RPC callback or a subscription callback and adds it to the collection of callbacks or subscriptions. See server // documentation for a summary of these criteria. diff --git a/signer/core/api.go b/signer/core/api.go index 45933284bf..1372646de0 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -432,13 +432,11 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (c } sig[64] -= 27 // Transform yellow paper V from 27/28 to 0/1 hash, _ := SignHash(data) - rpk, err := crypto.Ecrecover(hash, sig) + rpk, err := crypto.SigToPub(hash, sig) if err != nil { return common.Address{}, err } - pubKey := crypto.ToECDSAPub(rpk) - recoveredAddr := crypto.PubkeyToAddress(*pubKey) - return recoveredAddr, nil + return crypto.PubkeyToAddress(*rpk), nil } // SignHash is a helper function that calculates a hash for the given message that can be diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index b44e7878ef..acf3479e52 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -155,10 +155,6 @@ func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) { testDiscoverySimulation(t, nodes, conns, adapters.NewExecAdapter(baseDir)) } -func TestDiscoverySimulationSocketAdapter(t *testing.T) { - testDiscoverySimulationSocketAdapter(t, *nodeCount, *initCount) -} - func TestDiscoverySimulationSimAdapter(t *testing.T) { testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount) } @@ -175,10 +171,6 @@ func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services)) } -func testDiscoverySimulationSocketAdapter(t *testing.T, nodes, conns int) { - testDiscoverySimulation(t, nodes, conns, adapters.NewSocketAdapter(services)) -} - func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { startedAt := time.Now() result, err := discoverySimulation(nodes, conns, adapter) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 7c9fea3f4e..9d1f997f29 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -50,7 +50,7 @@ var ( stores map[discover.NodeID]storage.ChunkStore toAddr func(discover.NodeID) *network.BzzAddr peerCount func(discover.NodeID) int - adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + adapter = flag.String("adapter", "sim", "type of simulation: sim|exec|docker") loglevel = flag.Int("loglevel", 2, "verbosity of logs") nodes = flag.Int("nodes", 0, "number of nodes") chunks = flag.Int("chunks", 0, "number of chunks") diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 1cb9e9104e..ff1c39319d 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -604,8 +604,6 @@ func initNetWithSnapshot(nodeCount int) (*simulations.Network, error) { return nil, err } a = adapters.NewExecAdapter(dirname) - } else if *adapter == "socket" { - a = adapters.NewSocketAdapter(services) } else if *adapter == "tcp" { a = adapters.NewTCPAdapter(services) } else if *adapter == "sim" { diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index 08f94e44be..d584ec3974 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -79,8 +79,6 @@ func NewAdapter(adapterType string, services adapters.Services) (adapter adapter switch adapterType { case "sim": adapter = adapters.NewSimAdapter(services) - case "socket": - adapter = adapters.NewSocketAdapter(services) case "exec": baseDir, err0 := ioutil.TempDir("", "swarm-test") if err0 != nil { @@ -94,7 +92,7 @@ func NewAdapter(adapterType string, services adapters.Services) (adapter adapter return nil, teardown, err } default: - return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker") + return nil, teardown, errors.New("adapter needs to be one of sim, exec, docker") } return adapter, teardown, nil } diff --git a/swarm/pss/api.go b/swarm/pss/api.go index 74dce145de..eba7bb722c 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -118,9 +118,13 @@ func (pssapi *API) GetPublicKey() (keybytes hexutil.Bytes) { // Set Public key to associate with a particular Pss peer func (pssapi *API) SetPeerPublicKey(pubkey hexutil.Bytes, topic Topic, addr PssAddress) error { - err := pssapi.Pss.SetPeerPublicKey(crypto.ToECDSAPub(pubkey), topic, &addr) + pk, err := crypto.UnmarshalPubkey(pubkey) if err != nil { - return fmt.Errorf("Invalid key: %x", pubkey) + return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkey) + } + err = pssapi.Pss.SetPeerPublicKey(pk, topic, &addr) + if err != nil { + return fmt.Errorf("Invalid key: %x", pk) } return nil } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 294871f7b3..77191b25a0 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -725,9 +725,8 @@ func (p *Pss) SendSym(symkeyid string, topic Topic, msg []byte) error { // // Fails if the key id does not match any in of the stored public keys func (p *Pss) SendAsym(pubkeyid string, topic Topic, msg []byte) error { - pubkey := crypto.ToECDSAPub(common.FromHex(pubkeyid)) - if pubkey == nil { - return fmt.Errorf("Invalid public key id %x", pubkey) + if _, err := crypto.UnmarshalPubkey(common.FromHex(pubkeyid)); err != nil { + return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkeyid) } p.pubKeyPoolMu.Lock() psp, ok := p.pubKeyPool[pubkeyid][topic] @@ -770,7 +769,11 @@ func (p *Pss) send(to []byte, topic Topic, msg []byte, asymmetric bool, key []by Padding: padding, } if asymmetric { - wparams.Dst = crypto.ToECDSAPub(key) + pk, err := crypto.UnmarshalPubkey(key) + if err != nil { + return fmt.Errorf("Cannot unmarshal pubkey: %x", key) + } + wparams.Dst = pk } else { wparams.KeySym = key } diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index ac784c9904..e28af275bb 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -955,10 +955,10 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke func TestNetwork2000(t *testing.T) { //enableMetrics() - t.Run("3/2000/4/sock", testNetwork) - t.Run("4/2000/4/sock", testNetwork) - t.Run("8/2000/4/sock", testNetwork) - t.Run("16/2000/4/sock", testNetwork) + t.Run("3/2000/4/sim", testNetwork) + t.Run("4/2000/4/sim", testNetwork) + t.Run("8/2000/4/sim", testNetwork) + t.Run("16/2000/4/sim", testNetwork) } func TestNetwork5000(t *testing.T) { @@ -1010,8 +1010,6 @@ func testNetwork(t *testing.T) { t.Fatal(err) } a = adapters.NewExecAdapter(dirname) - } else if adapter == "sock" { - a = adapters.NewSocketAdapter(newServices(false)) } else if adapter == "tcp" { a = adapters.NewTCPAdapter(newServices(false)) } else if adapter == "sim" { diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index d4465a5bc1..91fa54fcf0 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -19,6 +19,7 @@ package swap import ( "context" "crypto/ecdsa" + "errors" "fmt" "math/big" "os" @@ -136,6 +137,11 @@ func NewSwap(localProfile *LocalProfile, remoteProfile *RemoteProfile, backend c out *chequebook.Outbox ) + remotekey, err := crypto.UnmarshalPubkey(common.FromHex(remoteProfile.PublicKey)) + if err != nil { + return nil, errors.New("invalid remote public key") + } + // check if remoteProfile chequebook is valid // insolvent chequebooks suicide so will signal as invalid // TODO: monitoring a chequebooks events @@ -144,7 +150,7 @@ func NewSwap(localProfile *LocalProfile, remoteProfile *RemoteProfile, backend c log.Info(fmt.Sprintf("invalid contract %v for peer %v: %v)", remoteProfile.Contract.Hex()[:8], proto, err)) } else { // remoteProfile contract valid, create inbox - in, err = chequebook.NewInbox(localProfile.privateKey, remoteProfile.Contract, localProfile.Beneficiary, crypto.ToECDSAPub(common.FromHex(remoteProfile.PublicKey)), backend) + in, err = chequebook.NewInbox(localProfile.privateKey, remoteProfile.Contract, localProfile.Beneficiary, remotekey, backend) if err != nil { log.Warn(fmt.Sprintf("unable to set up inbox for chequebook contract %v for peer %v: %v)", remoteProfile.Contract.Hex()[:8], proto, err)) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index a72799f6e2..2db47da57f 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -42,6 +42,7 @@ type BlockTest struct { json btJSON } +// UnmarshalJSON implements json.Unmarshaler interface. func (t *BlockTest) UnmarshalJSON(in []byte) error { return json.Unmarshal(in, &t.json) } diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 6006373009..20294cc9d5 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -27,7 +27,7 @@ import ( var ( mainnetChainConfig = params.ChainConfig{ - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(1150000), DAOForkBlock: big.NewInt(1920000), DAOForkSupport: true, diff --git a/tests/init.go b/tests/init.go index 0bea5ccd63..af65f92069 100644 --- a/tests/init.go +++ b/tests/init.go @@ -26,26 +26,26 @@ import ( // Forks table defines supported forks and their chain config. var Forks = map[string]*params.ChainConfig{ "Frontier": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), }, "Homestead": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), }, "EIP150": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), }, "EIP158": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), EIP158Block: big.NewInt(0), }, "Byzantium": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), @@ -54,22 +54,22 @@ var Forks = map[string]*params.ChainConfig{ ByzantiumBlock: big.NewInt(0), }, "FrontierToHomesteadAt5": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(5), }, "HomesteadToEIP150At5": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(5), }, "HomesteadToDaoAt5": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), DAOForkBlock: big.NewInt(5), DAOForkSupport: true, }, "EIP158ToByzantiumAt5": { - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), diff --git a/tests/transaction_test.go b/tests/transaction_test.go index c743996c2a..42ad81877e 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -35,7 +35,7 @@ func TestTransaction(t *testing.T) { EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), EIP158Block: big.NewInt(0), - ChainId: big.NewInt(1), + ChainID: big.NewInt(1), }) txt.config(`^Byzantium/`, params.ChainConfig{ HomesteadBlock: big.NewInt(0), diff --git a/trie/database.go b/trie/database.go index da36e72f98..468c139df4 100644 --- a/trie/database.go +++ b/trie/database.go @@ -23,6 +23,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) + memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) + memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) + + memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil) + memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) + memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) + + memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) + memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) + memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) ) // secureKeyPrefix is the database key prefix used to store trie node preimages. @@ -46,15 +61,22 @@ type DatabaseReader interface { type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes - nodes map[common.Hash]*cachedNode // Data and references relationships of a node - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys + nodes map[common.Hash]*cachedNode // Data and references relationships of a node + oldest common.Hash // Oldest tracked node, flush-list head + newest common.Hash // Newest tracked node, flush-list tail + + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit - nodesSize common.StorageSize // Storage size of the nodes cache + flushtime time.Duration // Time spent on data flushing since last commit + flushnodes uint64 // Nodes flushed since last commit + flushsize common.StorageSize // Data storage flushed since last commit + + nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist) preimagesSize common.StorageSize // Storage size of the preimages cache lock sync.RWMutex @@ -66,6 +88,9 @@ type cachedNode struct { blob []byte // Cached data block of the trie node parents int // Number of live nodes referencing this one children map[common.Hash]int // Children referenced by this nodes + + flushPrev common.Hash // Previous node in the flush-list + flushNext common.Hash // Next node in the flush-list } // NewDatabase creates a new trie database to store ephemeral trie content before @@ -96,12 +121,20 @@ func (db *Database) Insert(hash common.Hash, blob []byte) { // insert is the private locked version of Insert. func (db *Database) insert(hash common.Hash, blob []byte) { + // If the node's already cached, skip if _, ok := db.nodes[hash]; ok { return } db.nodes[hash] = &cachedNode{ - blob: common.CopyBytes(blob), - children: make(map[common.Hash]int), + blob: common.CopyBytes(blob), + children: make(map[common.Hash]int), + flushPrev: db.newest, + } + // Update the flush-list endpoints + if db.oldest == (common.Hash{}) { + db.oldest, db.newest = hash, hash + } else { + db.nodes[db.newest].flushNext, db.newest = hash, hash } db.nodesSize += common.StorageSize(common.HashLength + len(blob)) } @@ -208,6 +241,10 @@ func (db *Database) Dereference(child common.Hash, parent common.Hash) { db.gcsize += storage - db.nodesSize db.gctime += time.Since(start) + memcacheGCTimeTimer.Update(time.Since(start)) + memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize)) + memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes))) + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) } @@ -221,7 +258,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { if node.children[child] == 0 { delete(node.children, child) } - // If the node does not exist, it's a previously committed node. + // If the child does not exist, it's a previously committed node. node, ok := db.nodes[child] if !ok { return @@ -229,6 +266,14 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { // If there are no more references to the child, delete it and cascade node.parents-- if node.parents == 0 { + // Remove the node from the flush-list + if child == db.oldest { + db.oldest = node.flushNext + } else { + db.nodes[node.flushPrev].flushNext = node.flushNext + db.nodes[node.flushNext].flushPrev = node.flushPrev + } + // Dereference all children and delete the node for hash := range node.children { db.dereference(hash, child) } @@ -237,6 +282,107 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { } } +// Cap iteratively flushes old but still referenced trie nodes until the total +// memory usage goes below the given threshold. +func (db *Database) Cap(limit common.StorageSize) error { + // Create a database batch to flush persistent data out. It is important that + // outside code doesn't see an inconsistent state (referenced data removed from + // memory cache during commit but not yet in persistent storage). This is ensured + // by only uncaching existing data when the database write finalizes. + db.lock.RLock() + + nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + batch := db.diskdb.NewBatch() + + // db.nodesSize only contains the useful data in the cache, but when reporting + // the total memory consumption, the maintenance metadata is also needed to be + // counted. For every useful node, we track 2 extra hashes as the flushlist. + size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength) + + // If the preimage cache got large enough, push to disk. If it's still small + // leave for later to deduplicate writes. + flushPreimages := db.preimagesSize > 4*1024*1024 + if flushPreimages { + for hash, preimage := range db.preimages { + if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil { + log.Error("Failed to commit preimage from trie database", "err", err) + db.lock.RUnlock() + return err + } + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + db.lock.RUnlock() + return err + } + batch.Reset() + } + } + } + // Keep committing nodes from the flush-list until we're below allowance + oldest := db.oldest + for size > limit && oldest != (common.Hash{}) { + // Fetch the oldest referenced node and push into the batch + node := db.nodes[oldest] + if err := batch.Put(oldest[:], node.blob); err != nil { + db.lock.RUnlock() + return err + } + // If we exceeded the ideal batch size, commit and reset + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Error("Failed to write flush list to disk", "err", err) + db.lock.RUnlock() + return err + } + batch.Reset() + } + // Iterate to the next flush item, or abort if the size cap was achieved. Size + // is the total size, including both the useful cached data (hash -> blob), as + // well as the flushlist metadata (2*hash). When flushing items from the cache, + // we need to reduce both. + size -= common.StorageSize(3*common.HashLength + len(node.blob)) + oldest = node.flushNext + } + // Flush out any remainder data from the last batch + if err := batch.Write(); err != nil { + log.Error("Failed to write flush list to disk", "err", err) + db.lock.RUnlock() + return err + } + db.lock.RUnlock() + + // Write successful, clear out the flushed data + db.lock.Lock() + defer db.lock.Unlock() + + if flushPreimages { + db.preimages = make(map[common.Hash][]byte) + db.preimagesSize = 0 + } + for db.oldest != oldest { + node := db.nodes[db.oldest] + delete(db.nodes, db.oldest) + db.oldest = node.flushNext + + db.nodesSize -= common.StorageSize(common.HashLength + len(node.blob)) + } + if db.oldest != (common.Hash{}) { + db.nodes[db.oldest].flushPrev = common.Hash{} + } + db.flushnodes += uint64(nodes - len(db.nodes)) + db.flushsize += storage - db.nodesSize + db.flushtime += time.Since(start) + + memcacheFlushTimeTimer.Update(time.Since(start)) + memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize)) + memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes))) + + log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), + "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + + return nil +} + // Commit iterates over all the children of a particular node, writes them out // to disk, forcefully tearing down all references in both directions. // @@ -266,7 +412,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { } } // Move the trie itself into the batch, flushing if enough data is accumulated - nodes, storage := len(db.nodes), db.nodesSize+db.preimagesSize + nodes, storage := len(db.nodes), db.nodesSize if err := db.commit(node, batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() @@ -289,15 +435,20 @@ func (db *Database) Commit(node common.Hash, report bool) error { db.uncache(node) + memcacheCommitTimeTimer.Update(time.Since(start)) + memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize)) + memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes))) + logger := log.Info if !report { logger = log.Debug } - logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), + logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime, "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 + db.flushnodes, db.flushsize, db.flushtime = 0, 0, 0 return nil } @@ -317,7 +468,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { if err := batch.Put(hash[:], node.blob); err != nil { return err } - // If we've reached an optimal match size, commit and start over + // If we've reached an optimal batch size, commit and start over if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return err @@ -337,7 +488,14 @@ func (db *Database) uncache(hash common.Hash) { if !ok { return } - // Otherwise uncache the node's subtries and remove the node itself too + // Node still exists, remove it from the flush-list + if hash == db.oldest { + db.oldest = node.flushNext + } else { + db.nodes[node.flushPrev].flushNext = node.flushNext + db.nodes[node.flushNext].flushPrev = node.flushPrev + } + // Uncache the node's subtries and remove the node itself too for child := range node.children { db.uncache(child) } @@ -347,9 +505,13 @@ func (db *Database) uncache(hash common.Hash) { // Size returns the current storage size of the memory cache in front of the // persistent database layer. -func (db *Database) Size() common.StorageSize { +func (db *Database) Size() (common.StorageSize, common.StorageSize) { db.lock.RLock() defer db.lock.RUnlock() - return db.nodesSize + db.preimagesSize + // db.nodesSize only contains the useful data in the cache, but when reporting + // the total memory consumption, the maintenance metadata is also needed to be + // counted. For every useful node, we track 2 extra hashes as the flushlist. + var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength) + return db.nodesSize + flushlistSize, db.preimagesSize } diff --git a/trie/encoding.go b/trie/encoding.go index 221fa6d3aa..5f120de638 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -53,10 +53,9 @@ func hexToCompact(hex []byte) []byte { func compactToHex(compact []byte) []byte { base := keybytesToHex(compact) - base = base[:len(base)-1] - // apply terminator flag - if base[0] >= 2 { - base = append(base, 16) + // delete terminator flag + if base[0] < 2 { + base = base[:len(base)-1] } // apply odd flag chop := 2 - base[0]&1 diff --git a/trie/hasher.go b/trie/hasher.go index ff61e70922..47c6dd8f9d 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -17,7 +17,6 @@ package trie import ( - "bytes" "hash" "sync" @@ -27,17 +26,39 @@ import ( ) type hasher struct { - tmp *bytes.Buffer - sha hash.Hash + tmp sliceBuffer + sha keccakState cachegen uint16 cachelimit uint16 onleaf LeafCallback } +// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports +// Read to get a variable amount of data from the hash state. Read is faster than Sum +// because it doesn't copy the internal state, but also modifies the internal state. +type keccakState interface { + hash.Hash + Read([]byte) (int, error) +} + +type sliceBuffer []byte + +func (b *sliceBuffer) Write(data []byte) (n int, err error) { + *b = append(*b, data...) + return len(data), nil +} + +func (b *sliceBuffer) Reset() { + *b = (*b)[:0] +} + // hashers live in a global db. var hasherPool = sync.Pool{ New: func() interface{} { - return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} + return &hasher{ + tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewKeccak256().(keccakState), + } }, } @@ -157,26 +178,23 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { } // Generate the RLP encoding of the node h.tmp.Reset() - if err := rlp.Encode(h.tmp, n); err != nil { + if err := rlp.Encode(&h.tmp, n); err != nil { panic("encode error: " + err.Error()) } - if h.tmp.Len() < 32 && !force { + if len(h.tmp) < 32 && !force { return n, nil // Nodes smaller than 32 bytes are stored inside their parent } // Larger nodes are replaced by their hash and stored in the database. hash, _ := n.cache() if hash == nil { - h.sha.Reset() - h.sha.Write(h.tmp.Bytes()) - hash = hashNode(h.sha.Sum(nil)) + hash = h.makeHashNode(h.tmp) } + if db != nil { // We are pooling the trie nodes into an intermediate memory cache db.lock.Lock() - hash := common.BytesToHash(hash) - db.insert(hash, h.tmp.Bytes()) - + db.insert(hash, h.tmp) // Track all direct parent->child node references switch n := n.(type) { case *shortNode: @@ -210,3 +228,11 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { } return hash, nil } + +func (h *hasher) makeHashNode(data []byte) hashNode { + n := make(hashNode, h.sha.Size()) + h.sha.Reset() + h.sha.Write(data) + h.sha.Read(n) + return n +} diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index c56d139499..2ce4642202 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -252,8 +252,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er // Set asymmetric key that is used to encrypt the message if pubKeyGiven { - params.Dst = crypto.ToECDSAPub(req.PublicKey) - if !ValidatePublicKey(params.Dst) { + if params.Dst, err = crypto.UnmarshalPubkey(req.PublicKey); err != nil { return false, ErrInvalidPublicKey } } @@ -329,8 +328,7 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. } if len(crit.Sig) > 0 { - filter.Src = crypto.ToECDSAPub(crit.Sig) - if !ValidatePublicKey(filter.Src) { + if filter.Src, err = crypto.UnmarshalPubkey(crit.Sig); err != nil { return nil, ErrInvalidSigningPubKey } } @@ -513,8 +511,7 @@ func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { } if len(req.Sig) > 0 { - src = crypto.ToECDSAPub(req.Sig) - if !ValidatePublicKey(src) { + if src, err = crypto.UnmarshalPubkey(req.Sig); err != nil { return "", ErrInvalidSigningPubKey } } diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index 388962559e..1a54b86eb5 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -19,7 +19,6 @@ package whisperv5 import ( "bytes" "crypto/ecdsa" - "fmt" "net" "sync" "testing" @@ -108,8 +107,6 @@ func TestSimulation(t *testing.T) { func initialize(t *testing.T) { var err error - ip := net.IPv4(127, 0, 0, 1) - port0 := 30303 for i := 0; i < NumNodes; i++ { var node TestNode @@ -128,29 +125,15 @@ func initialize(t *testing.T) { if err != nil { t.Fatalf("failed convert the key: %s.", keys[i]) } - port := port0 + i - addr := fmt.Sprintf(":%d", port) // e.g. ":30303" name := common.MakeName("whisper-go", "2.0") - var peers []*discover.Node - if i > 0 { - peerNodeId := nodes[i-1].id - peerPort := uint16(port - 1) - peerNode := discover.PubkeyID(&peerNodeId.PublicKey) - peer := discover.NewNode(peerNode, ip, peerPort, peerPort) - peers = append(peers, peer) - } - node.server = &p2p.Server{ Config: p2p.Config{ - PrivateKey: node.id, - MaxPeers: NumNodes/2 + 1, - Name: name, - Protocols: node.shh.Protocols(), - ListenAddr: addr, - NAT: nat.Any(), - BootstrapNodes: peers, - StaticNodes: peers, - TrustedNodes: peers, + PrivateKey: node.id, + MaxPeers: NumNodes/2 + 1, + Name: name, + Protocols: node.shh.Protocols(), + ListenAddr: "127.0.0.1:0", + NAT: nat.Any(), }, } @@ -159,6 +142,15 @@ func initialize(t *testing.T) { t.Skipf("failed to start server %d (port may be taken, skipping since there is no handler in test for this, should be ported to simulation framework): error is %v", i, err) } + for j := 0; j < i; j++ { + peerNodeId := nodes[j].id + address, _ := net.ResolveTCPAddr("tcp", nodes[j].server.ListenAddr) + peerPort := uint16(address.Port) + peerNode := discover.PubkeyID(&peerNodeId.PublicKey) + peer := discover.NewNode(peerNode, address.IP, peerPort, peerPort) + node.server.AddPeer(peer) + } + nodes[i] = &node } } diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index c60bc46a13..d729e79c35 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -272,8 +272,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (hexutil. // Set asymmetric key that is used to encrypt the message if pubKeyGiven { - params.Dst = crypto.ToECDSAPub(req.PublicKey) - if !ValidatePublicKey(params.Dst) { + if params.Dst, err = crypto.UnmarshalPubkey(req.PublicKey); err != nil { return nil, ErrInvalidPublicKey } } @@ -360,8 +359,7 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. } if len(crit.Sig) > 0 { - filter.Src = crypto.ToECDSAPub(crit.Sig) - if !ValidatePublicKey(filter.Src) { + if filter.Src, err = crypto.UnmarshalPubkey(crit.Sig); err != nil { return nil, ErrInvalidSigningPubKey } } @@ -544,8 +542,7 @@ func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { } if len(req.Sig) > 0 { - src = crypto.ToECDSAPub(req.Sig) - if !ValidatePublicKey(src) { + if src, err = crypto.UnmarshalPubkey(req.Sig); err != nil { return "", ErrInvalidSigningPubKey } } diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 06389c8252..57c3811f7a 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -21,12 +21,13 @@ import ( "crypto/ecdsa" "fmt" mrand "math/rand" - "net" "sync" "sync/atomic" "testing" "time" + "net" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" @@ -173,8 +174,6 @@ func initialize(t *testing.T) { initBloom(t) var err error - ip := net.IPv4(127, 0, 0, 1) - port0 := 30303 for i := 0; i < NumNodes; i++ { var node TestNode @@ -199,40 +198,36 @@ func initialize(t *testing.T) { if err != nil { t.Fatalf("failed convert the key: %s.", keys[i]) } - port := port0 + i - addr := fmt.Sprintf(":%d", port) // e.g. ":30303" name := common.MakeName("whisper-go", "2.0") - var peers []*discover.Node - if i > 0 { - peerNodeID := nodes[i-1].id - peerPort := uint16(port - 1) - peerNode := discover.PubkeyID(&peerNodeID.PublicKey) - peer := discover.NewNode(peerNode, ip, peerPort, peerPort) - peers = append(peers, peer) - } node.server = &p2p.Server{ Config: p2p.Config{ - PrivateKey: node.id, - MaxPeers: NumNodes/2 + 1, - Name: name, - Protocols: node.shh.Protocols(), - ListenAddr: addr, - NAT: nat.Any(), - BootstrapNodes: peers, - StaticNodes: peers, - TrustedNodes: peers, + PrivateKey: node.id, + MaxPeers: NumNodes/2 + 1, + Name: name, + Protocols: node.shh.Protocols(), + ListenAddr: "127.0.0.1:0", + NAT: nat.Any(), }, } + go startServer(t, node.server) + nodes[i] = &node } + waitForServersToStart(t) + for i := 0; i < NumNodes; i++ { - go startServer(t, nodes[i].server) + for j := 0; j < i; j++ { + peerNodeId := nodes[j].id + address, _ := net.ResolveTCPAddr("tcp", nodes[j].server.ListenAddr) + peerPort := uint16(address.Port) + peerNode := discover.PubkeyID(&peerNodeId.PublicKey) + peer := discover.NewNode(peerNode, address.IP, peerPort, peerPort) + nodes[i].server.AddPeer(peer) + } } - - waitForServersToStart(t) } func startServer(t *testing.T, s *p2p.Server) {