Support multiple signing methods

adapters/prometheus/pusher.go

@@ -2,18 +2,20 @@ package prometheus
 
 import (
 	"context"
-	"github.com/axieinfinity/bridge-core/adapters"
 	"time"
 
+	"github.com/axieinfinity/bridge-core/adapters"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/push"
 )
 
 type Pusher struct {
-	counters map[string]prometheus.Counter
-	gauges   map[string]prometheus.Gauge
-	registry *prometheus.Registry
-	pusher   *push.Pusher
+	counters   map[string]prometheus.Counter
+	gauges     map[string]prometheus.Gauge
+	histograms map[string]prometheus.Histogram
+	registry   *prometheus.Registry
+	pusher     *push.Pusher
 }
 
 func (p *Pusher) AddCounter(name string, description string) *Pusher {
@@ -58,6 +60,35 @@ func (p *Pusher) IncrGauge(name string, value int) {
 	p.gauges[name].Add(float64(value))
 }
 
+func (p *Pusher) SetGauge(name string, value int) {
+	if _, ok := p.gauges[name]; !ok {
+		return
+	}
+
+	p.gauges[name].Set(float64(value))
+}
+
+func (p *Pusher) AddHistogram(name string, description string) *Pusher {
+	if _, ok := p.histograms[name]; ok {
+		return p
+	}
+
+	histogram := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name: name,
+		Help: description,
+	})
+	p.pusher.Collector(histogram)
+	return p
+}
+
+func (p *Pusher) ObserveHistogram(name string, value int) {
+	if _, ok := p.histograms[name]; !ok {
+		return
+	}
+
+	p.histograms[name].Observe(float64(value))
+}
+
 func (p *Pusher) Push() error {
 	return p.pusher.Push()
 }

controller.go

@@ -131,7 +131,7 @@ func New(cfg *Config, db *gorm.DB, helpers utils.Utils) (*Controller, error) {
 		if !ok {
 			continue
 		}
-		l := initFunc(c.ctx, lsConfig, c.store, helpers)
+		l := initFunc(c.ctx, lsConfig, c.store, c.utilWrapper)
 		if l == nil {
 			return nil, errors.New("listener is nil")
 		}

go.mod

@@ -5,7 +5,8 @@ replace github.com/ethereum/go-ethereum => ../ronin
 go 1.17
 
 require (
-	github.com/ethereum/go-ethereum v0.0.0-00010101000000-000000000000
+	github.com/axieinfinity/bridge-contracts v0.0.0-20220731110242-d273b578b657
+	github.com/ethereum/go-ethereum v1.10.21
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/prometheus/client_golang v1.12.2
 	golang.org/x/text v0.3.7
@@ -17,7 +18,7 @@ require (
 require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
-	github.com/axieinfinity/bridge-contracts v0.0.0-20220731110242-d273b578b657 // indirect
+	github.com/axieinfinity/ronin-kms-client v0.0.0-20220805072849-960e04981b70 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd v0.20.1-beta // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -71,8 +72,11 @@ require (
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
 	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
+	google.golang.org/grpc v1.48.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )

metrics/metrics.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"context"
+
 	"github.com/axieinfinity/bridge-core/adapters/prometheus"
 )
 
@@ -21,6 +22,12 @@ const (
 	WithdrawalTaskMetric    string = "Ronin/tasks/withdrawal"
 	DepositTaskMetric       string = "Ronin/tasks/deposit"
 	AckWithdrawalTaskMetric string = "Ronin/tasks/acknowledgeWithdrawal"
+
+	KmsSuccessSign     string = "kms/success"
+	KmsNetworkFailure  string = "kms/failure/network"
+	KmsInternalFailure string = "kms/failure/internal"
+	KmsSignLatency     string = "kms/latency"
+	KmsLastSuccess     string = "kms/lastSuccess"
 )
 
 var (
@@ -35,12 +42,17 @@ func RunPusher(ctx context.Context) {
 	Pusher.AddCounter(ProcessedSuccessJobMetric, "count number of processed jobs successfully")
 	Pusher.AddCounter(ProcessedFailedJobMetric, "count number of failed jobs")
 	Pusher.AddCounter(PendingTaskMetric, "count number of pending tasks in queue")
-	Pusher.AddCounter(ProcessingTaskMetric, "count number of processing tasks")
+	Pusher.AddGauge(ProcessingTaskMetric, "count number of processing tasks")
 	Pusher.AddCounter(ProcessedSuccessJobMetric, "count number of success tasks")
 	Pusher.AddCounter(ProcessedFailedJobMetric, "count number failed tasks")
 	Pusher.AddCounter(WithdrawalTaskMetric, "count number of ronin’s withdrawal tasks occurred")
 	Pusher.AddCounter(DepositTaskMetric, "count number of ronin’s deposit tasks occurred")
 	Pusher.AddCounter(AckWithdrawalTaskMetric, "count number of ronin acknowledge withdrawal tasks occurred")
+	Pusher.AddCounter(KmsSuccessSign, "count number of successful KMS signs")
+	Pusher.AddCounter(KmsNetworkFailure, "count number of failed KMS signs due to network")
+	Pusher.AddCounter(KmsInternalFailure, "count number of KMS server error responses")
+	Pusher.AddHistogram(KmsSignLatency, "the latency of signing request to KMS server in milliseconds")
+	Pusher.AddGauge(KmsLastSuccess, "timestamp of last KMS successful signs")
 
 	go Pusher.Start(ctx)
 }

types.go

@@ -2,14 +2,15 @@ package internal
 
 import (
 	"context"
+	"math/big"
+	"time"
+
 	"github.com/axieinfinity/bridge-core/models"
 	"github.com/axieinfinity/bridge-core/stores"
 	"github.com/axieinfinity/bridge-core/utils"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"math/big"
-	"time"
 )
 
 const (
@@ -65,6 +66,9 @@ type Listener interface {
 
 	IsUpTodate() bool
 	SetPrepareJobChan(chan JobHandler)
+
+	GetValidatorSign() utils.ISign
+	GetRelayerSign() utils.ISign
 }
 
 type Transaction interface {
@@ -179,8 +183,8 @@ type LsConfig struct {
 }
 
 type Secret struct {
-	Validator string `json:"validator"`
-	Relayer   string `json:"relayer"`
+	Validator *utils.SignMethodConfig `json:"validator"`
+	Relayer   *utils.SignMethodConfig `json:"relayer"`
 }
 
 type Subscribe struct {

utils/main.go

@@ -2,23 +2,23 @@ package utils
 
 import (
 	"context"
-	"crypto/ecdsa"
 	"errors"
 	"fmt"
+	"math/big"
+	"os"
+	"reflect"
+
+	kmsUtils "github.com/axieinfinity/ronin-kms-client/utils"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/signer/core"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
-	"math/big"
-	"os"
-	"reflect"
 )
 
 type EthClient interface {
@@ -40,8 +40,8 @@ type Utils interface {
 	UnpackToInterface(a abi.ABI, name string, data []byte, isInput bool, v interface{}) error
 	Title(text string) string
 	NewEthClient(url string) (EthClient, error)
-	SendContractTransaction(key *ecdsa.PrivateKey, chainId *big.Int, fn func(opts *bind.TransactOpts) (*types.Transaction, error)) (*types.Transaction, error)
-	SignTypedData(typedData core.TypedData, privateKey *ecdsa.PrivateKey) (hexutil.Bytes, error)
+	SendContractTransaction(signMethod ISign, chainId *big.Int, fn func(opts *bind.TransactOpts) (*types.Transaction, error)) (*types.Transaction, error)
+	SignTypedData(typedData core.TypedData, signMethod ISign) (hexutil.Bytes, error)
 	FilterLogs(client EthClient, opts *bind.FilterOpts, contractAddresses []common.Address, filteredMethods map[*abi.ABI]map[string]struct{}) ([]types.Log, error)
 }
 
@@ -140,11 +140,31 @@ func (u *utils) NewEthClient(url string) (EthClient, error) {
 	return ethclient.Dial(url)
 }
 
-func (u *utils) SendContractTransaction(key *ecdsa.PrivateKey, chainId *big.Int, fn func(opts *bind.TransactOpts) (*types.Transaction, error)) (*types.Transaction, error) {
-	if key == nil {
-		return nil, nil
-	}
-	opts, err := bind.NewKeyedTransactorWithChainID(key, chainId)
+func newKeyedTransactorWithChainID(signMethod ISign, chainID *big.Int) (*bind.TransactOpts, error) {
+	keyAddr := signMethod.GetAddress()
+	signer := types.LatestSignerForChainID(chainID)
+	return &bind.TransactOpts{
+		From: keyAddr,
+		Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) {
+			if address != keyAddr {
+				return nil, bind.ErrNotAuthorized
+			}
+			encodedTx, err := kmsUtils.RlpEncode(tx, chainID)
+			if err != nil {
+				return nil, err
+			}
+			signature, err := signMethod.Sign(encodedTx, "non-ether")
+			if err != nil {
+				return nil, err
+			}
+			return tx.WithSignature(signer, signature)
+		},
+		Context: context.Background(),
+	}, nil
+}
+
+func (u *utils) SendContractTransaction(signMethod ISign, chainId *big.Int, fn func(opts *bind.TransactOpts) (*types.Transaction, error)) (*types.Transaction, error) {
+	opts, err := newKeyedTransactorWithChainID(signMethod, chainId)
 	if err != nil {
 		return nil, err
 	}
@@ -156,12 +176,12 @@ func (u *utils) SendContractTransaction(key *ecdsa.PrivateKey, chainId *big.Int,
 // It returns
 // - the signature,
 // - and/or any error
-func (u *utils) SignTypedData(typedData core.TypedData, privateKey *ecdsa.PrivateKey) (hexutil.Bytes, error) {
-	return u.signTypedData(typedData, privateKey)
+func (u *utils) SignTypedData(typedData core.TypedData, signMethod ISign) (hexutil.Bytes, error) {
+	return u.signTypedData(typedData, signMethod)
 }
 
 // signTypedData is identical to the capitalized version
-func (u *utils) signTypedData(typedData core.TypedData, privateKey *ecdsa.PrivateKey) (hexutil.Bytes, error) {
+func (u *utils) signTypedData(typedData core.TypedData, signMethod ISign) (hexutil.Bytes, error) {
 	domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map())
 	if err != nil {
 		return nil, err
@@ -171,7 +191,7 @@ func (u *utils) signTypedData(typedData core.TypedData, privateKey *ecdsa.Privat
 		return nil, err
 	}
 	rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash)))
-	signature, err := crypto.Sign(crypto.Keccak256(rawData), privateKey)
+	signature, err := signMethod.Sign(rawData, "non-ether")
 	if err != nil {
 		return nil, err
 	}

utils/sign.go

@@ -0,0 +1,98 @@
+package utils
+
+import (
+	"crypto/ecdsa"
+	"time"
+
+	"github.com/axieinfinity/bridge-core/metrics"
+	kms "github.com/axieinfinity/ronin-kms-client"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type SignMethodConfig struct {
+	PlainPrivateKey string         `json:"plainPrivateKey,omitempty"`
+	KmsConfig       *kms.KmsConfig `json:"kmsConfig,omitempty"`
+}
+
+func NewSignMethod(config *SignMethodConfig) (ISign, error) {
+	if config.PlainPrivateKey != "" {
+		return NewPrivateKeySign(config.PlainPrivateKey)
+	} else if config.KmsConfig != nil {
+		return NewKmsSign(config.KmsConfig)
+	}
+
+	log.Warn("No sign methods provided")
+	return nil, nil
+}
+
+type ISign interface {
+	// sign function receives raw message, not hash of message
+	Sign(message []byte, dataType string) ([]byte, error)
+	GetAddress() common.Address
+}
+
+type PrivateKeySign struct {
+	privateKey *ecdsa.PrivateKey
+}
+
+func NewPrivateKeySign(plainPrivateKey string) (*PrivateKeySign, error) {
+	privateKey, err := crypto.HexToECDSA(plainPrivateKey)
+	if err != nil {
+		log.Error("[NewPrivateKeySign] error while getting plain private key", "err", err)
+		return nil, err
+	}
+
+	return &PrivateKeySign{
+		privateKey: privateKey,
+	}, nil
+}
+
+type PrivateKeyConfig struct {
+	PrivateKey string `json:"privateKey"`
+}
+
+func (privateKeySign *PrivateKeySign) Sign(message []byte, dataType string) ([]byte, error) {
+	return crypto.Sign(crypto.Keccak256(message), privateKeySign.privateKey)
+}
+
+func (privateKeySign *PrivateKeySign) GetAddress() common.Address {
+	return crypto.PubkeyToAddress(privateKeySign.privateKey.PublicKey)
+}
+
+type KmsSign struct {
+	*kms.KmsSign
+}
+
+func NewKmsSign(kmsConfig *kms.KmsConfig) (*KmsSign, error) {
+	kms, err := kms.NewKmsSign(kmsConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &KmsSign{
+		KmsSign: kms,
+	}, nil
+}
+
+func (kmsSign *KmsSign) Sign(message []byte, dataType string) ([]byte, error) {
+	start := time.Now().UnixMilli()
+
+	signature, err := kmsSign.KmsSign.Sign(message, dataType)
+	if err != nil {
+		if err == kms.ErrAccessDenied {
+			metrics.Pusher.IncrCounter(metrics.KmsInternalFailure, 1)
+		} else {
+			metrics.Pusher.IncrCounter(metrics.KmsNetworkFailure, 1)
+		}
+		return signature, err
+	}
+	metrics.Pusher.ObserveHistogram(metrics.KmsSignLatency, int(start-time.Now().UnixMilli()))
+	metrics.Pusher.SetGauge(metrics.KmsLastSuccess, int(time.Now().Unix()))
+	metrics.Pusher.IncrCounter(metrics.KmsSuccessSign, 1)
+	return signature, err
+}
+
+func (kmsSign *KmsSign) GetAddress() common.Address {
+	return kmsSign.KmsSign.Address
+}