fix: Switch from domainName to domainID (#47)

* switch from domain anme to domainID

* merge gas price oracle impl

* e2e test fix

* update config file

* update config file

* config structure changes

* update the e2e test config

* minor fix

* pr comments address

* update yaml

Makefile

@@ -19,12 +19,11 @@ check:
 	gosec -exclude-dir=e2e ./...
 
 start: install
-	$(GOPATH)/bin/sygma-fee-oracle server -c $(makeFileDir)config.yaml -d $(makeFileDir)domain.json -r $(makeFileDir)resource.json -k $(makeFileDir)keyfile.priv -t secp256k1
+	$(GOPATH)/bin/sygma-fee-oracle server -c $(makeFileDir)config.yaml -d $(makeFileDir)domain.json -k $(makeFileDir)keyfile.priv -t secp256k1
 
 genmocks:
 	mockgen -destination=./store/mock/store.go -source=./store/store.go
 	mockgen -destination=./oracle/mock/oracle.go github.com/ChainSafe/sygma-fee-oracle/oracle GasPriceOracle,ConversionRateOracle
-	mockgen -destination=./config/mock/remoteParamOperator.go -source=./remoteParam/base.go
 
 test:
 	go clean -testcache

README.md

@@ -30,7 +30,7 @@ Fee oracle needs three config files in the `./` dir of the codebase:
 Template of the config.yaml can be found in `./config/config.template.yaml`.
 
 ### Domain config
-For domain configuration, it is posible to use local file, or the [shared configuration](https://github.com/sygmaprotocol/sygma-shared-configuration) file that is uploaded to some remote service (eg. ipfs). Depending if the URL or file path is provided to the flag `domain_config_path`, the aplication will use the domain configuration from local file or from the remote service.
+For domain configuration, it is possible to use local file, or the [shared configuration](https://github.com/sygmaprotocol/sygma-shared-configuration) file that is uploaded to some remote service (eg. ipfs). Depending on if the URL or file path is provided to the flag `domain_config_path`, the application will use the domain configuration from local file or from the remote service.
 This file indicates all the domains and resources the fee oracle needs to fetch data for. Details need to be matched with
 Sygma core configuration, such as `id`.
 

api/api.go

@@ -69,7 +69,7 @@ func (h *Handler) getRate(c *gin.Context) {
 		return
 	}
 
-	gp, err := h.consensus.FilterLocalGasPriceData(h.gasPriceStore, toDomain.Name)
+	gp, err := h.consensus.FilterLocalGasPriceData(h.gasPriceStore, toDomain.ID)
 	if err != nil {
 		h.log.Errorf("get gasprice process failed: %v", err)
 		ginErrorReturn(c, http.StatusInternalServerError, newReturnErrorResp(&oracleErrors.InternalServerError, err))

app/app.go

@@ -49,25 +49,39 @@ type FeeOracleApp struct {
 }
 
 func NewFeeOracleApp(appBase *base.FeeOracleAppBase) *FeeOracleApp {
-	// init concrete oracle services
-	coinMarketCap := oracle.NewCoinMarketCap(appBase.GetConfig(), appBase.GetLogger())
-	etherscan := oracle.NewEtherscan(appBase.GetConfig(), appBase.GetLogger())
-	polygonscan := oracle.NewPolygonscan(appBase.GetConfig(), appBase.GetLogger())
-	moonscan := oracle.NewMoonscan(appBase.GetConfig(), appBase.GetLogger())
-
-	// register concrete oracle services in operator
-	coinMarketCapConversionRateOracle := oracle.NewConversionRateOracleOperator(appBase.GetLogger(), coinMarketCap)
-	etherscanGasPriceOracle := oracle.NewGasPriceOracleOperator(appBase.GetLogger(), etherscan)
-	polygonscanGasPriceOracle := oracle.NewGasPriceOracleOperator(appBase.GetLogger(), polygonscan)
-	moonscanGasPriceOracle := oracle.NewGasPriceOracleOperator(appBase.GetLogger(), moonscan)
+	// initialize gas price oracles and register concrete oracle services in operator
+	gasPriceOracles := make(map[string]*oracle.GasPriceOracleOperator)
+	for _, domain := range appBase.GetConfig().DomainsList {
+		for _, apiService := range domain.GasPriceApis {
+			if apiService.Enable {
+				var oracleInstance oracle.GasPriceOracle
+				switch apiService.Implementation {
+				case "etherscan":
+					oracleInstance = oracle.NewEtherscan(apiService.Source, appBase.GetConfig().GasPriceApikeyReload(domain.DomainID, apiService), domain.DomainID, appBase.GetLogger())
+				case "moonscan":
+					oracleInstance = oracle.NewMoonscan(apiService.Source, appBase.GetConfig().GasPriceApikeyReload(domain.DomainID, apiService), domain.DomainID, appBase.GetLogger())
+				default:
+					panic("unknown gas price oracle implementation")
+				}
+				gasPriceOracles[apiService.Source] = oracle.NewGasPriceOracleOperator(appBase.GetLogger(), oracleInstance)
+			}
+		}
+	}
 
+	// initialize conversion rate oracles and register concrete oracle services in operator
 	conversionRateOracles := make(map[string]*oracle.ConversionRateOracleOperator)
-	conversionRateOracles[coinMarketCap.Name()] = coinMarketCapConversionRateOracle
-
-	gasPriceOracles := make(map[string]*oracle.GasPriceOracleOperator)
-	gasPriceOracles[etherscan.Name()] = etherscanGasPriceOracle
-	gasPriceOracles[polygonscan.Name()] = polygonscanGasPriceOracle
-	gasPriceOracles[moonscan.Name()] = moonscanGasPriceOracle
+	for _, rateOracle := range appBase.GetConfig().ConversionRateApis {
+		if rateOracle.Enable {
+			var oracleInstance oracle.ConversionRateOracle
+			switch rateOracle.Implementation {
+			case "coinmarketcap":
+				oracleInstance = oracle.NewCoinMarketCap(rateOracle.Source, appBase.GetConfig().ConversionRateApikeyReload(rateOracle), appBase.GetLogger())
+			default:
+				panic("unknown conversion rate oracle implementation")
+			}
+			conversionRateOracles[rateOracle.Source] = oracle.NewConversionRateOracleOperator(appBase.GetLogger(), oracleInstance)
+		}
+	}
 
 	conversionRateStore := store.NewConversionRateStore(appBase.GetStore())
 	gasPriceStore := store.NewGasPriceStore(appBase.GetStore())

config.yaml

@@ -17,7 +17,7 @@ http_server:
 
 # time in second before server shutdown
 # this will allow server to finish running jobs before shutdown
-finish_up_time: 10
+finish_up_time: 3
 
 # internal scheduled cronjob
 cron_job:
@@ -36,32 +36,44 @@ cron_job:
 store:
   path: ./lvldbdata
 
-# supported oracles
-# api key is just a placeholder, replace with your valid key
-oracle:
-  etherscan:
-    enable: true
-    api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
-    apis:
-      gas_price: https://api.etherscan.io/api?module=gastracker&action=gasoracle&apikey=
-  polygonscan:
-    enable: true
-    api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
-    apis:
-      gas_price: https://api.polygonscan.com/api?module=gastracker&action=gasoracle&apikey=
-  moonscan:
-    enable: true
-    api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
-    apis:
-      gas_price: https://api-moonbeam.moonscan.io/api?module=proxy&action=eth_gasPrice&apikey=
-  coinmarketcap:
+# conversion_rate_apis contains the list of conversion rate api services
+# implementation indicates the implementation of the oracle
+# source is the source of the data, reflected the source of the api url
+# url is the api url of gas price of the oracle
+# api_key is the api key of the oracle
+# enable is the flag to enable/disable the current api service, fee oracle will not instantiate this service if it is disabled
+conversion_rate_apis:
+  - implementation: coinmarketcap
+    source: coinmarketcap
     enable: true
+    url: https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest?
     api_key: 1408daf0-0777-4916-9fe4-20da5ee77560
-    apis:
-      query_rate: https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest?
 
-gas_price_domains:
-  [ ethereum, polygon, moonbeam ]
+domain_list:
+  - domain_id: 0
+    gas_price_apis:
+      - implementation: etherscan
+        source: etherscan
+        enable: true
+        url: https://api.etherscan.io/api?module=gastracker&action=gasoracle&apikey=
+        api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
+        decimals: 9
+  - domain_id: 1
+    gas_price_apis:
+      - implementation: etherscan
+        source: polygonscan
+        enable: true
+        url: https://api.polygonscan.com/api?module=gastracker&action=gasoracle&apikey=
+        api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
+        decimals: 9
+  - domain_id: 2
+    gas_price_apis:
+      - implementation: moonscan
+        source: moonscan
+        enable: true
+        url: https://api-moonbeam.moonscan.io/api?module=proxy&action=eth_gasPrice&apikey=
+        api_key: BZM7P395BQS1YFKQMA2AK2ACQKWGEW4JB3
+        decimals:
 
 # conversion_rate_pairs contains price pair for conversion rate
 # must be paired and follow the format of [ base, foreign, base, foreign, ... ]
@@ -73,4 +85,4 @@ strategy:
   local: average
 
 # data_valid_interval defines how long the endpoint response data remains valid before sending it to fee handler contract
-data_valid_interval: 3600 # second
+data_valid_interval: 3600 # second

config/config.go

@@ -42,8 +42,8 @@ type Config struct {
 	FinishUpTime        int64            `mapstructure:"finish_up_time"`
 	CronJob             cronJobConfig    `mapstructure:"cron_job"`
 	Store               store            `mapstructure:"store"`
-	Oracle              oracle           `mapstructure:"oracle"`
-	GasPriceDomains     []string         `mapstructure:"gas_price_domains"`
+	ConversionRateApis  []ApiService     `mapstructure:"conversion_rate_apis"`
+	DomainsList         []domainList     `mapstructure:"domain_list"`
 	ConversionRatePairs []string         `mapstructure:"conversion_rate_pairs"`
 	Strategy            strategyConfig   `mapstructure:"strategy"`
 	DataValidInterval   int64            `mapstructure:"data_valid_interval"`
@@ -55,11 +55,18 @@ type strategyConfig struct {
 	Local string `mapstructure:"local"`
 }
 
-type oracle struct {
-	Etherscan     etherscan     `mapstructure:"etherscan"`
-	Polygonscan   polygonscan   `mapstructure:"polygonscan"`
-	CoinMarketCap coinMarketCap `mapstructure:"coinmarketcap"`
-	Moonscan      coinMarketCap `mapstructure:"moonscan"`
+type domainList struct {
+	DomainID     int          `mapstructure:"domain_id"`
+	GasPriceApis []ApiService `mapstructure:"gas_price_apis"`
+}
+
+type ApiService struct {
+	Implementation string `mapstructure:"implementation"`
+	Source         string `mapstructure:"source"`
+	Enable         bool   `mapstructure:"enable"`
+	URL            string `mapstructure:"url"`
+	ApiKey         string `mapstructure:"api_key"`
+	Decimals       int    `mapstructure:"decimals"`
 }
 
 type store struct {
@@ -83,29 +90,6 @@ type httpServerConfig struct {
 	Port string `mapstructure:"port"`
 }
 
-type polygonscan struct {
-	Enable bool    `mapstructure:"enable"`
-	ApiKey string  `mapstructure:"api_key"`
-	Apis   apiUrls `mapstructure:"apis"`
-}
-
-type etherscan struct {
-	Enable bool    `mapstructure:"enable"`
-	ApiKey string  `mapstructure:"api_key"`
-	Apis   apiUrls `mapstructure:"apis"`
-}
-
-type coinMarketCap struct {
-	Enable bool    `mapstructure:"enable"`
-	ApiKey string  `mapstructure:"api_key"`
-	Apis   apiUrls `mapstructure:"apis"`
-}
-
-type apiUrls struct {
-	GasPriceApiUrl string `mapstructure:"gas_price"`
-	QueryRate      string `mapstructure:"query_rate"`
-}
-
 func (c *Config) LogLevel() (logrus.Level, error) {
 	logLvl := os.Getenv("LOG_LEVEL")
 	if logLvl == "" {
@@ -175,28 +159,26 @@ func (c *Config) HttpServerConfig() httpServerConfig {
 	return httpConfig
 }
 
-func (c *Config) OracleConfig() oracle {
-	oracleConfig := c.Oracle
+// GasPriceApikeyReload dynamically builds the env var key string and replaces the api key if the env var is set
+// dynamic env var key string format: <SOURCE>_API_KEY_<DOMAIN_ID>
+func (c *Config) GasPriceApikeyReload(domainID int, apiService ApiService) ApiService {
+	loadedApiKey := os.Getenv(fmt.Sprintf("%s_API_KEY_%d", strings.ToUpper(apiService.Source), domainID))
+	if loadedApiKey != "" {
+		apiService.ApiKey = loadedApiKey
+	}
 
-	etherscanAPIKey := os.Getenv("ETHERSCAN_API_KEY")
-	polygonscanAPIKey := os.Getenv("POLYGONSCAN_API_KEY")
-	coinMarketCapAPIKey := os.Getenv("COINMARKETCAP_API_KEY")
-	moonscanAPIKey := os.Getenv("MOONSCAN_API_KEY")
+	return apiService
+}
 
-	if etherscanAPIKey != "" {
-		oracleConfig.Etherscan.ApiKey = etherscanAPIKey
-	}
-	if polygonscanAPIKey != "" {
-		oracleConfig.Polygonscan.ApiKey = polygonscanAPIKey
-	}
-	if coinMarketCapAPIKey != "" {
-		oracleConfig.CoinMarketCap.ApiKey = coinMarketCapAPIKey
-	}
-	if moonscanAPIKey != "" {
-		oracleConfig.Moonscan.ApiKey = moonscanAPIKey
+// ConversionRateApikeyReload dynamically builds the env var key string and replaces the api key if the env var is set
+// dynamic env var key string format: <SOURCE>_API_KEY
+func (c *Config) ConversionRateApikeyReload(apiService ApiService) ApiService {
+	loadedApiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(apiService.Source)))
+	if loadedApiKey != "" {
+		apiService.ApiKey = loadedApiKey
 	}
 
-	return oracleConfig
+	return apiService
 }
 
 func (c *Config) CronJobConfig() cronJobConfig {

config/config.template.yaml

@@ -36,25 +36,38 @@ cron_job:
 store:
   path: ./lvldbdata
 
-oracle:
-  etherscan:
+conversion_rate_apis:
+  - implementation: coinmarketcap
+    source: coinmarketcap
     enable: true
+    url:
     api_key:
-    apis:
-      gas_price:
-  polygonscan:
-    enable: true
-    api_key:
-    apis:
-      gas_price:
-  coinmarketcap:
-    enable: true
-    api_key:
-    apis:
-      query_rate:
 
-gas_price_domains:
-  [ ethereum ]
+domain_list:
+  - domain_id: 0
+    gas_price_apis:
+      - implementation: etherscan
+        source: etherscan
+        enable: true
+        url:
+        api_key:
+        decimals: 9
+  - domain_id: 1
+    gas_price_apis:
+      - implementation: etherscan
+        source: polygonscan
+        enable: true
+        url:
+        api_key:
+        decimals: 9
+  - domain_id: 2
+    gas_price_apis:
+      - implementation: moonscan
+        source: moonscan
+        enable: true
+        url:
+        api_key:
+        decimals:
 
 # conversion_rate_pairs contains price pair for conversion rate
 # must be paired and follow the format of [ base, foreign, base, foreign, ... ]
@@ -66,4 +79,4 @@ strategy:
   local: average
 
 # data_valid_interval defines how long the endpoint response data remains valid before sending it to fee handler contract
-data_valid_interval: 3600 # second
+data_valid_interval: 3600 # second

consensus/base.go

@@ -30,8 +30,8 @@ func (c *Consensus) GetStrategy() string {
 	return c.strategy.Name()
 }
 
-func (c *Consensus) FilterLocalGasPriceData(store *store.GasPriceStore, domainName string) (*types.GasPrices, error) {
-	return c.strategy.GasPrice(store, domainName)
+func (c *Consensus) FilterLocalGasPriceData(store *store.GasPriceStore, domainID int) (*types.GasPrices, error) {
+	return c.strategy.GasPrice(store, domainID)
 }
 
 func (c *Consensus) FilterLocalConversionRateData(store *store.ConversionRateStore, base, foreign string) (*types.ConversionRate, error) {

consensus/strategy/average.go

@@ -22,8 +22,8 @@ func (a *Average) Name() string {
 	return "average"
 }
 
-func (a *Average) GasPrice(store *store.GasPriceStore, domainName string) (*types.GasPrices, error) {
-	re, err := store.GetGasPricesByDomain(domainName)
+func (a *Average) GasPrice(store *store.GasPriceStore, domainID int) (*types.GasPrices, error) {
+	re, err := store.GetGasPricesByDomain(domainID)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +55,7 @@ func (a *Average) GasPrice(store *store.GasPriceStore, domainName string) (*type
 		SafeGasPrice:    fmt.Sprintf("%d", int(math.Round(safe/dataSize))),
 		ProposeGasPrice: fmt.Sprintf("%d", int(propose/dataSize)),
 		FastGasPrice:    fmt.Sprintf("%d", int(fast/dataSize)),
-		DomainName:      domainName,
+		DomainID:        domainID,
 		Time:            re[0].Time, // use the first data time for now
 	}, nil