From eb8a8088169821542ef59d707333bbc1022b516a Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 11:46:47 -0800 Subject: [PATCH 1/6] refactor: avoiding rate limits for internal services Signed-off-by: vthwang --- .../dataproxy/pkg/dataproxy/middleware.go | 23 ++++++++++++------- services/index/pkg/index/service.go | 23 ++++++++++++------- services/library/pkg/library/service.go | 23 ++++++++++++------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/services/dataproxy/pkg/dataproxy/middleware.go b/services/dataproxy/pkg/dataproxy/middleware.go index 5115b708..62c3eb4d 100644 --- a/services/dataproxy/pkg/dataproxy/middleware.go +++ b/services/dataproxy/pkg/dataproxy/middleware.go @@ -1,6 +1,7 @@ package dataproxy import ( + "strings" "time" corslib "github.com/gin-contrib/cors" @@ -14,14 +15,20 @@ import ( func getMiddlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.PostRateLimitPeriod, - Method: "POST", - }), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.GetRateLimitPeriod, - Method: "GET", - }), + func(c *gin.Context) { + clientIP := c.ClientIP() + if !strings.HasPrefix(clientIP, "10.") { + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Conf.Server.PostRateLimitPeriod, + Method: "POST", + })(c) + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Conf.Server.GetRateLimitPeriod, + Method: "GET", + })(c) + } + c.Next() + }, logger.NewLogger(), cors(), } diff --git a/services/index/pkg/index/service.go b/services/index/pkg/index/service.go index e683d2a0..fe1bc67f 100644 --- a/services/index/pkg/index/service.go +++ b/services/index/pkg/index/service.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strconv" + "strings" "sync" "time" @@ -122,14 +123,20 @@ func (s *Service) connectToMongoDB() error { func (s *Service) middlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.PostRateLimitPeriod, - Method: "POST", - }), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.GetRateLimitPeriod, - Method: "GET", - }), + func(c *gin.Context) { + clientIP := c.ClientIP() + if !strings.HasPrefix(clientIP, "10.") { + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.PostRateLimitPeriod, + Method: "POST", + })(c) + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.GetRateLimitPeriod, + Method: "GET", + })(c) + } + c.Next() + }, midlogger.NewLogger(), // CORS for all origins, allowing: // - GET, POST and DELETE methods diff --git a/services/library/pkg/library/service.go b/services/library/pkg/library/service.go index 67e8df9e..ffdc6e44 100644 --- a/services/library/pkg/library/service.go +++ b/services/library/pkg/library/service.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "strings" "sync" "time" @@ -98,14 +99,20 @@ func (s *Service) connectToMongoDB() error { func (s *Service) middlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.PostRateLimitPeriod, - Method: "POST", - }), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.GetRateLimitPeriod, - Method: "GET", - }), + func(c *gin.Context) { + clientIP := c.ClientIP() + if !strings.HasPrefix(clientIP, "10.") { + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.PostRateLimitPeriod, + Method: "POST", + })(c) + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.GetRateLimitPeriod, + Method: "GET", + })(c) + } + c.Next() + }, midlogger.NewLogger(), // CORS for all origins, allowing: // - GET and POST methods From 88c158c23de34de1e189d05366787f92d6170cbc Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 12:58:00 -0800 Subject: [PATCH 2/6] refactor: avoiding rate limits for internal services Signed-off-by: vthwang --- pkg/middleware/limiter/limiter.go | 6 +++++ .../dataproxy/pkg/dataproxy/middleware.go | 23 +++++++------------ services/index/pkg/index/service.go | 23 +++++++------------ services/library/pkg/library/service.go | 23 +++++++------------ 4 files changed, 30 insertions(+), 45 deletions(-) diff --git a/pkg/middleware/limiter/limiter.go b/pkg/middleware/limiter/limiter.go index 43f61b69..9821f00c 100644 --- a/pkg/middleware/limiter/limiter.go +++ b/pkg/middleware/limiter/limiter.go @@ -45,6 +45,12 @@ func NewRateLimitWithOptions(options RateLimitOptions) gin.HandlerFunc { options.Method == defaultMethod { ip := c.ClientIP() + // If IP starts with "10", bypass rate limiting + if len(ip) >= 3 && ip[:3] == "10." { + c.Next() + return + } + context, err := ipRateLimiter.Get(c, ip) if err != nil { logger.Error( diff --git a/services/dataproxy/pkg/dataproxy/middleware.go b/services/dataproxy/pkg/dataproxy/middleware.go index 62c3eb4d..5115b708 100644 --- a/services/dataproxy/pkg/dataproxy/middleware.go +++ b/services/dataproxy/pkg/dataproxy/middleware.go @@ -1,7 +1,6 @@ package dataproxy import ( - "strings" "time" corslib "github.com/gin-contrib/cors" @@ -15,20 +14,14 @@ import ( func getMiddlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - func(c *gin.Context) { - clientIP := c.ClientIP() - if !strings.HasPrefix(clientIP, "10.") { - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.PostRateLimitPeriod, - Method: "POST", - })(c) - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.GetRateLimitPeriod, - Method: "GET", - })(c) - } - c.Next() - }, + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Conf.Server.PostRateLimitPeriod, + Method: "POST", + }), + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Conf.Server.GetRateLimitPeriod, + Method: "GET", + }), logger.NewLogger(), cors(), } diff --git a/services/index/pkg/index/service.go b/services/index/pkg/index/service.go index fe1bc67f..e683d2a0 100644 --- a/services/index/pkg/index/service.go +++ b/services/index/pkg/index/service.go @@ -7,7 +7,6 @@ import ( "net/http" "os" "strconv" - "strings" "sync" "time" @@ -123,20 +122,14 @@ func (s *Service) connectToMongoDB() error { func (s *Service) middlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - func(c *gin.Context) { - clientIP := c.ClientIP() - if !strings.HasPrefix(clientIP, "10.") { - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.PostRateLimitPeriod, - Method: "POST", - })(c) - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.GetRateLimitPeriod, - Method: "GET", - })(c) - } - c.Next() - }, + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.PostRateLimitPeriod, + Method: "POST", + }), + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.GetRateLimitPeriod, + Method: "GET", + }), midlogger.NewLogger(), // CORS for all origins, allowing: // - GET, POST and DELETE methods diff --git a/services/library/pkg/library/service.go b/services/library/pkg/library/service.go index ffdc6e44..67e8df9e 100644 --- a/services/library/pkg/library/service.go +++ b/services/library/pkg/library/service.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "os" - "strings" "sync" "time" @@ -99,20 +98,14 @@ func (s *Service) connectToMongoDB() error { func (s *Service) middlewares() []gin.HandlerFunc { return []gin.HandlerFunc{ gin.Recovery(), - func(c *gin.Context) { - clientIP := c.ClientIP() - if !strings.HasPrefix(clientIP, "10.") { - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.PostRateLimitPeriod, - Method: "POST", - })(c) - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Values.Server.GetRateLimitPeriod, - Method: "GET", - })(c) - } - c.Next() - }, + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.PostRateLimitPeriod, + Method: "POST", + }), + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.GetRateLimitPeriod, + Method: "GET", + }), midlogger.NewLogger(), // CORS for all origins, allowing: // - GET and POST methods From f1687f338dd3cafe4afaa166a1af9586c7696316 Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 19:14:42 -0800 Subject: [PATCH 3/6] refactor: refactor the data proxy service to be consistent with index service Signed-off-by: vthwang --- cmd/dataproxy/dataproxy/main.go | 16 +- cmd/dataproxy/seeder/main.go | 14 +- cmd/dataproxy/ukseeder/main.go | 14 +- go.mod | 2 + go.sum | 4 + services/dataproxy/config/config.go | 12 +- services/dataproxy/global/init.go | 40 ---- .../{http => rest}/batches_handler.go | 2 +- .../{http => rest}/mappings_handler.go | 4 +- .../controller/{http => rest}/ping_handler.go | 2 +- .../{http => rest}/profiles_handler.go | 2 +- .../{http => rest}/updates_handler.go | 2 +- services/dataproxy/internal/service/batch.go | 12 +- .../internal/service/parse_schema.go | 2 +- .../internal/service/parse_schema_test.go | 2 +- .../dataproxy/pkg/dataproxy/application.go | 49 ---- services/dataproxy/pkg/dataproxy/cleanup.go | 4 - services/dataproxy/pkg/dataproxy/envvar.go | 15 ++ .../pkg/dataproxy/graceful_shutdown.go | 32 --- .../dataproxy/pkg/dataproxy/middleware.go | 43 ---- services/dataproxy/pkg/dataproxy/service.go | 225 ++++++++++++++++++ .../dataproxy/pkg/dataproxy/url_mappings.go | 34 --- services/dataproxyrefresher/config/config.go | 6 +- services/dataproxyrefresher/global/init.go | 8 +- .../internal/repository/mongo/profile.go | 14 +- .../dataproxyrefresher/pkg/fresher/service.go | 10 +- services/dataproxyupdater/config/config.go | 4 +- services/dataproxyupdater/global/init.go | 8 +- .../internal/repository/mongo/profile_db.go | 14 +- .../internal/repository/mongo/update.go | 8 +- .../pkg/dataproxyupdater/service.go | 8 +- services/nodecleaner/config/config.go | 4 +- .../internal/repository/mongo/node.go | 4 +- services/nodecleaner/internal/service/node.go | 4 +- .../nodecleaner/pkg/nodecleaner/cronjob.go | 10 +- services/revalidatenode/config/config.go | 4 +- .../internal/repository/mongo/node.go | 2 +- .../pkg/revalidatenode/cronjob.go | 10 +- 38 files changed, 348 insertions(+), 302 deletions(-) delete mode 100644 services/dataproxy/global/init.go rename services/dataproxy/internal/controller/{http => rest}/batches_handler.go (99%) rename services/dataproxy/internal/controller/{http => rest}/mappings_handler.go (98%) rename services/dataproxy/internal/controller/{http => rest}/ping_handler.go (95%) rename services/dataproxy/internal/controller/{http => rest}/profiles_handler.go (98%) rename services/dataproxy/internal/controller/{http => rest}/updates_handler.go (98%) delete mode 100644 services/dataproxy/pkg/dataproxy/application.go delete mode 100644 services/dataproxy/pkg/dataproxy/cleanup.go create mode 100644 services/dataproxy/pkg/dataproxy/envvar.go delete mode 100644 services/dataproxy/pkg/dataproxy/graceful_shutdown.go delete mode 100644 services/dataproxy/pkg/dataproxy/middleware.go create mode 100644 services/dataproxy/pkg/dataproxy/service.go delete mode 100644 services/dataproxy/pkg/dataproxy/url_mappings.go diff --git a/cmd/dataproxy/dataproxy/main.go b/cmd/dataproxy/dataproxy/main.go index e871798f..028dbb4f 100644 --- a/cmd/dataproxy/dataproxy/main.go +++ b/cmd/dataproxy/dataproxy/main.go @@ -1,7 +1,19 @@ package main -import "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/pkg/dataproxy" +import ( + "github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger" + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/pkg/dataproxy" +) func main() { - dataproxy.StartApplication() + logger.Info("Dataproxy service starting") + + s := dataproxy.NewService() + + go func() { + <-s.WaitUntilUp() + logger.Info("Dataproxy service started") + }() + + s.Run() } diff --git a/cmd/dataproxy/seeder/main.go b/cmd/dataproxy/seeder/main.go index 5d6127ba..2e2f1890 100644 --- a/cmd/dataproxy/seeder/main.go +++ b/cmd/dataproxy/seeder/main.go @@ -32,12 +32,12 @@ func Init() { func mongoInit() { uri := mongo.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - err := mongo.NewClient(uri, config.Conf.Mongo.DBName) + err := mongo.NewClient(uri, config.Values.Mongo.DBName) if err != nil { fmt.Println("Error when trying to connect to MongoDB.", err) os.Exit(1) @@ -164,7 +164,7 @@ func importData( } // Validate data - validateURL := config.Conf.Index.URL + "/v2/validate" + validateURL := config.Values.Index.URL + "/v2/validate" isValid, failureReasons, err := importutil.Validate( validateURL, profileJSON, @@ -193,8 +193,8 @@ func importData( } // Post to index service - postNodeURL := config.Conf.Index.URL + "/v2/nodes" - profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string) + postNodeURL := config.Values.Index.URL + "/v2/nodes" + profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string) nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { return false, fmt.Errorf( diff --git a/cmd/dataproxy/ukseeder/main.go b/cmd/dataproxy/ukseeder/main.go index 3f4b7062..3c462a90 100644 --- a/cmd/dataproxy/ukseeder/main.go +++ b/cmd/dataproxy/ukseeder/main.go @@ -30,12 +30,12 @@ func Init() { func mongoInit() { uri := mongo.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - err := mongo.NewClient(uri, config.Conf.Mongo.DBName) + err := mongo.NewClient(uri, config.Values.Mongo.DBName) if err != nil { fmt.Println("Error when trying to connect to MongoDB.", err) os.Exit(1) @@ -214,7 +214,7 @@ func importData(row int, file *excelize.File) (bool, error) { profile["metadata"] = metadata // Validate data - validateURL := config.Conf.Index.URL + "/v2/validate" + validateURL := config.Values.Index.URL + "/v2/validate" isValid, failureReasons, err := importutil.Validate(validateURL, profile) if err != nil { return false, fmt.Errorf( @@ -240,8 +240,8 @@ func importData(row int, file *excelize.File) (bool, error) { } // Post to index service - postNodeURL := config.Conf.Index.URL + "/v2/nodes" - profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profile["cuid"].(string) + postNodeURL := config.Values.Index.URL + "/v2/nodes" + profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profile["cuid"].(string) nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { return false, fmt.Errorf( diff --git a/go.mod b/go.mod index 1193b85e..3bf84573 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/MurmurationsNetwork/MurmurationsServices go 1.21 require ( + github.com/caarlos0/env v3.5.0+incompatible github.com/caarlos0/env/v10 v10.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/gin-contrib/cors v1.7.2 @@ -13,6 +14,7 @@ require ( github.com/olivere/elastic/v7 v7.0.32 github.com/redis/go-redis/v9 v9.7.0 github.com/stretchr/testify v1.10.0 + github.com/tevino/abool v1.2.0 github.com/tevino/abool/v2 v2.1.0 github.com/ulule/limiter/v3 v3.11.2 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index 50980981..f27ec557 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= +github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -116,6 +118,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA= +github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg= github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c= github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= diff --git a/services/dataproxy/config/config.go b/services/dataproxy/config/config.go index 8633cd2c..f0cc3e24 100644 --- a/services/dataproxy/config/config.go +++ b/services/dataproxy/config/config.go @@ -1,13 +1,10 @@ package config import ( - "log" "time" - - env "github.com/caarlos0/env/v10" ) -var Conf = config{} +var Values = config{} type config struct { Server serverConf @@ -44,10 +41,3 @@ type mongoConf struct { HOST string `env:"MONGO_HOST,required"` DBName string `env:"MONGO_DB_NAME,required"` } - -func Init() { - err := env.Parse(&Conf) - if err != nil { - log.Fatalf("Failed to decode environment variables: %s", err) - } -} diff --git a/services/dataproxy/global/init.go b/services/dataproxy/global/init.go deleted file mode 100644 index f58acdf3..00000000 --- a/services/dataproxy/global/init.go +++ /dev/null @@ -1,40 +0,0 @@ -package global - -import ( - "os" - - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger" - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/mongo" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" -) - -func Init() { - config.Init() - mongoInit() -} - -func mongoInit() { - uri := mongo.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, - ) - - err := mongo.NewClient(uri, config.Conf.Mongo.DBName) - if err != nil { - logger.Error("error when trying to connect to MongoDB", err) - os.Exit(1) - } - err = mongo.Client.Ping() - if err != nil { - logger.Error("error when trying to ping the MongoDB", err) - os.Exit(1) - } - - // Create an index on the `cuid` field in the `profiles` collection - err = mongo.Client.CreateUniqueIndex("profiles", "cuid") - if err != nil { - logger.Error("error when trying to create index on `cuid` field in `profiles` collection", err) - os.Exit(1) - } -} diff --git a/services/dataproxy/internal/controller/http/batches_handler.go b/services/dataproxy/internal/controller/rest/batches_handler.go similarity index 99% rename from services/dataproxy/internal/controller/http/batches_handler.go rename to services/dataproxy/internal/controller/rest/batches_handler.go index e2962107..6904c2fb 100644 --- a/services/dataproxy/internal/controller/http/batches_handler.go +++ b/services/dataproxy/internal/controller/rest/batches_handler.go @@ -1,4 +1,4 @@ -package http +package rest import ( "encoding/csv" diff --git a/services/dataproxy/internal/controller/http/mappings_handler.go b/services/dataproxy/internal/controller/rest/mappings_handler.go similarity index 98% rename from services/dataproxy/internal/controller/http/mappings_handler.go rename to services/dataproxy/internal/controller/rest/mappings_handler.go index 44d4391a..64173087 100644 --- a/services/dataproxy/internal/controller/http/mappings_handler.go +++ b/services/dataproxy/internal/controller/rest/mappings_handler.go @@ -1,4 +1,4 @@ -package http +package rest import ( "encoding/json" @@ -47,7 +47,7 @@ func (handler *mappingsHandler) Create(c *gin.Context) { // download mapping from cdn url := fmt.Sprintf( "%s/v2/schemas/%s", - config.Conf.Library.InternalURL, + config.Values.Library.InternalURL, schema, ) //var schemas map[string]interface{} diff --git a/services/dataproxy/internal/controller/http/ping_handler.go b/services/dataproxy/internal/controller/rest/ping_handler.go similarity index 95% rename from services/dataproxy/internal/controller/http/ping_handler.go rename to services/dataproxy/internal/controller/rest/ping_handler.go index 3aaf658b..b25624df 100644 --- a/services/dataproxy/internal/controller/http/ping_handler.go +++ b/services/dataproxy/internal/controller/rest/ping_handler.go @@ -1,4 +1,4 @@ -package http +package rest import ( "net/http" diff --git a/services/dataproxy/internal/controller/http/profiles_handler.go b/services/dataproxy/internal/controller/rest/profiles_handler.go similarity index 98% rename from services/dataproxy/internal/controller/http/profiles_handler.go rename to services/dataproxy/internal/controller/rest/profiles_handler.go index 3f28b9e4..b9376670 100644 --- a/services/dataproxy/internal/controller/http/profiles_handler.go +++ b/services/dataproxy/internal/controller/rest/profiles_handler.go @@ -1,4 +1,4 @@ -package http +package rest import ( "net/http" diff --git a/services/dataproxy/internal/controller/http/updates_handler.go b/services/dataproxy/internal/controller/rest/updates_handler.go similarity index 98% rename from services/dataproxy/internal/controller/http/updates_handler.go rename to services/dataproxy/internal/controller/rest/updates_handler.go index 0a790165..3c752223 100644 --- a/services/dataproxy/internal/controller/http/updates_handler.go +++ b/services/dataproxy/internal/controller/rest/updates_handler.go @@ -1,4 +1,4 @@ -package http +package rest import ( "net/http" diff --git a/services/dataproxy/internal/service/batch.go b/services/dataproxy/internal/service/batch.go index 9f91ddf2..f02ee2e9 100644 --- a/services/dataproxy/internal/service/batch.go +++ b/services/dataproxy/internal/service/batch.go @@ -338,8 +338,8 @@ func (s *batchService) Import( } // Post the profile to the index service. - postNodeURL := config.Conf.Index.URL + "/v2/nodes" - profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileCUID + postNodeURL := config.Values.Index.URL + "/v2/nodes" + profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileCUID nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { return batchID, -1, nil, fmt.Errorf( @@ -495,8 +495,8 @@ func (s *batchService) Edit( } // Import profile to Index - postNodeURL := config.Conf.Index.URL + "/v2/nodes" - profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileCuid + postNodeURL := config.Values.Index.URL + "/v2/nodes" + profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileCuid nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { return line, nil, errors.New( @@ -533,7 +533,7 @@ func (s *batchService) Edit( // Delete profiles from Index if profile["is_posted"].(bool) { nodeID := profile["node_id"].(string) - deleteNodeURL := config.Conf.Index.URL + "/v2/nodes/" + nodeID + deleteNodeURL := config.Values.Index.URL + "/v2/nodes/" + nodeID err := importutil.DeleteIndex(deleteNodeURL, nodeID) if err != nil { return -1, nil, errors.New( @@ -573,7 +573,7 @@ func (s *batchService) Delete(userID string, batchID string) error { for _, profile := range profiles { if profile["is_posted"].(bool) { nodeID := profile["node_id"].(string) - deleteNodeURL := config.Conf.Index.URL + "/v2/nodes/" + nodeID + deleteNodeURL := config.Values.Index.URL + "/v2/nodes/" + nodeID err := importutil.DeleteIndex(deleteNodeURL, nodeID) if err != nil { return errors.New( diff --git a/services/dataproxy/internal/service/parse_schema.go b/services/dataproxy/internal/service/parse_schema.go index 28cc406c..14c1901c 100644 --- a/services/dataproxy/internal/service/parse_schema.go +++ b/services/dataproxy/internal/service/parse_schema.go @@ -23,7 +23,7 @@ func ParseSchemas(schemas []string) (*SchemasResponse, error) { // Include default schema and append provided schemas. schemaNames := append([]string{DefaultSchema}, schemas...) jsonSchemas := make([]string, len(schemaNames)) - baseURL := fmt.Sprintf("%s/v2/schemas", config.Conf.Library.InternalURL) + baseURL := fmt.Sprintf("%s/v2/schemas", config.Values.Library.InternalURL) for i, schema := range schemaNames { schemaURL := fmt.Sprintf("%s/%s", baseURL, schema) diff --git a/services/dataproxy/internal/service/parse_schema_test.go b/services/dataproxy/internal/service/parse_schema_test.go index 32155513..13091b98 100644 --- a/services/dataproxy/internal/service/parse_schema_test.go +++ b/services/dataproxy/internal/service/parse_schema_test.go @@ -33,7 +33,7 @@ func TestParseSchemas(t *testing.T) { defer mockServer.Close() // Mock the config to use the test server's URL. - config.Conf.Library.InternalURL = mockServer.URL + config.Values.Library.InternalURL = mockServer.URL // Define test cases. tests := []struct { diff --git a/services/dataproxy/pkg/dataproxy/application.go b/services/dataproxy/pkg/dataproxy/application.go deleted file mode 100644 index 8dcc304d..00000000 --- a/services/dataproxy/pkg/dataproxy/application.go +++ /dev/null @@ -1,49 +0,0 @@ -package dataproxy - -import ( - "fmt" - "net/http" - "os" - - "github.com/gin-gonic/gin" - - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/global" -) - -func init() { - global.Init() -} - -func StartApplication() { - gin.SetMode(gin.ReleaseMode) - router := gin.New() - router.Use(getMiddlewares()...) - - mapURLs(router) - - server := getServer(router) - - closed := make(chan struct{}) - go waitForShutdown(server, closed) - - if err := server.ListenAndServe(); err != nil && - err != http.ErrServerClosed { - logger.Error("Error when trying to start the server", err) - os.Exit(1) - } - - <-closed -} - -func getServer(router *gin.Engine) *http.Server { - srv := &http.Server{ - Addr: fmt.Sprintf(":%s", config.Conf.Server.Port), - Handler: router, - ReadTimeout: config.Conf.Server.TimeoutRead, - WriteTimeout: config.Conf.Server.TimeoutWrite, - IdleTimeout: config.Conf.Server.TimeoutIdle, - } - return srv -} diff --git a/services/dataproxy/pkg/dataproxy/cleanup.go b/services/dataproxy/pkg/dataproxy/cleanup.go deleted file mode 100644 index 6b0eeb17..00000000 --- a/services/dataproxy/pkg/dataproxy/cleanup.go +++ /dev/null @@ -1,4 +0,0 @@ -package dataproxy - -func cleanup() { -} diff --git a/services/dataproxy/pkg/dataproxy/envvar.go b/services/dataproxy/pkg/dataproxy/envvar.go new file mode 100644 index 00000000..f55d61da --- /dev/null +++ b/services/dataproxy/pkg/dataproxy/envvar.go @@ -0,0 +1,15 @@ +package dataproxy + +import ( + "log" + + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" + env "github.com/caarlos0/env/v10" +) + +func init() { + err := env.Parse(&config.Values) + if err != nil { + log.Fatalf("Failed to decode environment variables: %s", err) + } +} diff --git a/services/dataproxy/pkg/dataproxy/graceful_shutdown.go b/services/dataproxy/pkg/dataproxy/graceful_shutdown.go deleted file mode 100644 index 605dee7a..00000000 --- a/services/dataproxy/pkg/dataproxy/graceful_shutdown.go +++ /dev/null @@ -1,32 +0,0 @@ -package dataproxy - -import ( - "context" - "net/http" - "os" - "os/signal" - "syscall" - - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" -) - -func waitForShutdown(server *http.Server, closed chan struct{}) { - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - <-quit - - cleanup() - - ctx, cancel := context.WithTimeout( - context.Background(), - config.Conf.Server.TimeoutIdle, - ) - defer cancel() - - if err := server.Shutdown(ctx); err != nil { - logger.Fatal("Data-Proxy service shutdown failure", err) - } - - close(closed) -} diff --git a/services/dataproxy/pkg/dataproxy/middleware.go b/services/dataproxy/pkg/dataproxy/middleware.go deleted file mode 100644 index 5115b708..00000000 --- a/services/dataproxy/pkg/dataproxy/middleware.go +++ /dev/null @@ -1,43 +0,0 @@ -package dataproxy - -import ( - "time" - - corslib "github.com/gin-contrib/cors" - "github.com/gin-gonic/gin" - - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/middleware/limiter" - "github.com/MurmurationsNetwork/MurmurationsServices/pkg/middleware/logger" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" -) - -func getMiddlewares() []gin.HandlerFunc { - return []gin.HandlerFunc{ - gin.Recovery(), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.PostRateLimitPeriod, - Method: "POST", - }), - limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ - Period: config.Conf.Server.GetRateLimitPeriod, - Method: "GET", - }), - logger.NewLogger(), - cors(), - } -} - -func cors() gin.HandlerFunc { - // CORS for all origins, allowing: - // - GET and POST methods - // - Origin, Authorization and Content-Type header - // - Credentials share - // - Preflight requests cached for 12 hours - return corslib.New(corslib.Config{ - AllowOrigins: []string{"*"}, - AllowHeaders: []string{"Origin", "Authorization", "Content-Type"}, - ExposeHeaders: []string{"Content-Length"}, - AllowCredentials: true, - MaxAge: 12 * time.Hour, - }) -} diff --git a/services/dataproxy/pkg/dataproxy/service.go b/services/dataproxy/pkg/dataproxy/service.go new file mode 100644 index 00000000..dbd82f19 --- /dev/null +++ b/services/dataproxy/pkg/dataproxy/service.go @@ -0,0 +1,225 @@ +package dataproxy + +import ( + "context" + "fmt" + "net/http" + "os" + "sync" + "time" + + corslib "github.com/gin-contrib/cors" + "github.com/gin-gonic/gin" + "github.com/tevino/abool" + "go.uber.org/zap/zapcore" + + "github.com/MurmurationsNetwork/MurmurationsServices/pkg/core" + "github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger" + "github.com/MurmurationsNetwork/MurmurationsServices/pkg/middleware/limiter" + midlogger "github.com/MurmurationsNetwork/MurmurationsServices/pkg/middleware/logger" + mongodb "github.com/MurmurationsNetwork/MurmurationsServices/pkg/mongo" + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/controller/rest" + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/repository/mongo" + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/service" +) + +// Service represents the dataproxy service. +type Service struct { + // HTTP server + server *http.Server + // Atomic boolean to manage service state + run *abool.AtomicBool + // HTTP router for the dataproxy service + router *gin.Engine + // Ensures cleanup is only run once + runCleanup sync.Once + // Context for shutdown + shutdownCtx context.Context + // Cancel function for shutdown context + shutdownCancelCtx context.CancelFunc +} + +// NewService initializes a new dataproxy service. +func NewService() *Service { + svc := &Service{ + run: abool.New(), + } + + svc.setupServer() + core.InstallShutdownHandler(svc.Shutdown) + + return svc +} + +// setupServer configures and initializes the HTTP server. +func (s *Service) setupServer() { + gin.SetMode(gin.ReleaseMode) + s.router = gin.New() + s.router.Use(s.middlewares()...) + s.registerRoutes() + + if err := s.connectToMongoDB(); err != nil { + s.panic("error when trying to connect to MongoDB", err) + } + + s.server = &http.Server{ + Addr: fmt.Sprintf(":%s", config.Values.Server.Port), + Handler: s.router, + ReadTimeout: config.Values.Server.TimeoutRead, + WriteTimeout: config.Values.Server.TimeoutWrite, + IdleTimeout: config.Values.Server.TimeoutIdle, + } + + s.shutdownCtx, s.shutdownCancelCtx = context.WithCancel( + context.Background(), + ) +} + +// connectToMongoDB establishes a connection to MongoDB. +func (s *Service) connectToMongoDB() error { + uri := mongodb.GetURI( + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, + ) + err := mongodb.NewClient(uri, config.Values.Mongo.DBName) + if err != nil { + return err + } + err = mongodb.Client.Ping() + if err != nil { + return err + } + // Create an index on the `cuid` field in the `profiles` collection + err = mongodb.Client.CreateUniqueIndex("profiles", "cuid") + if err != nil { + logger.Error("error when trying to create index on `cuid` field in `profiles` collection", err) + os.Exit(1) + } + return nil +} + +// middlewares returns the list of middlewares to be used by the HTTP server. +func (s *Service) middlewares() []gin.HandlerFunc { + return []gin.HandlerFunc{ + gin.Recovery(), + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.PostRateLimitPeriod, + Method: "POST", + }), + limiter.NewRateLimitWithOptions(limiter.RateLimitOptions{ + Period: config.Values.Server.GetRateLimitPeriod, + Method: "GET", + }), + midlogger.NewLogger(), + s.cors(), + } +} + +// cors returns the CORS middleware configuration. +func (s *Service) cors() gin.HandlerFunc { + // CORS for all origins, allowing: + // - GET and POST methods + // - Origin, Authorization and Content-Type header + // - Credentials share + // - Preflight requests cached for 12 hours + return corslib.New(corslib.Config{ + AllowOrigins: []string{"*"}, + AllowHeaders: []string{"Origin", "Authorization", "Content-Type"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + MaxAge: 12 * time.Hour, + }) +} + +// registerRoutes sets up the routes for the HTTP server. +func (s *Service) registerRoutes() { + pingHandler := rest.NewPingHandler() + mappingsHandler := rest.NewMappingsHandler(mongo.NewMappingRepository()) + profilesHandler := rest.NewProfilesHandler(mongo.NewProfileRepository()) + updatesHandler := rest.NewUpdatesHandler(mongo.NewUpdateRepository()) + batchesHandler := rest.NewBatchesHandler( + service.NewBatchService(mongo.NewBatchRepository()), + ) + + v1 := s.router.Group("/v1") + { + v1.GET("/ping", pingHandler.Ping) + v1.POST("/mappings", mappingsHandler.Create) + v1.GET("/profiles/:profileID", profilesHandler.Get) + v1.GET("/health/:schemaName", updatesHandler.Get) + + // for csv batch import + v1.GET("/batch/user", batchesHandler.GetBatchesByUserID) + v1.POST("/batch/validate", batchesHandler.Validate) + v1.POST("/batch/import", batchesHandler.Import) + v1.PUT("/batch/import", batchesHandler.Edit) + v1.DELETE("/batch/import", batchesHandler.Delete) + } +} + +// panic performs a cleanup and then emits the supplied message as the panic value. +func (s *Service) panic(msg string, err error, logFields ...zapcore.Field) { + s.cleanup() + logger.Error(msg, err, logFields...) + os.Exit(1) +} + +// Run starts the dataproxy service and will block until the service is shutdown. +func (s *Service) Run() { + s.run.Set() + + if err := s.server.ListenAndServe(); err != nil && + err != http.ErrServerClosed { + s.panic("Error when trying to start the server", err) + } +} + +// WaitUntilUp returns a channel which blocks until the dataproxy service is up. +func (s *Service) WaitUntilUp() <-chan struct{} { + initialized := make(chan struct{}) + timeout := time.After(1 * time.Minute) + go func() { + for { + select { + case <-timeout: + panic("Service startup timed out") + default: + resp, err := http.Get( + fmt.Sprintf( + "http://localhost:%s/v1/ping", + config.Values.Server.Port, + ), + ) + if err == nil && resp.StatusCode == http.StatusOK { + close(initialized) + return + } + logger.Info( + "Ping failed, waiting for service to finish starting...", + ) + time.Sleep(5 * time.Second) + } + } + }() + return initialized +} + +// Shutdown stops the dataproxy service. +func (s *Service) Shutdown() { + if s.run.IsSet() { + if err := s.server.Shutdown(s.shutdownCtx); err != nil { + logger.Error("Data-Proxy service shutdown failure", err) + } + } + s.cleanup() +} + +// cleanup will clean up the non-server resources associated with the service. +func (s *Service) cleanup() { + s.runCleanup.Do(func() { + // Shutdown the context. + s.shutdownCancelCtx() + }) +} diff --git a/services/dataproxy/pkg/dataproxy/url_mappings.go b/services/dataproxy/pkg/dataproxy/url_mappings.go deleted file mode 100644 index 391121f0..00000000 --- a/services/dataproxy/pkg/dataproxy/url_mappings.go +++ /dev/null @@ -1,34 +0,0 @@ -package dataproxy - -import ( - "github.com/gin-gonic/gin" - - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/controller/http" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/repository/mongo" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/internal/service" -) - -func mapURLs(router *gin.Engine) { - pingHandler := http.NewPingHandler() - mappingsHandler := http.NewMappingsHandler(mongo.NewMappingRepository()) - profilesHandler := http.NewProfilesHandler(mongo.NewProfileRepository()) - updatesHandler := http.NewUpdatesHandler(mongo.NewUpdateRepository()) - batchesHandler := http.NewBatchesHandler( - service.NewBatchService(mongo.NewBatchRepository()), - ) - - v1 := router.Group("/v1") - { - v1.GET("/ping", pingHandler.Ping) - v1.POST("/mappings", mappingsHandler.Create) - v1.GET("/profiles/:profileID", profilesHandler.Get) - v1.GET("/health/:schemaName", updatesHandler.Get) - - // for csv batch import - v1.GET("/batch/user", batchesHandler.GetBatchesByUserID) - v1.POST("/batch/validate", batchesHandler.Validate) - v1.POST("/batch/import", batchesHandler.Import) - v1.PUT("/batch/import", batchesHandler.Edit) - v1.DELETE("/batch/import", batchesHandler.Delete) - } -} diff --git a/services/dataproxyrefresher/config/config.go b/services/dataproxyrefresher/config/config.go index 778c1ae5..d8089694 100644 --- a/services/dataproxyrefresher/config/config.go +++ b/services/dataproxyrefresher/config/config.go @@ -6,7 +6,7 @@ import ( env "github.com/caarlos0/env/v10" ) -var Conf = config{} +var Values = config{} type config struct { Index indexConf @@ -31,8 +31,8 @@ type mongoConf struct { } func Init() { - err := env.Parse(&Conf) + err := env.Parse(&Values) if err != nil { log.Fatalf("Failed to decode environment variables: %s", err) } -} +} \ No newline at end of file diff --git a/services/dataproxyrefresher/global/init.go b/services/dataproxyrefresher/global/init.go index 62103765..f55b675a 100644 --- a/services/dataproxyrefresher/global/init.go +++ b/services/dataproxyrefresher/global/init.go @@ -15,12 +15,12 @@ func Init() { func mongoInit() { uri := mongo.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - err := mongo.NewClient(uri, config.Conf.Mongo.DBName) + err := mongo.NewClient(uri, config.Values.Mongo.DBName) if err != nil { logger.Error("error when trying to connect to MongoDB", err) os.Exit(1) diff --git a/services/dataproxyrefresher/internal/repository/mongo/profile.go b/services/dataproxyrefresher/internal/repository/mongo/profile.go index a1332182..827a1340 100644 --- a/services/dataproxyrefresher/internal/repository/mongo/profile.go +++ b/services/dataproxyrefresher/internal/repository/mongo/profile.go @@ -39,7 +39,7 @@ type profileRepository struct { func (r *profileRepository) Count(profileID string) (int64, error) { filter := bson.M{"oid": profileID} - count, err := r.client.Database(config.Conf.Mongo.DBName). + count, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). CountDocuments(context.Background(), filter) if err != nil { @@ -49,7 +49,7 @@ func (r *profileRepository) Count(profileID string) (int64, error) { } func (r *profileRepository) Add(profileJSON map[string]interface{}) error { - _, err := r.client.Database(config.Conf.Mongo.DBName). + _, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). InsertOne(context.Background(), profileJSON) @@ -68,7 +68,7 @@ func (r *profileRepository) Update( update := bson.M{"$set": profileJSON} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -93,7 +93,7 @@ func (r *profileRepository) UpdateNodeID( update := bson.M{"$set": bson.M{"node_id": nodeID, "is_posted": false}} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -118,7 +118,7 @@ func (r *profileRepository) FindLessThan( } var profiles []model.Profile - cursor, err := r.client.Database(config.Conf.Mongo.DBName). + cursor, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). Find(context.Background(), filter) if err != nil { @@ -140,7 +140,7 @@ func (r *profileRepository) UpdateAccessTime(profileID string) error { } opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -154,7 +154,7 @@ func (r *profileRepository) UpdateAccessTime(profileID string) error { func (r *profileRepository) Delete(profileID string) error { filter := bson.M{"cuid": profileID} - _, err := r.client.Database(config.Conf.Mongo.DBName). + _, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). DeleteOne(context.Background(), filter) if err != nil { diff --git a/services/dataproxyrefresher/pkg/fresher/service.go b/services/dataproxyrefresher/pkg/fresher/service.go index 99d1557c..f6c4bdb6 100644 --- a/services/dataproxyrefresher/pkg/fresher/service.go +++ b/services/dataproxyrefresher/pkg/fresher/service.go @@ -85,7 +85,7 @@ func (r *DataproxyRefresher) getProfiles( schemaName string, ) ([]model.Profile, error) { curTime := time.Now().Unix() - refreshBefore := curTime - config.Conf.RefreshTTL + refreshBefore := curTime - config.Values.RefreshTTL profiles, err := r.svc.FindLessThan(schemaName, refreshBefore) if err != nil { @@ -289,7 +289,7 @@ func (r *DataproxyRefresher) updateProfileIfValid( func (r *DataproxyRefresher) validateProfile( profileJSON map[string]interface{}, ) (bool, string, error) { - validateURL := config.Conf.Index.URL + APIValidatePath + validateURL := config.Values.Index.URL + APIValidatePath isValid, failureReasons, err := importutil.Validate( validateURL, @@ -340,8 +340,8 @@ func (r *DataproxyRefresher) UpdateProfile( } // Post the update to the Index - postNodeURL := config.Conf.Index.URL + APINodesPath - profileURL := config.Conf.DataProxy.URL + APIProfilesPath + "/" + + postNodeURL := config.Values.Index.URL + APINodesPath + profileURL := config.Values.DataProxy.URL + APIProfilesPath + "/" + profileJSON["cuid"].(string) nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { @@ -389,7 +389,7 @@ func (r *DataproxyRefresher) deleteProfile(profile model.Profile) error { } // Delete from the Index service. - deleteNodeURL := config.Conf.Index.URL + APINodesPath + "/" + profile.NodeID + deleteNodeURL := config.Values.Index.URL + APINodesPath + "/" + profile.NodeID client := &http.Client{} req, err := http.NewRequest(http.MethodDelete, deleteNodeURL, nil) diff --git a/services/dataproxyupdater/config/config.go b/services/dataproxyupdater/config/config.go index 4ab267c9..f6896f14 100644 --- a/services/dataproxyupdater/config/config.go +++ b/services/dataproxyupdater/config/config.go @@ -6,7 +6,7 @@ import ( env "github.com/caarlos0/env/v10" ) -var Conf = config{} +var Values = config{} type config struct { Index indexConf @@ -30,7 +30,7 @@ type mongoConf struct { } func Init() { - err := env.Parse(&Conf) + err := env.Parse(&Values) if err != nil { log.Fatalf("Failed to decode environment variables: %s", err) } diff --git a/services/dataproxyupdater/global/init.go b/services/dataproxyupdater/global/init.go index fc4c520e..d87db9ee 100644 --- a/services/dataproxyupdater/global/init.go +++ b/services/dataproxyupdater/global/init.go @@ -15,12 +15,12 @@ func Init() { func mongoInit() { uri := mongo.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - err := mongo.NewClient(uri, config.Conf.Mongo.DBName) + err := mongo.NewClient(uri, config.Values.Mongo.DBName) if err != nil { logger.Error("error when trying to connect to MongoDB", err) os.Exit(1) diff --git a/services/dataproxyupdater/internal/repository/mongo/profile_db.go b/services/dataproxyupdater/internal/repository/mongo/profile_db.go index 71bd32dd..6c0b7c68 100644 --- a/services/dataproxyupdater/internal/repository/mongo/profile_db.go +++ b/services/dataproxyupdater/internal/repository/mongo/profile_db.go @@ -38,7 +38,7 @@ type profileRepository struct { func (r *profileRepository) Count(profileID string) (int64, error) { filter := bson.M{"oid": profileID} - count, err := r.client.Database(config.Conf.Mongo.DBName). + count, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). CountDocuments(context.Background(), filter) if err != nil { @@ -48,7 +48,7 @@ func (r *profileRepository) Count(profileID string) (int64, error) { } func (r *profileRepository) Add(profileJSON map[string]interface{}) error { - _, err := r.client.Database(config.Conf.Mongo.DBName). + _, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). InsertOne(context.Background(), profileJSON) @@ -67,7 +67,7 @@ func (r *profileRepository) Update( update := bson.M{"$set": profileJSON} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -92,7 +92,7 @@ func (r *profileRepository) UpdateNodeID( update := bson.M{"$set": bson.M{"node_id": nodeID, "is_posted": false}} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -107,7 +107,7 @@ func (r *profileRepository) GetNotPosted() ([]model.Profile, error) { filter := bson.M{"is_posted": false} var profiles []model.Profile - cursor, err := r.client.Database(config.Conf.Mongo.DBName). + cursor, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). Find(context.Background(), filter) if err != nil { @@ -126,7 +126,7 @@ func (r *profileRepository) UpdateIsPosted(nodeID string) error { update := bson.M{"$set": bson.M{"is_posted": true}} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -140,7 +140,7 @@ func (r *profileRepository) UpdateIsPosted(nodeID string) error { func (r *profileRepository) Delete(profileID string) error { filter := bson.M{"cuid": profileID} - _, err := r.client.Database(config.Conf.Mongo.DBName). + _, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Profile). DeleteOne(context.Background(), filter) diff --git a/services/dataproxyupdater/internal/repository/mongo/update.go b/services/dataproxyupdater/internal/repository/mongo/update.go index 286c85b8..4e47f485 100644 --- a/services/dataproxyupdater/internal/repository/mongo/update.go +++ b/services/dataproxyupdater/internal/repository/mongo/update.go @@ -37,7 +37,7 @@ type updateRepository struct { func (r *updateRepository) Get(schemaName string) *model.Update { filter := bson.M{"schema": schemaName} - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Update). FindOne(context.Background(), filter) @@ -59,7 +59,7 @@ func (r *updateRepository) Save( "api_entry": apiEntry, } - _, err := r.client.Database(config.Conf.Mongo.DBName). + _, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Update). InsertOne(context.Background(), filter) @@ -75,7 +75,7 @@ func (r *updateRepository) Update(schemaName string, lastUpdated int64) error { update := bson.M{"$set": bson.M{"last_updated": lastUpdated}} opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Update). FindOneAndUpdate(context.Background(), filter, update, opt) @@ -102,7 +102,7 @@ func (r *updateRepository) SaveError( } opt := options.FindOneAndUpdate().SetUpsert(true) - result := r.client.Database(config.Conf.Mongo.DBName). + result := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Update). FindOneAndUpdate(context.Background(), filter, update, opt) diff --git a/services/dataproxyupdater/pkg/dataproxyupdater/service.go b/services/dataproxyupdater/pkg/dataproxyupdater/service.go index 01c8ce51..b4ca26da 100644 --- a/services/dataproxyupdater/pkg/dataproxyupdater/service.go +++ b/services/dataproxyupdater/pkg/dataproxyupdater/service.go @@ -166,7 +166,7 @@ func Run() { } // validate data - validateURL := config.Conf.Index.URL + "/v2/validate" + validateURL := config.Values.Index.URL + "/v2/validate" isValid, failureReasons, err := importutil.Validate( validateURL, profileJSON, @@ -208,8 +208,8 @@ func Run() { } // post update to Index - postNodeURL := config.Conf.Index.URL + "/v2/nodes" - profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string) + postNodeURL := config.Values.Index.URL + "/v2/nodes" + profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string) nodeID, err := importutil.PostIndex(postNodeURL, profileURL) if err != nil { errStr := "failed to post profile to Index, profile url is " + profileURL + ". error message: " + err.Error() @@ -262,7 +262,7 @@ func Run() { } for _, notPostedProfile := range notPostedProfiles { - getNodeURL := config.Conf.Index.URL + "/v2/nodes/" + notPostedProfile.NodeID + getNodeURL := config.Values.Index.URL + "/v2/nodes/" + notPostedProfile.NodeID res, err := http.Get(getNodeURL) if err != nil { errStr := "failed to get not posted nodes, node id is " + notPostedProfile.NodeID + err.Error() diff --git a/services/nodecleaner/config/config.go b/services/nodecleaner/config/config.go index ac8089d8..db9ec8c7 100644 --- a/services/nodecleaner/config/config.go +++ b/services/nodecleaner/config/config.go @@ -6,7 +6,7 @@ import ( env "github.com/caarlos0/env/v10" ) -var Conf = config{} +var Values = config{} type config struct { Mongo mongoConf @@ -31,7 +31,7 @@ type ttlConf struct { } func Init() { - err := env.Parse(&Conf) + err := env.Parse(&Values) if err != nil { log.Fatalf("Failed to decode environment variables: %s", err) } diff --git a/services/nodecleaner/internal/repository/mongo/node.go b/services/nodecleaner/internal/repository/mongo/node.go index 840b64ea..c271bde7 100644 --- a/services/nodecleaner/internal/repository/mongo/node.go +++ b/services/nodecleaner/internal/repository/mongo/node.go @@ -79,7 +79,7 @@ func (r *nodeRepository) removeNodes( }, } - result, err := r.client.Database(config.Conf.Mongo.DBName). + result, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Node). DeleteMany(ctx, filter) if err != nil { @@ -113,7 +113,7 @@ func (r *nodeRepository) UpdateStatusByExpiration( }, } - result, err := r.client.Database(config.Conf.Mongo.DBName). + result, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Node). UpdateMany(ctx, filter, update) if err != nil { diff --git a/services/nodecleaner/internal/service/node.go b/services/nodecleaner/internal/service/node.go index 715940df..8548628c 100644 --- a/services/nodecleaner/internal/service/node.go +++ b/services/nodecleaner/internal/service/node.go @@ -43,7 +43,7 @@ func NewNodeService( // before a calculated time from the MongoDB repository. func (svc *nodesService) RemoveValidationFailed(ctx context.Context) error { timeBefore := dateutil.NowSubtract( - time.Duration(config.Conf.TTL.ValidationFailedTTL) * time.Second, + time.Duration(config.Values.TTL.ValidationFailedTTL) * time.Second, ) return svc.mongoRepo.RemoveByCreatedAt( ctx, @@ -55,7 +55,7 @@ func (svc *nodesService) RemoveValidationFailed(ctx context.Context) error { // RemoveDeleted removes nodes with Deleted status updated before the calculated time. func (svc *nodesService) RemoveDeleted(ctx context.Context) error { timeBefore := dateutil.NowSubtract( - time.Duration(config.Conf.TTL.DeletedTTL) * time.Second, + time.Duration(config.Values.TTL.DeletedTTL) * time.Second, ) err := svc.mongoRepo.RemoveByLastUpdated( diff --git a/services/nodecleaner/pkg/nodecleaner/cronjob.go b/services/nodecleaner/pkg/nodecleaner/cronjob.go index 2aae22f0..1648c497 100644 --- a/services/nodecleaner/pkg/nodecleaner/cronjob.go +++ b/services/nodecleaner/pkg/nodecleaner/cronjob.go @@ -24,11 +24,11 @@ func NewCronJob() *NodeCleaner { config.Init() uri := mongodb.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - if err := mongodb.NewClient(uri, config.Conf.Mongo.DBName); err != nil { + if err := mongodb.NewClient(uri, config.Values.Mongo.DBName); err != nil { logger.Error("Failed to connect to MongoDB", err) os.Exit(1) } @@ -38,7 +38,7 @@ func NewCronJob() *NodeCleaner { os.Exit(1) } - if err := elastic.NewClient(config.Conf.ES.URL); err != nil { + if err := elastic.NewClient(config.Values.ES.URL); err != nil { logger.Error("Failed to connect to Elasticsearch", err) os.Exit(1) } diff --git a/services/revalidatenode/config/config.go b/services/revalidatenode/config/config.go index 4cf82e75..aa982f84 100644 --- a/services/revalidatenode/config/config.go +++ b/services/revalidatenode/config/config.go @@ -7,7 +7,7 @@ import ( ) // Conf holds the configuration settings for the application. -var Conf = config{} +var Values = config{} type config struct { // Mongo holds the configuration for MongoDB. @@ -38,7 +38,7 @@ type natsConf struct { // Init initializes the Conf variable by parsing environment variables. func Init() { - if err := env.Parse(&Conf); err != nil { + if err := env.Parse(&Values); err != nil { log.Fatalf("Failed to decode environment variables: %s", err) } } diff --git a/services/revalidatenode/internal/repository/mongo/node.go b/services/revalidatenode/internal/repository/mongo/node.go index cd9601dc..9971a1c5 100644 --- a/services/revalidatenode/internal/repository/mongo/node.go +++ b/services/revalidatenode/internal/repository/mongo/node.go @@ -40,7 +40,7 @@ func (r *nodeRepository) FindByStatuses( skip := (page - 1) * pageSize opts := options.Find().SetSkip(int64(skip)).SetLimit(int64(pageSize)) - cur, err := r.client.Database(config.Conf.Mongo.DBName). + cur, err := r.client.Database(config.Values.Mongo.DBName). Collection(constant.MongoIndex.Node). Find(ctx, filter, opts) if err != nil { diff --git a/services/revalidatenode/pkg/revalidatenode/cronjob.go b/services/revalidatenode/pkg/revalidatenode/cronjob.go index baecdb7e..c55bb7da 100644 --- a/services/revalidatenode/pkg/revalidatenode/cronjob.go +++ b/services/revalidatenode/pkg/revalidatenode/cronjob.go @@ -24,11 +24,11 @@ func NewCronJob() *NodeRevalidationCron { // Initialize MongoDB client. uri := mongodb.GetURI( - config.Conf.Mongo.USERNAME, - config.Conf.Mongo.PASSWORD, - config.Conf.Mongo.HOST, + config.Values.Mongo.USERNAME, + config.Values.Mongo.PASSWORD, + config.Values.Mongo.HOST, ) - if err := mongodb.NewClient(uri, config.Conf.Mongo.DBName); err != nil { + if err := mongodb.NewClient(uri, config.Values.Mongo.DBName); err != nil { logger.Error("error when trying to connect to MongoDB", err) os.Exit(1) } @@ -47,7 +47,7 @@ func NewCronJob() *NodeRevalidationCron { // setupNATS initializes Nats service. func setupNATS() { - err := natsclient.Initialize(config.Conf.Nats.URL) + err := natsclient.Initialize(config.Values.Nats.URL) if err != nil { logger.Error("Failed to create Nats client", err) os.Exit(1) From e20ca433ba5b22a7346d963dc22a637abe7da9be Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 19:15:50 -0800 Subject: [PATCH 4/6] fix: fix pacakge Signed-off-by: vthwang --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 3bf84573..62c1b368 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/MurmurationsNetwork/MurmurationsServices go 1.21 require ( - github.com/caarlos0/env v3.5.0+incompatible github.com/caarlos0/env/v10 v10.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/gin-contrib/cors v1.7.2 diff --git a/go.sum b/go.sum index f27ec557..124eaa14 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= -github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= From 1a9a218dc907f89c9b60e4f2ee79d7bd01f4c239 Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 19:20:26 -0800 Subject: [PATCH 5/6] fix: remove duplicate packages Signed-off-by: vthwang --- cmd/dataproxy/seeder/main.go | 1 - cmd/dataproxy/ukseeder/main.go | 1 - go.mod | 1 - go.sum | 2 -- services/dataproxy/pkg/dataproxy/service.go | 2 +- 5 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/dataproxy/seeder/main.go b/cmd/dataproxy/seeder/main.go index 2e2f1890..42eca5c7 100644 --- a/cmd/dataproxy/seeder/main.go +++ b/cmd/dataproxy/seeder/main.go @@ -26,7 +26,6 @@ const ( ) func Init() { - config.Init() mongoInit() } diff --git a/cmd/dataproxy/ukseeder/main.go b/cmd/dataproxy/ukseeder/main.go index 3c462a90..cba56671 100644 --- a/cmd/dataproxy/ukseeder/main.go +++ b/cmd/dataproxy/ukseeder/main.go @@ -24,7 +24,6 @@ const ( ) func Init() { - config.Init() mongoInit() } diff --git a/go.mod b/go.mod index 62c1b368..1193b85e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/olivere/elastic/v7 v7.0.32 github.com/redis/go-redis/v9 v9.7.0 github.com/stretchr/testify v1.10.0 - github.com/tevino/abool v1.2.0 github.com/tevino/abool/v2 v2.1.0 github.com/ulule/limiter/v3 v3.11.2 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index 124eaa14..50980981 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA= -github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg= github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c= github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= diff --git a/services/dataproxy/pkg/dataproxy/service.go b/services/dataproxy/pkg/dataproxy/service.go index dbd82f19..21b0fc71 100644 --- a/services/dataproxy/pkg/dataproxy/service.go +++ b/services/dataproxy/pkg/dataproxy/service.go @@ -10,7 +10,7 @@ import ( corslib "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" - "github.com/tevino/abool" + "github.com/tevino/abool/v2" "go.uber.org/zap/zapcore" "github.com/MurmurationsNetwork/MurmurationsServices/pkg/core" From 2c24ef1b26553636ea8ff9e9a033e34ae25fe89d Mon Sep 17 00:00:00 2001 From: vthwang Date: Mon, 16 Dec 2024 19:25:41 -0800 Subject: [PATCH 6/6] fix: fix the format Signed-off-by: vthwang --- services/dataproxy/pkg/dataproxy/envvar.go | 3 ++- services/dataproxyrefresher/config/config.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/dataproxy/pkg/dataproxy/envvar.go b/services/dataproxy/pkg/dataproxy/envvar.go index f55d61da..ee498c6a 100644 --- a/services/dataproxy/pkg/dataproxy/envvar.go +++ b/services/dataproxy/pkg/dataproxy/envvar.go @@ -3,8 +3,9 @@ package dataproxy import ( "log" - "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" env "github.com/caarlos0/env/v10" + + "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config" ) func init() { diff --git a/services/dataproxyrefresher/config/config.go b/services/dataproxyrefresher/config/config.go index d8089694..b3b3e61c 100644 --- a/services/dataproxyrefresher/config/config.go +++ b/services/dataproxyrefresher/config/config.go @@ -35,4 +35,4 @@ func Init() { if err != nil { log.Fatalf("Failed to decode environment variables: %s", err) } -} \ No newline at end of file +}