Skip to content

Commit f1687f3

Browse files
vthwangvthwang
committed
refactor: refactor the data proxy service to be consistent with index service
Signed-off-by: vthwang <[email protected]>
1 parent 88c158c commit f1687f3

File tree

38 files changed

+348
-302
lines changed

38 files changed

+348
-302
lines changed

‎cmd/dataproxy/dataproxy/main.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
package main
22

3-
import "github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/pkg/dataproxy"
3+
import (
4+
"github.com/MurmurationsNetwork/MurmurationsServices/pkg/logger"
5+
"github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/pkg/dataproxy"
6+
)
47

58
func main() {
6-
dataproxy.StartApplication()
9+
logger.Info("Dataproxy service starting")
10+
11+
s := dataproxy.NewService()
12+
13+
go func() {
14+
<-s.WaitUntilUp()
15+
logger.Info("Dataproxy service started")
16+
}()
17+
18+
s.Run()
719
}

‎cmd/dataproxy/seeder/main.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,12 @@ func Init() {
3232

3333
func mongoInit() {
3434
uri := mongo.GetURI(
35-
config.Conf.Mongo.USERNAME,
36-
config.Conf.Mongo.PASSWORD,
37-
config.Conf.Mongo.HOST,
35+
config.Values.Mongo.USERNAME,
36+
config.Values.Mongo.PASSWORD,
37+
config.Values.Mongo.HOST,
3838
)
3939

40-
err := mongo.NewClient(uri, config.Conf.Mongo.DBName)
40+
err := mongo.NewClient(uri, config.Values.Mongo.DBName)
4141
if err != nil {
4242
fmt.Println("Error when trying to connect to MongoDB.", err)
4343
os.Exit(1)
@@ -164,7 +164,7 @@ func importData(
164164
}
165165

166166
// Validate data
167-
validateURL := config.Conf.Index.URL + "/v2/validate"
167+
validateURL := config.Values.Index.URL + "/v2/validate"
168168
isValid, failureReasons, err := importutil.Validate(
169169
validateURL,
170170
profileJSON,
@@ -193,8 +193,8 @@ func importData(
193193
}
194194

195195
// Post to index service
196-
postNodeURL := config.Conf.Index.URL + "/v2/nodes"
197-
profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string)
196+
postNodeURL := config.Values.Index.URL + "/v2/nodes"
197+
profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileJSON["cuid"].(string)
198198
nodeID, err := importutil.PostIndex(postNodeURL, profileURL)
199199
if err != nil {
200200
return false, fmt.Errorf(

‎cmd/dataproxy/ukseeder/main.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ func Init() {
3030

3131
func mongoInit() {
3232
uri := mongo.GetURI(
33-
config.Conf.Mongo.USERNAME,
34-
config.Conf.Mongo.PASSWORD,
35-
config.Conf.Mongo.HOST,
33+
config.Values.Mongo.USERNAME,
34+
config.Values.Mongo.PASSWORD,
35+
config.Values.Mongo.HOST,
3636
)
3737

38-
err := mongo.NewClient(uri, config.Conf.Mongo.DBName)
38+
err := mongo.NewClient(uri, config.Values.Mongo.DBName)
3939
if err != nil {
4040
fmt.Println("Error when trying to connect to MongoDB.", err)
4141
os.Exit(1)
@@ -214,7 +214,7 @@ func importData(row int, file *excelize.File) (bool, error) {
214214
profile["metadata"] = metadata
215215

216216
// Validate data
217-
validateURL := config.Conf.Index.URL + "/v2/validate"
217+
validateURL := config.Values.Index.URL + "/v2/validate"
218218
isValid, failureReasons, err := importutil.Validate(validateURL, profile)
219219
if err != nil {
220220
return false, fmt.Errorf(
@@ -240,8 +240,8 @@ func importData(row int, file *excelize.File) (bool, error) {
240240
}
241241

242242
// Post to index service
243-
postNodeURL := config.Conf.Index.URL + "/v2/nodes"
244-
profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profile["cuid"].(string)
243+
postNodeURL := config.Values.Index.URL + "/v2/nodes"
244+
profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profile["cuid"].(string)
245245
nodeID, err := importutil.PostIndex(postNodeURL, profileURL)
246246
if err != nil {
247247
return false, fmt.Errorf(

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/MurmurationsNetwork/MurmurationsServices
33
go 1.21
44

55
require (
6+
github.com/caarlos0/env v3.5.0+incompatible
67
github.com/caarlos0/env/v10 v10.0.0
78
github.com/cenkalti/backoff/v4 v4.3.0
89
github.com/gin-contrib/cors v1.7.2
@@ -13,6 +14,7 @@ require (
1314
github.com/olivere/elastic/v7 v7.0.32
1415
github.com/redis/go-redis/v9 v9.7.0
1516
github.com/stretchr/testify v1.10.0
17+
github.com/tevino/abool v1.2.0
1618
github.com/tevino/abool/v2 v2.1.0
1719
github.com/ulule/limiter/v3 v3.11.2
1820
github.com/xeipuuv/gojsonschema v1.2.0

‎go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc
66
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
77
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
88
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
9+
github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs=
10+
github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y=
911
github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
1012
github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
1113
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -116,6 +118,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
116118
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
117119
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
118120
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
121+
github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
122+
github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
119123
github.com/tevino/abool/v2 v2.1.0 h1:7w+Vf9f/5gmKT4m4qkayb33/92M+Um45F2BkHOR+L/c=
120124
github.com/tevino/abool/v2 v2.1.0/go.mod h1:+Lmlqk6bHDWHqN1cbxqhwEAwMPXgc8I1SDEamtseuXY=
121125
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=

‎services/dataproxy/config/config.go

+1-11
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
package config
22

33
import (
4-
"log"
54
"time"
6-
7-
env "github.com/caarlos0/env/v10"
85
)
96

10-
var Conf = config{}
7+
var Values = config{}
118

129
type config struct {
1310
Server serverConf
@@ -44,10 +41,3 @@ type mongoConf struct {
4441
HOST string `env:"MONGO_HOST,required"`
4542
DBName string `env:"MONGO_DB_NAME,required"`
4643
}
47-
48-
func Init() {
49-
err := env.Parse(&Conf)
50-
if err != nil {
51-
log.Fatalf("Failed to decode environment variables: %s", err)
52-
}
53-
}

‎services/dataproxy/global/init.go

-40
This file was deleted.

‎services/dataproxy/internal/controller/http/batches_handler.go renamed to ‎services/dataproxy/internal/controller/rest/batches_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package http
1+
package rest
22

33
import (
44
"encoding/csv"

‎services/dataproxy/internal/controller/http/mappings_handler.go renamed to ‎services/dataproxy/internal/controller/rest/mappings_handler.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package http
1+
package rest
22

33
import (
44
"encoding/json"
@@ -47,7 +47,7 @@ func (handler *mappingsHandler) Create(c *gin.Context) {
4747
// download mapping from cdn
4848
url := fmt.Sprintf(
4949
"%s/v2/schemas/%s",
50-
config.Conf.Library.InternalURL,
50+
config.Values.Library.InternalURL,
5151
schema,
5252
)
5353
//var schemas map[string]interface{}

‎services/dataproxy/internal/controller/http/ping_handler.go renamed to ‎services/dataproxy/internal/controller/rest/ping_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package http
1+
package rest
22

33
import (
44
"net/http"

‎services/dataproxy/internal/controller/http/profiles_handler.go renamed to ‎services/dataproxy/internal/controller/rest/profiles_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package http
1+
package rest
22

33
import (
44
"net/http"

‎services/dataproxy/internal/controller/http/updates_handler.go renamed to ‎services/dataproxy/internal/controller/rest/updates_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package http
1+
package rest
22

33
import (
44
"net/http"

‎services/dataproxy/internal/service/batch.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -338,8 +338,8 @@ func (s *batchService) Import(
338338
}
339339

340340
// Post the profile to the index service.
341-
postNodeURL := config.Conf.Index.URL + "/v2/nodes"
342-
profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileCUID
341+
postNodeURL := config.Values.Index.URL + "/v2/nodes"
342+
profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileCUID
343343
nodeID, err := importutil.PostIndex(postNodeURL, profileURL)
344344
if err != nil {
345345
return batchID, -1, nil, fmt.Errorf(
@@ -495,8 +495,8 @@ func (s *batchService) Edit(
495495
}
496496

497497
// Import profile to Index
498-
postNodeURL := config.Conf.Index.URL + "/v2/nodes"
499-
profileURL := config.Conf.DataProxy.URL + "/v1/profiles/" + profileCuid
498+
postNodeURL := config.Values.Index.URL + "/v2/nodes"
499+
profileURL := config.Values.DataProxy.URL + "/v1/profiles/" + profileCuid
500500
nodeID, err := importutil.PostIndex(postNodeURL, profileURL)
501501
if err != nil {
502502
return line, nil, errors.New(
@@ -533,7 +533,7 @@ func (s *batchService) Edit(
533533
// Delete profiles from Index
534534
if profile["is_posted"].(bool) {
535535
nodeID := profile["node_id"].(string)
536-
deleteNodeURL := config.Conf.Index.URL + "/v2/nodes/" + nodeID
536+
deleteNodeURL := config.Values.Index.URL + "/v2/nodes/" + nodeID
537537
err := importutil.DeleteIndex(deleteNodeURL, nodeID)
538538
if err != nil {
539539
return -1, nil, errors.New(
@@ -573,7 +573,7 @@ func (s *batchService) Delete(userID string, batchID string) error {
573573
for _, profile := range profiles {
574574
if profile["is_posted"].(bool) {
575575
nodeID := profile["node_id"].(string)
576-
deleteNodeURL := config.Conf.Index.URL + "/v2/nodes/" + nodeID
576+
deleteNodeURL := config.Values.Index.URL + "/v2/nodes/" + nodeID
577577
err := importutil.DeleteIndex(deleteNodeURL, nodeID)
578578
if err != nil {
579579
return errors.New(

‎services/dataproxy/internal/service/parse_schema.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func ParseSchemas(schemas []string) (*SchemasResponse, error) {
2323
// Include default schema and append provided schemas.
2424
schemaNames := append([]string{DefaultSchema}, schemas...)
2525
jsonSchemas := make([]string, len(schemaNames))
26-
baseURL := fmt.Sprintf("%s/v2/schemas", config.Conf.Library.InternalURL)
26+
baseURL := fmt.Sprintf("%s/v2/schemas", config.Values.Library.InternalURL)
2727

2828
for i, schema := range schemaNames {
2929
schemaURL := fmt.Sprintf("%s/%s", baseURL, schema)

‎services/dataproxy/internal/service/parse_schema_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func TestParseSchemas(t *testing.T) {
3333
defer mockServer.Close()
3434

3535
// Mock the config to use the test server's URL.
36-
config.Conf.Library.InternalURL = mockServer.URL
36+
config.Values.Library.InternalURL = mockServer.URL
3737

3838
// Define test cases.
3939
tests := []struct {

‎services/dataproxy/pkg/dataproxy/application.go

-49
This file was deleted.

‎services/dataproxy/pkg/dataproxy/cleanup.go

-4
This file was deleted.

‎services/dataproxy/pkg/dataproxy/envvar.go

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package dataproxy
2+
3+
import (
4+
"log"
5+
6+
"github.com/MurmurationsNetwork/MurmurationsServices/services/dataproxy/config"
7+
env "github.com/caarlos0/env/v10"
8+
)
9+
10+
func init() {
11+
err := env.Parse(&config.Values)
12+
if err != nil {
13+
log.Fatalf("Failed to decode environment variables: %s", err)
14+
}
15+
}

‎services/dataproxy/pkg/dataproxy/graceful_shutdown.go

-32
This file was deleted.

0 commit comments

Comments
 (0)