Merge pull request #474 from RTradeLtd/refac

ReCAPTCHA + Rate Limit Refactor + Deduped Pin Cost Calculation Where Possible

.travis.yml

@@ -9,6 +9,7 @@ env:
     - SSL_MODE_DISABLE=true
   global:
     - GO111MODULE=on
+    - RECAPTCHA_KEY=cantbeblankortestsfail
     # decodes into SENGRID_API_KEY environment variable
     - secure: GXKJ+1Wa+pqUm1PKHJMbDrejSfIsqhsZ6l88i0paTvVou9T/mv03mx9hzrSLsYXFFVjinfbKysxMtEorTFDl0YBSNxuPc7eaiJinnoptubBh++bvEzF/A8wmWHgIBxroFIrK4SNsnreCXPqT1YBQZWi3JenMkzS68j1oa/uPa5ODZ3rjx1Wmu6hSANSZPqzOoGO6lnRP8G30oiFzbawEnB/52iBcHhPtffUihLFnn3k1wXyL8fpi4JYuyNQe84Br+w91KSS2nMmG+RVvJWumHNGXFyjEVv0n7HbirjCBI1iZo50bTdFtujkGSOCoHqM1hwa2yLWISipkb19Ls4eeWcWZBFpcdBhIIeHEmz9iCXxn0ksIGzuw00Xof/HTcWUpnzzQFq4E8iVE4tmMrFbYEcD5vcGB3S7bURjTD38uZ3/7I4Oyuo33/WQGdDNIzugU/dycGkSNzwTeIB3yJfpr/wbC6lU5RnADh5Ej5hGreAfkng/iuC9T36Fnn9u4fTocj++smOxZSjbbKsyekNNdadYmU0sfd6Ka9mFcv8H6sDs7zwqyZLTDu6P57CN5VFz4Yjc1emupFX5NFWor5jull9cr+ilOmfnBSCQPV0ArKrpKjrL9K9NJAhRKOqxmpLKx3pozGnrQMFKM0xsSuTcOCdyTdZF8JEQESeENHzwyoB4=
 install:

api/middleware/jwt.go

@@ -48,16 +48,21 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 					return "", false
 				}
 			}
+			// email enabled implies they have verified their email
+			if !usr.EmailEnabled {
+				return "", false
+			}
 			lAuth.Info("successful login", "username", usr.UserName)
 			return usr.UserName, true
 		},
 		Authorizator: func(userId string, c *gin.Context) bool {
 			// as a final security step, ensure that we can find the user in our database
 			userManager := models.NewUserManager(db)
-			if _, err := userManager.FindByUserName(userId); err != nil {
+			usr, err := userManager.FindByUserName(userId)
+			if err != nil {
 				return false
 			}
-			return true
+			return usr.EmailEnabled && usr.AccountEnabled
 		},
 		Unauthorized: func(c *gin.Context, code int, message string) {
 			l.Error("invalid login detected")

api/v2/api.go

@@ -9,22 +9,21 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/streadway/amqp"
-
+	"github.com/RTradeLtd/ChainRider-Go/dash"
+	"github.com/RTradeLtd/Temporal/queue"
 	"github.com/RTradeLtd/Temporal/rtfscluster"
 	pbLens "github.com/RTradeLtd/grpc/lensv2"
 	pbOrch "github.com/RTradeLtd/grpc/nexus"
 	pbSigner "github.com/RTradeLtd/grpc/pay"
-	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
-
 	"github.com/RTradeLtd/kaas/v2"
-	"go.uber.org/zap"
-
-	"github.com/RTradeLtd/ChainRider-Go/dash"
-	"github.com/RTradeLtd/Temporal/queue"
 	"github.com/RTradeLtd/rtfs/v2"
-
-	limit "github.com/aviddiviner/gin-limit"
+	recaptcha "github.com/ezzarghili/recaptcha-go"
+	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
+	"github.com/streadway/amqp"
+	"github.com/ulule/limiter/v3"
+	mgin "github.com/ulule/limiter/v3/drivers/middleware/gin"
+	"github.com/ulule/limiter/v3/drivers/store/memory"
+	"go.uber.org/zap"
 
 	"github.com/RTradeLtd/config/v2"
 	stats "github.com/semihalev/gin-stats"
@@ -62,8 +61,10 @@ type API struct {
 	dc          *dash.Client
 	queues      queues
 	service     string
-	cmcAPIKey   string
 	version     string
+
+	captcha        recaptcha.ReCAPTCHA
+	captchaEnabled bool
 }
 
 // Initialize is used ot initialize our API service. debug = true is useful
@@ -83,6 +84,8 @@ func Initialize(
 		err    error
 		router = gin.Default()
 	)
+	// if we dont set this, rate limiting wont work properly
+	router.ForwardedByClientIP = true
 	// update dev mode
 	dev = opts.DevMode
 	l = l.Named("api")
@@ -108,7 +111,14 @@ func Initialize(
 		return nil, err
 	}
 	api.version = version
-
+	if api.getCaptchaKey() != "" {
+		captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*20)
+		if err != nil {
+			return nil, err
+		}
+		api.captcha = captcha
+		api.captchaEnabled = true
+	}
 	// init routes
 	if err = api.setupRoutes(opts.DebugLogging); err != nil {
 		return nil, err
@@ -124,7 +134,6 @@ func new(cfg *config.TemporalConfig, router *gin.Engine, l *zap.SugaredLogger, c
 		dbm *database.Manager
 		err error
 	)
-
 	// set up database manager
 	dbm, err = database.New(cfg, database.Options{LogMode: debug})
 	if err != nil {
@@ -379,6 +388,18 @@ func (api *API) setupRoutes(debug bool) error {
 			return err
 		}
 	}
+	// make sure we dont throttle dev too much
+	var rateLimit string
+	if dev {
+		rateLimit = "100000-H"
+	} else {
+		rateLimit = fmt.Sprintf("%v-H", connLimit)
+	}
+	rate, err := limiter.NewRateFromFormatted(rateLimit)
+	if err != nil {
+		return err
+	}
+
 	// ensure we have valid cors configuration, otherwise default to allow all
 	var allowedOrigins []string
 	if len(api.cfg.API.Connection.CORS.AllowedOrigins) > 0 {
@@ -392,7 +413,7 @@ func (api *API) setupRoutes(debug bool) error {
 		// greater than what can be configured with HTTP Headers
 		xssMdlwr.RemoveXss(),
 		// rate limiting
-		limit.MaxAllowed(connLimit),
+		mgin.NewMiddleware(limiter.New(memory.NewStore(), rate)),
 		// security middleware
 		middleware.NewSecWare(dev),
 		// request id middleware
@@ -650,6 +671,13 @@ func (api *API) setupRoutes(debug bool) error {
 		ens.POST("/claim", api.ClaimENSName)
 		ens.POST("/update", api.UpdateContentHash)
 	}
+	if api.captchaEnabled {
+		recap := v2.Group("/captcha")
+		{
+			recap.POST("/verify", api.verifyCaptcha)
+		}
+
+	}
 
 	api.l.Info("Routes initialized")
 	return nil

api/v2/api_test.go

@@ -256,7 +256,8 @@ func Test_API_Setup(t *testing.T) {
 		args     args
 		wantCode int
 	}{
-		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 200},
+		// this test should fail due to invalid email
+		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 401},
 		{"Login-testuser", args{"POST", "/v2/auth/login", "testuser", "admin", ""}, 200},
 		// tests login via the email instead of using
 		{"Login-TestUser-Email", args{"POST", "/v2/auth/login", "[email protected]", "admin", ""}, 200},
@@ -272,7 +273,7 @@ func Test_API_Setup(t *testing.T) {
 			)
 			api.r.ServeHTTP(testRecorder, req)
 			if testRecorder.Code != tt.wantCode {
-				t.Fatalf("bad http status code from %s", tt.args.call)
+				t.Fatalf("bad http status code from %s. got %v, want %v", tt.args.call, testRecorder.Code, tt.wantCode)
 			}
 			bodBytes, err := ioutil.ReadAll(testRecorder.Result().Body)
 			if err != nil {

api/v2/routes_frontend.go

@@ -35,7 +35,7 @@ func (api *API) calculatePinCost(c *gin.Context) {
 		return
 	}
 	// calculate pin cost
-	totalCost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	totalCost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		Fail(c, err)

api/v2/routes_ipns.go

@@ -167,14 +167,8 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get size of object
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// get the cost of this object
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -184,7 +178,7 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		api.LogError(c, err, eh.InvalidBalanceError)(http.StatusPaymentRequired)
 		return
 	}
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -196,13 +190,13 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
 		CreditCost:       cost,
-		Size:             int64(stats.CumulativeSize),
+		Size:             int64(size),
 	}
 	// send message for processing
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log and return

api/v2/routes_ipns_test.go

@@ -188,6 +188,12 @@ func Test_API_Routes_IPNS_Pin(t *testing.T) {
 			fakeManager.ResolveReturnsOnCall(0, validResolveResult, nil)
 			fakeManager.StatReturnsOnCall(0, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
 			fakeManager.StatReturnsOnCall(1, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(2, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(3, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.RefsReturnsOnCall(0, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(1, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(2, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(3, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
 			var apiResp apiResponse
 			urlValues := url.Values{}
 			urlValues.Add("hold_time", tt.args.holdTime)

api/v2/routes_rtfs.go

@@ -54,14 +54,8 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get object size
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// determine cost of upload
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -72,7 +66,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		return
 	}
 	// update their data usage
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -83,15 +77,15 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		NetworkName:      "public",
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
-		Size:             int64(stats.CumulativeSize),
+		Size:             size,
 		CreditCost:       cost,
 		FileName:         c.PostForm("file_name"),
 	}
 	// sent pin message
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log success and return
@@ -381,7 +375,7 @@ func (api *API) extendPin(c *gin.Context) {
 		return
 	}
 	// calculate cost of hold time extension
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return

api/v2/routes_utils.go

@@ -17,6 +17,7 @@ import (
 	pb "github.com/RTradeLtd/grpc/krab"
 	"github.com/RTradeLtd/rtfs/v2"
 	"github.com/RTradeLtd/rtfs/v2/beam"
+	"github.com/ezzarghili/recaptcha-go"
 	"github.com/gin-gonic/gin"
 	gocid "github.com/ipfs/go-cid"
 )
@@ -388,3 +389,15 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
 	},
 	})
 }
+
+func (api *API) verifyCaptcha(c *gin.Context) {
+	err := api.captcha.VerifyWithOptions(
+		c.PostForm("g-recaptcha-response"),
+		// require a threshold of 0.8, default is 0.5
+		recaptcha.VerifyOption{Threshold: 0.8},
+	)
+	if err != nil {
+		Fail(c, errors.New("captcha validation failed"))
+	}
+	Respond(c, http.StatusOK, gin.H{"response": "captcha validation succeeded"})
+}

api/v2/utils.go

@@ -292,5 +292,12 @@ func (api *API) getCMCKey() string {
 	if os.Getenv("CMC_API") != "" {
 		return os.Getenv("CMC_API")
 	}
-	return api.cmcAPIKey
+	return api.cfg.APIKeys.CoinMarketCap
+}
+
+func (api *API) getCaptchaKey() string {
+	if os.Getenv("RECAPTCHA_KEY") != "" {
+		return os.Getenv("RECAPTCHA_KEY")
+	}
+	return api.cfg.APIKeys.ReCAPTCHA
 }

go.mod

@@ -5,9 +5,9 @@ go 1.14
 require (
 	github.com/RTradeLtd/ChainRider-Go v1.0.8
 	github.com/RTradeLtd/cmd/v2 v2.1.0
-	github.com/RTradeLtd/config/v2 v2.2.0
+	github.com/RTradeLtd/config/v2 v2.2.1-rc1
 	github.com/RTradeLtd/crypto/v2 v2.1.1
-	github.com/RTradeLtd/database/v2 v2.7.5-rc1
+	github.com/RTradeLtd/database/v2 v2.7.6-rc1
 	github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636
 	github.com/RTradeLtd/go-ipfs-api v0.0.0-20190522213636-8e3700e602fd
 	github.com/RTradeLtd/gpaginator v0.0.4
@@ -17,12 +17,12 @@ require (
 	github.com/RTradeLtd/rtns v0.0.19
 	github.com/appleboy/gin-jwt v2.3.1+incompatible
 	github.com/appleboy/gofight/v2 v2.1.1 // indirect
-	github.com/aviddiviner/gin-limit v0.0.0-20170918012823-43b5f79762c1
 	github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dvwright/xss-mw v0.0.0-20191029162136-7a0dab86d8f6
+	github.com/ezzarghili/recaptcha-go v4.0.0+incompatible
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/gcash/bchutil v0.0.0-20191012211144-98e73ec336ba
 	github.com/gcash/bchwallet v0.8.2
@@ -57,7 +57,6 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.2 // indirect
 	github.com/multiformats/go-multiaddr v0.2.0
 	github.com/multiformats/go-multihash v0.0.13
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 // indirect
 	github.com/prometheus/client_golang v1.4.1 // indirect
 	github.com/rs/cors v1.7.0
@@ -67,6 +66,7 @@ require (
 	github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94
 	github.com/stripe/stripe-go v60.0.1+incompatible
 	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c // indirect
+	github.com/ulule/limiter/v3 v3.5.0
 	go.bobheadxi.dev/res v0.2.0
 	go.bobheadxi.dev/zapx/zapx v0.6.8
 	go.bobheadxi.dev/zapx/ztest v0.6.4