analytics feature

.env.example

@@ -1,3 +1,6 @@
 ENVIRONMENT=
 BOT_TOKEN=
 SENTRY_DSN=
+REDIS_URL=
+DATABASE_URL=
+TZ=

.github/workflows/deploy.yml

@@ -10,15 +10,24 @@ jobs:
     name: CI
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    container: golang:1.17.3
+    services:
+      db:
+        image: postgres:14-alpine
+        ports:
+          - 5432:5432
+        emv:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+          POSTGRES_DB: captcha
+      cache:
+        image: redis:6-alpine
+        ports:
+          - 6379:6379
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.17.x
-
       - name: Installling dependencies
         run: go mod download
 
@@ -29,6 +38,9 @@ jobs:
         run: go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
         env:
           ENVIRONMENT: development
+          DATABASE_URL: postgres://postgres:password@db:5432/captcha?sslmode=disable
+          REDIS_URL: redis://@cache:6379/
+          TZ: UTC
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v1
@@ -52,6 +64,7 @@ jobs:
       - uses: superfly/flyctl-actions@master
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          CERT_URL: ${{ secrets.CERT_URL }}
         with:
           args: 'deploy'
 

.github/workflows/pr.yml

@@ -1,4 +1,4 @@
-name: Deploy
+name: Check
 
 on:
   pull_request:
@@ -10,15 +10,24 @@ jobs:
     name: CI
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    container: golang:1.17.3
+    services:
+      db:
+        image: postgres:14-alpine
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+          POSTGRES_DB: captcha
+      cache:
+        image: redis:6-alpine
+        ports:
+          - 6379:6379
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.17.x
-
       - name: Installling dependencies
         run: go mod download
 
@@ -29,6 +38,9 @@ jobs:
         run: go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
         env:
           ENVIRONMENT: development
+          DATABASE_URL: postgres://postgres:password@db:5432/captcha?sslmode=disable
+          REDIS_URL: redis://@cache:6379/
+          TZ: UTC
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v1

Dockerfile

@@ -1,5 +1,7 @@
 FROM golang:1.17.1-buster
 
+RUN curl --create-dirs -o $HOME/.postgresql/root.crt -O ${CERT_URL}
+
 WORKDIR /usr/app
 
 COPY . .
@@ -8,4 +10,4 @@ RUN go mod download
 
 RUN go build main.go
 
-CMD [ "./main" ]
+CMD [ "./main" ]

analytics/analytics.go

@@ -0,0 +1,22 @@
+package analytics
+
+// On this package we have 2 main keys on redis:
+// analytics:hour and analytics:counter
+
+import (
+	"github.com/allegro/bigcache/v3"
+	"github.com/bsm/redislock"
+	"github.com/getsentry/sentry-go"
+	"github.com/go-redis/redis/v8"
+	"github.com/jmoiron/sqlx"
+	tb "gopkg.in/tucnak/telebot.v2"
+)
+
+type Dependency struct {
+	Memory *bigcache.BigCache
+	Redis  *redis.Client
+	Locker *redislock.Client
+	Bot    *tb.Bot
+	Logger *sentry.Client
+	DB     *sqlx.DB
+}

analytics/analytics_test.go

@@ -0,0 +1,101 @@
+package analytics_test
+
+import (
+	"context"
+	"database/sql"
+	"log"
+	"os"
+	"teknologi-umum-bot/analytics"
+	"testing"
+	"time"
+
+	"github.com/allegro/bigcache/v3"
+	"github.com/go-redis/redis/v8"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+)
+
+var db *sqlx.DB
+var cache *redis.Client
+var memory *bigcache.BigCache
+
+func TestMain(m *testing.M) {
+	Setup()
+
+	defer Teardown()
+
+	os.Exit(m.Run())
+}
+
+func Cleanup() {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	c, err := db.Connx(ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer c.Close()
+
+	tx, err := c.BeginTxx(ctx, &sql.TxOptions{})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = tx.ExecContext(ctx, "TRUNCATE TABLE analytics")
+	if err != nil {
+		tx.Rollback()
+		log.Fatal(err)
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		tx.Rollback()
+		log.Fatal(err)
+	}
+
+	err = cache.FlushAll(ctx).Err()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = memory.Reset()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func Setup() {
+	dbURL, err := pq.ParseURL(os.Getenv("DATABASE_URL"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	db, err = sqlx.Open("postgres", dbURL)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	redisURL, err := redis.ParseURL(os.Getenv("REDIS_URL"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	cache = redis.NewClient(redisURL)
+
+	memory, err = bigcache.NewBigCache(bigcache.DefaultConfig(time.Hour * 1))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = analytics.MustMigrate(db)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func Teardown() {
+	memory.Close()
+	cache.Close()
+	db.Close()
+}

analytics/incr.go

@@ -0,0 +1,101 @@
+package analytics
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/aldy505/decrr"
+	"github.com/go-redis/redis/v8"
+)
+
+func (d *Dependency) IncrementUsrDB(ctx context.Context, users []UserMap) error {
+	c, err := d.DB.Connx(ctx)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	t, err := c.BeginTxx(ctx, &sql.TxOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, user := range users {
+		now := time.Now()
+
+		_, err = t.ExecContext(
+			ctx,
+			`INSERT INTO analytics
+				(user_id, username, display_name, counter, created_at, joined_at, updated_at)
+				VALUES
+				($1, $2, $3, $4, $5, $6, $7)
+				ON CONFLICT (user_id)
+				DO UPDATE
+				SET counter = (SELECT counter FROM analytics WHERE user_id = $1)+$4,
+					username = $2,
+					display_name = $3,
+					updated_at = $7`,
+			user.UserID,
+			user.Username,
+			user.DisplayName,
+			user.Counter,
+			now,
+			now,
+			now,
+		)
+		if err != nil {
+			t.Rollback()
+			return err
+		}
+	}
+
+	err = t.Commit()
+	if err != nil {
+		t.Rollback()
+		return decrr.Wrap(err)
+	}
+
+	return nil
+}
+
+func (d *Dependency) IncrementUsrRedis(ctx context.Context, user UserMap) error {
+	p := d.Redis.TxPipeline()
+	defer p.Close()
+
+	usrID := strconv.FormatInt(user.UserID, 10)
+
+	exists, err := d.Redis.HExists(ctx, "analytics:"+usrID, "count").Result()
+	if err != nil && !errors.Is(err, redis.Nil) {
+		return decrr.Wrap(err)
+	}
+
+	if !exists {
+		p.HSet(
+			ctx,
+			"analytics:"+usrID,
+			"counter",
+			0,
+			"username",
+			user.Username,
+			"display_name",
+			user.DisplayName,
+		)
+	}
+
+	// Per Redis' documentation, INCR will create a new key
+	// if the named key does not exists in the first place.
+	p.HIncrBy(ctx, "analytics:"+usrID, "counter", 1)
+
+	// Add the user ID into the Sets of users
+	p.SAdd(ctx, "analytics:users", usrID)
+
+	_, err = p.Exec(ctx)
+	if err != nil {
+		return decrr.Wrap(err)
+	}
+
+	return nil
+}