feat: implementing the database for tinyurls (#7)

* feat: schema design with mermaid

* feat: setting up postgres connection, schema and migrations

* fix: set fetch-depth and use checkout@v4

Author: sanathkumarbs

.github/workflows/makefile.yml

@@ -12,16 +12,28 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
     - uses: actions/setup-go@v4
       with:
         go-version: '^1.21.3' # The Go version to download (if necessary) and use.
 
     - name: Run OAPI-Generate
       run: make gen
 
-    - name: Run Lint
-      run: make lint
+    - name: Run Lint Go
+      run: make lint-go
+
+    - name: Find modified migrations
+      run: |
+          modified_migrations=$(git diff --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'migrations/*.sql')
+          echo "$modified_migrations"
+          echo "::set-output name=file_names::$modified_migrations"
+      id: modified-migrations
+    - uses: sbdchd/squawk-action@v1
+      with:
+        pattern: ${{ steps.modified-migrations.outputs.file_names }}
 
     - name: Run Format
       run: make fmt

Makefile

@@ -22,6 +22,12 @@ ifeq (, $(shell which goimports))
 	go install -v golang.org/x/tools/cmd/goimports@v$(GO_IMPORTS_VERSION)
 endif
 
+_install-squawk-cli: 
+ifeq (, $(shell which squawk))
+	npm install -g [email protected]
+endif
+
+
 all: gen fmt lint build
 
 ru: run-up
@@ -43,6 +49,11 @@ run-down-volumes:
 	docker compose -f devenv/docker-compose.yaml down --volumes
 	rm -rf tmp/*
 
+docker-prune:
+	$(info Running docker-prune...)
+	docker system prune -a --force
+	docker builder prune -a --force
+
 psql:
 	$(info Running exec into postgres...)
 	docker exec -it devenv-postgres-1 psql -d tiny -U postgres
@@ -70,9 +81,19 @@ fmt:
 
 lint: 
 	$(info Running lint...)
+	@$(MAKE) lint-go
+	@$(MAKE) lint-sql
+
+lint-go:
+	$(info Running lint-go...)
 	@$(MAKE) --quiet _install-golangci-lint	
 	time $(GO_LINT_BIN) run ./...
 
+lint-sql:
+	$(info Running lint-sql...)
+	@$(MAKE) --quiet _install-squawk-cli
+	time squawk internal/tiny/db/migrations/*
+
 gen:
 	@$(MAKE) generate
 

cmd/tiny/main.go

@@ -1,9 +1,71 @@
 package main
 
 import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/spf13/cobra"
 	"sanathk.com/tinyurl/internal/tiny"
+	"sanathk.com/tinyurl/internal/tiny/db"
+	"sanathk.com/tinyurl/pkg/postgres"
 )
 
 func main() {
-	tiny.Hello()
+	if err := cmd(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func cmd() error {
+	rootCmd := &cobra.Command{
+		Use:   "tiny",
+		Short: "TinyURL Service",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return run()
+		},
+	}
+	return rootCmd.Execute()
+}
+
+func run() error {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
+	defer cancel()
+
+	// TODO: remove hardcoded url, get from env vars and into a config
+	rawURL := "postgres://postgres:admin@postgres:5432/tiny"
+	pgURL, err := url.Parse(rawURL)
+	if err != nil {
+		return err
+	}
+
+	pg, err := postgres.NewPostgres(ctx, pgURL)
+	if err != nil {
+		return fmt.Errorf("creating postgres: %w", err)
+	}
+
+	err = pg.Migrate(db.Migrations, db.MigrationsPath)
+	if err != nil {
+		return err
+	}
+
+	svc, err := tiny.NewService(ctx, pg)
+	if err != nil {
+		return err
+	}
+
+	ticker := time.NewTicker(10 * time.Second)
+	for {
+		select {
+		case <-ticker.C:
+			svc.Hello()
+		case <-time.After(5 * time.Minute):
+			ticker.Stop()
+			return nil
+		}
+	}
 }

devenv/docker-compose.yaml

@@ -1,17 +1,25 @@
 services:
   tiny:
     working_dir: /app
-    image: devenv-tiny
-    build:
-      context: ./..
-      dockerfile: ./packaging/services/tiny/Dockerfile
+    image: tiny
     volumes:
       - ../:/app
+    depends_on: 
+      postgres:
+        condition: service_healthy
   postgres:
     image: postgres:16.1
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-d", "tiny"]
+      interval: 5s
+      # timeout for each health check request, should be lower than interval
+      timeout: 2s
+      retries: 5
+      start_period: 10s
     volumes:
       - postgres-persisted:/var/lib/postgresql/data
     environment:
+      POSTGRES_USER: postgres
       POSTGRES_PASSWORD: admin
       POSTGRES_DB: tiny
     ports:

docs/design.md

@@ -86,28 +86,57 @@ TBD: Use OpenAPI Schema definition if possible?
 * Methods: POST
 * Encoding: JSON
 
-Request Data:
+**Request Body**
+
+    TinyURLRequest:
+      type: object
+      required:
+        - original
+      properties:
+        original:
+          type: string
+          example: "https://www.sanathk.com/some/random/page?with=queryparams"
+        expiry:
+          type: string
+          format: date
+          example: "2025-01-01"
+
+**Response**
+
+    TinyURLResponse:
+      type: object
+      required:
+        - original
+        - tinyurl
+        - expiry
+      properties:
+        original:
+          type: string
+          example: "https://www.sanathk.com/some/random/page?with=queryparams"
+        tinyurl:
+          type: string
+          example: "https://tinyurl.sanath.com/randHash"
+        expiry:
+          type: string
+          format: date
+          example: "2025-01-01"
 
-	request: 
-		object: generateURLRequest
-
-	generateURLRequest:
-		required:
-			url: string
-		optional:
-			expiry: date 
+### Persistence
+The TinyURL data will be persisted in the `postgres` datastore for long-term storage. A table schema design will be added here for reference. 
+The go application will interact with the database using `pgx` and `sqlc`, the reasons behind choosing this combination is eloquently written [here](https://brandur.org/sqlc) and [here](https://jbrandhorst.com/post/postgres/).
 
-	response: 
-		object: generateURLResponse
+**Database Schema**
 
-		generateURLResponse:
-			required:
-				url: string
-				short: string
-				expiry: date 
+The datastore will keep a mapping of original URLs to their hashed URLs with their expirations.
 
-### Persistence
-The TinyURL data will be persisted in the `postgres` datastore for long-term storage. A table schema design will be added here for reference. 
+```mermaid
+erDiagram
+    tinyurls {
+        original varchar
+        hash varchar(PK)
+        expiry date
+    }
+```
 
 We will use various caching strategies to provide low latency environment, and, details around the same will be provided here as well. 
 

go.mod

@@ -4,26 +4,40 @@ go 1.21.3
 
 require (
 	github.com/getkin/kin-openapi v0.122.0
+	github.com/golang-migrate/migrate/v4 v4.17.0
+	github.com/jackc/pgx/v5 v5.5.1
 	github.com/labstack/echo/v4 v4.11.4
 	github.com/oapi-codegen/runtime v1.1.1
+	github.com/spf13/cobra v1.8.0
 )
 
 require (
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/google/uuid v1.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/invopop/yaml v0.2.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect