Refactor

.github/workflows/gobuild.yml

@@ -9,10 +9,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.20
-
+        go-version: '1.20'
+        check-latest: true
+    - run: go version     
     - name: Build
       run: |
         go mod tidy

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.20 AS builder
 WORKDIR /
 COPY . .
-RUN CGO_ENABLED=1 go build -o quackpipe quackpipe.go
+RUN CGO_ENABLED=1 go build -o quackpipe .
 RUN strip quackpipe
 RUN apt update && apt install -y libgrpc-dev
 

controller/root/root.go

@@ -0,0 +1,31 @@
+package root
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"quackpipe/model"
+	"quackpipe/service/db"
+	"quackpipe/utils"
+)
+
+func QueryOperation(flagInformation *model.CommandLineFlags, query string, r *http.Request, defaultPath string, defaultFormat string, defaultParams string) (string, error) {
+	// auth to hash based temp file storage
+	username, password, ok := r.BasicAuth()
+	hashdb := ""
+	if ok && len(password) > 0 {
+		hash := sha256.Sum256([]byte(username + password))
+		hashdb = fmt.Sprintf("%s/%x.db", defaultPath, hash)
+	}
+	rows, duration, err := db.Quack(*flagInformation, query, false, defaultParams, hashdb)
+	if err != nil {
+		return "", err
+	} else {
+		result, err := utils.ConversationOfRows(rows, defaultFormat, duration)
+		if err != nil {
+			return "", err
+		}
+		return result, nil
+	}
+
+}

go.mod

@@ -2,7 +2,10 @@ module quackpipe
 
 go 1.20
 
-require github.com/marcboeker/go-duckdb v1.7.0
+require (
+	github.com/gorilla/mux v1.8.1
+	github.com/marcboeker/go-duckdb v1.7.0
+)
 
 require (
 	github.com/apache/arrow/go/v14 v14.0.2 // indirect

go.sum

@@ -1,11 +1,15 @@
 github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw=
 github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
 github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
@@ -17,14 +21,19 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -33,4 +42,6 @@ golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
+gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handler/api_handler.go

@@ -0,0 +1,77 @@
+package handlers
+
+import (
+	_ "embed"
+	"fmt"
+	"io"
+	"net/http"
+	"quackpipe/controller/root"
+	"quackpipe/model"
+	"quackpipe/utils"
+)
+
+//go:embed play.html
+var staticPlay string
+
+type Handler struct {
+	FlagInformation *model.CommandLineFlags
+}
+
+func (u *Handler) Handlers(w http.ResponseWriter, r *http.Request) {
+	var bodyBytes []byte
+	var query string
+	var err error
+	defaultFormat := *u.FlagInformation.Format
+	defaultParams := *u.FlagInformation.Params
+	defaultPath := *u.FlagInformation.DBPath
+	// handle query parameter
+	if r.URL.Query().Get("query") != "" {
+		query = r.URL.Query().Get("query")
+	} else if r.Body != nil {
+		bodyBytes, err = io.ReadAll(r.Body)
+		if err != nil {
+			fmt.Printf("Body reading error: %v", err)
+			return
+		}
+		defer r.Body.Close()
+		query = string(bodyBytes)
+	}
+
+	switch r.Header.Get("Accept") {
+	case "application/json":
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	case "application/xml":
+		w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+	case "text/css":
+		w.Header().Set("Content-Type", "text/css; charset=utf-8")
+	default:
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	}
+	// format handling
+	if r.URL.Query().Get("default_format") != "" {
+		defaultFormat = r.URL.Query().Get("default_format")
+	}
+	// param handling
+	if r.URL.Query().Get("default_params") != "" {
+		defaultParams = r.URL.Query().Get("default_params")
+	}
+
+	// extract FORMAT from query and override the current `default_format`
+	cleanQuery, format := utils.ExtractAndRemoveFormat(query)
+	if len(format) > 0 {
+		query = cleanQuery
+		defaultFormat = format
+	}
+	if len(query) == 0 {
+		_, _ = w.Write([]byte(staticPlay))
+
+	} else {
+		result, err := root.QueryOperation(u.FlagInformation, query, r, defaultPath, defaultFormat, defaultParams)
+		if err != nil {
+			_, _ = w.Write([]byte(err.Error()))
+		} else {
+			_, _ = w.Write([]byte(result))
+		}
+	}
+
+}

main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"quackpipe/model"
+	"quackpipe/router"
+	"quackpipe/utils"
+)
+
+// initFlags initializes the command line flags
+func initFlags() *model.CommandLineFlags {
+
+	appFlags := &model.CommandLineFlags{}
+	appFlags.Host = flag.String("host", "0.0.0.0", "API host. Default 0.0.0.0")
+	appFlags.Port = flag.String("port", "8123", "API port. Default 8123")
+	appFlags.Format = flag.String("format", "JSONCompact", "API port. Default JSONCompact")
+	appFlags.Params = flag.String("params", "", "DuckDB optional parameters. Default to none.")
+	appFlags.DBPath = flag.String("dbpath", "/tmp/", "DuckDB DB storage path. Default to /tmp/")
+	appFlags.Stdin = flag.Bool("stdin", false, "STDIN query. Default false")
+	appFlags.Alias = flag.Bool("alias", true, "Built-in CH Aliases. Default true")
+	flag.Parse()
+
+	return appFlags
+}
+
+var appFlags *model.CommandLineFlags
+
+func main() {
+	appFlags = initFlags()
+	if *appFlags.Stdin {
+		rows, duration, format, err := utils.ReadFromScanner(*appFlags)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		results, err := utils.ConversationOfRows(rows, format, duration)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		} else {
+			fmt.Println(results)
+		}
+
+	} else {
+		r := router.NewRouter(appFlags)
+		fmt.Printf("QuackPipe API Running: %s:%s\n", *appFlags.Host, *appFlags.Port)
+		if err := http.ListenAndServe(*appFlags.Host+":"+*appFlags.Port, r); err != nil {
+			panic(err)
+		}
+
+	}
+
+}

model/flags.go

@@ -0,0 +1,12 @@
+package model
+
+// params for Flags
+type CommandLineFlags struct {
+	Host   *string `json:"host"`
+	Port   *string `json:"port"`
+	Stdin  *bool   `json:"stdin"`
+	Alias  *bool   `json:"alias"`
+	Format *string `json:"format"`
+	Params *string `json:"params"`
+	DBPath *string `json:"dbpath"`
+}

model/internal.go

@@ -0,0 +1,23 @@
+package model
+
+// Metadata is the metadata for a column
+type Metadata struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
+// Statistics is the statistics for a query
+type Statistics struct {
+	Elapsed   float64 `json:"elapsed"`
+	RowsRead  int     `json:"rows_read"`
+	BytesRead int     `json:"bytes_read"`
+}
+
+// OutputJSON is the JSON output for a query
+type OutputJSON struct {
+	Meta                   []Metadata      `json:"meta"`
+	Data                   [][]interface{} `json:"data"`
+	Rows                   int             `json:"rows"`
+	RowsBeforeLimitAtLeast int             `json:"rows_before_limit_at_least"`
+	Statistics             Statistics      `json:"statistics"`
+}