Merge pull request #79 from sdellang/httpserver

HTTP support #67

Author: ugol

go.mod

@@ -11,12 +11,14 @@ require (
 	github.com/spf13/viper v1.15.0
 	github.com/squeeze69/generacodicefiscale v1.0.3
 	golang.org/x/text v0.9.0
+	golang.org/x/tools v0.6.0
 )
 
 require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/heetch/avro v0.4.4 // indirect
 	github.com/iancoleman/orderedmap v0.2.0 // indirect

go.sum

@@ -1123,6 +1123,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -2077,6 +2079,7 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/cmd/createTopic.go

@@ -22,12 +22,10 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/ugol/jr/pkg/functions"
 	"github.com/ugol/jr/pkg/producers/kafka"
 )
 
-const DEFAULT_PARTITIONS = 6
-const DEFAULT_REPLICA = 3
-
 var createTopicCmd = &cobra.Command{
 	Use:   "createTopic [topic]",
 	Short: "simple command to create a Kafka Topic",
@@ -47,7 +45,7 @@ var createTopicCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(createTopicCmd)
-	createTopicCmd.Flags().IntP("partitions", "p", DEFAULT_PARTITIONS, "Number of partitions")
-	createTopicCmd.Flags().IntP("replica", "r", DEFAULT_REPLICA, "Replica Factor")
-	createTopicCmd.Flags().StringP("kafkaConfig", "F", KAFKA_CONFIG, "Kafka configuration")
+	createTopicCmd.Flags().IntP("partitions", "p", functions.DEFAULT_PARTITIONS, "Number of partitions")
+	createTopicCmd.Flags().IntP("replica", "r", functions.DEFAULT_REPLICA, "Replica Factor")
+	createTopicCmd.Flags().StringP("kafkaConfig", "F", functions.KAFKA_CONFIG, "Kafka configuration")
 }

pkg/cmd/list.go

@@ -23,12 +23,13 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/ugol/jr/pkg/functions"
 	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
+
+	"github.com/spf13/cobra"
+	"github.com/ugol/jr/pkg/functions"
 )
 
 var listCmd = &cobra.Command{
@@ -104,6 +105,6 @@ func isValidTemplate(t []byte) (bool, error) {
 
 func init() {
 	rootCmd.AddCommand(listCmd)
-	listCmd.Flags().String("templateDir", TEMPLATEDIR, "directory containing templates")
+	listCmd.Flags().String("templateDir", functions.TEMPLATEDIR, "directory containing templates")
 	listCmd.Flags().BoolP("fullPath", "f", false, "Print full path")
 }

pkg/cmd/run.go

@@ -21,26 +21,12 @@
 package cmd
 
 import (
-	"github.com/spf13/cobra"
-	"github.com/ugol/jr/pkg/functions"
 	"os"
-	"strings"
 	"time"
-)
 
-const NUM = 1
-const LOCALE = "US"
-const FREQUENCY = -1
-const DURATION = 0
-const TEMPLATEDIR = "$HOME/.jr/templates"
-const DEFAULT_KEY = "null"
-const DEFAULT_OUTPUT = "stdout"
-const DEFAULT_OUTPUT_TEMPLATE = "{{.V}}\n"
-const DEFAULT_SERIALIZER = "json-schema"
-const KAFKA_CONFIG = "./kafka/config.properties"
-const REGISTRY_CONFIG = "./kafka/registry.properties"
-const REDIS_TTL = 1 * time.Minute
-const REDIS_CONFIG = "./redis/config.json"
+	"github.com/spf13/cobra"
+	"github.com/ugol/jr/pkg/functions"
+)
 
 var runCmd = &cobra.Command{
 	Use:   "run [template]",
@@ -117,52 +103,41 @@ jr run --templateFileName ~/.jr/templates/net_device.tpl
 			Preload:          preload,
 			PreloadSize:      preloadSize,
 		}
-		configureJrContext(conf)
-		functions.DoTemplates(conf)
-	},
-}
 
-func configureJrContext(conf functions.Configuration) {
-	functions.Random.Seed(conf.Seed)
-	functions.JrContext.Num = conf.Num
-	functions.JrContext.Range = make([]int, conf.Num)
-	functions.JrContext.Frequency = conf.Frequency
-	functions.JrContext.Locale = strings.ToLower(conf.Locale)
-	functions.JrContext.Seed = conf.Seed
-	functions.JrContext.TemplateDir = conf.TemplateDir
-	functions.JrContext.CountryIndex = functions.IndexOf(strings.ToUpper(conf.Locale), "country")
+		functions.DoTemplates(conf, nil)
+	},
 }
 
 func init() {
 	rootCmd.AddCommand(runCmd)
-	runCmd.Flags().IntP("num", "n", NUM, "Number of elements to create for each pass")
-	runCmd.Flags().DurationP("frequency", "f", FREQUENCY, "how much time to wait for next generation pass")
-	runCmd.Flags().DurationP("duration", "d", DURATION, "If frequency is enabled, with Duration you can set a finite amount of time")
+	runCmd.Flags().IntP("num", "n", functions.NUM, "Number of elements to create for each pass")
+	runCmd.Flags().DurationP("frequency", "f", functions.FREQUENCY, "how much time to wait for next generation pass")
+	runCmd.Flags().DurationP("duration", "d", functions.DURATION, "If frequency is enabled, with Duration you can set a finite amount of time")
 
 	runCmd.Flags().Int64("seed", time.Now().UTC().UnixNano(), "Seed to init pseudorandom generator")
 
-	runCmd.Flags().String("templateDir", os.ExpandEnv(TEMPLATEDIR), "directory containing templates")
-	runCmd.Flags().StringP("kafkaConfig", "F", KAFKA_CONFIG, "Kafka configuration")
-	runCmd.Flags().String("registryConfig", REGISTRY_CONFIG, "Kafka configuration")
+	runCmd.Flags().String("templateDir", os.ExpandEnv(functions.TEMPLATEDIR), "directory containing templates")
+	runCmd.Flags().StringP("kafkaConfig", "F", functions.KAFKA_CONFIG, "Kafka configuration")
+	runCmd.Flags().String("registryConfig", functions.REGISTRY_CONFIG, "Kafka configuration")
 	runCmd.Flags().Bool("templateFileName", false, "If enabled, [template] must be a template file")
 	runCmd.Flags().Bool("template", false, "If enabled, [template] must be a string containing a template, to be embedded directly in the script")
 
 	runCmd.Flags().StringSliceP("preload", "p", []string{""}, "Array of templates to preload")
 	runCmd.Flags().IntSlice("preloadSize", []int{}, "Array of template sizes to preload")
 
-	runCmd.Flags().StringP("key", "k", DEFAULT_KEY, "A template to generate a key")
+	runCmd.Flags().StringP("key", "k", functions.DEFAULT_KEY, "A template to generate a key")
 	runCmd.Flags().StringSliceP("topic", "t", []string{"test"}, "Array of Kafka topic names")
 
 	runCmd.Flags().Bool("kcat", false, "If you want to pipe jr with kcat, use this flag: it is equivalent to --output stdout --outputTemplate '{{key}},{{value}}' --oneline")
-	runCmd.Flags().StringP("output", "o", DEFAULT_OUTPUT, "can be one of stdout, kafka, redis, mongo")
-	runCmd.Flags().String("outputTemplate", DEFAULT_OUTPUT_TEMPLATE, "Formatting of K,V on standard output")
+	runCmd.Flags().StringP("output", "o", functions.DEFAULT_OUTPUT, "can be one of stdout, kafka, redis, mongo")
+	runCmd.Flags().String("outputTemplate", functions.DEFAULT_OUTPUT_TEMPLATE, "Formatting of K,V on standard output")
 	runCmd.Flags().BoolP("oneline", "l", false, "strips /n from output, for example to be pipelined to tools like kcat")
 	runCmd.Flags().BoolP("autocreate", "a", false, "if enabled, autocreate topics")
-	runCmd.Flags().String("locale", LOCALE, "Locale")
+	runCmd.Flags().String("locale", functions.LOCALE, "Locale")
 
 	runCmd.Flags().BoolP("schemaRegistry", "s", false, "If you want to use Confluent Schema Registry")
-	runCmd.Flags().String("serializer", DEFAULT_SERIALIZER, "Type of serializer: json-schema, avro-generic, avro, protobuf")
-	runCmd.Flags().Duration("redis.ttl", REDIS_TTL, "If output is redis, ttl of the object")
-	runCmd.Flags().String("redisConfig", REDIS_CONFIG, "Redis configuration")
+	runCmd.Flags().String("serializer", functions.DEFAULT_SERIALIZER, "Type of serializer: json-schema, avro-generic, avro, protobuf")
+	runCmd.Flags().Duration("redis.ttl", functions.REDIS_TTL, "If output is redis, ttl of the object")
+	runCmd.Flags().String("redisConfig", functions.REDIS_CONFIG, "Redis configuration")
 
 }

pkg/cmd/server.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/spf13/cobra"
+	"github.com/ugol/jr/pkg/functions"
+)
+
+type JsonConfig struct {
+	URL         string `json:"url"`
+	Template    string `json:"template"`
+	Key         string `json:"key"`
+	TemplateDir string `json:"templatedir"`
+	Locale      string `json:"locale"`
+	Num         int    `json:"num"`
+}
+
+var savedConfigurations map[string]functions.Configuration
+var outputTemplate = "{{.K}},{{.V}}"
+
+var serverCmd = &cobra.Command{
+	Use:   "server",
+	Short: "Starts the jr http server",
+	Long: `Start the jr web server and takes the port flag. Default port is 8080.
+	To configure a new http generator jr expect a configuration post on the /jr/configure address, 
+	the post body should be a json with those parameters:
+	Name: the name of the configuration 
+	URL: the URL to expose in the form http://domain/jr/data/{URL}. URL should be unique among all the configuration otherwise an existing one will be updated.
+	Template: the name of the template to use to generate the data
+	Num: the number of element to create for each http get
+	You can do multiple configurations using different URLs and different templates`,
+	Run: func(cmd *cobra.Command, args []string) {
+		//initialise the global map for configurations
+		savedConfigurations = make(map[string]functions.Configuration)
+		port, err := cmd.Flags().GetInt("port")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		router := mux.NewRouter()
+		router.HandleFunc("/jr/configure", handleConfiguration).Methods("POST")
+		router.HandleFunc("/jr/data/{URL}", handleData).Methods("GET")
+
+		// Use the specified port, or default to 8080
+		addr := fmt.Sprintf(":%d", port)
+		log.Printf("Starting server on port %d\n", port)
+		log.Fatal(http.ListenAndServe(addr, router))
+	},
+}
+
+func handleConfiguration(w http.ResponseWriter, r *http.Request) {
+	// Parse request body
+	var jsonconf JsonConfig
+
+	err := json.NewDecoder(r.Body).Decode(&jsonconf)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if jsonconf.Key == "" {
+		jsonconf.Key = functions.DEFAULT_KEY
+	}
+
+	if jsonconf.TemplateDir == "" {
+		jsonconf.TemplateDir = functions.JrContext.TemplateDir
+	}
+
+	if jsonconf.Locale == "" {
+		jsonconf.Locale = functions.LOCALE
+	}
+
+	if jsonconf.Num == 0 {
+		jsonconf.Num = 1
+	}
+
+	conf := functions.Configuration{
+		TemplateNames:  []string{jsonconf.Template},
+		KeyTemplate:    jsonconf.Key,
+		OutputTemplate: outputTemplate,
+		Output:         "http",
+		Oneline:        true,
+		Locale:         jsonconf.Locale,
+		Num:            jsonconf.Num,
+		Frequency:      -1,
+		Duration:       0,
+		Seed:           time.Now().UTC().UnixNano(),
+		TemplateDir:    jsonconf.TemplateDir,
+		Autocreate:     false,
+		SchemaRegistry: false,
+		Serializer:     functions.DEFAULT_SERIALIZER,
+		Url:            jsonconf.URL,
+	}
+	// Save Configuration in the global map to handle it later
+	savedConfigurations[jsonconf.URL] = conf
+	// Respond with success message
+	response := fmt.Sprintf("Configuration %s saved successfully", jsonconf.URL)
+	w.Write([]byte(response))
+}
+
+func handleData(w http.ResponseWriter, r *http.Request) {
+	// Get the URL parameter from the request
+	vars := mux.Vars(r)
+	url := vars["URL"]
+	configuration, ok := savedConfigurations[url]
+
+	if ok {
+		functions.DoTemplates(configuration, &w)
+	} else {
+		w.WriteHeader(http.StatusNotFound)
+	}
+}
+
+func init() {
+	rootCmd.AddCommand(serverCmd)
+	serverCmd.Flags().IntP("port", "p", 8080, "Port for the server")
+}

pkg/cmd/show.go

@@ -22,11 +22,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
 	"log"
 	"os"
 	"runtime"
 	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/ugol/jr/pkg/functions"
 )
 
 var showCmd = &cobra.Command{
@@ -66,6 +68,6 @@ var showCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(showCmd)
-	showCmd.Flags().String("templateDir", TEMPLATEDIR, "directory containing templates")
+	showCmd.Flags().String("templateDir", functions.TEMPLATEDIR, "directory containing templates")
 	showCmd.Flags().BoolP("nocolor", "n", false, "disable colorized output")
 }

pkg/functions/constants.go

@@ -0,0 +1,20 @@
+package functions
+
+import "time"
+
+const NUM = 1
+const LOCALE = "US"
+const FREQUENCY = -1
+const DURATION = 0
+const TEMPLATEDIR = "$HOME/.jr/templates"
+const DEFAULT_KEY = "null"
+const DEFAULT_OUTPUT = "stdout"
+const DEFAULT_OUTPUT_TEMPLATE = "{{.V}}\n"
+const DEFAULT_SERIALIZER = "json-schema"
+const KAFKA_CONFIG = "./kafka/config.properties"
+const REGISTRY_CONFIG = "./kafka/registry.properties"
+const REDIS_TTL = 1 * time.Minute
+const REDIS_CONFIG = "./redis/config.json"
+const NUM_TEMPLATES = 5
+const DEFAULT_PARTITIONS = 6
+const DEFAULT_REPLICA = 3

pkg/functions/context.go

@@ -27,12 +27,6 @@ import (
 
 var JrContext Context
 
-const NUM = 1
-const NUM_TEMPLATES = 5
-const FREQUENCY = 0
-const DURATION = 0
-const TEMPLATEDIR = "$HOME/.jr/templates"
-
 type Configuration struct {
 	TemplateNames    []string
 	KeyTemplate      string
@@ -58,6 +52,7 @@ type Configuration struct {
 	Serializer       string
 	RedisTtl         time.Duration
 	RedisConfig      string
+	Url              string
 }
 
 // Context is the object passed on the templates which contains all the needed details.