From 0b63bcdad5d5f18d6c505b3e7a9c645283a1658a Mon Sep 17 00:00:00 2001 From: Shaun Verch Date: Tue, 19 May 2020 16:36:16 -0400 Subject: [PATCH] Make it easier to test multiple endpoints with aya This cleans up the interface of are you alive to pass it a config file with endpoint configuration rather than command line options that make are you alive construct the connection strings in different ways. This makes it easier to test a larger number of endpoints, so that, for example, you could easily configure it to write to one endpoint while reading from 11 others, at any rate you want. Also removes environment from the labels since that could be added later by the deployment, and makes the rate configuration a true rate instead of a delay. Signed-off-by: Shaun Verch --- examples/are-you-alive/README.md | 8 + examples/are-you-alive/build/dev/Dockerfile | 19 +- examples/are-you-alive/build/dev/reflex.conf | 5 +- .../are-you-alive/build/release/Dockerfile | 3 +- .../are-you-alive/cmd/are-you-alive/main.go | 255 +++++++++--------- examples/are-you-alive/docker-compose.yml | 2 - examples/are-you-alive/go.mod | 9 +- examples/are-you-alive/go.sum | 29 ++ examples/are-you-alive/pkg/client/client.go | 67 +++-- 9 files changed, 215 insertions(+), 182 deletions(-) diff --git a/examples/are-you-alive/README.md b/examples/are-you-alive/README.md index 65bff90cad6..7da5460c64d 100644 --- a/examples/are-you-alive/README.md +++ b/examples/are-you-alive/README.md @@ -73,6 +73,14 @@ After you run docker compose, navigate to `http://localhost:9090` to see Prometheus and `http://localhost:8080/metrics` to see the raw metrics being exported. +## Test Specific Tablet Types + +See [this vitess +documentation](https://vitess.io/docs/user-guides/faq/#how-do-i-choose-between-master-vs-replica-for-queries) +for how to target specific tablet types. In the configuration file you'll want +to, for example, put "@master" or "@replica" on the ends of your connection +strings. + ## Push to Registry If you have push access to the [planetscale public diff --git a/examples/are-you-alive/build/dev/Dockerfile b/examples/are-you-alive/build/dev/Dockerfile index 3b079cfcf8a..7a6a06d590b 100644 --- a/examples/are-you-alive/build/dev/Dockerfile +++ b/examples/are-you-alive/build/dev/Dockerfile @@ -1,13 +1,6 @@ -# Use a [multi stage -# build](https://docs.docker.com/develop/develop-images/multistage-build/) to -# build [reflex](https://github.com/cespare/reflex), a tool that will allow us -# to automatically rerun the project when any files change. -FROM golang:1.12.5 AS build -# Build reflex as a static binary (CGO_ENABLED=0) so we can run it in our final -# container. -RUN CGO_ENABLED=0 go get -v github.com/cespare/reflex - -FROM golang:1.12.5-alpine AS runtime -COPY --from=build /go/bin/reflex /go/bin/reflex -COPY reflex.conf / -ENTRYPOINT ["/go/bin/reflex", "-c", "/reflex.conf"] +FROM golang:1.14 +RUN go get -v github.com/cespare/reflex +COPY reflex.conf /reflex.conf +COPY entrypoint.sh /entrypoint.sh +COPY endpoints.yaml /endpoints.yaml +ENTRYPOINT ["/entrypoint.sh"] diff --git a/examples/are-you-alive/build/dev/reflex.conf b/examples/are-you-alive/build/dev/reflex.conf index 9e92ab1a7ca..6985f66d979 100644 --- a/examples/are-you-alive/build/dev/reflex.conf +++ b/examples/are-you-alive/build/dev/reflex.conf @@ -1,2 +1,5 @@ +# Get all dependencies when go.mod changes. +-r '(\.mod$)' -s -- go get vitess.io/vitess/examples/are-you-alive/cmd/are-you-alive/... + # Rerun "go run" every time a ".go" file changes. --r '(\.go$)' -s -- go run vitess.io/vitess/examples/are-you-alive/cmd/are-you-alive --initialize +-r '(\.go$)' -s -- go run vitess.io/vitess/examples/are-you-alive/cmd/are-you-alive --initialize --endpoints_config /endpoints.yaml diff --git a/examples/are-you-alive/build/release/Dockerfile b/examples/are-you-alive/build/release/Dockerfile index 67310da895a..55b8c9f66a1 100644 --- a/examples/are-you-alive/build/release/Dockerfile +++ b/examples/are-you-alive/build/release/Dockerfile @@ -1,5 +1,6 @@ -FROM golang:1.12.5 AS build +FROM golang:1.14 AS build COPY . /go/src/vitess.io/vitess/examples/are-you-alive +RUN go get vitess.io/vitess/examples/are-you-alive/cmd/are-you-alive/... RUN CGO_ENABLED=0 go install vitess.io/vitess/examples/are-you-alive/cmd/are-you-alive FROM debian:stretch-slim AS runtime diff --git a/examples/are-you-alive/cmd/are-you-alive/main.go b/examples/are-you-alive/cmd/are-you-alive/main.go index fd747468558..4744c68efd4 100644 --- a/examples/are-you-alive/cmd/are-you-alive/main.go +++ b/examples/are-you-alive/cmd/are-you-alive/main.go @@ -2,6 +2,7 @@ package main import ( "database/sql" + "errors" "flag" "fmt" "github.com/go-sql-driver/mysql" @@ -9,12 +10,13 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "go.uber.org/ratelimit" + "gopkg.in/yaml.v2" "math/rand" "net/http" "os" "os/signal" "sync" - "time" "vitess.io/vitess/examples/are-you-alive/pkg/client" ) @@ -38,10 +40,10 @@ var ( ) ) -func writeNextRecord(environmentName string, connectionString string) error { +func writeNextRecord(connectionString string) error { // 1. Call monitored client - err := client.Write(environmentName, connectionString, maxPage) + err := client.Write(connectionString, maxPage) if err != nil { // Check to see if this is a duplicate key error. We've seen this // sometimes happen, and when it does this client app gets stuck in an @@ -70,7 +72,7 @@ func writeNextRecord(environmentName string, connectionString string) error { return nil } -func readRandomRecord(environmentName string, connectionString string) error { +func readRandomRecord(connectionString string) error { // 1. Pick Random Number Between "minPage" and "maxPage" if minPage == maxPage { @@ -80,7 +82,7 @@ func readRandomRecord(environmentName string, connectionString string) error { page := (rand.Int() % (maxPage - minPage)) + minPage // 2. Read Record - readID, readMsg, err := client.Read(environmentName, connectionString, page) + readID, readMsg, err := client.Read(connectionString, page) if err != nil { if err == sql.ErrNoRows { // This races with deletion, but if our page is greater than minPage @@ -125,10 +127,10 @@ func readRandomRecord(environmentName string, connectionString string) error { return nil } -func runCount(environmentName string, connectionString string) error { +func runCount(connectionString string) error { // 1. Run Count - count, err := client.Count(environmentName, connectionString) + count, err := client.Count(connectionString) if err != nil { logrus.WithError(err).Error("Error counting records") return err @@ -141,7 +143,7 @@ func runCount(environmentName string, connectionString string) error { return nil } -func deleteLastRecordIfNecessary(environmentName string, connectionString string) error { +func deleteLastRecordIfNecessary(connectionString string) error { // 1. Compare "maxPage" - "minPage" to Desired Dataset Size if (maxPage - minPage) < *datasetSize { @@ -153,7 +155,7 @@ func deleteLastRecordIfNecessary(environmentName string, connectionString string }).Debug("Deleting last record") // 2. Delete Record If We Are Above Desired Size - err := client.Delete(environmentName, connectionString, minPage) + err := client.Delete(connectionString, minPage) if err != nil { logrus.WithError(err).Error("Error deleting record") return err @@ -165,33 +167,27 @@ func deleteLastRecordIfNecessary(environmentName string, connectionString string } var ( - mysqlConnectionString = flag.String( - "mysql_connection_string", "", "Connection string for db to test") prometheusMetricsAddress = flag.String( "prometheus_metrics_address", ":8080", "Address on which to serve prometheus metrics") - debug = flag.Bool("debug", false, "Enable debug logging") - useVtgate = flag.Bool("vtgate", false, "Using vtgate (for @master and @replica)") - readFromReplica = flag.Bool("replica", false, "Read from replica") - readFromReadOnly = flag.Bool("rdonly", false, "Read from rdonly") - initialize = flag.Bool("initialize", false, "Initialize database (for testing)") - sleepTime = flag.Int("delay", 1*1000*1000*1000, "Delay in nanoseconds between ops") - datasetSize = flag.Int("dataset_size", 10, "Number of total records in database") - environmentName = flag.String("environment_name", "prod", - "Environment the database is deployed in that this client is pointing at") + debug = flag.Bool("debug", false, "Enable debug logging") + useVtgate = flag.Bool("vtgate", false, "Using vtgate (for @master and @replica)") + initialize = flag.Bool("initialize", false, "Initialize database (for testing)") + datasetSize = flag.Int("dataset_size", 10, "Number of total records in database") + endpointsConfigFilename = flag.String("endpoints_config", "", "Endpoint and load configuration.") ) type runner struct { - connString string - envName string - fn func(string, string) error - errMessage string - sleepTime time.Duration + connString string + fn func(string) error + errMessage string + opsPerSecond int } func (r *runner) run() { + rl := ratelimit.New(r.opsPerSecond) for { - time.Sleep(r.sleepTime) - err := r.fn(r.envName, r.connString) + _ = rl.Take() + err := r.fn(r.connString) if err != nil { logrus.WithError(err).Error(r.errMessage) } @@ -216,6 +212,35 @@ func runPrometheus() { logrus.Fatal(http.ListenAndServe(*prometheusMetricsAddress, nil)) } +func loadEndpointsConfig(endpointsConfigFilename string) (endpointsConfig, error) { + if endpointsConfigFilename == "" { + return endpointsConfig{}, errors.New("You must pass an endpoints configuration file") + } + + f, err := os.Open(endpointsConfigFilename) + if err != nil { + return endpointsConfig{}, err + } + defer f.Close() + + var cfg endpointsConfig + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&cfg) + if err != nil { + return endpointsConfig{}, err + } + return cfg, nil +} + +type endpointsConfig struct { + Endpoints []struct { + ConnectionString string `yaml:"connectionString"` + TargetCountsPerSecond int `yaml:"targetCountsPerSecond"` + TargetQueriesPerSecond int `yaml:"targetQueriesPerSecond"` + TargetWritesPerSecond int `yaml:"targetWritesPerSecond"` + } `yaml:"endpoints"` +} + func main() { // 0. Handle Arguments @@ -224,24 +249,31 @@ func main() { logrus.SetLevel(logrus.DebugLevel) } logrus.WithFields(logrus.Fields{ - "mysqlConnectionString": *mysqlConnectionString, + "endpointsConfigFilename": *endpointsConfigFilename, "prometheusMetricsAddress": *prometheusMetricsAddress, "debug": *debug, }).Debug("Command line arguments") - connectionString := "" - if *mysqlConnectionString != "" { - connectionString = *mysqlConnectionString - } else if os.Getenv("MYSQL_CONN_STRING") != "" { - connectionString = os.Getenv("MYSQL_CONN_STRING") + var endpoints endpointsConfig + endpoints, err := loadEndpointsConfig(*endpointsConfigFilename) + if err != nil { + logrus.WithError(err).Error("Failed to load endpoints config.") + os.Exit(1) } - masterConnectionString := connectionString - replicaConnectionString := connectionString - rdonlyConnectionString := connectionString - // When using vtgate, we want to append @master and @replica to the DSN, but - // this will fail against normal mysql which we're using for testing. See: - // https://vitess.io/docs/user-guides/faq/#how-do-i-choose-between-master-vs-replica-for-queries - if *useVtgate { + + // 0. Set Up Prometheus Metrics + logrus.Info("Prometheus Go") + go runPrometheus() + + // 1. Pass "interpolateParams" + for _, endpoint := range endpoints.Endpoints { + logrus.WithFields(logrus.Fields{ + "connectionString": endpoint.ConnectionString, + "targetCountsPerSecond": endpoint.TargetCountsPerSecond, + "targetQueriesPerSecond": endpoint.TargetQueriesPerSecond, + "targetWritesPerSecond": endpoint.TargetWritesPerSecond, + }).Info("Found endpoint configuration") + // We need to pass interpolateParams when using a vtgate because // prepare is not supported. // @@ -249,101 +281,72 @@ func main() { // - https://github.com/go-sql-driver/mysql/blob/master/README.md#interpolateparams // - https://github.com/src-d/go-mysql-server/issues/428 // - https://github.com/vitessio/vitess/pull/3862 - masterConnectionString = fmt.Sprintf("%s@master?interpolateParams=true", connectionString) - replicaConnectionString = fmt.Sprintf("%s@replica?interpolateParams=true", connectionString) - rdonlyConnectionString = fmt.Sprintf("%s@rdonly?interpolateParams=true", connectionString) + endpoint.ConnectionString = fmt.Sprintf("%s?interpolateParams=true", endpoint.ConnectionString) } - fmt.Println("masterConnectionString:", masterConnectionString) - fmt.Println("replicaConnectionString:", replicaConnectionString) - fmt.Println("rdonlyConnectionString:", rdonlyConnectionString) - - // 1. Set Up Prometheus Metrics - logrus.Info("Prometheus Go") - go runPrometheus() // 2. Initialize Database - logrus.Info("Initializing database") - // For local testing, does not initialize vschema - if *initialize { - client.InitializeDatabase(*environmentName, masterConnectionString, "are_you_alive_messages") + for _, endpoint := range endpoints.Endpoints { + logrus.WithFields(logrus.Fields{ + "connectionString": endpoint.ConnectionString, + "targetCountsPerSecond": endpoint.TargetCountsPerSecond, + "targetQueriesPerSecond": endpoint.TargetQueriesPerSecond, + "targetWritesPerSecond": endpoint.TargetWritesPerSecond, + }).Info("Found endpoint configuration") + + if endpoint.TargetWritesPerSecond > 0 { + logrus.Info("Initializing database") + // For local testing, does not initialize vschema + if *initialize { + client.InitializeDatabase(endpoint.ConnectionString, "are_you_alive_messages") + } + client.WipeTestTable(endpoint.ConnectionString, "are_you_alive_messages") + } } - client.WipeTestTable(*environmentName, masterConnectionString, "are_you_alive_messages") - // 3. Start goroutines to do various things + // 3. Start Client Goroutines logrus.Info("Starting client goroutines") - deleter := runner{ - connString: masterConnectionString, - envName: *environmentName, - fn: deleteLastRecordIfNecessary, - errMessage: "Recieved error deleting last record", - sleepTime: time.Duration(*sleepTime), - } - go deleter.run() - writer := runner{ - connString: masterConnectionString, - envName: *environmentName, - fn: writeNextRecord, - errMessage: "Recieved error writing next record", - sleepTime: time.Duration(*sleepTime), - } - go writer.run() - reader := runner{ - connString: masterConnectionString, - envName: *environmentName, - fn: readRandomRecord, - errMessage: "Recieved error reading record", - sleepTime: time.Duration(*sleepTime), - } - go reader.run() - counter := runner{ - connString: masterConnectionString, - envName: *environmentName, - fn: runCount, - errMessage: "Recieved error running count", - sleepTime: time.Duration(*sleepTime), - } - go counter.run() - - // Only bother starting a replica reader/counter if we are using a vtgate - // and actually are asking to do replica reads - if *useVtgate && *readFromReplica { - replicaReader := runner{ - connString: replicaConnectionString, - envName: *environmentName, - fn: readRandomRecord, - errMessage: "Recieved error reading record from replica", - sleepTime: time.Duration(*sleepTime), - } - go replicaReader.run() - replicaRowCounter := runner{ - connString: replicaConnectionString, - envName: *environmentName, - fn: runCount, - errMessage: "Recieved error running count on replica", - sleepTime: time.Duration(*sleepTime), + for _, endpoint := range endpoints.Endpoints { + logrus.WithFields(logrus.Fields{ + "connectionString": endpoint.ConnectionString, + "targetCountsPerSecond": endpoint.TargetCountsPerSecond, + "targetQueriesPerSecond": endpoint.TargetQueriesPerSecond, + "targetWritesPerSecond": endpoint.TargetWritesPerSecond, + }).Info("Found endpoint configuration") + + if endpoint.TargetWritesPerSecond > 0 { + writer := runner{ + connString: endpoint.ConnectionString, + fn: writeNextRecord, + errMessage: "Recieved error writing next record", + opsPerSecond: endpoint.TargetWritesPerSecond, + } + go writer.run() + deleter := runner{ + connString: endpoint.ConnectionString, + fn: deleteLastRecordIfNecessary, + errMessage: "Recieved error deleting last record", + opsPerSecond: 100, // This is based on target "dataset_size", and will not make a query if not needed. TODO: Actually tune this in a reasonable way after redesigning the schema? + } + go deleter.run() } - go replicaRowCounter.run() - } - - // Only bother starting a rdonly reader/counter if we are using a vtgate and - // actually are asking to do rdonly reads - if *useVtgate && *readFromReadOnly { - replicaReader := runner{ - connString: rdonlyConnectionString, - envName: *environmentName, - fn: readRandomRecord, - errMessage: "Recieved error reading record from rdonly", - sleepTime: time.Duration(*sleepTime), + if endpoint.TargetQueriesPerSecond > 0 { + reader := runner{ + connString: endpoint.ConnectionString, + fn: readRandomRecord, + errMessage: "Recieved error reading record", + opsPerSecond: endpoint.TargetQueriesPerSecond, + } + go reader.run() } - go replicaReader.run() - replicaRowCounter := runner{ - connString: rdonlyConnectionString, - envName: *environmentName, - fn: runCount, - errMessage: "Recieved error running count on rdonly", - sleepTime: time.Duration(*sleepTime), + if endpoint.TargetCountsPerSecond > 0 { + counter := runner{ + connString: endpoint.ConnectionString, + fn: runCount, + errMessage: "Recieved error running count", + opsPerSecond: endpoint.TargetCountsPerSecond, + } + go counter.run() } - go replicaRowCounter.run() } logrus.Info("Press Ctrl+C to end\n") diff --git a/examples/are-you-alive/docker-compose.yml b/examples/are-you-alive/docker-compose.yml index 603353dcb89..700d840b0c2 100644 --- a/examples/are-you-alive/docker-compose.yml +++ b/examples/are-you-alive/docker-compose.yml @@ -7,8 +7,6 @@ services: volumes: - .:/go/src/vitess.io/vitess/examples/are-you-alive working_dir: /go/src/vitess.io/vitess/examples/are-you-alive - environment: - MYSQL_CONN_STRING: root:mysql@tcp(mysql)/testfixture depends_on: - mysql ports: diff --git a/examples/are-you-alive/go.mod b/examples/are-you-alive/go.mod index 9bb69e1307b..79c3fc251bf 100644 --- a/examples/are-you-alive/go.mod +++ b/examples/are-you-alive/go.mod @@ -1,9 +1,12 @@ module vitess.io/vitess/examples/are-you-alive -go 1.12 +go 1.14 require ( github.com/go-sql-driver/mysql v1.5.0 - github.com/prometheus/client_golang v1.4.1 - github.com/sirupsen/logrus v1.4.2 + github.com/prometheus/client_golang v1.6.0 + github.com/sirupsen/logrus v1.6.0 + go.uber.org/ratelimit v0.1.0 + golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 // indirect + gopkg.in/yaml.v2 v2.2.5 ) diff --git a/examples/are-you-alive/go.sum b/examples/are-you-alive/go.sum index 7c05c83b99b..409a0ce1d02 100644 --- a/examples/are-you-alive/go.sum +++ b/examples/are-you-alive/go.sum @@ -22,6 +22,13 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -30,6 +37,8 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -48,6 +57,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= +github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= @@ -59,14 +70,20 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= +go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -78,15 +95,27 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= diff --git a/examples/are-you-alive/pkg/client/client.go b/examples/are-you-alive/pkg/client/client.go index e4f35d3e882..95a564af678 100644 --- a/examples/are-you-alive/pkg/client/client.go +++ b/examples/are-you-alive/pkg/client/client.go @@ -22,70 +22,70 @@ var ( Help: "Latency to recieve a count error", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) readErrorLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_read_error_latency_seconds", Help: "Latency to recieve a read error", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) deleteErrorLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_delete_error_latency_seconds", Help: "Latency to recieve a delete error", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name"}, + []string{"database_name"}, ) writeErrorLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_write_error_latency_seconds", Help: "Latency to recieve a write error", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name"}, + []string{"database_name"}, ) connectErrorLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_connect_error_latency_seconds", Help: "Latency to recieve a connect error", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) countLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_count_latency_seconds", Help: "Time it takes to count to the database", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) readLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_read_latency_seconds", Help: "Time it takes to read to the database", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) deleteLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_delete_latency_seconds", Help: "Time it takes to delete to the database", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name"}, + []string{"database_name"}, ) writeLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_write_latency_seconds", Help: "Time it takes to write to the database", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name"}, + []string{"database_name"}, ) connectLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "are_you_alive_connect_latency_seconds", Help: "Time it takes to connect to the database", Buckets: defaultBuckets, }, - []string{"environment_name", "database_name", "tablet_type"}, + []string{"database_name", "tablet_type"}, ) ) @@ -115,7 +115,7 @@ func ParseTabletType(connectionString string) string { } } -func openDatabase(environmentName string, connectionString string) (*sql.DB, error) { +func openDatabase(connectionString string) (*sql.DB, error) { databaseName := ParseDBName(connectionString) tabletType := ParseTabletType(connectionString) // NOTE: This is probably not measuring open connections. I think they @@ -125,9 +125,8 @@ func openDatabase(environmentName string, connectionString string) (*sql.DB, err // happening locally. We should just see everything complete within // milliseconds. labels := prometheus.Labels{ - "environment_name": environmentName, - "database_name": databaseName, - "tablet_type": tabletType} + "database_name": databaseName, + "tablet_type": tabletType} connectTimer := prometheus.NewTimer(connectLatency.With(labels)) connectErrorTimer := prometheus.NewTimer(connectErrorLatency.With(labels)) db, err := sql.Open("mysql", connectionString) @@ -144,13 +143,13 @@ func openDatabase(environmentName string, connectionString string) (*sql.DB, err // given tableName, and recreate it with the schema that the rest of the client // expects. This is not something any normal client would do but is convenient // here because we are just using this client for monitoring. -func InitializeDatabase(environmentName string, connectionString string, tableName string) error { +func InitializeDatabase(connectionString string, tableName string) error { // 0. Create logger log := logrus.WithField("connection_string", connectionString) // 1. Open client to database - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") return err @@ -176,13 +175,13 @@ func InitializeDatabase(environmentName string, connectionString string, tableNa // everything in the table given by tableName because this client expects the // table to be empty. No client would normally do this, but it's convenient for // testing. -func WipeTestTable(environmentName string, connectionString string, tableName string) error { +func WipeTestTable(connectionString string, tableName string) error { // 0. Create logger log := logrus.WithField("connection_string", connectionString) // 1. Open client to database - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") return err @@ -198,14 +197,14 @@ func WipeTestTable(environmentName string, connectionString string, tableName st // Write will write the record given by page to the test table in the database // referenced by connectionString. -func Write(environmentName string, connectionString string, page int) error { +func Write(connectionString string, page int) error { // 0. Create logger log := logrus.WithField("connection_string", connectionString) // 1. Open client to database databaseName := ParseDBName(connectionString) - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") return err @@ -214,8 +213,7 @@ func Write(environmentName string, connectionString string, page int) error { // 2. Write record labels := prometheus.Labels{ - "environment_name": environmentName, - "database_name": databaseName} + "database_name": databaseName} writeTimer := prometheus.NewTimer(writeLatency.With(labels)) writeErrorTimer := prometheus.NewTimer(writeErrorLatency.With(labels)) if _, err := db.Exec("INSERT INTO are_you_alive_messages (page, message) VALUES (?, ?)", page, "foo"); err != nil { @@ -229,14 +227,14 @@ func Write(environmentName string, connectionString string, page int) error { // Read will read the record given by page from the test table in the database // referenced by connectionString. -func Read(environmentName string, connectionString string, page int) (int, string, error) { +func Read(connectionString string, page int) (int, string, error) { // 0. Create logger log := logrus.WithField("connection_string", connectionString) // 1. Open client to database databaseName := ParseDBName(connectionString) - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") return 0, "", err @@ -246,9 +244,8 @@ func Read(environmentName string, connectionString string, page int) (int, strin // 2. Read record tabletType := ParseTabletType(connectionString) labels := prometheus.Labels{ - "environment_name": environmentName, - "database_name": databaseName, - "tablet_type": tabletType} + "database_name": databaseName, + "tablet_type": tabletType} readTimer := prometheus.NewTimer(readLatency.With(labels)) readErrorTimer := prometheus.NewTimer(readErrorLatency.With(labels)) row := db.QueryRow("SELECT * FROM are_you_alive_messages WHERE page=?", page) @@ -275,14 +272,14 @@ func Read(environmentName string, connectionString string, page int) (int, strin // Count will count all the documents in the test table in the database // referenced by connectionString. -func Count(environmentName string, connectionString string) (int, error) { +func Count(connectionString string) (int, error) { // 0. Create logger log := logrus.WithField("connection_string", connectionString) // 1. Open client to database databaseName := ParseDBName(connectionString) - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") return 0, err @@ -292,9 +289,8 @@ func Count(environmentName string, connectionString string) (int, error) { // 2. Run Count tabletType := ParseTabletType(connectionString) labels := prometheus.Labels{ - "environment_name": environmentName, - "database_name": databaseName, - "tablet_type": tabletType} + "database_name": databaseName, + "tablet_type": tabletType} countTimer := prometheus.NewTimer(countLatency.With(labels)) countErrorTimer := prometheus.NewTimer(countErrorLatency.With(labels)) row := db.QueryRow("SELECT COUNT(*) FROM are_you_alive_messages") @@ -313,7 +309,7 @@ func Count(environmentName string, connectionString string) (int, error) { // Delete will delete the record given by page from the test table in the // database referenced by connectionString. -func Delete(environmentName string, connectionString string, page int) error { +func Delete(connectionString string, page int) error { // 0. Create logger log := logrus.WithFields(logrus.Fields{ @@ -324,11 +320,10 @@ func Delete(environmentName string, connectionString string, page int) error { // 1. Open client to database databaseName := ParseDBName(connectionString) labels := prometheus.Labels{ - "environment_name": environmentName, - "database_name": databaseName} + "database_name": databaseName} deleteTimer := prometheus.NewTimer(deleteLatency.With(labels)) deleteErrorTimer := prometheus.NewTimer(deleteErrorLatency.With(labels)) - db, err := openDatabase(environmentName, connectionString) + db, err := openDatabase(connectionString) if err != nil { log.WithError(err).Error("Error opening database") deleteErrorTimer.ObserveDuration()