update to work with cosmosdb

cmd/mt-split-metrics-by-ttl/main.go

@@ -32,6 +32,8 @@ var (
 	cassandraUsername = flag.String("cassandra-username", "cassandra", "username for authentication")
 	cassandraPassword = flag.String("cassandra-password", "cassandra", "password for authentication")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	// hard coded to default because those have no effect in the case of this tool anyway
 	windowFactor             = 20
 	cassandraOmitReadTimeout = 60
@@ -70,7 +72,7 @@ func main() {
 		panic(fmt.Sprintf("Error creating directory: %s", err))
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, ttls)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, cassandraReadConcurrency, cassandraReadConcurrency, cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, windowFactor, cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, ttls, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to instantiate cassandra: %s", err))
 	}

cmd/mt-store-cat/main.go

@@ -53,6 +53,8 @@ var (
 	cassandraPassword        = flag.String("cassandra-password", "cassandra", "password for authentication")
 	cassandraOmitReadTimeout = flag.Int("cassandra-omit-read-timeout", 60, "if a read is older than this, it will directly be omitted without executing")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	// our own flags
 	from         = flag.String("from", "-24h", "get data from (inclusive). only for points and points-summary format")
 	to           = flag.String("to", "now", "get data until (exclusive). only for points and points-summary format")
@@ -163,7 +165,7 @@ func main() {
 		}
 	}
 
-	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
+	store, err := cassandra.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, *cassandraOmitReadTimeout, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		log.Fatal(4, "failed to initialize cassandra. %s", err)
 	}

cmd/mt-update-ttl/main.go

@@ -36,6 +36,8 @@ var (
 	cassandraUsername = flag.String("cassandra-username", "cassandra", "username for authentication")
 	cassandraPassword = flag.String("cassandra-password", "cassandra", "password for authentication")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	startTs    = flag.Int("start-timestamp", 0, "timestamp at which to start, defaults to 0")
 	endTs      = flag.Int("end-timestamp", math.MaxInt32, "timestamp at which to stop, defaults to int max")
 	numThreads = flag.Int("threads", 1, "number of workers to use to process data")
@@ -100,6 +102,7 @@ func NewCassandraStore() (*gocql.Session, error) {
 	cluster.ProtoVersion = *cqlProtocolVersion
 	cluster.Keyspace = *cassandraKeyspace
 	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cassandraRetries}
+	cluster.DisableInitialHostLookup = *cassandraDisableInitialHostLookup
 
 	switch *cassandraHostSelectionPolicy {
 	case "roundrobin":

cmd/mt-whisper-importer-writer/main.go

@@ -94,6 +94,8 @@ var (
 
 	cassandraSchemaFile = flag.String("cassandra-schema-file", "/etc/metrictank/schema-store-cassandra.toml", "File containing the needed schemas in case database needs initializing")
 
+	cassandraDisableInitialHostLookup = flag.Bool("cassandra-disable-initial-host-lookup", false, "instruct the driver to not attempt to get host info from the system.peers table")
+
 	gitHash = "(none)"
 )
 
@@ -153,7 +155,7 @@ func main() {
 		log.SetLevel(log.InfoLevel)
 	}
 
-	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil)
+	store, err := cassandraStore.NewCassandraStore(*cassandraAddrs, *cassandraKeyspace, *cassandraConsistency, *cassandraCaPath, *cassandraUsername, *cassandraPassword, *cassandraHostSelectionPolicy, *cassandraTimeout, *cassandraReadConcurrency, *cassandraReadConcurrency, *cassandraReadQueueSize, 0, *cassandraRetries, *cqlProtocolVersion, *windowFactor, 60, *cassandraSSL, *cassandraAuth, *cassandraHostVerification, *cassandraCreateKeyspace, *cassandraSchemaFile, nil, *cassandraDisableInitialHostLookup)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to initialize cassandra: %q", err))
 	}

docker/docker-chaos/metrictank.ini

@@ -72,6 +72,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 
 ## Profiling and logging ##
 
@@ -356,6 +358,8 @@ password = cassandra
 create-keyspace = false
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 
 ### in-memory only
 [memory-idx]

docker/docker-cluster/metrictank.ini

@@ -72,6 +72,8 @@ cassandra-auth = false
 cassandra-username = cassandra
 # password for authentication
 cassandra-password = cassandra
+# instruct the driver to not attempt to get host info from the system.peers table
+cassandra-disable-initial-host-lookup = false
 
 ## Profiling and logging ##
 
@@ -356,6 +358,8 @@ password = cassandra
 create-keyspace = false
 # File containing the needed schemas in case database needs initializing
 schema-file = /etc/metrictank/schema-idx-cassandra.toml
+# instruct the driver to not attempt to get host info from the system.peers table
+disable-initial-host-lookup = false
 
 ### in-memory only
 [memory-idx]

docker/docker-cosmosdb/docker-compose.yml

@@ -0,0 +1,72 @@
+version: '2'
+
+services:
+  metrictank:
+    hostname: metrictank
+    image: grafana/metrictank
+    ports:
+     - "6060:6060"
+     - "2003:2003"
+    volumes:
+      - ../../build/metrictank:/usr/bin/metrictank
+      - ./metrictank.ini:/etc/metrictank/metrictank.ini
+      - ./storage-schemas.conf:/etc/metrictank/storage-schemas.conf
+      - ./storage-aggregation.conf:/etc/metrictank/storage-aggregation.conf
+    environment:
+     WAIT_HOSTS: kafka:9092
+     WAIT_TIMEOUT: 60
+     MT_STATS_ADDR: graphite:2003
+     MT_CASSANDRA_ADDRS: <addr>
+     MT_CASSANDRA_SSL: "true"
+     MT_CASSANDRA_HOST_VERIFICATION: "false"
+     MT_CASSANDRA_AUTH: "true"
+     MT_CASSANDRA_USERNAME: <user>
+     MT_CASSANDRA_PASSWORD: <pass>
+     MT_CASSANDRA_IDX_HOSTS: <addr>
+     MT_CASSANDRA_IDX_NUM_CONNS: 1
+     MT_CASSANDRA_IDX_SSL: "true"
+     MT_CASSANDRA_IDX_HOST_VERIFICATION: "false"
+     MT_CASSANDRA_IDX_AUTH: "true"
+     MT_CASSANDRA_IDX_USERNAME: <user>
+     MT_CASSANDRA_IDX_PASSWORD: <pass>
+    links:
+     - kafka
+
+  kafka:
+    hostname: kafka
+    image: raintank/kafka:latest
+    environment:
+      ADVERTISED_HOST: kafka
+      NUM_PARTITIONS: 8
+    ports:
+      - "2181:2181"
+      - "9092:9092"
+      - "9999:9999"
+    volumes:
+      - /tmp/kafka-logs
+
+  graphite:
+    hostname: graphite
+    image: graphiteapp/graphite-statsd:latest
+    ports:
+     - "8080:80"
+
+  grafana:
+    hostname: grafana
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+
+  tsdb-gw:
+    hostname: tsdb-gw
+    image: raintank/tsdb-gw:latest
+    ports:
+      - "8081:80"
+    environment:
+      GW_GRAPHITE_URL: http://metrictank:6060
+      GW_METRICS_PUBLISH: "true"
+      GW_METRICS_KAFKA_COMP: snappy
+      GW_KAFKA_TCP_ADDR: kafka:9092
+      GW_STATS_ENABLED: "true"
+      GW_STATS_PREFIX: "tsdb-gw.stats.dev.tsdbgw_tsdb-gw_1"
+      GW_STATS_ADDR: "graphite:2003"