Merge branch 'master' into javier/issue2798_custom_tokenizer_in_bulk_…

…loader

README.md

@@ -113,6 +113,10 @@ The Dgraph team maintain a number of [officially supported client libraries](htt
 * [https://github.com/AlexandreDaSilva/DgraphNet](https://github.com/AlexandreDaSilva/DgraphNet)
 * [https://github.com/MichaelJCompton/Dgraph-dotnet](https://github.com/MichaelJCompton/Dgraph-dotnet)
 
+**Dart**
+
+* [https://github.com/katutz/dgraph](https://github.com/katutz/dgraph)
+
 **Elixir**
 
 * [https://github.com/ospaarmann/exdgraph](https://github.com/ospaarmann/exdgraph)

contrib/config/kubernetes/dgraph-ha.yaml

@@ -61,6 +61,7 @@ spec:
     name: alpha-http
   selector:
     statefulset.kubernetes.io/pod-name: dgraph-alpha-0
+---
 apiVersion: v1
 kind: Service
 metadata:

dgraph/cmd/alpha/run.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -54,6 +55,11 @@ import (
 	hapi "google.golang.org/grpc/health/grpc_health_v1"
 )
 
+const (
+	tlsNodeCert = "node.crt"
+	tlsNodeKey  = "node.key"
+)
+
 var (
 	bindall bool
 	tlsConf x.TLSHelperConfig
@@ -120,6 +126,12 @@ they form a Raft group and provide synchronous replication.
 		"If set, all Alter requests to Dgraph would need to have this token."+
 			" The token can be passed as follows: For HTTP requests, in X-Dgraph-AuthToken header."+
 			" For Grpc, in auth-token key in the context.")
+	flag.String("hmac_secret_file", "", "The file storing the HMAC secret"+
+		" that is used for signing the JWT. Enterprise feature.")
+	flag.Duration("access_jwt_ttl", 6*time.Hour, "The TTL for the access jwt. "+
+		"Enterprise feature.")
+	flag.Duration("refresh_jwt_ttl", 30*24*time.Hour, "The TTL for the refresh jwt. "+
+		"Enterprise feature.")
 	flag.Float64P("lru_mb", "l", -1,
 		"Estimated memory the LRU cache can take. "+
 			"Actual usage by the process would be more than specified here.")
@@ -380,7 +392,7 @@ var shutdownCh chan struct{}
 func run() {
 	bindall = Alpha.Conf.GetBool("bindall")
 
-	edgraph.SetConfiguration(edgraph.Options{
+	opts := edgraph.Options{
 		BadgerTables: Alpha.Conf.GetString("badger.tables"),
 		BadgerVlog:   Alpha.Conf.GetString("badger.vlog"),
 
@@ -390,7 +402,22 @@ func run() {
 		Nomutations:    Alpha.Conf.GetBool("nomutations"),
 		AuthToken:      Alpha.Conf.GetString("auth_token"),
 		AllottedMemory: Alpha.Conf.GetFloat64("lru_mb"),
-	})
+	}
+
+	secretFile := Alpha.Conf.GetString("hmac_secret_file")
+	if secretFile != "" {
+		hmacSecret, err := ioutil.ReadFile(secretFile)
+		if err != nil {
+			glog.Fatalf("Unable to read HMAC secret from file: %v", secretFile)
+		}
+
+		opts.HmacSecret = hmacSecret
+		opts.AccessJwtTtl = Alpha.Conf.GetDuration("access_jwt_ttl")
+		opts.RefreshJwtTtl = Alpha.Conf.GetDuration("refresh_jwt_ttl")
+
+		glog.Info("HMAC secret loaded successfully.")
+	}
+	edgraph.SetConfiguration(opts)
 
 	ips, err := parseIPsFromString(Alpha.Conf.GetString("whitelist"))
 	x.Check(err)
@@ -406,7 +433,7 @@ func run() {
 		MaxRetries:          Alpha.Conf.GetInt("max_retries"),
 	}
 
-	x.LoadTLSConfig(&tlsConf, Alpha.Conf)
+	x.LoadTLSConfig(&tlsConf, Alpha.Conf, tlsNodeCert, tlsNodeKey)
 	tlsConf.ClientAuth = Alpha.Conf.GetString("tls_client_auth")
 
 	setupCustomTokenizers()

dgraph/cmd/alpha/run_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/query"
 	"github.com/dgraph-io/dgraph/schema"
+	"github.com/dgraph-io/dgraph/x"
 )
 
 var q0 = `
@@ -272,7 +273,12 @@ func TestDeletePredicate(t *testing.T) {
 
 	output, err = runQuery(`schema{}`)
 	require.NoError(t, err)
-	require.JSONEq(t, `{"data":{"schema":[{"predicate":"_predicate_","type":"string","list":true},{"predicate":"age","type":"default"},{"predicate":"name","type":"string","index":true, "tokenizer":["term"]}]}}`, output)
+	require.JSONEq(t, `{"data":{"schema":[`+
+		`{"predicate":"_predicate_","type":"string","list":true},`+
+		`{"predicate":"age","type":"default"},`+
+		x.AclPredsJson+","+
+		`{"predicate":"name","type":"string","index":true, "tokenizer":["term"]}`+
+		`]}}`, output)
 
 	output, err = runQuery(q1)
 	require.NoError(t, err)
@@ -1273,7 +1279,10 @@ func TestListTypeSchemaChange(t *testing.T) {
 	q = `schema{}`
 	res, err = runQuery(q)
 	require.NoError(t, err)
-	require.JSONEq(t, `{"data":{"schema":[{"predicate":"_predicate_","type":"string","list":true},{"predicate":"occupations","type":"string"}]}}`, res)
+	require.JSONEq(t, `{"data":{"schema":[`+
+		`{"predicate":"_predicate_","type":"string","list":true},`+
+		x.AclPredsJson+","+
+		`{"predicate":"occupations","type":"string"}]}}`, res)
 
 }
 
@@ -1364,7 +1373,9 @@ func TestDropAll(t *testing.T) {
 	output, err = runQuery(q3)
 	require.NoError(t, err)
 	require.JSONEq(t,
-		`{"data":{"schema":[{"predicate":"_predicate_","type":"string","list":true}]}}`, output)
+		`{"data":{"schema":[{"predicate":"_predicate_","type":"string","list":true},`+
+			x.AclPredsJson+
+			`]}}`, output)
 
 	// Reinstate schema so that we can re-run the original query.
 	err = alterSchemaWithRetry(s)

dgraph/cmd/live/run.go

@@ -34,8 +34,6 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 
 	"github.com/dgraph-io/badger"
@@ -48,11 +46,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const (
-	tlsLiveCert = "client.live.crt"
-	tlsLiveKey  = "client.live.key"
-)
-
 type options struct {
 	files               string
 	schemaFile          string
@@ -239,34 +232,6 @@ func (l *loader) processFile(ctx context.Context, file string) error {
 	return nil
 }
 
-func setupConnection(host string, insecure bool) (*grpc.ClientConn, error) {
-	if insecure {
-		return grpc.Dial(host,
-			grpc.WithDefaultCallOptions(
-				grpc.MaxCallRecvMsgSize(x.GrpcMaxSize),
-				grpc.MaxCallSendMsgSize(x.GrpcMaxSize)),
-			grpc.WithInsecure(),
-			grpc.WithBlock(),
-			grpc.WithTimeout(10*time.Second))
-	}
-
-	tlsConf.ConfigType = x.TLSClientConfig
-	tlsConf.Cert = filepath.Join(tlsConf.CertDir, tlsLiveCert)
-	tlsConf.Key = filepath.Join(tlsConf.CertDir, tlsLiveKey)
-	tlsCfg, _, err := x.GenerateTLSConfig(tlsConf)
-	if err != nil {
-		return nil, err
-	}
-
-	return grpc.Dial(host,
-		grpc.WithDefaultCallOptions(
-			grpc.MaxCallRecvMsgSize(x.GrpcMaxSize),
-			grpc.MaxCallSendMsgSize(x.GrpcMaxSize)),
-		grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)),
-		grpc.WithBlock(),
-		grpc.WithTimeout(10*time.Second))
-}
-
 func fileList(files string) []string {
 	if len(files) == 0 {
 		return []string{}
@@ -285,7 +250,7 @@ func setup(opts batchMutationOptions, dc *dgo.Dgraph) *loader {
 	kv, err := badger.Open(o)
 	x.Checkf(err, "Error while creating badger KV posting store")
 
-	connzero, err := setupConnection(opt.zero, true)
+	connzero, err := x.SetupConnection(opt.zero, &tlsConf)
 	x.Checkf(err, "Unable to connect to zero, Is it running at %s?", opt.zero)
 
 	alloc := xidmap.New(
@@ -329,7 +294,7 @@ func run() error {
 		ignoreIndexConflict: Live.Conf.GetBool("ignore_index_conflict"),
 		authToken:           Live.Conf.GetString("auth_token"),
 	}
-	x.LoadTLSConfig(&tlsConf, Live.Conf)
+	x.LoadTLSConfig(&tlsConf, Live.Conf, x.TlsClientCert, x.TlsClientKey)
 	tlsConf.ServerName = Live.Conf.GetString("tls_server_name")
 
 	go http.ListenAndServe("localhost:6060", nil)
@@ -345,7 +310,7 @@ func run() error {
 	ds := strings.Split(opt.dgraph, ",")
 	var clients []api.DgraphClient
 	for _, d := range ds {
-		conn, err := setupConnection(d, !tlsConf.CertRequired)
+		conn, err := x.SetupConnection(d, &tlsConf)
 		x.Checkf(err, "While trying to setup connection to Dgraph alpha.")
 		defer conn.Close()
 

dgraph/cmd/root.go

@@ -29,6 +29,7 @@ import (
 	"github.com/dgraph-io/dgraph/dgraph/cmd/live"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/version"
 	"github.com/dgraph-io/dgraph/dgraph/cmd/zero"
+	"github.com/dgraph-io/dgraph/ee/acl/cmd"
 	"github.com/dgraph-io/dgraph/x"
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
@@ -86,7 +87,7 @@ func init() {
 
 	var subcommands = []*x.SubCommand{
 		&bulk.Bulk, &cert.Cert, &conv.Conv, &live.Live, &alpha.Alpha, &zero.Zero,
-		&version.Version, &debug.Debug,
+		&version.Version, &debug.Debug, &acl.CmdAcl,
 	}
 	for _, sc := range subcommands {
 		RootCmd.AddCommand(sc.Cmd)

dgraph/docker-compose-single.yml

@@ -0,0 +1,86 @@
+# Docker compose file for testing. Use it with:
+# docker-compose up --force-recreate
+# This would pick up dgraph binary from $GOPATH.
+
+version: "3.5"
+services:
+  zero1:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg0.1
+    working_dir: /data/dg0.1
+    ports:
+      - 5080:5080
+      - 6080:6080
+    environment:
+      - GOPATH=$GOPATH
+    command: /gobin/dgraph zero --my=zero1:5080 --replicas 3 --idx 1 --bindall --expose_trace --profile_mode block --block_rate 10 --logtostderr -v=2 --jaeger.collector http://jaeger:14268
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+
+  dg1:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg1
+    working_dir: /data/dg1
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+    ports:
+      - 8180:8180
+      - 9180:9180
+      - 9999:9999
+    security_opt:
+      - seccomp:unconfined
+    command: /gobin/dgraph alpha --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 --expose_trace -o 100 --trace 1.0 --profile_mode block --block_rate 10 --logtostderr -v=3 --jaeger.collector http://jaeger:14268
+    #command: /gobin/dlv --listen=:9999 --headless=true --api-version=2 --init $GOPATH/src/github.com/dgraph-io/dgraph/dgraph/dlv.init exec /gobin/dgraph -- alpha --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 -o 100 --expose_trace  --trace 1.0 --profile_mode block --block_rate 10 --logtostderr -v=3 --jaeger.collector http://jaeger:14268
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    container_name: jaeger
+    hostname: jaeger
+    ports:
+      - "5775:5775/udp"
+      - "6831:6831/udp"
+      - "6832:6832/udp"
+      - "5778:5778"
+      - "16686:16686"
+      - "14268:14268"
+      - "9411:9411"
+    environment:
+      - COLLECTOR_ZIPKIN_HTTP_PORT=9411
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter
+    container_name: node-exporter
+    pid: "host"
+    volumes:
+      - type: bind
+        source: /
+        target: /host
+        read_only: true
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    hostname: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - type: bind
+        source: $GOPATH/src/github.com/dgraph-io/dgraph/dgraph/prometheus.yml
+        target: /etc/prometheus/prometheus.yml
+        read_only: true
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    hostname: grafana
+    ports:
+     - "3000:3000"  
+
+
+

dgraph/prometheus.yml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 15s
+scrape_configs:
+  - job_name: 'dgraph'
+    scrape_interval: 15s
+    metrics_path: '/debug/prometheus_metrics'
+    static_configs:
+      - targets:
+        - 'bank-dg1:8180'
+        - 'bank-dg0.1:6080'
+  - job_name: 'node-exporter'
+    scrape_interval: 15s
+    metrics_path: '/metrics'
+    static_configs:
+      - targets:
+        - 'node-exporter:9100'
+

dgraph/run-single.sh

@@ -0,0 +1,2 @@
+md5sum ~/go/bin/dgraph; go build . && go install . && md5sum dgraph ~/go/bin/dgraph
+docker-compose -f docker-compose-single.yml down && docker-compose -f docker-compose-single.yml up --force-recreate --remove-orphans

edgraph/access.go

@@ -0,0 +1,34 @@
+// +build oss
+
+/*
+ * Copyright 2018 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edgraph
+
+import (
+	"context"
+
+	"github.com/dgraph-io/dgo/protos/api"
+	"github.com/dgraph-io/dgraph/x"
+	"github.com/golang/glog"
+)
+
+func (s *Server) Login(ctx context.Context,
+	request *api.LoginRequest) (*api.Response, error) {
+
+	glog.Warningf("Login failed: %s", x.ErrNotSupported)
+	return &api.Response{}, x.ErrNotSupported
+}