Tablet move and group removal (#2880)

* send error when removing last node in a group would orphan tablets.

* reusing backup test data, moved to top systest dir.

* tests for tablet move and node removal.

* removed cnt reset.

* restructured tests to fail in order, and observing all errors from http content.

* created new group-delete dir for custom cluster testing

* docker-compose file for group-delete tests

dgraph/cmd/zero/zero.go

@@ -377,6 +377,10 @@ func (s *Server) removeNode(ctx context.Context, nodeId uint64, groupId uint32)
 	if _, ok := s.state.Groups[groupId].Members[nodeId]; !ok {
 		return x.Errorf("No node with nodeId %d found in group %d", nodeId, groupId)
 	}
+	if len(s.state.Groups[groupId].Members) == 1 && len(s.state.Groups[groupId].Tablets) > 0 {
+		return x.Errorf("Move all tablets from group %d before removing the last node", groupId)
+	}
+
 	return s.Node.proposeAndWait(ctx, zp)
 }
 

ee/backup/systest/backup_test.go

@@ -59,7 +59,7 @@ func BackupSetup(t *testing.T, c *dgo.Dgraph) {
 	require.NoError(t, err)
 	require.NoError(t, c.Alter(ctx, &api.Operation{Schema: string(schema)}))
 
-	fp, err := os.Open(`data/goldendata_export.rdf.gz`)
+	fp, err := os.Open(`../../../systest/data/goldendata_export.rdf.gz`)
 	x.Check(err)
 	defer fp.Close()
 

systest/group-delete/README.md

@@ -0,0 +1,14 @@
+# Group Delete Test
+
+This test runs a scenario where nodes are removed from groups to make zero delete those
+groups. At every stage we check the zero state to make sure the nodes and groups are deleted.
+
+Process:
+
+1. Bring up a cluster with 3 groups, 1 node each.
+2. Delete node from group 3
+3. Check that zero deleted group 3
+4. Run a query to test that the cluster is viable
+5. Delete node from group 2
+6. Check that zero deleted group 2
+7. Run a query to test that the cluster is viable

systest/group-delete/docker-compose.yml

@@ -0,0 +1,70 @@
+# This file sets up the cluster required by the tests in this directory.
+version: "3.5"
+services:
+  zero1:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg0.1
+    working_dir: /data/dg0.1
+    ports:
+      - 5080:5080
+      - 6080:6080
+    labels:
+      cluster: group-delete-test
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+    command: /gobin/dgraph zero --my=zero1:5080 --bindall --logtostderr
+
+  dg1:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg1
+    working_dir: /data/dg1
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+    ports:
+      - 8180:8180
+      - 9180:9180
+    labels:
+      cluster: group-delete-test
+    command: /gobin/dgraph alpha --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 -o 100 --logtostderr
+
+  dg2:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg2
+    working_dir: /data/dg2
+    depends_on:
+      - dg1
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+    ports:
+      - 8182:8182
+      - 9182:9182
+    labels:
+      cluster: group-delete-test
+    command: /gobin/dgraph alpha --my=dg2:7182 --lru_mb=1024 --zero=zero1:5080 -o 102 --logtostderr
+
+  dg3:
+    image: dgraph/dgraph:latest
+    container_name: bank-dg3
+    working_dir: /data/dg3
+    depends_on:
+      - dg2
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+    ports:
+      - 8183:8183
+      - 9183:9183
+    labels:
+      cluster: group-delete-test
+    command: /gobin/dgraph alpha --my=dg3:7183 --lru_mb=1024 --zero=zero1:5080 -o 103 --logtostderr

systest/group-delete/group_delete_test.go

@@ -0,0 +1,251 @@
+/*
+ * Copyright 2018 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/dgraph-io/dgo"
+	"github.com/dgraph-io/dgo/protos/api"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+func TestNodes(t *testing.T) {
+	wrap := func(fn func(*testing.T, *dgo.Dgraph)) func(*testing.T) {
+		return func(t *testing.T) {
+			conn, err := grpc.Dial("localhost:9180", grpc.WithInsecure())
+			require.NoError(t, err)
+			dg := dgo.NewDgraphClient(api.NewDgraphClient(conn))
+			fn(t, dg)
+		}
+	}
+
+	tests := []struct {
+		name string
+		fn   func(*testing.T, *dgo.Dgraph)
+	}{
+		{name: "setup test data", fn: NodesSetup},
+		{name: "move tablets from 3", fn: NodesMoveTablets3},
+		{name: "test query 1", fn: NodesTestQuery},
+		{name: "move tablets from 2", fn: NodesMoveTablets2},
+		{name: "test query 2", fn: NodesTestQuery},
+	}
+	for _, tc := range tests {
+		if !t.Run(tc.name, wrap(tc.fn)) {
+			break
+		}
+	}
+	t.Run("cleanup", wrap(NodesCleanup))
+}
+
+func NodesSetup(t *testing.T, c *dgo.Dgraph) {
+	ctx := context.Background()
+
+	require.NoError(t, c.Alter(ctx, &api.Operation{DropAll: true}))
+
+	schema, err := ioutil.ReadFile(`data/goldendata.schema`)
+	require.NoError(t, err)
+	require.NoError(t, c.Alter(ctx, &api.Operation{Schema: string(schema)}))
+
+	fp, err := os.Open(`data/goldendata_export.rdf.gz`)
+	require.NoError(t, err)
+	defer fp.Close()
+
+	gz, err := gzip.NewReader(fp)
+	require.NoError(t, err)
+	defer gz.Close()
+
+	var (
+		cnt int
+		bb  bytes.Buffer
+	)
+
+	reader := bufio.NewReader(gz)
+	for {
+		b, err := reader.ReadBytes('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			require.NoError(t, err)
+		}
+		bb.Write(b)
+		cnt++
+		if cnt%100 == 0 {
+			_, err = c.NewTxn().Mutate(ctx, &api.Mutation{
+				CommitNow: true,
+				SetNquads: bb.Bytes(),
+			})
+			require.NoError(t, err)
+			bb.Reset()
+		}
+	}
+}
+
+func NodesCleanup(t *testing.T, c *dgo.Dgraph) {
+	require.NoError(t, c.Alter(context.Background(), &api.Operation{DropAll: true}))
+}
+
+type response struct {
+	Groups map[string]struct {
+		Members map[string]interface{} `json:"members"`
+		Tablets map[string]struct {
+			GroupID   int    `json:"groupId"`
+			Predicate string `json:"predicate"`
+		} `json:"tablets"`
+	} `json:"groups"`
+}
+
+func getState() (*response, error) {
+	resp, err := http.Get("http://localhost:6080/state")
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if bytes.Contains(b, []byte("Error")) {
+		return nil, fmt.Errorf("Failed to get state: %s", string(b))
+	}
+
+	var st response
+	if err := json.Unmarshal(b, &st); err != nil {
+		return nil, err
+	}
+	return &st, nil
+}
+
+func getError(rc io.ReadCloser) error {
+	defer rc.Close()
+	b, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return fmt.Errorf("Read failed: %v", err)
+	}
+	if bytes.Contains(b, []byte("Error")) {
+		return fmt.Errorf("%s", string(b))
+	}
+	return nil
+}
+
+func NodesMoveTablets3(t *testing.T, c *dgo.Dgraph) {
+	state1, err := getState()
+	require.NoError(t, err)
+
+	for pred := range state1.Groups["3"].Tablets {
+		url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=2", pred)
+		resp, err := http.Get(url)
+		require.NoError(t, err)
+		require.NoError(t, getError(resp.Body))
+		time.Sleep(time.Second)
+	}
+
+	state2, err := getState()
+	require.NoError(t, err)
+
+	if len(state2.Groups["3"].Tablets) > 0 {
+		t.Errorf("moving tablets failed")
+	}
+
+	resp, err := http.Get("http://localhost:6080/removeNode?group=3&id=3")
+	require.NoError(t, err)
+	require.NoError(t, getError(resp.Body))
+
+	state2, err = getState()
+	require.NoError(t, err)
+
+	if _, ok := state2.Groups["3"]; ok {
+		t.Errorf("node removal failed")
+	}
+}
+
+func NodesMoveTablets2(t *testing.T, c *dgo.Dgraph) {
+	state1, err := getState()
+	require.NoError(t, err)
+
+	for pred := range state1.Groups["2"].Tablets {
+		url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=1", pred)
+		resp, err := http.Get(url)
+		require.NoError(t, err)
+		require.NoError(t, getError(resp.Body))
+		time.Sleep(time.Second)
+	}
+
+	state2, err := getState()
+	require.NoError(t, err)
+
+	if len(state2.Groups["2"].Tablets) > 0 {
+		t.Errorf("moving tablets failed")
+	}
+
+	resp, err := http.Get("http://localhost:6080/removeNode?group=2&id=2")
+	require.NoError(t, err)
+	require.NoError(t, getError(resp.Body))
+
+	state2, err = getState()
+	require.NoError(t, err)
+
+	if _, ok := state2.Groups["2"]; ok {
+		t.Errorf("node removal failed")
+	}
+}
+
+func NodesTestQuery(t *testing.T, c *dgo.Dgraph) {
+	resp, err := c.NewTxn().Query(context.Background(), `
+  {
+  q(func:anyofterms(name@en, "good bad"), first: -5) {
+    name@en
+  }
+  }`)
+	require.NoError(t, err)
+
+	CompareJSON(t, `
+  {
+    "q": [
+      {
+        "name@en": "Good Grief"
+      },
+      {
+        "name@en": "Half Good Killer"
+      },
+      {
+        "name@en": "Bad Friend"
+      },
+      {
+        "name@en": "Ace of Spades: Bad Destiny"
+      },
+      {
+        "name@en": "Bad Girls 6"
+      }
+    ]
+  }`, string(resp.GetJson()))
+}