From a51e479cc29b3636735f60ffba15767f0fb6b598 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=AE=AA=E0=AE=BE=E0=AE=B2=E0=AE=BE=E0=AE=9C=E0=AE=BF?=
<balaji@dgraph.io>
Date: Fri, 7 Aug 2020 12:05:14 +0530
Subject: [PATCH 1/8] add query cost to the header
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: பாலாஜி <balaji@dgraph.io>
---
dgraph/cmd/alpha/http.go | 5 ++
dgraph/cmd/alpha/http_test.go | 118 ++++++++++++++++++++++++++++++++++
edgraph/server.go | 5 +-
x/x.go | 1 +
4 files changed, 128 insertions(+), 1 deletion(-)
diff --git a/dgraph/cmd/alpha/http.go b/dgraph/cmd/alpha/http.go
index 2e64e6f14de..263a4b8277e 100644
--- a/dgraph/cmd/alpha/http.go
+++ b/dgraph/cmd/alpha/http.go
@@ -21,6 +21,7 @@ import (
"compress/gzip"
"context"
"encoding/json"
+ "fmt"
"io"
"io/ioutil"
"net/http"
@@ -239,6 +240,8 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
x.SetStatusWithData(w, x.ErrorInvalidRequest, err.Error())
return
}
+ // Add cost to the header.
+ w.Header().Set(x.DgraphCostHeader, fmt.Sprint(resp.Metrics.NumUids["_total"]))
e := query.Extensions{
Txn: resp.Txn,
@@ -391,6 +394,8 @@ func mutationHandler(w http.ResponseWriter, r *http.Request) {
x.SetStatusWithData(w, x.ErrorInvalidRequest, err.Error())
return
}
+ // Add cost to the header.
+ w.Header().Set(x.DgraphCostHeader, fmt.Sprint(resp.Metrics.NumUids["_total"]))
resp.Latency.ParsingNs = uint64(parseEnd.Sub(parseStart).Nanoseconds())
e := query.Extensions{
diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go
index b39446a05d3..7b1400338d8 100644
--- a/dgraph/cmd/alpha/http_test.go
+++ b/dgraph/cmd/alpha/http_test.go
@@ -196,6 +196,37 @@ func queryWithTs(queryText, contentType, debug string, ts uint64) (string, uint6
return string(output), startTs, err
}
+func queryWithTsForCost(queryText, contentType, debug string, ts uint64) (string,
+ uint64, string, error) {
+ params := make([]string, 0, 2)
+ if debug != "" {
+ params = append(params, "debug="+debug)
+ }
+ if ts != 0 {
+ params = append(params, fmt.Sprintf("startTs=%v", strconv.FormatUint(ts, 10)))
+ }
+ url := addr + "/query?" + strings.Join(params, "&")
+
+ _, body, cost, err := runWithRetriesForCost("POST", contentType, url, queryText)
+ if err != nil {
+ return "", 0, cost, err
+ }
+
+ var r res
+ if err := json.Unmarshal(body, &r); err != nil {
+ return "", 0, cost, err
+ }
+ startTs := r.Extensions.Txn.StartTs
+
+ // Remove the extensions.
+ r2 := res{
+ Data: r.Data,
+ }
+ output, err := json.Marshal(r2)
+
+ return string(output), startTs, cost, err
+}
+
type mutationResponse struct {
keys []string
preds []string
@@ -314,6 +345,61 @@ func runRequest(req *http.Request) (*x.QueryResWithData, []byte, error) {
return qr, body, nil
}
+func runWithRetriesForCost(method, contentType, url string, body string) (
+ *x.QueryResWithData, []byte, string, error) {
+
+ req, err := createRequest(method, contentType, url, body)
+ if err != nil {
+ return nil, nil, "", err
+ }
+
+ qr, respBody, cost, err := runRequestForCost(req)
+ if err != nil && strings.Contains(err.Error(), "Token is expired") {
+ grootAccessJwt, grootRefreshJwt, err = testutil.HttpLogin(&testutil.LoginParams{
+ Endpoint: addr + "/admin",
+ RefreshJwt: grootRefreshJwt,
+ })
+ if err != nil {
+ return nil, nil, "", err
+ }
+
+ // create a new request since the previous request would have been closed upon the err
+ retryReq, err := createRequest(method, contentType, url, body)
+ if err != nil {
+ return nil, nil, "", err
+ }
+
+ return runRequestForCost(retryReq)
+ }
+ return qr, respBody, cost, err
+}
+
+// attach the grootAccessJWT to the request and sends the http request
+func runRequestForCost(req *http.Request) (*x.QueryResWithData, []byte, string, error) {
+ client := &http.Client{}
+ req.Header.Set("X-Dgraph-AccessToken", grootAccessJwt)
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, nil, "", err
+ }
+ if status := resp.StatusCode; status != http.StatusOK {
+ return nil, nil, "", errors.Errorf("Unexpected status code: %v", status)
+ }
+
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, nil, "", errors.Errorf("unable to read from body: %v", err)
+ }
+
+ qr := new(x.QueryResWithData)
+ json.Unmarshal(body, qr) // Don't check error.
+ if len(qr.Errors) > 0 {
+ return nil, nil, "", errors.New(qr.Errors[0].Message)
+ }
+ return qr, body, resp.Header.Get(x.DgraphCostHeader), nil
+}
+
func commitWithTs(keys, preds []string, ts uint64) error {
url := addr + "/commit"
if ts != 0 {
@@ -452,6 +538,38 @@ func TestTransactionBasicNoPreds(t *testing.T) {
require.NoError(t, err)
require.Equal(t, `{"data":{"balances":[{"name":"Bob","balance":"110"}]}}`, data)
}
+func TestTransactionForCost(t *testing.T) {
+ require.NoError(t, dropAll())
+ require.NoError(t, alterSchema(`name: string @index(term) .`))
+
+ q1 := `
+ {
+ balances(func: anyofterms(name, "Alice Bob")) {
+ name
+ balance
+ }
+ }
+ `
+ _, _, err := queryWithTs(q1, "application/graphql+-", "", 0)
+ require.NoError(t, err)
+
+ m1 := `
+ {
+ set {
+ _:alice <name> "Bob" .
+ _:alice <balance> "110" .
+ _:bob <balance> "60" .
+ }
+ }
+ `
+
+ _, err = mutationWithTs(m1, "application/rdf", false, true, 0)
+ require.NoError(t, err)
+
+ _, _, cost, err := queryWithTsForCost(q1, "application/graphql+-", "", 0)
+ require.NoError(t, err)
+ fmt.Println(cost)
+}
func TestTransactionBasicOldCommitFormat(t *testing.T) {
require.NoError(t, dropAll())
diff --git a/edgraph/server.go b/edgraph/server.go
index b46494c89bc..ec2e04d756c 100644
--- a/edgraph/server.go
+++ b/edgraph/server.go
@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "fmt"
"math"
"net"
"sort"
@@ -36,6 +37,7 @@ import (
"go.opencensus.io/tag"
"go.opencensus.io/trace"
otrace "go.opencensus.io/trace"
+ "google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
@@ -997,7 +999,8 @@ func (s *Server) doQuery(ctx context.Context, req *api.Request, doAuth AuthMode)
EncodingNs: uint64(l.Json.Nanoseconds()),
TotalNs: uint64((time.Since(l.Start)).Nanoseconds()),
}
-
+ md := metadata.Pairs(x.DgraphCostHeader, fmt.Sprint(resp.Metrics.NumUids["_total"]))
+ grpc.SendHeader(ctx, md)
return resp, nil
}
diff --git a/x/x.go b/x/x.go
index 53db9638dee..7646613aa26 100644
--- a/x/x.go
+++ b/x/x.go
@@ -146,6 +146,7 @@ const (
AccessControlAllowedHeaders = "X-Dgraph-AccessToken, " +
"Content-Type, Content-Length, Accept-Encoding, Cache-Control, " +
"X-CSRF-Token, X-Auth-Token, X-Requested-With"
+ DgraphCostHeader = "X-Dgraph-Cost"
// GraphqlPredicates is the json representation of the predicate reserved for graphql system.
GraphqlPredicates = `
From 8dec1ffabff9c8388c89200e87353e98d128302a Mon Sep 17 00:00:00 2001
From: Balaji <rbalajis25@gmail.com>
Date: Fri, 7 Aug 2020 07:03:20 +0000
Subject: [PATCH 2/8] fix panic
Signed-off-by: Balaji <rbalajis25@gmail.com>
---
dgraph/cmd/alpha/http_test.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go
index 7b1400338d8..7bd3e837dce 100644
--- a/dgraph/cmd/alpha/http_test.go
+++ b/dgraph/cmd/alpha/http_test.go
@@ -568,7 +568,7 @@ func TestTransactionForCost(t *testing.T) {
_, _, cost, err := queryWithTsForCost(q1, "application/graphql+-", "", 0)
require.NoError(t, err)
- fmt.Println(cost)
+ require.Equal(t, "2", cost)
}
func TestTransactionBasicOldCommitFormat(t *testing.T) {
From 622a4c763be70e0966ea5a23e4cffb3a56aca8c9 Mon Sep 17 00:00:00 2001
From: Balaji <rbalajis25@gmail.com>
Date: Fri, 7 Aug 2020 07:03:42 +0000
Subject: [PATCH 3/8] fix minor
Signed-off-by: Balaji <rbalajis25@gmail.com>
---
edgraph/server.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/edgraph/server.go b/edgraph/server.go
index ec2e04d756c..8a4c1cdea8f 100644
--- a/edgraph/server.go
+++ b/edgraph/server.go
@@ -1007,6 +1007,10 @@ func (s *Server) doQuery(ctx context.Context, req *api.Request, doAuth AuthMode)
func processQuery(ctx context.Context, qc *queryContext) (*api.Response, error) {
resp := &api.Response{}
if len(qc.req.Query) == 0 {
+ // No query, so make the query cost 0.
+ resp.Metrics = &api.Metrics{
+ NumUids: map[string]uint64{"_total": 0},
+ }
return resp, nil
}
if ctx.Err() != nil {
From e23efa506b2d4f75a60658147a5f2722324adaf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=AE=AA=E0=AE=BE=E0=AE=B2=E0=AE=BE=E0=AE=9C=E0=AE=BF?=
<balaji@dgraph.io>
Date: Fri, 7 Aug 2020 14:17:22 +0530
Subject: [PATCH 4/8] change header name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: பாலாஜி <balaji@dgraph.io>
---
x/x.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/x/x.go b/x/x.go
index 7646613aa26..0bdb09cb94d 100644
--- a/x/x.go
+++ b/x/x.go
@@ -146,7 +146,7 @@ const (
AccessControlAllowedHeaders = "X-Dgraph-AccessToken, " +
"Content-Type, Content-Length, Accept-Encoding, Cache-Control, " +
"X-CSRF-Token, X-Auth-Token, X-Requested-With"
- DgraphCostHeader = "X-Dgraph-Cost"
+ DgraphCostHeader = "numUids"
// GraphqlPredicates is the json representation of the predicate reserved for graphql system.
GraphqlPredicates = `
From d184b13e1de491b952b0468d3f9340b3c1ed37c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=AE=AA=E0=AE=BE=E0=AE=B2=E0=AE=BE=E0=AE=9C=E0=AE=BF?=
<balaji@dgraph.io>
Date: Fri, 7 Aug 2020 14:25:45 +0530
Subject: [PATCH 5/8] return resp instead of cost
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: பாலாஜி <balaji@dgraph.io>
---
dgraph/cmd/alpha/http_test.go | 36 +++++++++++++++++------------------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go
index 7bd3e837dce..22f1856ce47 100644
--- a/dgraph/cmd/alpha/http_test.go
+++ b/dgraph/cmd/alpha/http_test.go
@@ -196,8 +196,8 @@ func queryWithTs(queryText, contentType, debug string, ts uint64) (string, uint6
return string(output), startTs, err
}
-func queryWithTsForCost(queryText, contentType, debug string, ts uint64) (string,
- uint64, string, error) {
+func queryWithTsForResp(queryText, contentType, debug string, ts uint64) (string,
+ uint64, *http.Response, error) {
params := make([]string, 0, 2)
if debug != "" {
params = append(params, "debug="+debug)
@@ -345,59 +345,59 @@ func runRequest(req *http.Request) (*x.QueryResWithData, []byte, error) {
return qr, body, nil
}
-func runWithRetriesForCost(method, contentType, url string, body string) (
- *x.QueryResWithData, []byte, string, error) {
+func runWithRetriesForResp(method, contentType, url string, body string) (
+ *x.QueryResWithData, []byte, *http.Response, error) {
req, err := createRequest(method, contentType, url, body)
if err != nil {
- return nil, nil, "", err
+ return nil, nil, nil, err
}
- qr, respBody, cost, err := runRequestForCost(req)
+ qr, respBody, resp, err := runRequestForResp(req)
if err != nil && strings.Contains(err.Error(), "Token is expired") {
grootAccessJwt, grootRefreshJwt, err = testutil.HttpLogin(&testutil.LoginParams{
Endpoint: addr + "/admin",
RefreshJwt: grootRefreshJwt,
})
if err != nil {
- return nil, nil, "", err
+ return nil, nil, nil, err
}
// create a new request since the previous request would have been closed upon the err
retryReq, err := createRequest(method, contentType, url, body)
if err != nil {
- return nil, nil, "", err
+ return nil, nil, resp, err
}
- return runRequestForCost(retryReq)
+ return runRequestForResp(retryReq)
}
- return qr, respBody, cost, err
+ return qr, respBody, resp, err
}
// attach the grootAccessJWT to the request and sends the http request
-func runRequestForCost(req *http.Request) (*x.QueryResWithData, []byte, string, error) {
+func runRequestForResp(req *http.Request) (*x.QueryResWithData, []byte, *http.Response, error) {
client := &http.Client{}
req.Header.Set("X-Dgraph-AccessToken", grootAccessJwt)
resp, err := client.Do(req)
if err != nil {
- return nil, nil, "", err
+ return nil, nil, resp, err
}
if status := resp.StatusCode; status != http.StatusOK {
- return nil, nil, "", errors.Errorf("Unexpected status code: %v", status)
+ return nil, nil, resp, errors.Errorf("Unexpected status code: %v", status)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- return nil, nil, "", errors.Errorf("unable to read from body: %v", err)
+ return nil, nil, resp, errors.Errorf("unable to read from body: %v", err)
}
qr := new(x.QueryResWithData)
json.Unmarshal(body, qr) // Don't check error.
if len(qr.Errors) > 0 {
- return nil, nil, "", errors.New(qr.Errors[0].Message)
+ return nil, nil, resp, errors.New(qr.Errors[0].Message)
}
- return qr, body, resp.Header.Get(x.DgraphCostHeader), nil
+ return qr, body, resp, nil
}
func commitWithTs(keys, preds []string, ts uint64) error {
@@ -566,9 +566,9 @@ func TestTransactionForCost(t *testing.T) {
_, err = mutationWithTs(m1, "application/rdf", false, true, 0)
require.NoError(t, err)
- _, _, cost, err := queryWithTsForCost(q1, "application/graphql+-", "", 0)
+ _, _, resp, err := queryWithTsForResp(q1, "application/graphql+-", "", 0)
require.NoError(t, err)
- require.Equal(t, "2", cost)
+ require.Equal(t, "2", resp.Header.Get(x.DgraphCostHeader))
}
func TestTransactionBasicOldCommitFormat(t *testing.T) {
From 78e9b6b1fe902f9bec28c7a303c7d8b723e16d36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=AE=AA=E0=AE=BE=E0=AE=B2=E0=AE=BE=E0=AE=9C=E0=AE=BF?=
<balaji@dgraph.io>
Date: Fri, 7 Aug 2020 14:59:03 +0530
Subject: [PATCH 6/8] minor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: பாலாஜி <balaji@dgraph.io>
---
dgraph/cmd/alpha/http_test.go | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go
index 22f1856ce47..e49d2fa2c6d 100644
--- a/dgraph/cmd/alpha/http_test.go
+++ b/dgraph/cmd/alpha/http_test.go
@@ -207,14 +207,14 @@ func queryWithTsForResp(queryText, contentType, debug string, ts uint64) (string
}
url := addr + "/query?" + strings.Join(params, "&")
- _, body, cost, err := runWithRetriesForCost("POST", contentType, url, queryText)
+ _, body, resp, err := runWithRetriesForResp("POST", contentType, url, queryText)
if err != nil {
- return "", 0, cost, err
+ return "", 0, resp, err
}
var r res
if err := json.Unmarshal(body, &r); err != nil {
- return "", 0, cost, err
+ return "", 0, resp, err
}
startTs := r.Extensions.Txn.StartTs
@@ -224,7 +224,7 @@ func queryWithTsForResp(queryText, contentType, debug string, ts uint64) (string
}
output, err := json.Marshal(r2)
- return string(output), startTs, cost, err
+ return string(output), startTs, resp, err
}
type mutationResponse struct {
From d0fac7477158b5b0963de859b39b617872b9e3ee Mon Sep 17 00:00:00 2001
From: Tejas Dinkar <tejas@gja.in>
Date: Fri, 7 Aug 2020 16:29:02 +0530
Subject: [PATCH 7/8] Update x.go
---
x/x.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/x/x.go b/x/x.go
index 0bdb09cb94d..c54d198590e 100644
--- a/x/x.go
+++ b/x/x.go
@@ -146,7 +146,7 @@ const (
AccessControlAllowedHeaders = "X-Dgraph-AccessToken, " +
"Content-Type, Content-Length, Accept-Encoding, Cache-Control, " +
"X-CSRF-Token, X-Auth-Token, X-Requested-With"
- DgraphCostHeader = "numUids"
+ DgraphCostHeader = "Dgraph-TouchedUids"
// GraphqlPredicates is the json representation of the predicate reserved for graphql system.
GraphqlPredicates = `
From f15546357760732f0b4efd4c069a951509ea44f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=AE=AA=E0=AE=BE=E0=AE=B2=E0=AE=BE=E0=AE=9C=E0=AE=BF?=
<balaji@dgraph.io>
Date: Sun, 9 Aug 2020 17:48:36 +0530
Subject: [PATCH 8/8] add comment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: பாலாஜி <balaji@dgraph.io>
---
dgraph/cmd/alpha/http_test.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go
index e49d2fa2c6d..8d5316fbd9e 100644
--- a/dgraph/cmd/alpha/http_test.go
+++ b/dgraph/cmd/alpha/http_test.go
@@ -196,6 +196,7 @@ func queryWithTs(queryText, contentType, debug string, ts uint64) (string, uint6
return string(output), startTs, err
}
+// queryWithTsForResp query the dgraph and returns it's http response and result.
func queryWithTsForResp(queryText, contentType, debug string, ts uint64) (string,
uint64, *http.Response, error) {
params := make([]string, 0, 2)