From a153ed5ae533c5cc4760404bcf4d56ceb4d7edfd Mon Sep 17 00:00:00 2001 From: Anurag Date: Wed, 8 Feb 2023 19:25:44 +0530 Subject: [PATCH] feat(cloud): add shared-instance flag in limit superflag in alpha (#7770) (#8625) This PR adds a shared-instance flag to --limit superflag. When set to true (false by default), it will: - Restrict access to any of the ACL operations like Login, add/remove/update user from non-galaxy namespaces. - Prevent the leaking of environment variables for minio and aws. (cherry picked from commit 5f3cece75da375077c45ab64228cb7053cf03d02) --- dgraph/cmd/alpha/run.go | 7 +- edgraph/access_ee.go | 79 ++++++--- systest/acl/restore/acl_restore_test.go | 10 +- systest/backup/multi-tenancy/backup_test.go | 6 +- systest/cloud/cloud_test.go | 178 ++++++++++++++++++++ systest/cloud/docker-compose.yml | 53 ++++++ systest/multi-tenancy/basic_test.go | 70 ++++---- testutil/client.go | 45 +++-- testutil/docker.go | 2 +- testutil/multi_tenancy.go | 41 ++++- testutil/utils.go | 25 +++ worker/server_state.go | 2 +- x/config.go | 3 + x/minioclient.go | 12 +- 14 files changed, 438 insertions(+), 95 deletions(-) create mode 100644 systest/cloud/cloud_test.go create mode 100644 systest/cloud/docker-compose.yml diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 62846356d54..6966f494237 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -1,5 +1,5 @@ /* - * Copyright 2017-2022 Dgraph Labs, Inc. and Contributors + * Copyright 2017-2023 Dgraph Labs, Inc. and Contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -205,6 +205,9 @@ they form a Raft group and provide synchronous replication. "worker in a failed state. Use -1 to retry infinitely."). Flag("txn-abort-after", "Abort any pending transactions older than this duration."+ " The liveness of a transaction is determined by its last mutation."). + Flag("shared-instance", "When set to true, it disables ACLs for non-galaxy users. "+ + "It expects the access JWT to be constructed outside dgraph for non-galaxy users as "+ + "login is denied to them. Additionally, this disables access to environment variables for minio, aws, etc."). String()) flag.String("graphql", worker.GraphQLDefaults, z.NewSuperFlagHelp(worker.GraphQLDefaults). @@ -628,7 +631,6 @@ func run() { pstoreBlockCacheSize, pstoreIndexCacheSize) bopts := badger.DefaultOptions("").FromSuperFlag(worker.BadgerDefaults + cacheOpts). FromSuperFlag(Alpha.Conf.GetString("badger")) - security := z.NewSuperFlag(Alpha.Conf.GetString("security")).MergeAndCheckDefault( worker.SecurityDefaults) conf := audit.GetAuditConf(Alpha.Conf.GetString("audit")) @@ -717,6 +719,7 @@ func run() { x.Config.LimitNormalizeNode = int(x.Config.Limit.GetInt64("normalize-node")) x.Config.QueryTimeout = x.Config.Limit.GetDuration("query-timeout") x.Config.MaxRetries = x.Config.Limit.GetInt64("max-retries") + x.Config.SharedInstance = x.Config.Limit.GetBool("shared-instance") x.Config.GraphQL = z.NewSuperFlag(Alpha.Conf.GetString("graphql")).MergeAndCheckDefault( worker.GraphQLDefaults) diff --git a/edgraph/access_ee.go b/edgraph/access_ee.go index bbb927b205e..509de97289a 100644 --- a/edgraph/access_ee.go +++ b/edgraph/access_ee.go @@ -49,6 +49,10 @@ type predsAndvars struct { func (s *Server) Login(ctx context.Context, request *api.LoginRequest) (*api.Response, error) { + if !shouldAllowAcls(request.GetNamespace()) { + return nil, errors.New("operation is not allowed in shared cloud mode") + } + if err := x.HealthCheck(); err != nil { return nil, err } @@ -628,18 +632,15 @@ type authPredResult struct { } func authorizePreds(ctx context.Context, userData *userData, preds []string, - aclOp *acl.Operation) (*authPredResult, error) { + aclOp *acl.Operation) *authPredResult { - ns, err := x.ExtractNamespace(ctx) - if err != nil { - return nil, errors.Wrapf(err, "While authorizing preds") - } if !worker.AclCachePtr.Loaded() { RefreshACLs(ctx) } userId := userData.userId groupIds := userData.groupIds + ns := userData.namespace blockedPreds := make(map[string]struct{}) for _, pred := range preds { nsPred := x.NamespaceAttr(ns, pred) @@ -651,15 +652,13 @@ func authorizePreds(ctx context.Context, userData *userData, preds []string, operation: aclOp, allowed: false, }) - blockedPreds[pred] = struct{}{} } } - if worker.HasAccessToAllPreds(ns, groupIds, aclOp) { // Setting allowed to nil allows access to all predicates. Note that the access to ACL // predicates will still be blocked. - return &authPredResult{allowed: nil, blocked: blockedPreds}, nil + return &authPredResult{allowed: nil, blocked: blockedPreds} } // User can have multiple permission for same predicate, add predicate allowedPreds := make([]string, 0, len(worker.AclCachePtr.GetUserPredPerms(userId))) @@ -669,7 +668,7 @@ func authorizePreds(ctx context.Context, userData *userData, preds []string, allowedPreds = append(allowedPreds, predicate) } } - return &authPredResult{allowed: allowedPreds, blocked: blockedPreds}, nil + return &authPredResult{allowed: allowedPreds, blocked: blockedPreds} } // authorizeAlter parses the Schema in the operation and authorizes the operation @@ -724,10 +723,7 @@ func authorizeAlter(ctx context.Context, op *api.Operation) error { "only guardians are allowed to drop all data, but the current user is %s", userId) } - result, err := authorizePreds(ctx, userData, preds, acl.Modify) - if err != nil { - return nil - } + result := authorizePreds(ctx, userData, preds, acl.Modify) if len(result.blocked) > 0 { var msg strings.Builder for key := range result.blocked { @@ -836,12 +832,17 @@ func authorizeMutation(ctx context.Context, gmu *dql.Mutation) error { case isAclPredMutation(gmu.Del): return errors.Errorf("ACL predicates can't be deleted") } + if !shouldAllowAcls(userData.namespace) { + for _, pred := range preds { + if x.IsAclPredicate(pred) { + return status.Errorf(codes.PermissionDenied, + "unauthorized to mutate acl predicates: %s\n", pred) + } + } + } return nil } - result, err := authorizePreds(ctx, userData, preds, acl.Write) - if err != nil { - return err - } + result := authorizePreds(ctx, userData, preds, acl.Write) if len(result.blocked) > 0 { var msg strings.Builder for key := range result.blocked { @@ -949,6 +950,21 @@ func logAccess(log *accessEntry) { } } +func blockedPreds(preds []string) map[string]struct{} { + blocked := make(map[string]struct{}) + for _, pred := range preds { + if x.IsAclPredicate(pred) { + blocked[pred] = struct{}{} + } + } + return blocked +} + +// With shared instance enabled, we don't allow ACL operations from any of the non-galaxy namespace. +func shouldAllowAcls(ns uint64) bool { + return !x.Config.SharedInstance || ns == x.GalaxyNamespace +} + // authorizeQuery authorizes the query using the aclCachePtr. It will silently drop all // unauthorized predicates from query. // At this stage, namespace is not attached in the predicates. @@ -960,6 +976,7 @@ func authorizeQuery(ctx context.Context, parsedReq *dql.Result, graphql bool) er var userId string var groupIds []string + var namespace uint64 predsAndvars := parsePredsFromQuery(parsedReq.Query) preds := predsAndvars.preds varsToPredMap := predsAndvars.vars @@ -979,14 +996,18 @@ func authorizeQuery(ctx context.Context, parsedReq *dql.Result, graphql bool) er userId = userData.userId groupIds = userData.groupIds + namespace = userData.namespace if x.IsGuardian(groupIds) { - // Members of guardian groups are allowed to query anything. - return nil, nil, nil + if shouldAllowAcls(userData.namespace) { + // Members of guardian groups are allowed to query anything. + return nil, nil, nil + } + return blockedPreds(preds), nil, nil } - result, err := authorizePreds(ctx, userData, preds, acl.Read) - return result.blocked, result.allowed, err + result := authorizePreds(ctx, userData, preds, acl.Read) + return result.blocked, result.allowed, nil } blockedPreds, allowedPreds, err := doAuthorizeQuery() @@ -1007,7 +1028,7 @@ func authorizeQuery(ctx context.Context, parsedReq *dql.Result, graphql bool) er if len(blockedPreds) != 0 { // For GraphQL requests, we allow filtered access to the ACL predicates. // Filter for user_id and group_id is applied for the currently logged in user. - if graphql { + if graphql && shouldAllowAcls(namespace) { for _, gq := range parsedReq.Query { addUserFilterToQuery(gq, userId, groupIds) } @@ -1068,11 +1089,14 @@ func authorizeSchemaQuery(ctx context.Context, er *query.ExecutionResult) error groupIds := userData.groupIds if x.IsGuardian(groupIds) { - // Members of guardian groups are allowed to query anything. - return nil, nil + if shouldAllowAcls(userData.namespace) { + // Members of guardian groups are allowed to query anything. + return nil, nil + } + return blockedPreds(preds), nil } - result, err := authorizePreds(ctx, userData, preds, acl.Read) - return result.blocked, err + result := authorizePreds(ctx, userData, preds, acl.Read) + return result.blocked, nil } // find the predicates which are blocked for the schema query @@ -1115,8 +1139,7 @@ func AuthGuardianOfTheGalaxy(ctx context.Context) error { } ns, err := x.ExtractNamespaceFrom(ctx) if err != nil { - return status.Error(codes.Unauthenticated, - "AuthGuardianOfTheGalaxy: extracting jwt token, error: "+err.Error()) + return errors.Wrap(err, "Authorize guardian of the galaxy, extracting jwt token, error:") } if ns != 0 { return status.Error( diff --git a/systest/acl/restore/acl_restore_test.go b/systest/acl/restore/acl_restore_test.go index e4102a53adf..43700aba193 100644 --- a/systest/acl/restore/acl_restore_test.go +++ b/systest/acl/restore/acl_restore_test.go @@ -36,7 +36,8 @@ func disableDraining(t *testing.T) { b, err := json.Marshal(params) require.NoError(t, err) - token := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: 0}) + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: 0}) + require.NoError(t, err, "login failed") client := &http.Client{} req, err := http.NewRequest("POST", testutil.AdminUrl(), bytes.NewBuffer(b)) @@ -70,7 +71,9 @@ func sendRestoreRequest(t *testing.T, location, backupId string, backupNum int) }, } - token := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: 0}) + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: 0}) + require.NoError(t, err, "login failed") + resp := testutil.MakeGQLRequestWithAccessJwt(t, params, token.AccessJwt) resp.RequireNoGraphQLErrors(t) @@ -97,8 +100,9 @@ func TestAclCacheRestore(t *testing.T) { sendRestoreRequest(t, "/backups", "vibrant_euclid5", 1) testutil.WaitForRestore(t, dg, testutil.SockAddrHttp) - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "alice1", Passwd: "password", Namespace: 0}) + require.NoError(t, err, "login failed") params := &common.GraphQLParams{ Query: `query{ queryPerson{ diff --git a/systest/backup/multi-tenancy/backup_test.go b/systest/backup/multi-tenancy/backup_test.go index e669e38013e..9663e4140ea 100644 --- a/systest/backup/multi-tenancy/backup_test.go +++ b/systest/backup/multi-tenancy/backup_test.go @@ -50,9 +50,9 @@ func TestBackupMultiTenancy(t *testing.T) { dg := testutil.DgClientWithLogin(t, "groot", "password", x.GalaxyNamespace) testutil.DropAll(t, dg) - galaxyCreds := &testutil.LoginParams{ - UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace} - galaxyToken := testutil.Login(t, galaxyCreds) + galaxyCreds := &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace} + galaxyToken, err := testutil.Login(t, galaxyCreds) + require.NoError(t, err, "login failed") // Create a new namespace ns1, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) diff --git a/systest/cloud/cloud_test.go b/systest/cloud/cloud_test.go new file mode 100644 index 00000000000..be2c5d1178c --- /dev/null +++ b/systest/cloud/cloud_test.go @@ -0,0 +1,178 @@ +/* + * Copyright 2023 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dgraph-io/dgo/v210/protos/api" + "github.com/dgraph-io/dgraph/graphql/e2e/common" + "github.com/dgraph-io/dgraph/testutil" + "github.com/dgraph-io/dgraph/x" +) + +func setup(t *testing.T) { + dc := testutil.DgClientWithLogin(t, "groot", "password", x.GalaxyNamespace) + require.NoError(t, dc.Alter(context.Background(), &api.Operation{DropAll: true})) +} + +func readFile(t *testing.T, path string) []byte { + data, err := ioutil.ReadFile(path) + require.NoError(t, err) + return data +} + +func getHttpToken(t *testing.T, user, password string, ns uint64) *testutil.HttpToken { + jwt := testutil.GetAccessJwt(t, testutil.JwtParams{ + User: user, + Groups: []string{"guardians"}, + Ns: ns, + Exp: time.Hour, + Secret: readFile(t, "../../ee/acl/hmac-secret"), + }) + + return &testutil.HttpToken{ + UserId: user, + Password: password, + AccessJwt: jwt, + } +} + +func graphqlHelper(t *testing.T, query string, headers http.Header, + expectedResult string) { + params := &common.GraphQLParams{ + Query: query, + Headers: headers, + } + queryResult := params.ExecuteAsPost(t, common.GraphqlURL) + common.RequireNoGQLErrors(t, queryResult) + testutil.CompareJSON(t, expectedResult, string(queryResult.Data)) +} + +func TestDisallowNonGalaxy(t *testing.T) { + setup(t) + + galaxyToken := getHttpToken(t, "groot", "password", x.GalaxyNamespace) + // Create a new namespace + ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) + require.NoError(t, err) + require.Greater(t, int(ns), 0) + + nsToken := getHttpToken(t, "groot", "password", ns) + header := http.Header{} + header.Set("X-Dgraph-AccessToken", nsToken.AccessJwt) + + // User from namespace ns should be able to query/mutate. + schema := ` + type Author { + id: ID! + name: String + }` + common.SafelyUpdateGQLSchema(t, common.Alpha1HTTP, schema, header) + + graphqlHelper(t, ` + mutation { + addAuthor(input:{name: "Alice"}) { + author{ + name + } + } + }`, header, + `{ + "addAuthor": { + "author":[{ + "name":"Alice" + }] + } + }`) + + query := ` + query { + queryAuthor { + name + } + }` + graphqlHelper(t, query, header, + `{ + "queryAuthor": [ + { + "name":"Alice" + } + ] + }`) + + // Login to namespace 1 via groot and create new user alice. Non-galaxy namespace user should + // not be able to do so in cloud mode. + _, err = testutil.HttpLogin(&testutil.LoginParams{ + Endpoint: testutil.AdminUrl(), + UserID: "groot", + Passwd: "password", + Namespace: ns, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "operation is not allowed in shared cloud mode") + + // Ns guardian should not be able to create user. + resp := testutil.CreateUser(t, nsToken, "alice", "newpassword") + require.Greater(t, len(resp.Errors), 0) + require.Contains(t, resp.Errors.Error(), "unauthorized to mutate acl predicates") + + // Galaxy guardian should be able to create user. + resp = testutil.CreateUser(t, galaxyToken, "alice", "newpassword") + require.Equal(t, 0, len(resp.Errors)) +} + +func TestEnvironmentAccess(t *testing.T) { + setup(t) + + galaxyToken := getHttpToken(t, "groot", "password", x.GalaxyNamespace) + // Create a new namespace + ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) + require.NoError(t, err) + require.Greater(t, int(ns), 0) + + nsToken := getHttpToken(t, "groot", "password", ns) + header := http.Header{} + header.Set("X-Dgraph-AccessToken", nsToken.AccessJwt) + + // Create a minio bucket. + bucketname := "dgraph-export" + mc, err := testutil.NewMinioClient() + require.NoError(t, err) + require.NoError(t, mc.MakeBucket(bucketname, "")) + minioDest := "minio://minio:9001/dgraph-export?secure=false" + + // Export without the minio creds should fail for non-galaxy. + resp := testutil.Export(t, nsToken, minioDest, "", "") + require.Greater(t, len(resp.Errors), 0) + require.Contains(t, resp.Errors.Error(), "task failed") + + // Export with the minio creds should work for non-galaxy. + resp = testutil.Export(t, nsToken, minioDest, "accesskey", "secretkey") + require.Zero(t, len(resp.Errors)) + + // Galaxy guardian should provide the credentials as well. + resp = testutil.Export(t, galaxyToken, minioDest, "accesskey", "secretkey") + require.Zero(t, len(resp.Errors)) + +} diff --git a/systest/cloud/docker-compose.yml b/systest/cloud/docker-compose.yml new file mode 100644 index 00000000000..d1ab7dd7650 --- /dev/null +++ b/systest/cloud/docker-compose.yml @@ -0,0 +1,53 @@ +version: "3.5" +services: + zero1: + image: dgraph/dgraph:local + working_dir: /data/zero1 + ports: + - 5080 + - 6080 + labels: + cluster: test + service: zero + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + command: /gobin/dgraph zero --my=zero1:5080 --raft="idx=1" --logtostderr -v=2 --bindall --expose_trace --profile_mode block --block_rate 10 + + alpha1: + image: dgraph/dgraph:local + working_dir: /data/alpha1 + env_file: + - ./../../dgraph/minio.env + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + - type: bind + source: ../../ee/acl/hmac-secret + target: /dgraph-acl/hmac-secret + read_only: true + ports: + - 8080 + - 9080 + labels: + cluster: test + service: alpha + command: /gobin/dgraph alpha --my=alpha1:7080 --zero=zero1:5080,zero2:5080,zero3:5080 --expose_trace --profile_mode block --block_rate 10 --logtostderr -v=2 + --security "whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16;" + --acl "secret-file=/dgraph-acl/hmac-secret; access-ttl=20s;" + --limit "shared-instance=true" + + minio: + image: minio/minio:latest + env_file: + - ./../../dgraph/minio.env + working_dir: /data/minio + ports: + - 9001 + labels: + cluster: test + command: minio server /data/minio --address :9001 diff --git a/systest/multi-tenancy/basic_test.go b/systest/multi-tenancy/basic_test.go index cda5eb1b273..1d0379069aa 100644 --- a/systest/multi-tenancy/basic_test.go +++ b/systest/multi-tenancy/basic_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 Dgraph Labs, Inc. and Contributors + * Copyright 2023 Dgraph Labs, Inc. and Contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,9 +48,11 @@ var timeout = 5 * time.Second // this file can me made common to the other acl tests as well. Needs some refactoring as well. func TestAclBasic(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) + require.NotNil(t, galaxyToken, "galaxy token is nil") + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) @@ -80,7 +82,8 @@ func TestAclBasic(t *testing.T) { testutil.CompareJSON(t, `{"me": []}`, string(resp)) // Login to namespace 1 via groot and create new user alice. - token := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) + require.NoError(t, err, "login failed") testutil.CreateUser(t, token, "alice", "newpassword") // Alice should not be able to see data added by groot in namespace 1 @@ -100,8 +103,9 @@ func TestAclBasic(t *testing.T) { } func createGroupAndSetPermissions(t *testing.T, namespace uint64, group, user, predicate string) { - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: namespace}) + require.NoError(t, err, "login failed") testutil.CreateGroup(t, token, group) testutil.AddToGroup(t, token, user, group) testutil.AddRulesToGroup(t, token, group, @@ -110,9 +114,9 @@ func createGroupAndSetPermissions(t *testing.T, namespace uint64, group, user, p func TestTwoPermissionSetsInNameSpacesWithAcl(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) - + require.NoError(t, err, "login failed") query := ` { me(func: has(name)) { @@ -130,8 +134,9 @@ func TestTwoPermissionSetsInNameSpacesWithAcl(t *testing.T) { testutil.AddData(t, dc) // Create user alice - token1 := testutil.Login(t, + token1, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns1}) + require.NoError(t, err, "login failed") testutil.CreateUser(t, token1, "alice", "newpassword") // Create a new group, add alice to that group and give read access of to dev group. @@ -150,8 +155,9 @@ func TestTwoPermissionSetsInNameSpacesWithAcl(t *testing.T) { testutil.AddData(t, dc) // Create user bob - token2 := testutil.Login(t, + token2, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns2}) + require.NoError(t, err, "login failed") testutil.CreateUser(t, token2, "bob", "newpassword") // Create a new group, add bob to that group and give read access of to dev group. @@ -167,8 +173,9 @@ func TestTwoPermissionSetsInNameSpacesWithAcl(t *testing.T) { testutil.CompareJSON(t, `{"me": [{"name":"guy2"}, {"name":"guy1"}]}`, string(resp)) // Change permissions in namespace-2 - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns2}) + require.NoError(t, err, "login failed") testutil.AddRulesToGroup(t, token, "dev", []testutil.Rule{{Predicate: "name", Permission: acl.Read.Code}}, false) @@ -185,16 +192,16 @@ func TestTwoPermissionSetsInNameSpacesWithAcl(t *testing.T) { func TestCreateNamespace(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) - + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) - + require.NoError(t, err, "login failed") // Create a new namespace using guardian of other namespace. _, err = testutil.CreateNamespaceWithRetry(t, token) require.Error(t, err) @@ -204,9 +211,9 @@ func TestCreateNamespace(t *testing.T) { func TestResetPassword(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) - + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) @@ -216,23 +223,24 @@ func TestResetPassword(t *testing.T) { require.NoError(t, err) // Try and Fail with old password for groot - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) + require.Error(t, err, "expected error because incorrect login") require.Nil(t, token, "nil token because incorrect login") // Try and success with new password for groot - token = testutil.Login(t, + token, err = testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "newpassword", Namespace: ns}) - + require.NoError(t, err, "login failed") require.Equal(t, token.Password, "newpassword", "new password matches the reset password") } func TestDeleteNamespace(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) - + require.NoError(t, err, "login failed") dg := make(map[uint64]*dgo.Dgraph) dg[x.GalaxyNamespace] = testutil.DgClientWithLogin(t, "groot", "password", x.GalaxyNamespace) // Create a new namespace @@ -320,8 +328,8 @@ func TestLiveLoadMulti(t *testing.T) { prepare(t) dc0 := testutil.DgClientWithLogin(t, "groot", "password", x.GalaxyNamespace) galaxyCreds := &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace} - galaxyToken := testutil.Login(t, galaxyCreds) - + galaxyToken, err := testutil.Login(t, galaxyCreds) + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) @@ -493,16 +501,16 @@ func postPersistentQuery(t *testing.T, query, sha, accessJwt string) *common.Gra func TestPersistentQuery(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) - + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) - + require.NoError(t, err, "login failed") sch := `type Product { productID: ID! name: String @search(by: [term]) @@ -536,18 +544,22 @@ func TestPersistentQuery(t *testing.T) { func TestTokenExpired(t *testing.T) { prepare(t) - galaxyToken := testutil.Login(t, + galaxyToken, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: x.GalaxyNamespace}) + require.NoError(t, err, "login failed") // Create a new namespace ns, err := testutil.CreateNamespaceWithRetry(t, galaxyToken) require.NoError(t, err) - token := testutil.Login(t, + token, err := testutil.Login(t, &testutil.LoginParams{UserID: "groot", Passwd: "password", Namespace: ns}) + require.NoError(t, err, "login failed") // Relogin using refresh JWT. - token = testutil.Login(t, + token, err = testutil.Login(t, &testutil.LoginParams{RefreshJwt: token.RefreshToken}) + require.NoError(t, err, "login failed") + _, err = testutil.CreateNamespaceWithRetry(t, token) require.Error(t, err) require.Contains(t, err.Error(), "Only guardian of galaxy is allowed to do this operation") diff --git a/testutil/client.go b/testutil/client.go index c34322d03b5..345126ae781 100644 --- a/testutil/client.go +++ b/testutil/client.go @@ -349,46 +349,45 @@ func HttpLogin(params *LoginParams) (*HttpToken, error) { return nil, errors.New(fmt.Sprintf("got non 200 response from the server with %s ", string(respBody))) } - var outputJson map[string]interface{} - if err := json.Unmarshal(respBody, &outputJson); err != nil { - var errOutputJson map[string]interface{} - if err := json.Unmarshal(respBody, &errOutputJson); err == nil { - if _, ok := errOutputJson["errors"]; ok { - return nil, errors.Errorf("response error: %v", string(respBody)) - } - } - return nil, errors.Wrapf(err, "unable to unmarshal the output to get JWTs") + + var gqlResp GraphQLResponse + if err := json.Unmarshal(respBody, &gqlResp); err != nil { + return nil, err } - data, found := outputJson["data"].(map[string]interface{}) - if !found { - return nil, errors.Wrapf(err, "data entry found in the output") + if len(gqlResp.Errors) > 0 { + return nil, errors.Errorf(gqlResp.Errors.Error()) } - l, found := data["login"].(map[string]interface{}) - if !found { + if gqlResp.Data == nil { return nil, errors.Wrapf(err, "data entry found in the output") } - response, found := l["response"].(map[string]interface{}) - if !found { - return nil, errors.Wrapf(err, "data entry found in the output") + type Response struct { + Login struct { + Response struct { + AccessJWT string + RefreshJwt string + } + } + } + var r Response + if err := json.Unmarshal(gqlResp.Data, &r); err != nil { + return nil, err } - newAccessJwt, found := response["accessJWT"].(string) - if !found || newAccessJwt == "" { + if r.Login.Response.AccessJWT == "" { return nil, errors.Errorf("no access JWT found in the output") } - newRefreshJwt, found := response["refreshJWT"].(string) - if !found || newRefreshJwt == "" { + if r.Login.Response.RefreshJwt == "" { return nil, errors.Errorf("no refresh JWT found in the output") } return &HttpToken{ UserId: params.UserID, Password: params.Passwd, - AccessJwt: newAccessJwt, - RefreshToken: newRefreshJwt, + AccessJwt: r.Login.Response.AccessJWT, + RefreshToken: r.Login.Response.RefreshJwt, }, nil } diff --git a/testutil/docker.go b/testutil/docker.go index 4c470eb81ac..bce3cc603d9 100644 --- a/testutil/docker.go +++ b/testutil/docker.go @@ -81,7 +81,7 @@ func (in ContainerInstance) BestEffortWaitForHealthy(privatePort uint16) error { fmt.Printf("Health for %s failed: %v. Response: %q. Retrying...\n", in, err, body) time.Sleep(time.Second) } - return nil + return fmt.Errorf("did not pass health check on %s", "http://localhost:"+port+"/health\n") } func (in ContainerInstance) publicPort(privatePort uint16) string { diff --git a/testutil/multi_tenancy.go b/testutil/multi_tenancy.go index fa8a04ef1c1..9f885c85397 100644 --- a/testutil/multi_tenancy.go +++ b/testutil/multi_tenancy.go @@ -56,7 +56,7 @@ func MakeRequest(t *testing.T, token *HttpToken, params GraphQLParams) *GraphQLR return MakeGQLRequestWithAccessJwt(t, ¶ms, token.AccessJwt) } -func Login(t *testing.T, loginParams *LoginParams) *HttpToken { +func Login(t *testing.T, loginParams *LoginParams) (*HttpToken, error) { if loginParams.Endpoint == "" { loginParams.Endpoint = AdminUrl() } @@ -66,8 +66,7 @@ func Login(t *testing.T, loginParams *LoginParams) *HttpToken { token, err = HttpLogin(loginParams) return err }) - require.NoError(t, err, "login failed") - return token + return token, err } func ResetPassword(t *testing.T, token *HttpToken, userID, newPass string, nsID uint64) (string, error) { @@ -177,7 +176,7 @@ func DeleteNamespace(t *testing.T, token *HttpToken, nsID uint64) error { } func CreateUser(t *testing.T, token *HttpToken, username, - password string) { + password string) *GraphQLResponse { addUser := ` mutation addUser($name: String!, $pass: String!) { addUser(input: [{name: $name, password: $pass}]) { @@ -205,6 +204,7 @@ func CreateUser(t *testing.T, token *HttpToken, username, var r Response err := json.Unmarshal(resp.Data, &r) require.NoError(t, err) + return resp } func CreateGroup(t *testing.T, token *HttpToken, name string) { @@ -378,3 +378,36 @@ func QueryData(t *testing.T, dg *dgo.Dgraph, query string) []byte { require.NoError(t, err) return resp.GetJson() } + +func Export(t *testing.T, token *HttpToken, dest, accessKey, secretKey string) *GraphQLResponse { + exportRequest := `mutation export($dst: String!, $f: String!, $acc: String!, $sec: String!){ +export(input: {destination: $dst, format: $f, accessKey: $acc, secretKey: $sec}) { + response { + message + } + } + }` + + params := GraphQLParams{ + Query: exportRequest, + Variables: map[string]interface{}{ + "dst": dest, + "f": "rdf", + "acc": accessKey, + "sec": secretKey, + }, + } + + resp := MakeRequest(t, token, params) + type Response struct { + Export struct { + Response struct { + Message string + } + } + } + var r Response + err := json.Unmarshal(resp.Data, &r) + require.NoError(t, err) + return resp +} diff --git a/testutil/utils.go b/testutil/utils.go index 27813537ac5..17a96579dde 100644 --- a/testutil/utils.go +++ b/testutil/utils.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/require" "github.com/dgraph-io/dgo/v210" @@ -60,6 +61,30 @@ func GalaxyCountKey(attr string, count uint32, reverse bool) []byte { return x.CountKey(attr, count, reverse) } +type JwtParams struct { + User string + Groups []string + Ns uint64 + Exp time.Duration + Secret []byte +} + +// GetAccessJwt constructs an access jwt with the given user id, groupIds, namespace +// and expiration TTL. +func GetAccessJwt(t *testing.T, params JwtParams) string { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "userid": params.User, + "groups": params.Groups, + "namespace": params.Ns, + // set the jwt exp according to the ttl + "exp": time.Now().Add(params.Exp).Unix(), + }) + + jwtString, err := token.SignedString(params.Secret) + require.NoError(t, err) + return jwtString +} + func WaitForTask(t *testing.T, taskId string, useHttps bool, socketAddrHttp string) { const query = `query task($id: String!) { task(input: {id: $id}) { diff --git a/worker/server_state.go b/worker/server_state.go index 5e05b0cdfbb..fa25dff9f95 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -48,7 +48,7 @@ const ( `client_key=; sasl-mechanism=PLAIN; tls=false;` LimitDefaults = `mutations=allow; query-edge=1000000; normalize-node=10000; ` + `mutations-nquad=1000000; disallow-drop=false; query-timeout=0ms; txn-abort-after=5m; ` + - ` max-retries=10;max-pending-queries=10000` + ` max-retries=10;max-pending-queries=10000;shared-instance=false` ZeroLimitsDefaults = `uid-lease=0; refill-interval=30s; disable-admin-http=false;` GraphQLDefaults = `introspection=true; debug=false; extensions=true; poll-interval=1s; ` + `lambda-url=;` diff --git a/x/config.go b/x/config.go index f42d6dcbde3..6d699daf63e 100644 --- a/x/config.go +++ b/x/config.go @@ -39,6 +39,8 @@ type Options struct { // mutations-nquad int - maximum number of nquads that can be inserted in a mutation request // BlockDropAll bool - if set to true, the drop all operation will be rejected by the server. // query-timeout duration - Maximum time after which a query execution will fail. + // max-retries int64 - maximum number of retries made by dgraph to commit a transaction to disk. + // shared-instance bool - if set to true, ACLs will be disabled for non-galaxy users. Limit *z.SuperFlag LimitMutationsNquad int LimitQueryEdge uint64 @@ -46,6 +48,7 @@ type Options struct { LimitNormalizeNode int QueryTimeout time.Duration MaxRetries int64 + SharedInstance bool // GraphQL options: // diff --git a/x/minioclient.go b/x/minioclient.go index 8ec62b81c5f..ea23b237132 100644 --- a/x/minioclient.go +++ b/x/minioclient.go @@ -46,6 +46,11 @@ func (creds *MinioCredentials) isAnonymous() bool { return creds.Anonymous } +func MinioCredentialsProviderWithoutEnv(requestCreds credentials.Value) credentials.Provider { + providers := []credentials.Provider{&credentials.Static{Value: requestCreds}} + return &credentials.Chain{Providers: providers} +} + func MinioCredentialsProvider(scheme string, requestCreds credentials.Value) credentials.Provider { providers := []credentials.Provider{&credentials.Static{Value: requestCreds}} @@ -104,7 +109,12 @@ func NewMinioClient(uri *url.URL, creds *MinioCredentials) (*MinioClient, error) return &MinioClient{mc}, nil } - credsProvider := credentials.New(MinioCredentialsProvider(uri.Scheme, requestCreds(creds))) + var credsProvider *credentials.Credentials + if Config.SharedInstance { + credsProvider = credentials.New(MinioCredentialsProviderWithoutEnv(requestCreds(creds))) + } else { + credsProvider = credentials.New(MinioCredentialsProvider(uri.Scheme, requestCreds(creds))) + } mc, err := minio.NewWithCredentials(uri.Host, credsProvider, secure, "")