Add RBAC, gNOI, API Versioning and Bulk Set operations. (#61)

* Add gnoi infra with only time rpc implemented

* Remove unimplemeneted functions from gnoi_client

* Remove unimplemented sonic proto, will submit seperate PR for those rpc's

* Make gNOI active only on writable mode

* Add basic auth changes

* Fix Makefile

* Fixes

* Remove allow_no_client_auth option and instead use ca option to determine if client cert is required

* Add autheticate call to gNOI function calls

* Add client certificate authentication mode

* Add JWT based authentication and authorization mechanism

* Add ShowTechSupport gNOI function

* Register gNOI System service on gRPC server

* Register Sonic gnoi service. Remove other Sonic gNOI RPCs until a later PR

* Update proto and add missing deps

* Add translib RPC helper function to call from gNOI RPCs

* Forgot to add Refresh RPC to proto

* Add authenticate and refresh rpcs to gnoi client

* Add showTechSupport rpc to gnoi client

* Add copyConfig gNOI RPC

* Add image mgmt gNOI RPCs

* Add image clear neighbors gNOI RPC

* Add client patches

* Add yang versioning feature

* Pass auth info to translib API calls

* Add Bulk Set support

* Add context when creating transl data client for passing authentication

* Add extensions list to NewTranslClient

* Check yang bundle version in all actions

* Add yang bundle version check to Bulk set

* Update error message

* Use correct operations and fix error messages

* Add yang version check in TranslSubscribe function

* Add tlerr library

* Add yang version info to Capabilities response extensions

* Add support for UpdatesOnly subscribe option

* Remove duplicate gnoi service registration

* In order to support JWT Auth in read-only mode, separate out JWT RPCs into another proto package

* Address review comments, fix build issue

* Fix bulk bundle version check

* Add unit tests for new features

* Add error message to audit log

* Change defaults for auth and client cert back

* Fix missing jwt mode in client_auth options

* Enable default auth modes only in read/write mode

* Fix merge build issue

Makefile

@@ -32,12 +32,17 @@ go.mod:
 $(GO_DEPS): go.mod $(PATCHES)
 	# FIXME temporary workaround for crypto not downloading..
 	$(GO) get golang.org/x/crypto/ssh/terminal@e9b2fee46413
-	$(GO) get github.com/openconfig/gnmi@d2b4e6a45802a75b3571a627519cae85a197fdda
+	$(GO) get github.com/jipanyang/[email protected]
+	$(GO) get github.com/openconfig/[email protected]
 	$(GO) mod vendor
 	$(MGMT_COMMON_DIR)/patches/apply.sh vendor
-	cp -r $(GOPATH)/pkg/mod/golang.org/x/[email protected] vendor/golang.org/x/crypto
+	cp -r $(GOPATH)/pkg/mod/golang.org/x/[email protected]/* vendor/golang.org/x/crypto/
+	mkdir -p vendor/github.com/jipanyang/gnxi/
+	cp -r $(GOPATH)/pkg/mod/github.com/jipanyang/[email protected]/* vendor/github.com/jipanyang/gnxi/
 	chmod -R u+w vendor
-	patch -d vendor -p0 <patches/gnmi_cli.all.patch
+	patch -d vendor -p0 < patches/gnmi_cli.all.patch
+	patch -d vendor -p0 < patches/gnmi_set.patch
+	patch -d vendor -p0 < patches/gnmi_get.patch
 	touch $@
 
 go-deps: $(GO_DEPS)
@@ -48,9 +53,10 @@ go-deps-clean:
 sonic-telemetry: $(GO_DEPS)
 	$(GO) install -mod=vendor $(BLD_FLAGS) github.com/Azure/sonic-telemetry/telemetry
 	$(GO) install -mod=vendor $(BLD_FLAGS) github.com/Azure/sonic-telemetry/dialout/dialout_client_cli
-	$(GO) install github.com/jipanyang/gnxi/gnmi_get
-	$(GO) install github.com/jipanyang/gnxi/gnmi_set
+	$(GO) install -mod=vendor github.com/jipanyang/gnxi/gnmi_get
+	$(GO) install -mod=vendor github.com/jipanyang/gnxi/gnmi_set
 	$(GO) install -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
+	$(GO) install -mod=vendor github.com/Azure/sonic-telemetry/gnoi_client
 
 check:
 	sudo mkdir -p ${DBDIR}
@@ -75,12 +81,14 @@ install:
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_get $(DESTDIR)/usr/sbin/gnmi_get
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_set $(DESTDIR)/usr/sbin/gnmi_set
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_cli $(DESTDIR)/usr/sbin/gnmi_cli
+	$(INSTALL) -D $(BUILD_DIR)/gnoi_client $(DESTDIR)/usr/sbin/gnoi_client
 
 
 deinstall:
 	rm $(DESTDIR)/usr/sbin/telemetry
 	rm $(DESTDIR)/usr/sbin/dialout_client_cli
 	rm $(DESTDIR)/usr/sbin/gnmi_get
 	rm $(DESTDIR)/usr/sbin/gnmi_set
+	rm $(DESTDIR)/usr/sbin/gnoi_client
 
 

common_utils/context.go

@@ -0,0 +1,62 @@
+package common_utils
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+)
+
+
+// AuthInfo holds data about the authenticated user
+type AuthInfo struct {
+	// Username
+	User string
+	AuthEnabled bool
+	// Roles
+	Roles []string
+}
+
+// RequestContext holds metadata about REST request.
+type RequestContext struct {
+
+	// Unique reqiest id
+	ID string
+
+	// Auth contains the authorized user information
+	Auth AuthInfo
+
+	//Bundle Version is the release yang models version.
+	BundleVersion *string
+}
+
+type contextkey int
+
+const requestContextKey contextkey = 0
+
+// Request Id generator
+var requestCounter uint64
+
+// GetContext function returns the RequestContext object for a
+// gRPC request. RequestContext is maintained as a context value of
+// the request. Creates a new RequestContext object is not already
+// available.
+func GetContext(ctx context.Context) (*RequestContext, context.Context) {
+	cv := ctx.Value(requestContextKey)
+	if cv != nil {
+		return cv.(*RequestContext), ctx
+	}
+
+	rc := new(RequestContext)
+	rc.ID = fmt.Sprintf("TELEMETRY-%v", atomic.AddUint64(&requestCounter, 1))
+
+	ctx = context.WithValue(ctx, requestContextKey, rc)
+	return rc, ctx
+}
+
+func GetUsername(ctx context.Context, username *string) {
+        rc, ctx := GetContext(ctx)
+        if rc != nil {
+            *username = rc.Auth.User
+        }
+}
+

gnmi_server/basicAuth.go

@@ -0,0 +1,42 @@
+package gnmi
+
+import (
+	"github.com/Azure/sonic-telemetry/common_utils"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+func BasicAuthenAndAuthor(ctx context.Context) (context.Context, error) {
+	rc, ctx := common_utils.GetContext(ctx)
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return ctx, status.Errorf(codes.Unknown, "Invalid context")
+	}
+
+	var username string
+	var passwd string
+	if username_a, ok := md["username"]; ok {
+		username = username_a[0]
+	} else {
+		return ctx, status.Errorf(codes.Unauthenticated, "No Username Provided")
+	}
+
+	if passwd_a, ok := md["password"]; ok {
+		passwd = passwd_a[0]
+	} else {
+		return ctx, status.Errorf(codes.Unauthenticated, "No Password Provided")
+	}
+	if err := PopulateAuthStruct(username, &rc.Auth, nil); err != nil {
+		glog.Infof("[%s] Failed to retrieve authentication information; %v", rc.ID, err)
+		return ctx, status.Errorf(codes.Unauthenticated, "")
+	}
+	auth_success, _ := UserPwAuth(username, passwd)
+	if auth_success == false {
+		return ctx, status.Errorf(codes.PermissionDenied, "Invalid Password")
+	}
+
+	return ctx, nil
+}

gnmi_server/clientCertAuth.go

@@ -0,0 +1,41 @@
+package gnmi
+
+import (
+	"github.com/Azure/sonic-telemetry/common_utils"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+)
+
+func ClientCertAuthenAndAuthor(ctx context.Context) (context.Context, error) {
+	rc, ctx := common_utils.GetContext(ctx)
+	p, ok := peer.FromContext(ctx)
+	if !ok {
+		return ctx, status.Error(codes.Unauthenticated, "no peer found")
+	}
+	tlsAuth, ok := p.AuthInfo.(credentials.TLSInfo)
+	if !ok {
+		return ctx, status.Error(codes.Unauthenticated, "unexpected peer transport credentials")
+	}
+	if len(tlsAuth.State.VerifiedChains) == 0 || len(tlsAuth.State.VerifiedChains[0]) == 0 {
+		return ctx, status.Error(codes.Unauthenticated, "could not verify peer certificate")
+	}
+
+	var username string
+
+	username = tlsAuth.State.VerifiedChains[0][0].Subject.CommonName
+
+	if len(username) == 0 {
+		return ctx, status.Error(codes.Unauthenticated, "invalid username in certificate common name.")
+	}
+
+	if err := PopulateAuthStruct(username, &rc.Auth, nil); err != nil {
+		glog.Infof("[%s] Failed to retrieve authentication information; %v", rc.ID, err)
+		return ctx, status.Errorf(codes.Unauthenticated, "")
+	}
+
+	return ctx, nil
+}

gnmi_server/client_subscribe.go

@@ -73,6 +73,7 @@ func (c *Client) populateDbPathSubscrition(sublist *gnmipb.SubscriptionList) ([]
 // internally after sync until a Poll request is made to the server.
 func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 	defer log.V(1).Infof("Client %s shutdown", c)
+	ctx := stream.Context()
 
 	if stream == nil {
 		return grpc.Errorf(codes.FailedPrecondition, "cannot start client: stream is nil")
@@ -96,6 +97,8 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 	log.V(2).Infof("Client %s recieved initial query %v", c, query)
 
 	c.subscribe = query.GetSubscribe()
+	extensions := query.GetExtension()
+
 	if c.subscribe == nil {
 		return grpc.Errorf(codes.InvalidArgument, "first message must be SubscriptionList: %q", query)
 	}
@@ -124,7 +127,7 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 		dc, err = sdc.NewDbClient(paths, prefix)
 	} else {
 		/* For any other target or no target create new Transl Client. */
-		dc, err = sdc.NewTranslClient(prefix, paths)
+		dc, err = sdc.NewTranslClient(prefix, paths, ctx, extensions)
 	}
 
 	if err != nil {
@@ -140,12 +143,12 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 		c.polled = make(chan struct{}, 1)
 		c.polled <- struct{}{}
 		c.w.Add(1)
-		go dc.PollRun(c.q, c.polled, &c.w)
+		go dc.PollRun(c.q, c.polled, &c.w, c.subscribe)
 	case gnmipb.SubscriptionList_ONCE:
 		c.once = make(chan struct{}, 1)
 		c.once <- struct{}{}
 		c.w.Add(1)
-		go dc.OnceRun(c.q, c.once, &c.w)
+		go dc.OnceRun(c.q, c.once, &c.w, c.subscribe)
 	default:
 		return grpc.Errorf(codes.InvalidArgument, "Unkown subscription mode: %q", query)
 	}