From eb0b4437d38017c054e93fda790bb487d3e7de0f Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Mon, 27 Jun 2022 23:13:31 +0800 Subject: [PATCH 01/18] cert provider --- remoting/xds/xds_client_factory.go | 182 +++++++- remoting/xds/xds_client_factory_test.go | 15 + xds/client/bootstrap/bootstrap.go | 14 +- xds/credentials/certgenerate/crypto.go | 166 +++++++ xds/credentials/certgenerate/dual_use.go | 43 ++ xds/credentials/certgenerate/generate_cert.go | 436 ++++++++++++++++++ xds/credentials/certgenerate/generate_csr.go | 134 ++++++ xds/credentials/certgenerate/san.go | 213 +++++++++ xds/credentials/certprovider/distributor.go | 118 +++++ .../certprovider/pemfile/builder.go | 107 +++++ .../certprovider/pemfile/watcher.go | 262 +++++++++++ xds/credentials/certprovider/provider.go | 104 +++++ .../certprovider/remote/istioca_client.go | 200 ++++++++ .../certprovider/remote/v1alpha1/ca.pb.go | 278 +++++++++++ .../remote/v1alpha1/ca_grpc.pb.go | 108 +++++ xds/credentials/certprovider/store.go | 161 +++++++ xds/utils/envconfig/xds.go | 4 + 17 files changed, 2539 insertions(+), 6 deletions(-) create mode 100644 remoting/xds/xds_client_factory_test.go create mode 100644 xds/credentials/certgenerate/crypto.go create mode 100644 xds/credentials/certgenerate/dual_use.go create mode 100644 xds/credentials/certgenerate/generate_cert.go create mode 100644 xds/credentials/certgenerate/generate_csr.go create mode 100644 xds/credentials/certgenerate/san.go create mode 100644 xds/credentials/certprovider/distributor.go create mode 100644 xds/credentials/certprovider/pemfile/builder.go create mode 100644 xds/credentials/certprovider/pemfile/watcher.go create mode 100644 xds/credentials/certprovider/provider.go create mode 100644 xds/credentials/certprovider/remote/istioca_client.go create mode 100644 xds/credentials/certprovider/remote/v1alpha1/ca.pb.go create mode 100644 xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go create mode 100644 xds/credentials/certprovider/store.go diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go index f038e6467a..b182200676 100644 --- a/remoting/xds/xds_client_factory.go +++ b/remoting/xds/xds_client_factory.go @@ -18,10 +18,23 @@ package xds import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + log "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" + certprovider2 "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "encoding/pem" + "fmt" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" + "io/ioutil" + "os" + "strings" ) import ( @@ -44,10 +57,15 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd Metadata: mapping.GetDubboGoMetadata(""), } + certificates, err := buildCertificate() + if err != nil { + return nil, err + } + nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ ServerURI: istioAddr.String(), - Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + Creds: grpc.WithTransportCredentials(certificates), TransportAPI: version.TransportV3, NodeProto: v3NodeProto, }, @@ -60,3 +78,161 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd } return newClient, nil } + +var buildCertificate = func() (credentials.TransportCredentials, error) { + + bootstrapPath := os.Getenv(envconfig.XDSBootstrapFileNameEnv) + //agent exist, get cert from file + if "" != bootstrapPath { + config, err := bootstrap.NewConfig() + if err != nil { + return nil, err + } + certProvider, err := buildProviderFunc(config.CertProviderConfigs, "default", "rootCA", false, true) + + material, err := certProvider.KeyMaterial(context.Background()) + + if err != nil { + log.Errorf("failed to get root ca :%s", err.Error()) + return nil, err + } + + cred := credentials.NewTLS(&tls.Config{ + RootCAs: material.Roots, + Certificates: material.Certs, + GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + keyMaterial, err := certProvider.KeyMaterial(info.Context()) + if err != nil { + log.Errorf("failed to get keyMaterial :%s", err.Error()) + return nil, err + } + return &keyMaterial.Certs[0], nil + }, + }) + return cred, nil + + //agent not exist, get cert from istio citadel + } else { + tokenProvider, err := NewSaTokenProvider() + if err != nil { + return nil, err + } + citadelClient, err := remote.NewCitadelClient(&remote.Options{ + CAEndpoint: envconfig.IstioCAEndpoint, + TokenProvider: tokenProvider, + }) + + //use default + options := certgenerate.CertOptions{ + Host: "dubbo-go", + RSAKeySize: 2048, + PKCS8Key: true, + ECSigAlg: certgenerate.EcdsaSigAlg, + } + + // Generate the cert/key, send CSR to CA. + csrPEM, keyPEM, err := certgenerate.GenCSR(options) + if err != nil { + log.Errorf("failed to generate key and certificate for CSR: %v", err) + return nil, err + } + sign, err := citadelClient.CSRSign(csrPEM, int64(0)) + + if err != nil { + return nil, err + } + + cert, err := ParseCert(concatCerts(sign), keyPEM) + if err != nil { + return nil, err + } + + cred := credentials.NewTLS(&tls.Config{ + RootCAs: nil, //todo fetch rootCA + Certificates: []tls.Certificate{*cert}, + GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { + //todo check cert expired + return cert, nil + }, + }) + return cred, nil + } + +} + +//provide k8s service account +type saTokenProvider struct { + token string +} + +func NewSaTokenProvider() (*saTokenProvider, error) { + sa, err := ioutil.ReadFile(envconfig.KubernetesServiceAccountPath) + if err != nil { + return nil, err + } + return &saTokenProvider{ + token: string(sa), + }, nil +} + +func (s *saTokenProvider) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + + meta := make(map[string]string) + + meta["authorization"] = "Bearer " + s.token + return meta, nil +} + +func (s *saTokenProvider) RequireTransportSecurity() bool { + return false +} + +func buildProviderFunc(configs map[string]*certprovider2.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider2.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider2.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} +func concatCerts(certsPEM []string) []byte { + if len(certsPEM) == 0 { + return []byte{} + } + var certChain bytes.Buffer + for i, c := range certsPEM { + certChain.WriteString(c) + if i < len(certsPEM)-1 && !strings.HasSuffix(c, "\n") { + certChain.WriteString("\n") + } + } + return certChain.Bytes() +} + +func ParseCert(certByte []byte, keyByte []byte) (*tls.Certificate, error) { + block, _ := pem.Decode(certByte) + if block == nil { + return nil, fmt.Errorf("failed to decode certificate") + } + cert, err := x509.ParseCertificate(block.Bytes) + + expired := cert.NotAfter + log.Infof("cert expired after:" + expired.String()) + + pair, err := tls.X509KeyPair(certByte, keyByte) + if err != nil { + return nil, fmt.Errorf("failed to parse certificate: %v", err) + } + return &pair, nil +} diff --git a/remoting/xds/xds_client_factory_test.go b/remoting/xds/xds_client_factory_test.go new file mode 100644 index 0000000000..bf6a0d8492 --- /dev/null +++ b/remoting/xds/xds_client_factory_test.go @@ -0,0 +1,15 @@ +package xds + +import ( + "fmt" + "testing" +) + +func TestBuildCertificate(t *testing.T) { + fmt.Println("test begin ") + certificate, err := buildCertificate() + + fmt.Println(err) + fmt.Println(certificate) + +} diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index ebb76ba853..e6729298db 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -27,6 +27,8 @@ package bootstrap import ( "bytes" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/pemfile" "encoding/json" "fmt" "io/ioutil" @@ -43,13 +45,12 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/credentials/tls/certprovider" ) import ( dubboLogger "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" + internal "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) @@ -67,6 +68,13 @@ const ( clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) +func init() { + //init file_watcher builder + certprovider.Register(&pemfile.PluginBuilder{}) + //init builder func + internal.GetCertificateProviderBuilder = certprovider.GetBuilder +} + var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version) // For overriding in unit tests. @@ -345,7 +353,7 @@ func NewConfigFromContents(data []byte) (*Config, error) { return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) } configs := make(map[string]*certprovider.BuildableConfig) - getBuilder := internal2.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) for instance, data := range providerInstances { var nameAndConfig struct { PluginName string `json:"plugin_name"` diff --git a/xds/credentials/certgenerate/crypto.go b/xds/credentials/certgenerate/crypto.go new file mode 100644 index 0000000000..00ab1b10d1 --- /dev/null +++ b/xds/credentials/certgenerate/crypto.go @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright Istio Authors + * + */ + +package certgenerate + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "reflect" + "strings" +) + +const ( + blockTypeECPrivateKey = "EC PRIVATE KEY" + blockTypeRSAPrivateKey = "RSA PRIVATE KEY" // PKCS#1 private key + blockTypePKCS8PrivateKey = "PRIVATE KEY" // PKCS#8 plain private key +) + +// ParsePemEncodedCertificate constructs a `x509.Certificate` object using the +// given a PEM-encoded certificate. +func ParsePemEncodedCertificate(certBytes []byte) (*x509.Certificate, error) { + cb, _ := pem.Decode(certBytes) + if cb == nil { + return nil, fmt.Errorf("invalid PEM encoded certificate") + } + + cert, err := x509.ParseCertificate(cb.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse X.509 certificate") + } + + return cert, nil +} + +// ParsePemEncodedCertificateChain constructs a slice of `x509.Certificate` +// objects using the given a PEM-encoded certificate chain. +func ParsePemEncodedCertificateChain(certBytes []byte) ([]*x509.Certificate, error) { + var ( + certs []*x509.Certificate + cb *pem.Block + ) + for { + cb, certBytes = pem.Decode(certBytes) + if cb == nil { + break + } + cert, err := x509.ParseCertificate(cb.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse X.509 certificate") + } + certs = append(certs, cert) + } + if len(certs) == 0 { + return nil, fmt.Errorf("no PEM encoded X.509 certificates parsed") + } + return certs, nil +} + +// ParsePemEncodedCSR constructs a `x509.CertificateRequest` object using the +// given PEM-encoded certificate signing request. +func ParsePemEncodedCSR(csrBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(csrBytes) + if block == nil { + return nil, fmt.Errorf("certificate signing request is not properly encoded") + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse X.509 certificate signing request") + } + return csr, nil +} + +// ParsePemEncodedKey takes a PEM-encoded key and parsed the bytes into a `crypto.PrivateKey`. +func ParsePemEncodedKey(keyBytes []byte) (crypto.PrivateKey, error) { + kb, _ := pem.Decode(keyBytes) + if kb == nil { + return nil, fmt.Errorf("invalid PEM-encoded key") + } + + switch kb.Type { + case blockTypeECPrivateKey: + key, err := x509.ParseECPrivateKey(kb.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse the ECDSA private key") + } + return key, nil + case blockTypeRSAPrivateKey: + key, err := x509.ParsePKCS1PrivateKey(kb.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse the RSA private key") + } + return key, nil + case blockTypePKCS8PrivateKey: + key, err := x509.ParsePKCS8PrivateKey(kb.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse the PKCS8 private key") + } + return key, nil + default: + return nil, fmt.Errorf("unsupported PEM block type for a private key: %s", kb.Type) + } +} + +// GetRSAKeySize returns the size if it is RSA key, otherwise it returns an error. +func GetRSAKeySize(privKey crypto.PrivateKey) (int, error) { + if t := reflect.TypeOf(privKey); t != reflect.TypeOf(&rsa.PrivateKey{}) { + return 0, fmt.Errorf("key type is not RSA: %v", t) + } + pkey := privKey.(*rsa.PrivateKey) + return pkey.N.BitLen(), nil +} + +// IsSupportedECPrivateKey is a predicate returning true if the private key is EC based +func IsSupportedECPrivateKey(privKey *crypto.PrivateKey) bool { + switch (*privKey).(type) { + // this should agree with var SupportedECSignatureAlgorithms + case *ecdsa.PrivateKey: + return true + default: + return false + } +} + +// PemCertBytestoString: takes an array of PEM certs in bytes and returns a string array in the same order with +// trailing newline characters removed +func PemCertBytestoString(caCerts []byte) []string { + certs := []string{} + var cert string + pemBlock := caCerts + for block, rest := pem.Decode(pemBlock); block != nil && len(block.Bytes) != 0; block, rest = pem.Decode(pemBlock) { + if len(rest) == 0 { + cert = strings.TrimPrefix(strings.TrimSuffix(string(pemBlock), "\n"), "\n") + certs = append(certs, cert) + break + } + cert = string(pemBlock[0 : len(pemBlock)-len(rest)]) + cert = strings.TrimPrefix(strings.TrimSuffix(cert, "\n"), "\n") + certs = append(certs, cert) + pemBlock = rest + } + return certs +} diff --git a/xds/credentials/certgenerate/dual_use.go b/xds/credentials/certgenerate/dual_use.go new file mode 100644 index 0000000000..86cb16ceb7 --- /dev/null +++ b/xds/credentials/certgenerate/dual_use.go @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright Istio Authors + * + */ + +package certgenerate + +import ( + "fmt" + "strings" +) + +// DualUseCommonName extracts a valid CommonName from a comma-delimited host string +// for dual-use certificates. +func DualUseCommonName(host string) (string, error) { + // cn uses one hostname, drop the rest + first := strings.SplitN(host, ",", 2)[0] + + // cn max length is 64 (ub-common-name @ https://tools.ietf.org/html/rfc5280) + if l := len(first); l > 64 { + return "", fmt.Errorf("certificate CN upper bound exceeded (%v>64): %s", l, first) + } + + return first, nil +} diff --git a/xds/credentials/certgenerate/generate_cert.go b/xds/credentials/certgenerate/generate_cert.go new file mode 100644 index 0000000000..e9b264cecb --- /dev/null +++ b/xds/credentials/certgenerate/generate_cert.go @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright Istio Authors + * + */ + +package certgenerate + +import ( + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + log "dubbo.apache.org/dubbo-go/v3/common/logger" + "encoding/pem" + "errors" + "fmt" + "math/big" + "os" + "strings" + "time" +) + +// SupportedECSignatureAlgorithms are the types of EC Signature Algorithms +// to be used in key generation (e.g. ECDSA or ED2551) +type SupportedECSignatureAlgorithms string + +const ( + // only ECDSA using P256 is currently supported + EcdsaSigAlg SupportedECSignatureAlgorithms = "ECDSA" +) + +// CertOptions contains options for generating a new certificate. +type CertOptions struct { + // Comma-separated hostnames and IPs to generate a certificate for. + // This can also be set to the identity running the workload, + // like kubernetes service account. + Host string + + // The NotBefore field of the issued certificate. + NotBefore time.Time + + // TTL of the certificate. NotAfter - NotBefore. + TTL time.Duration + + // Signer certificate. + SignerCert *x509.Certificate + + // Signer private key. + SignerPriv crypto.PrivateKey + + // Signer private key (PEM encoded). + SignerPrivPem []byte + + // Organization for this certificate. + Org string + + // The size of RSA private key to be generated. + RSAKeySize int + + // Whether this certificate is used as signing cert for CA. + IsCA bool + + // Whether this certificate is self-signed. + IsSelfSigned bool + + // Whether this certificate is for a client. + IsClient bool + + // Whether this certificate is for a server. + IsServer bool + + // Whether this certificate is for dual-use clients (SAN+CN). + IsDualUse bool + + // If true, the private key is encoded with PKCS#8. + PKCS8Key bool + + // The type of Elliptical Signature algorithm to use + // when generating private keys. Currently only ECDSA is supported. + // If empty, RSA is used, otherwise ECC is used. + ECSigAlg SupportedECSignatureAlgorithms + + // Subjective Alternative Name values. + DNSNames string +} + +// GenCertKeyFromOptions generates a X.509 certificate and a private key with the given options. +func GenCertKeyFromOptions(options CertOptions) (pemCert []byte, pemKey []byte, err error) { + // Generate the appropriate private&public key pair based on options. + // The public key will be bound to the certificate generated below. The + // private key will be used to sign this certificate in the self-signed + // case, otherwise the certificate is signed by the signer private key + // as specified in the CertOptions. + if options.ECSigAlg != "" { + var ecPriv *ecdsa.PrivateKey + + switch options.ECSigAlg { + case EcdsaSigAlg: + ecPriv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at EC key generation (%v)", err) + } + default: + return nil, nil, errors.New("cert generation fails due to unsupported EC signature algorithm") + } + return genCert(options, ecPriv, &ecPriv.PublicKey) + } + + if options.RSAKeySize < minimumRsaKeySize { + return nil, nil, fmt.Errorf("requested key size does not meet the minimum requied size of %d (requested: %d)", minimumRsaKeySize, options.RSAKeySize) + } + rsaPriv, err := rsa.GenerateKey(rand.Reader, options.RSAKeySize) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at RSA key generation (%v)", err) + } + return genCert(options, rsaPriv, &rsaPriv.PublicKey) +} + +func genCert(options CertOptions, priv interface{}, key interface{}) ([]byte, []byte, error) { + template, err := genCertTemplateFromOptions(options) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at cert template creation (%v)", err) + } + signerCert, signerKey := template, crypto.PrivateKey(priv) + if !options.IsSelfSigned { + signerCert, signerKey = options.SignerCert, options.SignerPriv + } + certBytes, err := x509.CreateCertificate(rand.Reader, template, signerCert, key, signerKey) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at X509 cert creation (%v)", err) + } + + pemCert, pemKey, err := encodePem(false, certBytes, priv, options.PKCS8Key) + return pemCert, pemKey, err +} + +func publicKey(priv interface{}) interface{} { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + case ed25519.PrivateKey: + return k.Public().(ed25519.PublicKey) + default: + return nil + } +} + +// GenRootCertFromExistingKey generates a X.509 certificate using existing +// CA private key. Only called by a self-signed Citadel. +func GenRootCertFromExistingKey(options CertOptions) (pemCert []byte, pemKey []byte, err error) { + if !options.IsSelfSigned || len(options.SignerPrivPem) == 0 { + return nil, nil, fmt.Errorf("skip cert " + + "generation. Citadel is not in self-signed mode or CA private key is not " + + "available") + } + + template, err := genCertTemplateFromOptions(options) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at cert template creation (%v)", err) + } + caPrivateKey, err := ParsePemEncodedKey(options.SignerPrivPem) + if err != nil { + return nil, nil, fmt.Errorf("unrecogniazed CA "+ + "private key, skip root cert rotation: %s", err.Error()) + } + certBytes, err := x509.CreateCertificate(rand.Reader, template, template, publicKey(caPrivateKey), caPrivateKey) + if err != nil { + return nil, nil, fmt.Errorf("cert generation fails at X509 cert creation (%v)", err) + } + + pemCert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes}) + return pemCert, options.SignerPrivPem, nil +} + +// GetCertOptionsFromExistingCert parses cert and generates a CertOptions +// that contains information about the cert. This is the reverse operation of +// genCertTemplateFromOptions(), and only called by a self-signed Citadel. +func GetCertOptionsFromExistingCert(certBytes []byte) (opts CertOptions, err error) { + cert, certErr := ParsePemEncodedCertificate(certBytes) + if certErr != nil { + return opts, certErr + } + + orgs := cert.Subject.Organization + if len(orgs) > 0 { + opts.Org = orgs[0] + } + // TODO(JimmyCYJ): parse other fields from certificate, e.g. CommonName. + return opts, nil +} + +// MergeCertOptions merges deltaOpts into defaultOpts and returns the merged +// CertOptions. Only called by a self-signed Citadel. +func MergeCertOptions(defaultOpts, deltaOpts CertOptions) CertOptions { + if len(deltaOpts.Org) > 0 { + defaultOpts.Org = deltaOpts.Org + } + // TODO(JimmyCYJ): merge other fields, e.g. Host, IsDualUse, etc. + return defaultOpts +} + +// GenCertFromCSR generates a X.509 certificate with the given CSR. +func GenCertFromCSR(csr *x509.CertificateRequest, signingCert *x509.Certificate, publicKey interface{}, + signingKey crypto.PrivateKey, subjectIDs []string, ttl time.Duration, isCA bool, +) (cert []byte, err error) { + tmpl, err := genCertTemplateFromCSR(csr, subjectIDs, ttl, isCA) + if err != nil { + return nil, err + } + return x509.CreateCertificate(rand.Reader, tmpl, signingCert, publicKey, signingKey) +} + +// LoadSignerCredsFromFiles loads the signer cert&key from the given files. +// +// signerCertFile: cert file name +// signerPrivFile: private key file name +func LoadSignerCredsFromFiles(signerCertFile string, signerPrivFile string) (*x509.Certificate, crypto.PrivateKey, error) { + signerCertBytes, err := os.ReadFile(signerCertFile) + if err != nil { + return nil, nil, fmt.Errorf("certificate file reading failure (%v)", err) + } + + signerPrivBytes, err := os.ReadFile(signerPrivFile) + if err != nil { + return nil, nil, fmt.Errorf("private key file reading failure (%v)", err) + } + + cert, err := ParsePemEncodedCertificate(signerCertBytes) + if err != nil { + return nil, nil, fmt.Errorf("pem encoded cert parsing failure (%v)", err) + } + + key, err := ParsePemEncodedKey(signerPrivBytes) + if err != nil { + return nil, nil, fmt.Errorf("pem encoded key parsing failure (%v)", err) + } + + return cert, key, nil +} + +// ClockSkewGracePeriod defines the period of time a certificate will be valid before its creation. +// This is meant to handle cases where we have clock skew between the CA and workloads. +const ClockSkewGracePeriod = time.Minute * 2 + +// genCertTemplateFromCSR generates a certificate template with the given CSR. +// The NotBefore value of the cert is set to current time. +func genCertTemplateFromCSR(csr *x509.CertificateRequest, subjectIDs []string, ttl time.Duration, isCA bool) ( + *x509.Certificate, error, +) { + subjectIDsInString := strings.Join(subjectIDs, ",") + var keyUsage x509.KeyUsage + extKeyUsages := []x509.ExtKeyUsage{} + if isCA { + // If the cert is a CA cert, the private key is allowed to sign other certificates. + keyUsage = x509.KeyUsageCertSign + } else { + // Otherwise the private key is allowed for digital signature and key encipherment. + keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + // For now, we do not differentiate non-CA certs to be used on client auth or server auth. + extKeyUsages = append(extKeyUsages, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth) + } + + // Build cert extensions with the subjectIDs. + ext, err := BuildSubjectAltNameExtension(subjectIDsInString) + if err != nil { + return nil, err + } + exts := []pkix.Extension{*ext} + + subject := pkix.Name{} + // Dual use mode if common name in CSR is not empty. + // In this case, set CN as determined by DualUseCommonName(subjectIDsInString). + if len(csr.Subject.CommonName) != 0 { + if cn, err := DualUseCommonName(subjectIDsInString); err != nil { + // log and continue + log.Errorf("dual-use failed for cert template - omitting CN (%v)", err) + } else { + subject.CommonName = cn + } + } + + now := time.Now() + + serialNum, err := genSerialNum() + if err != nil { + return nil, err + } + // SignatureAlgorithm will use the default algorithm. + // See https://golang.org/src/crypto/x509/x509.go?s=5131:5158#L1965 . + return &x509.Certificate{ + SerialNumber: serialNum, + Subject: subject, + NotBefore: now.Add(-ClockSkewGracePeriod), + NotAfter: now.Add(ttl), + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsages, + IsCA: isCA, + BasicConstraintsValid: true, + ExtraExtensions: exts, + }, nil +} + +// genCertTemplateFromoptions generates a certificate template with the given options. +func genCertTemplateFromOptions(options CertOptions) (*x509.Certificate, error) { + var keyUsage x509.KeyUsage + if options.IsCA { + // If the cert is a CA cert, the private key is allowed to sign other certificates. + keyUsage = x509.KeyUsageCertSign + } else { + // Otherwise the private key is allowed for digital signature and key encipherment. + keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + } + + extKeyUsages := []x509.ExtKeyUsage{} + if options.IsServer { + extKeyUsages = append(extKeyUsages, x509.ExtKeyUsageServerAuth) + } + if options.IsClient { + extKeyUsages = append(extKeyUsages, x509.ExtKeyUsageClientAuth) + } + + notBefore := time.Now() + if !options.NotBefore.IsZero() { + notBefore = options.NotBefore + } + + serialNum, err := genSerialNum() + if err != nil { + return nil, err + } + + subject := pkix.Name{ + Organization: []string{options.Org}, + } + + exts := []pkix.Extension{} + if h := options.Host; len(h) > 0 { + s, err := BuildSubjectAltNameExtension(h) + if err != nil { + return nil, err + } + if options.IsDualUse { + cn, err := DualUseCommonName(h) + if err != nil { + // log and continue + log.Errorf("dual-use failed for cert template - omitting CN (%v)", err) + } else { + subject.CommonName = cn + } + } + exts = []pkix.Extension{*s} + } + + dnsNames := strings.Split(options.DNSNames, ",") + if len(dnsNames[0]) == 0 { + dnsNames = nil + } + + return &x509.Certificate{ + SerialNumber: serialNum, + Subject: subject, + NotBefore: notBefore, + NotAfter: notBefore.Add(options.TTL), + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsages, + IsCA: options.IsCA, + BasicConstraintsValid: true, + ExtraExtensions: exts, + DNSNames: dnsNames, + }, nil +} + +func genSerialNum() (*big.Int, error) { + serialNumLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNum, err := rand.Int(rand.Reader, serialNumLimit) + if err != nil { + return nil, fmt.Errorf("serial number generation failure (%v)", err) + } + return serialNum, nil +} + +func encodePem(isCSR bool, csrOrCert []byte, priv interface{}, pkcs8 bool) ( + csrOrCertPem []byte, privPem []byte, err error, +) { + encodeMsg := "CERTIFICATE" + if isCSR { + encodeMsg = "CERTIFICATE REQUEST" + } + csrOrCertPem = pem.EncodeToMemory(&pem.Block{Type: encodeMsg, Bytes: csrOrCert}) + + var encodedKey []byte + if pkcs8 { + if encodedKey, err = x509.MarshalPKCS8PrivateKey(priv); err != nil { + return nil, nil, err + } + privPem = pem.EncodeToMemory(&pem.Block{Type: blockTypePKCS8PrivateKey, Bytes: encodedKey}) + } else { + switch k := priv.(type) { + case *rsa.PrivateKey: + encodedKey = x509.MarshalPKCS1PrivateKey(k) + privPem = pem.EncodeToMemory(&pem.Block{Type: blockTypeRSAPrivateKey, Bytes: encodedKey}) + case *ecdsa.PrivateKey: + encodedKey, err = x509.MarshalECPrivateKey(k) + if err != nil { + return nil, nil, err + } + privPem = pem.EncodeToMemory(&pem.Block{Type: blockTypeECPrivateKey, Bytes: encodedKey}) + } + } + err = nil + return +} diff --git a/xds/credentials/certgenerate/generate_csr.go b/xds/credentials/certgenerate/generate_csr.go new file mode 100644 index 0000000000..43eec682f0 --- /dev/null +++ b/xds/credentials/certgenerate/generate_csr.go @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright Istio Authors + * + */ + +package certgenerate + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + log "dubbo.apache.org/dubbo-go/v3/common/logger" + "errors" + "fmt" + "os" + "strings" +) + +// minimumRsaKeySize is the minimum RSA key size to generate certificates +// to ensure proper security +const minimumRsaKeySize = 2048 + +// GenCSR generates a X.509 certificate sign request and private key with the given options. +func GenCSR(options CertOptions) ([]byte, []byte, error) { + var priv interface{} + var err error + if options.ECSigAlg != "" { + switch options.ECSigAlg { + case EcdsaSigAlg: + priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("EC key generation failed (%v)", err) + } + default: + return nil, nil, errors.New("csr cert generation fails due to unsupported EC signature algorithm") + } + } else { + if options.RSAKeySize < minimumRsaKeySize { + return nil, nil, fmt.Errorf("requested key size does not meet the minimum requied size of %d (requested: %d)", minimumRsaKeySize, options.RSAKeySize) + } + + priv, err = rsa.GenerateKey(rand.Reader, options.RSAKeySize) + if err != nil { + return nil, nil, fmt.Errorf("RSA key generation failed (%v)", err) + } + } + template, err := GenCSRTemplate(options) + if err != nil { + return nil, nil, fmt.Errorf("CSR template creation failed (%v)", err) + } + + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, template, crypto.PrivateKey(priv)) + if err != nil { + return nil, nil, fmt.Errorf("CSR creation failed (%v)", err) + } + + csr, privKey, err := encodePem(true, csrBytes, priv, options.PKCS8Key) + return csr, privKey, err +} + +// GenCSRTemplate generates a certificateRequest template with the given options. +func GenCSRTemplate(options CertOptions) (*x509.CertificateRequest, error) { + template := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{options.Org}, + }, + } + + if h := options.Host; len(h) > 0 { + s, err := BuildSubjectAltNameExtension(h) + if err != nil { + return nil, err + } + if options.IsDualUse { + cn, err := DualUseCommonName(h) + if err != nil { + // log and continue + //log.Errorf("dual-use failed for CSR template - omitting CN (%v)", err) + } else { + template.Subject.CommonName = cn + } + } + template.ExtraExtensions = []pkix.Extension{*s} + } + + return template, nil +} + +// AppendRootCerts appends root certificates in RootCertFile to the input certificate. +func AppendRootCerts(pemCert []byte, rootCertFile string) ([]byte, error) { + rootCerts := pemCert + if len(rootCertFile) > 0 { + log.Debugf("append root certificates from %v", rootCertFile) + certBytes, err := os.ReadFile(rootCertFile) + if err != nil { + return rootCerts, fmt.Errorf("failed to read root certificates (%v)", err) + } + rootCerts = AppendCertByte(pemCert, certBytes) + } + return rootCerts, nil +} + +// AppendCertByte: Append x.509 rootCert in bytes to existing certificate chain (in bytes) +func AppendCertByte(pemCert []byte, rootCert []byte) []byte { + rootCerts := []byte{} + if len(pemCert) > 0 { + // Copy the input certificate + rootCerts = []byte(strings.TrimSuffix(string(pemCert), "\n") + "\n") + } + rootCerts = append(rootCerts, rootCert...) + return rootCerts +} diff --git a/xds/credentials/certgenerate/san.go b/xds/credentials/certgenerate/san.go new file mode 100644 index 0000000000..7728681416 --- /dev/null +++ b/xds/credentials/certgenerate/san.go @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright Istio Authors + * + */ + +package certgenerate + +import ( + "crypto/x509/pkix" + "encoding/asn1" + "fmt" + "net" + "strings" +) + +// IdentityType represents type of an identity. This is used to properly encode +// an identity into a SAN extension. +type IdentityType int + +const ( + // TypeDNS represents a DNS name. + TypeDNS IdentityType = iota + // TypeIP represents an IP address. + TypeIP + // TypeURI represents a universal resource identifier. + TypeURI + + Scheme = "spiffe" + URIPrefix = Scheme + "://" + URIPrefixLen = len(URIPrefix) +) + +var ( + // Mapping from the type of an identity to the OID tag value for the X.509 + // SAN field (see https://tools.ietf.org/html/rfc5280#appendix-A.2) + // + // SubjectAltName ::= GeneralNames + // + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + // + // GeneralName ::= CHOICE { + // dNSName [2] IA5String, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // } + oidTagMap = map[IdentityType]int{ + TypeDNS: 2, + TypeURI: 6, + TypeIP: 7, + } + + // A reversed map that maps from an OID tag to the corresponding identity + // type. + identityTypeMap = generateReversedMap(oidTagMap) + + // The OID for the SAN extension (See + // http://www.alvestrand.no/objectid/2.5.29.17.html). + oidSubjectAlternativeName = asn1.ObjectIdentifier{2, 5, 29, 17} +) + +// Identity is an object holding both the encoded identifier bytes as well as +// the type of the identity. +type Identity struct { + Type IdentityType + Value []byte +} + +// BuildSubjectAltNameExtension builds the SAN extension for the certificate. +func BuildSubjectAltNameExtension(hosts string) (*pkix.Extension, error) { + ids := []Identity{} + for _, host := range strings.Split(hosts, ",") { + if ip := net.ParseIP(host); ip != nil { + // Use the 4-byte representation of the IP address when possible. + if eip := ip.To4(); eip != nil { + ip = eip + } + ids = append(ids, Identity{Type: TypeIP, Value: ip}) + } else if strings.HasPrefix(host, URIPrefix) { + ids = append(ids, Identity{Type: TypeURI, Value: []byte(host)}) + } else { + ids = append(ids, Identity{Type: TypeDNS, Value: []byte(host)}) + } + } + + san, err := BuildSANExtension(ids) + if err != nil { + return nil, fmt.Errorf("SAN extension building failure (%v)", err) + } + + return san, nil +} + +// BuildSANExtension builds a `pkix.Extension` of type "Subject +// Alternative Name" based on the given identities. +func BuildSANExtension(identites []Identity) (*pkix.Extension, error) { + rawValues := []asn1.RawValue{} + for _, i := range identites { + tag, ok := oidTagMap[i.Type] + if !ok { + return nil, fmt.Errorf("unsupported identity type: %v", i.Type) + } + + rawValues = append(rawValues, asn1.RawValue{ + Bytes: i.Value, + Class: asn1.ClassContextSpecific, + Tag: tag, + }) + } + + bs, err := asn1.Marshal(rawValues) + if err != nil { + return nil, fmt.Errorf("failed to marshal the raw values for SAN field (err: %s)", err) + } + + // SAN is Critical because the subject is empty. This is compliant with X.509 and SPIFFE standards. + return &pkix.Extension{Id: oidSubjectAlternativeName, Critical: true, Value: bs}, nil +} + +// ExtractIDsFromSAN takes a SAN extension and extracts the identities. +// The logic is mostly borrowed from +// https://github.com/golang/go/blob/master/src/crypto/x509/x509.go, with the +// addition of supporting extracting URIs. +func ExtractIDsFromSAN(sanExt *pkix.Extension) ([]Identity, error) { + if !sanExt.Id.Equal(oidSubjectAlternativeName) { + return nil, fmt.Errorf("the input is not a SAN extension") + } + + var sequence asn1.RawValue + if rest, err := asn1.Unmarshal(sanExt.Value, &sequence); err != nil { + return nil, err + } else if len(rest) != 0 { + return nil, fmt.Errorf("the SAN extension is incorrectly encoded") + } + + // Check the rawValue is a sequence. + if !sequence.IsCompound || sequence.Tag != asn1.TagSequence || sequence.Class != asn1.ClassUniversal { + return nil, fmt.Errorf("the SAN extension is incorrectly encoded") + } + + ids := []Identity{} + for bytes := sequence.Bytes; len(bytes) > 0; { + var rawValue asn1.RawValue + var err error + + bytes, err = asn1.Unmarshal(bytes, &rawValue) + if err != nil { + return nil, err + } + ids = append(ids, Identity{Type: identityTypeMap[rawValue.Tag], Value: rawValue.Bytes}) + } + + return ids, nil +} + +// ExtractSANExtension extracts the "Subject Alternative Name" externsion from +// the given PKIX extension set. +func ExtractSANExtension(exts []pkix.Extension) *pkix.Extension { + for _, ext := range exts { + if ext.Id.Equal(oidSubjectAlternativeName) { + // We don't need to examine other extensions anymore since a certificate + // must not include more than one instance of a particular extension. See + // https://tools.ietf.org/html/rfc5280#section-4.2. + return &ext + } + } + return nil +} + +// ExtractIDs first finds the SAN extension from the given extension set, then +// extract identities from the SAN extension. +func ExtractIDs(exts []pkix.Extension) ([]string, error) { + sanExt := ExtractSANExtension(exts) + if sanExt == nil { + return nil, fmt.Errorf("the SAN extension does not exist") + } + + idsWithType, err := ExtractIDsFromSAN(sanExt) + if err != nil { + return nil, fmt.Errorf("failed to extract identities from SAN extension (error %v)", err) + } + + ids := []string{} + for _, id := range idsWithType { + ids = append(ids, string(id.Value)) + } + return ids, nil +} + +func generateReversedMap(m map[IdentityType]int) map[int]IdentityType { + reversed := make(map[int]IdentityType) + for key, value := range m { + reversed[value] = key + } + return reversed +} diff --git a/xds/credentials/certprovider/distributor.go b/xds/credentials/certprovider/distributor.go new file mode 100644 index 0000000000..88e7a81f4b --- /dev/null +++ b/xds/credentials/certprovider/distributor.go @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright 2020 gRPC authors. + * + */ + +package certprovider + +import ( + "context" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "sync" +) + +// Distributor makes it easy for provider implementations to furnish new key +// materials by handling synchronization between the producer and consumers of +// the key material. +// +// Provider implementations which choose to use a Distributor should do the +// following: +// - create a new Distributor using the NewDistributor() function. +// - invoke the Set() method whenever they have new key material or errors to +// report. +// - delegate to the distributor when handing calls to KeyMaterial(). +// - invoke the Stop() method when they are done using the distributor. +type Distributor struct { + // mu protects the underlying key material. + mu sync.Mutex + km *KeyMaterial + pErr error + + // ready channel to unblock KeyMaterial() invocations blocked on + // availability of key material. + ready *grpcsync.Event + // done channel to notify provider implementations and unblock any + // KeyMaterial() calls, once the Distributor is closed. + closed *grpcsync.Event +} + +// NewDistributor returns a new Distributor. +func NewDistributor() *Distributor { + return &Distributor{ + ready: grpcsync.NewEvent(), + closed: grpcsync.NewEvent(), + } +} + +// Set updates the key material in the distributor with km. +// +// Provider implementations which use the distributor must not modify the +// contents of the KeyMaterial struct pointed to by km. +// +// A non-nil err value indicates the error that the provider implementation ran +// into when trying to fetch key material, and makes it possible to surface the +// error to the user. A non-nil error value passed here causes distributor's +// KeyMaterial() method to return nil key material. +func (d *Distributor) Set(km *KeyMaterial, err error) { + d.mu.Lock() + d.km = km + d.pErr = err + if err != nil { + // If a non-nil err is passed, we ignore the key material being passed. + d.km = nil + } + d.ready.Fire() + d.mu.Unlock() +} + +// KeyMaterial returns the most recent key material provided to the Distributor. +// If no key material was provided at the time of this call, it will block until +// the deadline on the context expires or fresh key material arrives. +func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) { + if d.closed.HasFired() { + return nil, errProviderClosed + } + + if d.ready.HasFired() { + return d.keyMaterial() + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-d.closed.Done(): + return nil, errProviderClosed + case <-d.ready.Done(): + return d.keyMaterial() + } +} + +func (d *Distributor) keyMaterial() (*KeyMaterial, error) { + d.mu.Lock() + defer d.mu.Unlock() + return d.km, d.pErr +} + +// Stop turns down the distributor, releases allocated resources and fails any +// active KeyMaterial() call waiting for new key material. +func (d *Distributor) Stop() { + d.closed.Fire() +} diff --git a/xds/credentials/certprovider/pemfile/builder.go b/xds/credentials/certprovider/pemfile/builder.go new file mode 100644 index 0000000000..932ab186f4 --- /dev/null +++ b/xds/credentials/certprovider/pemfile/builder.go @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright 2020 gRPC authors. + * + */ + +package pemfile + +import ( + "encoding/json" + "fmt" + "time" +) + +import ( + "google.golang.org/protobuf/encoding/protojson" + + "google.golang.org/protobuf/types/known/durationpb" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" +) + +const ( + pluginName = "file_watcher" + defaultRefreshInterval = 10 * time.Minute +) + +type PluginBuilder struct{} + +func (p *PluginBuilder) ParseConfig(c interface{}) (*certprovider.BuildableConfig, error) { + data, ok := c.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("meshca: unsupported config type: %T", c) + } + opts, err := pluginConfigFromJSON(data) + if err != nil { + return nil, err + } + fmt.Println("option") + fmt.Println(opts) + return certprovider.NewBuildableConfig(pluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return newProvider(opts) + }), nil +} + +func (p *PluginBuilder) Name() string { + return pluginName +} + +func pluginConfigFromJSON(jd json.RawMessage) (Options, error) { + // The only difference between this anonymous struct and the Options struct + // is that the refresh_interval is represented here as a duration proto, + // while in the latter a time.Duration is used. + cfg := &struct { + CertificateFile string `json:"certificate_file,omitempty"` + PrivateKeyFile string `json:"private_key_file,omitempty"` + CACertificateFile string `json:"ca_certificate_file,omitempty"` + RefreshInterval json.RawMessage `json:"refresh_interval,omitempty"` + }{} + if err := json.Unmarshal(jd, cfg); err != nil { + return Options{}, fmt.Errorf("pemfile: json.Unmarshal(%s) failed: %v", string(jd), err) + } + + opts := Options{ + CertFile: cfg.CertificateFile, + KeyFile: cfg.PrivateKeyFile, + RootFile: cfg.CACertificateFile, + // Refresh interval is the only field in the configuration for which we + // support a default value. We cannot possibly have valid defaults for + // file paths to watch. Also, it is valid to specify an empty path for + // some of those fields if the user does not want to watch them. + RefreshDuration: defaultRefreshInterval, + } + if cfg.RefreshInterval != nil { + dur := &durationpb.Duration{} + if err := protojson.Unmarshal(cfg.RefreshInterval, dur); err != nil { + return Options{}, fmt.Errorf("pemfile: protojson.Unmarshal(%+v) failed: %v", cfg.RefreshInterval, err) + } + opts.RefreshDuration = dur.AsDuration() + } + fmt.Println("pluginConfigFromJSON") + fmt.Println(opts) + + if err := opts.validate(); err != nil { + return Options{}, err + } + return opts, nil +} diff --git a/xds/credentials/certprovider/pemfile/watcher.go b/xds/credentials/certprovider/pemfile/watcher.go new file mode 100644 index 0000000000..1599275481 --- /dev/null +++ b/xds/credentials/certprovider/pemfile/watcher.go @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright 2020 gRPC authors. + * + */ + +// Package pemfile provides a file watching certificate provider plugin +// implementation which works for files with PEM contents. +package pemfile + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "time" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" + "github.com/dubbogo/grpc-go/grpclog" +) + +const defaultCertRefreshDuration = 1 * time.Hour + +var ( + // For overriding from unit tests. + newDistributor = func() distributor { return certprovider.NewDistributor() } + + logger = grpclog.Component("pemfile") +) + +// Options configures a certificate provider plugin that watches a specified set +// of files that contain certificates and keys in PEM format. +type Options struct { + // CertFile is the file that holds the identity certificate. + // Optional. If this is set, KeyFile must also be set. + CertFile string + // KeyFile is the file that holds identity private key. + // Optional. If this is set, CertFile must also be set. + KeyFile string + // RootFile is the file that holds trusted root certificate(s). + // Optional. + RootFile string + // RefreshDuration is the amount of time the plugin waits before checking + // for updates in the specified files. + // Optional. If not set, a default value (1 hour) will be used. + RefreshDuration time.Duration +} + +func (o Options) canonical() []byte { + return []byte(fmt.Sprintf("%s:%s:%s:%s", o.CertFile, o.KeyFile, o.RootFile, o.RefreshDuration)) +} + +func (o Options) validate() error { + if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" { + return fmt.Errorf("pemfile: at least one credential file needs to be specified") + } + if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified { + return fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified") + } + // C-core has a limitation that they cannot verify that a certificate file + // matches a key file. So, the only way to get around this is to make sure + // that both files are in the same directory and that they do an atomic + // read. Even though Java/Go do not have this limitation, we want the + // overall plugin behavior to be consistent across languages. + if certDir, keyDir := filepath.Dir(o.CertFile), filepath.Dir(o.KeyFile); certDir != keyDir { + return errors.New("pemfile: certificate and key file must be in the same directory") + } + return nil +} + +// NewProvider returns a new certificate provider plugin that is configured to +// watch the PEM files specified in the passed in options. +func NewProvider(o Options) (certprovider.Provider, error) { + if err := o.validate(); err != nil { + return nil, err + } + return newProvider(o), nil +} + +// newProvider is used to create a new certificate provider plugin after +// validating the options, and hence does not return an error. +func newProvider(o Options) certprovider.Provider { + if o.RefreshDuration == 0 { + o.RefreshDuration = defaultCertRefreshDuration + } + + provider := &watcher{opts: o} + if o.CertFile != "" && o.KeyFile != "" { + provider.identityDistributor = newDistributor() + } + if o.RootFile != "" { + provider.rootDistributor = newDistributor() + } + + ctx, cancel := context.WithCancel(context.Background()) + provider.cancel = cancel + go provider.run(ctx) + return provider +} + +// watcher is a certificate provider plugin that implements the +// certprovider.Provider interface. It watches a set of certificate and key +// files and provides the most up-to-date key material for consumption by +// credentials implementation. +type watcher struct { + identityDistributor distributor + rootDistributor distributor + opts Options + certFileContents []byte + keyFileContents []byte + rootFileContents []byte + cancel context.CancelFunc +} + +// distributor wraps the methods on certprovider.Distributor which are used by +// the plugin. This is very useful in tests which need to know exactly when the +// plugin updates its key material. +type distributor interface { + KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) + Set(km *certprovider.KeyMaterial, err error) + Stop() +} + +// updateIdentityDistributor checks if the cert/key files that the plugin is +// watching have changed, and if so, reads the new contents and updates the +// identityDistributor with the new key material. +// +// Skips updates when file reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateIdentityDistributor() { + if w.identityDistributor == nil { + return + } + + certFileContents, err := ioutil.ReadFile(w.opts.CertFile) + if err != nil { + logger.Warningf("certFile (%s) read failed: %v", w.opts.CertFile, err) + return + } + keyFileContents, err := ioutil.ReadFile(w.opts.KeyFile) + if err != nil { + logger.Warningf("keyFile (%s) read failed: %v", w.opts.KeyFile, err) + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.certFileContents, certFileContents) && bytes.Equal(w.keyFileContents, keyFileContents) { + return + } + + cert, err := tls.X509KeyPair(certFileContents, keyFileContents) + if err != nil { + logger.Warningf("tls.X509KeyPair(%q, %q) failed: %v", certFileContents, keyFileContents, err) + return + } + w.certFileContents = certFileContents + w.keyFileContents = keyFileContents + w.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}, nil) +} + +// updateRootDistributor checks if the root cert file that the plugin is +// watching hs changed, and if so, updates the rootDistributor with the new key +// material. +// +// Skips updates when root cert reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateRootDistributor() { + if w.rootDistributor == nil { + return + } + + rootFileContents, err := ioutil.ReadFile(w.opts.RootFile) + if err != nil { + logger.Warningf("rootFile (%s) read failed: %v", w.opts.RootFile, err) + return + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + logger.Warning("failed to parse root certificate") + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.rootFileContents, rootFileContents) { + return + } + + w.rootFileContents = rootFileContents + w.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil) +} + +// run is a long running goroutine which watches the configured files for +// changes, and pushes new key material into the appropriate distributors which +// is returned from calls to KeyMaterial(). +func (w *watcher) run(ctx context.Context) { + ticker := time.NewTicker(w.opts.RefreshDuration) + for { + w.updateIdentityDistributor() + w.updateRootDistributor() + select { + case <-ctx.Done(): + ticker.Stop() + if w.identityDistributor != nil { + w.identityDistributor.Stop() + } + if w.rootDistributor != nil { + w.rootDistributor.Stop() + } + return + case <-ticker.C: + } + } +} + +// KeyMaterial returns the key material sourced by the watcher. +// Callers are expected to use the returned value as read-only. +func (w *watcher) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) { + km := &certprovider.KeyMaterial{} + if w.identityDistributor != nil { + identityKM, err := w.identityDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.Certs = identityKM.Certs + } + if w.rootDistributor != nil { + rootKM, err := w.rootDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.Roots = rootKM.Roots + } + return km, nil +} + +// Close cleans up resources allocated by the watcher. +func (w *watcher) Close() { + w.cancel() +} diff --git a/xds/credentials/certprovider/provider.go b/xds/credentials/certprovider/provider.go new file mode 100644 index 0000000000..4f6f805130 --- /dev/null +++ b/xds/credentials/certprovider/provider.go @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright 2020 gRPC authors. + * + */ + +// Package certprovider defines APIs for Certificate Providers in gRPC. + +package certprovider + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" +) + +var ( + // errProviderClosed is returned by Distributor.KeyMaterial when it is + // closed. + errProviderClosed = errors.New("provider instance is closed") + + // m is a map from name to Provider builder. + m = make(map[string]Builder) +) + +// Register registers the Provider builder, whose name as returned by its Name() +// method will be used as the name registered with this builder. Registered +// Builders are used by the Store to create Providers. +func Register(b Builder) { + m[b.Name()] = b +} + +// GetBuilder returns the Provider builder registered with the given name. +// If no builder is registered with the provided name, nil will be returned. +func GetBuilder(name string) Builder { + if b, ok := m[name]; ok { + return b + } + return nil +} + +// Builder creates a Provider. +type Builder interface { + // ParseConfig parses the given config, which is in a format specific to individual + // implementations, and returns a BuildableConfig on success. + ParseConfig(interface{}) (*BuildableConfig, error) + + // Name returns the name of providers built by this builder. + Name() string +} + +// Provider makes it possible to keep channel credential implementations up to +// date with secrets that they rely on to secure communications on the +// underlying channel. +// +// Provider implementations are free to rely on local or remote sources to fetch +// the latest secrets, and free to share any state between different +// instantiations as they deem fit. +type Provider interface { + // KeyMaterial returns the key material sourced by the Provider. + // Callers are expected to use the returned value as read-only. + KeyMaterial(ctx context.Context) (*KeyMaterial, error) + + // Close cleans up resources allocated by the Provider. + Close() +} + +// KeyMaterial wraps the certificates and keys returned by a Provider instance. +type KeyMaterial struct { + // Certs contains a slice of cert/key pairs used to prove local identity. + Certs []tls.Certificate + // Roots contains the set of trusted roots to validate the peer's identity. + Roots *x509.CertPool +} + +// BuildOptions contains parameters passed to a Provider at build time. +type BuildOptions struct { + // CertName holds the certificate name, whose key material is of interest to + // the caller. + CertName string + // WantRoot indicates if the caller is interested in the root certificate. + WantRoot bool + // WantIdentity indicates if the caller is interested in the identity + // certificate. + WantIdentity bool +} diff --git a/xds/credentials/certprovider/remote/istioca_client.go b/xds/credentials/certprovider/remote/istioca_client.go new file mode 100644 index 0000000000..ee9e8761e2 --- /dev/null +++ b/xds/credentials/certprovider/remote/istioca_client.go @@ -0,0 +1,200 @@ +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "crypto/tls" + "crypto/x509" + v1alpha1 "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote/v1alpha1" + "errors" + "fmt" + structpb "github.com/golang/protobuf/ptypes/struct" + "log" + "path/filepath" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" +) + +const ( + // CertSigner info + CertSigner = "CertSigner" +) + +type Options struct { + CAEndpoint string + CAEndpointSAN string + + TokenProvider credentials.PerRPCCredentials + GRPCOptions []grpc.DialOption + + CertSigner string + ClusterID string + + TrustedRoots *x509.CertPool + + // ProvCert contains a long-lived 'provider' certificate that will be + // exchanged with the workload certificate. + // It is a cert signed by same CA (or a CA trusted by Istiod). + // It is still exchanged because Istiod may add info to the cert. + ProvCert string +} + +type CitadelClient struct { + enableTLS bool + client v1alpha1.IstioCertificateServiceClient + conn *grpc.ClientConn + opts *Options +} + +// NewCitadelClient create a CA client for Citadel. +func NewCitadelClient(opts *Options) (*CitadelClient, error) { + c := &CitadelClient{ + enableTLS: true, + opts: opts, + } + + conn, err := c.buildConnection() + + if err != nil { + log.Printf("Failed to connect to endpoint %s: %v", opts.CAEndpoint, err) + return nil, fmt.Errorf("failed to connect to endpoint %s", opts.CAEndpoint) + } + c.conn = conn + c.client = v1alpha1.NewIstioCertificateServiceClient(conn) + return c, nil +} + +func (c *CitadelClient) Close() { + if c.conn != nil { + c.conn.Close() + } +} + +// CSR Sign calls Citadel to sign a CSR. +func (c *CitadelClient) CSRSign(csrPEM []byte, certValidTTLInSec int64) ([]string, error) { + crMetaStruct := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + CertSigner: { + Kind: &structpb.Value_StringValue{StringValue: c.opts.CertSigner}, + }, + }, + } + req := &v1alpha1.IstioCertificateRequest{ + Csr: string(csrPEM), + ValidityDuration: certValidTTLInSec, + Metadata: crMetaStruct, + } + ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("ClusterID", c.opts.ClusterID)) + resp, err := c.client.CreateCertificate(ctx, req) + if err != nil { + return nil, fmt.Errorf("create certificate: %v", err) + } + + if len(resp.CertChain) <= 1 { + return nil, errors.New("invalid empty CertChain") + } + + return resp.CertChain, nil +} + +func (c *CitadelClient) getTLSDialOption() (grpc.DialOption, error) { + // Load the TLS root certificate from the specified file. + // Create a certificate pool + var certPool *x509.CertPool + var err error + if c.opts.TrustedRoots == nil { + // No explicit certificate - assume the citadel-compatible server uses a public cert + certPool, err = x509.SystemCertPool() + if err != nil { + return nil, err + } + } else { + certPool = c.opts.TrustedRoots + } + var certificate tls.Certificate + config := tls.Config{ + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + if c.opts.ProvCert != "" { + // Load the certificate from disk + certificate, err = tls.LoadX509KeyPair( + filepath.Join(c.opts.ProvCert, "cert-chain.pem"), + filepath.Join(c.opts.ProvCert, "key.pem")) + + if err != nil { + // we will return an empty cert so that when user sets the Prov cert path + // but not have such cert in the file path we use the token to provide verification + // instead of just broken the workflow + log.Printf("cannot load key pair, using token instead: %v", err) + return &certificate, nil + } + if certificate.Leaf.NotAfter.Before(time.Now()) { + log.Printf("cannot parse the cert chain, using token instead: %v", err) + return &tls.Certificate{}, nil + } + } + return &certificate, nil + }, + } + config.RootCAs = certPool + + // For debugging on localhost (with port forward) + // TODO: remove once istiod is stable and we have a way to validate JWTs locally + if strings.Contains(c.opts.CAEndpoint, "localhost") { + config.ServerName = "istiod.istio-system.svc" + } + if c.opts.CAEndpointSAN != "" { + config.ServerName = c.opts.CAEndpointSAN + } + + transportCreds := credentials.NewTLS(&config) + return grpc.WithTransportCredentials(transportCreds), nil +} + +func (c *CitadelClient) buildConnection() (*grpc.ClientConn, error) { + var ol []grpc.DialOption + var opts grpc.DialOption + var err error + if c.enableTLS { + opts, err = c.getTLSDialOption() + if err != nil { + return nil, err + } + ol = append(ol, opts) + } else { + opts = grpc.WithInsecure() + ol = append(ol, opts) + } + ol = append(ol, grpc.WithPerRPCCredentials(c.opts.TokenProvider)) + ol = append(ol, c.opts.GRPCOptions...) + //security.CARetryInterceptor()) + + conn, err := grpc.Dial(c.opts.CAEndpoint, ol...) + if err != nil { + log.Printf("Failed to connect to endpoint %s: %v", c.opts.CAEndpoint, err) + return nil, fmt.Errorf("failed to connect to endpoint %s", c.opts.CAEndpoint) + } + + return conn, nil +} + +// GetRootCertBundle: Citadel (Istiod) CA doesn't publish any endpoint to retrieve CA certs +func (c *CitadelClient) GetRootCertBundle() ([]string, error) { + return []string{}, nil +} diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go new file mode 100644 index 0000000000..8be6b75a54 --- /dev/null +++ b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go @@ -0,0 +1,278 @@ +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc (unknown) +// source: security/v1alpha1/ca.proto + +// Keep this package for backward compatibility. + +package v1alpha1 + +import ( + _struct "github.com/golang/protobuf/ptypes/struct" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Certificate request message. The authentication should be based on: +// 1. Bearer tokens carried in the side channel; +// 2. Client-side certificate via Mutual TLS handshake. +// Note: the service implementation is REQUIRED to verify the authenticated caller is authorize to +// all SANs in the CSR. The server side may overwrite any requested certificate field based on its +// policies. +type IstioCertificateRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // PEM-encoded certificate request. + // The public key in the CSR is used to generate the certificate, + // and other fields in the generated certificate may be overwritten by the CA. + Csr string `protobuf:"bytes,1,opt,name=csr,proto3" json:"csr,omitempty"` + // Optional: requested certificate validity period, in seconds. + ValidityDuration int64 `protobuf:"varint,3,opt,name=validity_duration,json=validityDuration,proto3" json:"validity_duration,omitempty"` + // $hide_from_docs + // Optional: Opaque metadata provided by the XDS node to Istio. + // Supported metadata: WorkloadName, WorkloadIP, ClusterID + Metadata *_struct.Struct `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` +} + +func (x *IstioCertificateRequest) Reset() { + *x = IstioCertificateRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_security_v1alpha1_ca_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IstioCertificateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IstioCertificateRequest) ProtoMessage() {} + +func (x *IstioCertificateRequest) ProtoReflect() protoreflect.Message { + mi := &file_security_v1alpha1_ca_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IstioCertificateRequest.ProtoReflect.Descriptor instead. +func (*IstioCertificateRequest) Descriptor() ([]byte, []int) { + return file_security_v1alpha1_ca_proto_rawDescGZIP(), []int{0} +} + +func (x *IstioCertificateRequest) GetCsr() string { + if x != nil { + return x.Csr + } + return "" +} + +func (x *IstioCertificateRequest) GetValidityDuration() int64 { + if x != nil { + return x.ValidityDuration + } + return 0 +} + +func (x *IstioCertificateRequest) GetMetadata() *_struct.Struct { + if x != nil { + return x.Metadata + } + return nil +} + +// Certificate response message. +type IstioCertificateResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // PEM-encoded certificate chain. + // The leaf cert is the first element, and the root cert is the last element. + CertChain []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"` +} + +func (x *IstioCertificateResponse) Reset() { + *x = IstioCertificateResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_security_v1alpha1_ca_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IstioCertificateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IstioCertificateResponse) ProtoMessage() {} + +func (x *IstioCertificateResponse) ProtoReflect() protoreflect.Message { + mi := &file_security_v1alpha1_ca_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IstioCertificateResponse.ProtoReflect.Descriptor instead. +func (*IstioCertificateResponse) Descriptor() ([]byte, []int) { + return file_security_v1alpha1_ca_proto_rawDescGZIP(), []int{1} +} + +func (x *IstioCertificateResponse) GetCertChain() []string { + if x != nil { + return x.CertChain + } + return nil +} + +var File_security_v1alpha1_ca_proto protoreflect.FileDescriptor + +var file_security_v1alpha1_ca_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2f, 0x63, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x69, 0x73, + 0x74, 0x69, 0x6f, 0x2e, 0x76, 0x31, 0x2e, 0x61, 0x75, 0x74, 0x68, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8d, 0x01, 0x0a, 0x17, 0x49, 0x73, + 0x74, 0x69, 0x6f, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x73, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x63, 0x73, 0x72, 0x12, 0x2b, 0x0a, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x69, 0x74, 0x79, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x10, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x69, 0x74, 0x79, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x39, 0x0a, 0x18, 0x49, 0x73, 0x74, + 0x69, 0x6f, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x32, 0x81, 0x01, 0x0a, 0x17, 0x49, 0x73, 0x74, 0x69, 0x6f, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x66, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x26, 0x2e, 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2e, 0x76, 0x31, + 0x2e, 0x61, 0x75, 0x74, 0x68, 0x2e, 0x49, 0x73, 0x74, 0x69, 0x6f, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, + 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2e, 0x76, 0x31, 0x2e, 0x61, 0x75, 0x74, 0x68, 0x2e, 0x49, 0x73, + 0x74, 0x69, 0x6f, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x20, 0x5a, 0x1e, 0x69, 0x73, 0x74, 0x69, + 0x6f, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_security_v1alpha1_ca_proto_rawDescOnce sync.Once + file_security_v1alpha1_ca_proto_rawDescData = file_security_v1alpha1_ca_proto_rawDesc +) + +func file_security_v1alpha1_ca_proto_rawDescGZIP() []byte { + file_security_v1alpha1_ca_proto_rawDescOnce.Do(func() { + file_security_v1alpha1_ca_proto_rawDescData = protoimpl.X.CompressGZIP(file_security_v1alpha1_ca_proto_rawDescData) + }) + return file_security_v1alpha1_ca_proto_rawDescData +} + +var file_security_v1alpha1_ca_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_security_v1alpha1_ca_proto_goTypes = []interface{}{ + (*IstioCertificateRequest)(nil), // 0: istio.v1.auth.IstioCertificateRequest + (*IstioCertificateResponse)(nil), // 1: istio.v1.auth.IstioCertificateResponse + (*_struct.Struct)(nil), // 2: google.protobuf.Struct +} +var file_security_v1alpha1_ca_proto_depIdxs = []int32{ + 2, // 0: istio.v1.auth.IstioCertificateRequest.metadata:type_name -> google.protobuf.Struct + 0, // 1: istio.v1.auth.IstioCertificateService.CreateCertificate:input_type -> istio.v1.auth.IstioCertificateRequest + 1, // 2: istio.v1.auth.IstioCertificateService.CreateCertificate:output_type -> istio.v1.auth.IstioCertificateResponse + 2, // [2:3] is the sub-list for method output_type + 1, // [1:2] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_security_v1alpha1_ca_proto_init() } +func file_security_v1alpha1_ca_proto_init() { + if File_security_v1alpha1_ca_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_security_v1alpha1_ca_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IstioCertificateRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_security_v1alpha1_ca_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IstioCertificateResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_security_v1alpha1_ca_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_security_v1alpha1_ca_proto_goTypes, + DependencyIndexes: file_security_v1alpha1_ca_proto_depIdxs, + MessageInfos: file_security_v1alpha1_ca_proto_msgTypes, + }.Build() + File_security_v1alpha1_ca_proto = out.File + file_security_v1alpha1_ca_proto_rawDesc = nil + file_security_v1alpha1_ca_proto_goTypes = nil + file_security_v1alpha1_ca_proto_depIdxs = nil +} diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go new file mode 100644 index 0000000000..9ae2cb9e7e --- /dev/null +++ b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go @@ -0,0 +1,108 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc (unknown) +// source: security/v1alpha1/ca.proto + +package v1alpha1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// IstioCertificateServiceClient is the client API for IstioCertificateService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type IstioCertificateServiceClient interface { + // Using provided CSR, returns a signed certificate. + CreateCertificate(ctx context.Context, in *IstioCertificateRequest, opts ...grpc.CallOption) (*IstioCertificateResponse, error) +} + +type istioCertificateServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewIstioCertificateServiceClient(cc grpc.ClientConnInterface) IstioCertificateServiceClient { + return &istioCertificateServiceClient{cc} +} + +func (c *istioCertificateServiceClient) CreateCertificate(ctx context.Context, in *IstioCertificateRequest, opts ...grpc.CallOption) (*IstioCertificateResponse, error) { + out := new(IstioCertificateResponse) + err := c.cc.Invoke(ctx, "/istio.v1.auth.IstioCertificateService/CreateCertificate", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// IstioCertificateServiceServer is the server API for IstioCertificateService service. +// All implementations must embed UnimplementedIstioCertificateServiceServer +// for forward compatibility +type IstioCertificateServiceServer interface { + // Using provided CSR, returns a signed certificate. + CreateCertificate(context.Context, *IstioCertificateRequest) (*IstioCertificateResponse, error) + mustEmbedUnimplementedIstioCertificateServiceServer() +} + +// UnimplementedIstioCertificateServiceServer must be embedded to have forward compatible implementations. +type UnimplementedIstioCertificateServiceServer struct { +} + +func (UnimplementedIstioCertificateServiceServer) CreateCertificate(context.Context, *IstioCertificateRequest) (*IstioCertificateResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented") +} +func (UnimplementedIstioCertificateServiceServer) mustEmbedUnimplementedIstioCertificateServiceServer() { +} + +// UnsafeIstioCertificateServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to IstioCertificateServiceServer will +// result in compilation errors. +type UnsafeIstioCertificateServiceServer interface { + mustEmbedUnimplementedIstioCertificateServiceServer() +} + +func RegisterIstioCertificateServiceServer(s grpc.ServiceRegistrar, srv IstioCertificateServiceServer) { + s.RegisterService(&IstioCertificateService_ServiceDesc, srv) +} + +func _IstioCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IstioCertificateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IstioCertificateServiceServer).CreateCertificate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/istio.v1.auth.IstioCertificateService/CreateCertificate", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IstioCertificateServiceServer).CreateCertificate(ctx, req.(*IstioCertificateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// IstioCertificateService_ServiceDesc is the grpc.ServiceDesc for IstioCertificateService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var IstioCertificateService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "istio.v1.auth.IstioCertificateService", + HandlerType: (*IstioCertificateServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateCertificate", + Handler: _IstioCertificateService_CreateCertificate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "security/v1alpha1/ca.proto", +} diff --git a/xds/credentials/certprovider/store.go b/xds/credentials/certprovider/store.go new file mode 100644 index 0000000000..c5709cbf99 --- /dev/null +++ b/xds/credentials/certprovider/store.go @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Copyright 2020 gRPC authors. + * + */ + +package certprovider + +import ( + "fmt" + "sync" +) + +// provStore is the global singleton certificate provider store. +var provStore = &store{ + providers: make(map[storeKey]*wrappedProvider), +} + +// storeKey acts as the key to the map of providers maintained by the store. A +// combination of provider name and configuration is used to uniquely identify +// every provider instance in the store. Go maps need to be indexed by +// comparable types, so the provider configuration is converted from +// `interface{}` to string using the ParseConfig method while creating this key. +type storeKey struct { + // name of the certificate provider. + name string + // configuration of the certificate provider in string form. + config string + // opts contains the certificate name and other keyMaterial options. + opts BuildOptions +} + +// wrappedProvider wraps a provider instance with a reference count. +type wrappedProvider struct { + Provider + refCount int + + // A reference to the key and store are also kept here to override the + // Close method on the provider. + storeKey storeKey + store *store +} + +// store is a collection of provider instances, safe for concurrent access. +type store struct { + mu sync.Mutex + providers map[storeKey]*wrappedProvider +} + +// Close overrides the Close method of the embedded provider. It releases the +// reference held by the caller on the underlying provider and if the +// provider's reference count reaches zero, it is removed from the store, and +// its Close method is also invoked. +func (wp *wrappedProvider) Close() { + ps := wp.store + ps.mu.Lock() + defer ps.mu.Unlock() + + wp.refCount-- + if wp.refCount == 0 { + wp.Provider.Close() + delete(ps.providers, wp.storeKey) + } +} + +// BuildableConfig wraps parsed provider configuration and functionality to +// instantiate provider instances. +type BuildableConfig struct { + name string + config []byte + starter func(BuildOptions) Provider + pStore *store +} + +// NewBuildableConfig creates a new BuildableConfig with the given arguments. +// Provider implementations are expected to invoke this function after parsing +// the given configuration as part of their ParseConfig() method. +// Equivalent configurations are expected to invoke this function with the same +// config argument. +func NewBuildableConfig(name string, config []byte, starter func(BuildOptions) Provider) *BuildableConfig { + return &BuildableConfig{ + name: name, + config: config, + starter: starter, + pStore: provStore, + } +} + +// Build kicks off a provider instance with the wrapped configuration. Multiple +// invocations of this method with the same opts will result in provider +// instances being reused. +func (bc *BuildableConfig) Build(opts BuildOptions) (Provider, error) { + provStore.mu.Lock() + defer provStore.mu.Unlock() + + sk := storeKey{ + name: bc.name, + config: string(bc.config), + opts: opts, + } + if wp, ok := provStore.providers[sk]; ok { + wp.refCount++ + return wp, nil + } + + provider := bc.starter(opts) + if provider == nil { + return nil, fmt.Errorf("provider(%q, %q).Build(%v) failed", sk.name, sk.config, opts) + } + wp := &wrappedProvider{ + Provider: provider, + refCount: 1, + storeKey: sk, + store: provStore, + } + provStore.providers[sk] = wp + return wp, nil +} + +// String returns the provider name and config as a colon separated string. +func (bc *BuildableConfig) String() string { + return fmt.Sprintf("%s:%s", bc.name, string(bc.config)) +} + +// ParseConfig is a convenience function to create a BuildableConfig given a +// provider name and configuration. Returns an error if there is no registered +// builder for the given name or if the config parsing fails. +func ParseConfig(name string, config interface{}) (*BuildableConfig, error) { + parser := GetBuilder(name) + if parser == nil { + return nil, fmt.Errorf("no certificate provider builder found for %q", name) + } + return parser.ParseConfig(config) +} + +// GetProvider is a convenience function to create a provider given the name, +// config and build options. +func GetProvider(name string, config interface{}, opts BuildOptions) (Provider, error) { + bc, err := ParseConfig(name, config) + if err != nil { + return nil, err + } + return bc.Build(opts) +} diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index 90ff78f083..868a980ce5 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -50,6 +50,10 @@ const ( rlsInXDSEnv = "GRPC_EXPERIMENTAL_XDS_RLS_LB" c2pResolverTestOnlyTrafficDirectorURIEnv = "GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI" + + KubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + IstioCAEndpoint = "istiod.istio-system.svc:15012" ) var ( From d6a4f1e98564d5d9da1cf6570015d29ca0ddab5d Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Tue, 12 Jul 2022 23:50:20 +0800 Subject: [PATCH 02/18] adjust code --- remoting/xds/client_test.go | 799 ------------------ remoting/xds/xds_client_factory.go | 180 +--- remoting/xds/xds_client_factory_test.go | 15 - xds/credentials/cert_manager.go | 244 ++++++ xds/credentials/cert_manager_test.go | 14 + .../certprovider/pemfile/builder.go | 2 - xds/credentials/env.go | 25 + xds/credentials/token_provider.go | 52 ++ xds/utils/envconfig/xds.go | 4 - 9 files changed, 336 insertions(+), 999 deletions(-) delete mode 100644 remoting/xds/client_test.go delete mode 100644 remoting/xds/xds_client_factory_test.go create mode 100644 xds/credentials/cert_manager.go create mode 100644 xds/credentials/cert_manager_test.go create mode 100644 xds/credentials/env.go create mode 100644 xds/credentials/token_provider.go diff --git a/remoting/xds/client_test.go b/remoting/xds/client_test.go deleted file mode 100644 index 23bd845e81..0000000000 --- a/remoting/xds/client_test.go +++ /dev/null @@ -1,799 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package xds - -import ( - "testing" - "time" -) - -import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "go.uber.org/atomic" -) - -import ( - "dubbo.apache.org/dubbo-go/v3/common/constant" - "dubbo.apache.org/dubbo-go/v3/registry" - registryMocks "dubbo.apache.org/dubbo-go/v3/registry/mocks" - "dubbo.apache.org/dubbo-go/v3/remoting" - "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/mocks" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" -) - -const ( - dubbogoPortFoo = "20000" - istioXDSPortFoo = "15010" - - podNameFoo = "mockPodName" - localNamespaceFoo = "default" - - localIPFoo = "172.16.100.1" - istioIPFoo = "172.16.100.2" - - localAddrFoo = localIPFoo + ":" + dubbogoPortFoo - istioXDSAddrFoo = istioIPFoo + ":" + istioXDSPortFoo - - istioHostNameFoo = "istiod.istio-system.svc.cluster.local" - istioHostAddrFoo = istioHostNameFoo + ":" + istioXDSPortFoo - localHostNameFoo = "dubbo-go-app." + localNamespaceFoo + ".svc.cluster.local" - localHostAddrFoo = localHostNameFoo + ":" + dubbogoPortFoo - - istioClusterNameFoo = "outbound|" + istioXDSPortFoo + "||" + istioHostNameFoo - localClusterNameFoo = "outbound|" + dubbogoPortFoo + "||" + localHostNameFoo -) - -const ( - providerIPFoo = "172.16.100.3" - providerV1IPFoo = "172.16.100.4" - providerV2IPFoo = "172.16.100.5" - providerAddrFoo = providerIPFoo + ":" + dubbogoPortFoo - providerV1AddrFoo = providerV1IPFoo + ":" + dubbogoPortFoo - providerV2AddrFoo = providerV2IPFoo + ":" + dubbogoPortFoo - - dubbogoProviderHostName = "dubbo-go-app-provider." + localNamespaceFoo + ".svc.cluster.local" - dubbogoProviderHostAddrFoo = dubbogoProviderHostName + ":" + dubbogoPortFoo - - dubbogoProviderInterfaceNameFoo = "api.Greeter" - dubbogoProviderserivceUniqueKeyFoo = "provider::" + dubbogoProviderInterfaceNameFoo - - providerClusterNameFoo = "outbound|" + dubbogoPortFoo + "||" + dubbogoProviderHostName - providerClusterV1NameFoo = "outbound|" + dubbogoPortFoo + "|v1|" + dubbogoProviderHostName - providerClusterV2NameFoo = "outbound|" + dubbogoPortFoo + "|v2|" + dubbogoProviderHostName -) - -func TestWrappedClientImpl(t *testing.T) { - // test New WrappedClientImpl - testFailedWithIstioCDS(t) - testFailedWithLocalCDS(t) - testFailedWithNoneCDS(t) - - testFailedWithLocalEDSFailed(t) - testFailedWithIstioEDSFailed(t) - - testWithDiscoverySuccess(t) - - assert.NotNil(t, GetXDSWrappedClient()) - - // test Subscription - testSubscribe(t) -} - -func testWithDiscoverySuccess(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - // istioClusterName CDS, this and next calling must be async because testify.Mock.MethodCalled() has calling lock - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: istioClusterNameFoo, - }, nil) - - // localClusterName CDS - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: localClusterNameFoo, - }, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Nil(t, err) - assert.NotNil(t, xdsWrappedClient) - - // assert eds cancel is called - assert.Equal(t, int32(2), cancelCalledCounter.Load()) - // discovery istiod pod ip - assert.Equal(t, istioIPFoo, xdsWrappedClient.GetIstioPodIP()) - address := xdsWrappedClient.GetHostAddress() - assert.Equal(t, localHostAddrFoo, address.String()) -} - -func testFailedWithIstioCDS(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - // do not send istioClusterName message, which cause discover istio ip failed - //go clusterUpdateHandler(resource.ClusterUpdate{ - // ClusterName: istioClusterNameFoo, - //}, nil) - - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: localClusterNameFoo, - }, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Equal(t, DiscoverIstiodPodIpError, err) - assert.Nil(t, xdsWrappedClient) - assert.Equal(t, int32(1), cancelCalledCounter.Load()) -} - -func testFailedWithLocalCDS(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: istioClusterNameFoo, - }, nil) - - // do not send localClusterNameFoo cds message, which cause discover local addr failed - //go clusterUpdateHandler(resource.ClusterUpdate{ - // ClusterName: localClusterNameFoo, - //}, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Equal(t, DiscoverLocalError, err) - assert.Nil(t, xdsWrappedClient) - assert.Equal(t, int32(1), cancelCalledCounter.Load()) -} - -func testFailedWithNoneCDS(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - // do not send any cds message, which cause discover failed - //go clusterUpdateHandler(resource.ClusterUpdate{ - // ClusterName: istioClusterNameFoo, - //}, nil) - - //go clusterUpdateHandler(resource.ClusterUpdate{ - // ClusterName: localClusterNameFoo, - //}, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Equal(t, DiscoverIstiodPodIpError, err) - assert.Nil(t, xdsWrappedClient) - assert.Equal(t, int32(0), cancelCalledCounter.Load()) -} - -func testFailedWithLocalEDSFailed(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: istioClusterNameFoo, - }, nil) - - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: localClusterNameFoo, - }, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - // do not send local eds message - //clusterUpdateHandler(resource.EndpointsUpdate{ - // Localities: []resource.Locality{ - // { - // Endpoints: []resource.Endpoint{ - // { - // Address: localAddrFoo, - // }, - // }, - // }, - // }, - //}, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Equal(t, DiscoverLocalError, err) - assert.Nil(t, xdsWrappedClient) - assert.Equal(t, int32(2), cancelCalledCounter.Load()) -} - -func testFailedWithIstioEDSFailed(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - cancelCalledCounter := &atomic.Int32{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: istioClusterNameFoo, - }, nil) - - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: localClusterNameFoo, - }, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == istioClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - // do not send istio eds message - //clusterUpdateHandler(resource.EndpointsUpdate{ - // Localities: []resource.Locality{ - // { - // Endpoints: []resource.Endpoint{ - // { - // Address: istioXDSAddrFoo, - // }, - // }, - // }, - // }, - //}, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - return clusterName == localClusterNameFoo - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() { - // return cancel function - cancelCalledCounter.Inc() - }) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Equal(t, DiscoverIstiodPodIpError, err) - assert.Nil(t, xdsWrappedClient) - assert.Equal(t, int32(2), cancelCalledCounter.Load()) -} - -func testSubscribe(t *testing.T) { - mockXDSClient := &mocks.XDSClient{} - // all cluster CDS - mockXDSClient.On("WatchCluster", "*", - mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { - // istio ClusterName CDS, this and next calling must be async because testify.Mock.MethodCalled() has calling lock - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: istioClusterNameFoo, - }, nil) - - // local ClusterName CDS - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: localClusterNameFoo, - }, nil) - - // provider Cluster Name CDS - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: providerClusterNameFoo, - }, nil) - - // provider Cluster Name v1 CDS - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: providerClusterV1NameFoo, - }, nil) - - // provider Cluster Name v2 CDS - go clusterUpdateHandler(resource.ClusterUpdate{ - ClusterName: providerClusterV2NameFoo, - }, nil) - return true - })). - Return(func() {}) - - // istio cluster EDS - clusterMatch := false - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - clusterMatch = clusterName == istioClusterNameFoo - return clusterMatch - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - if !clusterMatch { - return false - } - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: istioXDSAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() {}) - - // local cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - clusterMatch = clusterName == localClusterNameFoo - return clusterMatch - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - if !clusterMatch { - return false - } - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: localAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() {}) - - // provider cluster EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - clusterMatch = clusterName == providerClusterNameFoo - return clusterMatch - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - if !clusterMatch { - return false - } - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: providerAddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() {}) - - // provider cluster v1 EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - clusterMatch = clusterName == providerClusterV1NameFoo - return clusterMatch - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - if !clusterMatch { - return false - } - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: providerV1AddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() {}) - - // todo test router RDS - mockXDSClient.On("WatchRouteConfig", mock.Anything, mock.Anything).Return(func() {}) - - // provider cluster v2 EDS - mockXDSClient.On("WatchEndpoints", - mock.MatchedBy(func(clusterName string) bool { - clusterMatch = clusterName == providerClusterV2NameFoo - return clusterMatch - }), - mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { - if !clusterMatch { - return false - } - clusterUpdateHandler(resource.EndpointsUpdate{ - Localities: []resource.Locality{ - { - Endpoints: []resource.Endpoint{ - { - Address: providerV2AddrFoo, - }, - }, - }, - }, - }, nil) - return true - })). - Return(func() {}) - - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { - return mockXDSClient, nil - } - - xdsWrappedClient = nil - xdsWrappedClient, err := NewXDSWrappedClient(Config{ - PodName: podNameFoo, - Namespace: localNamespaceFoo, - LocalIP: localIPFoo, - IstioAddr: common.NewHostNameOrIPAddr(istioHostAddrFoo), - }) - assert.Nil(t, err) - assert.NotNil(t, xdsWrappedClient) - - notifyListener := ®istryMocks.NotifyListener{} - notifyListener.On("Notify", mock.Anything).Return(nil) - - go xdsWrappedClient.Subscribe(dubbogoProviderserivceUniqueKeyFoo, dubbogoProviderInterfaceNameFoo, dubbogoProviderHostAddrFoo, notifyListener) - - time.Sleep(time.Second) - notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { - if event.Action == remoting.EventTypeUpdate && - event.Service.Ip == providerV2IPFoo && - event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterV2NameFoo { - return true - } - return false - })) - - notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { - if event.Action == remoting.EventTypeUpdate && - event.Service.Ip == providerV1IPFoo && - event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterV1NameFoo { - return true - } - return false - })) - - notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { - if event.Action == remoting.EventTypeUpdate && - event.Service.Ip == providerIPFoo && - event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterNameFoo { - return true - } - return false - })) -} - -// todo TestDestroy -// todo TestRDS diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go index b182200676..4c04242d0c 100644 --- a/remoting/xds/xds_client_factory.go +++ b/remoting/xds/xds_client_factory.go @@ -18,23 +18,8 @@ package xds import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - log "dubbo.apache.org/dubbo-go/v3/common/logger" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" - certprovider2 "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "encoding/pem" - "fmt" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "io/ioutil" - "os" - "strings" ) import ( @@ -57,15 +42,10 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd Metadata: mapping.GetDubboGoMetadata(""), } - certificates, err := buildCertificate() - if err != nil { - return nil, err - } - nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ ServerURI: istioAddr.String(), - Creds: grpc.WithTransportCredentials(certificates), + Creds: grpc.WithInsecure(), TransportAPI: version.TransportV3, NodeProto: v3NodeProto, }, @@ -78,161 +58,3 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd } return newClient, nil } - -var buildCertificate = func() (credentials.TransportCredentials, error) { - - bootstrapPath := os.Getenv(envconfig.XDSBootstrapFileNameEnv) - //agent exist, get cert from file - if "" != bootstrapPath { - config, err := bootstrap.NewConfig() - if err != nil { - return nil, err - } - certProvider, err := buildProviderFunc(config.CertProviderConfigs, "default", "rootCA", false, true) - - material, err := certProvider.KeyMaterial(context.Background()) - - if err != nil { - log.Errorf("failed to get root ca :%s", err.Error()) - return nil, err - } - - cred := credentials.NewTLS(&tls.Config{ - RootCAs: material.Roots, - Certificates: material.Certs, - GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - keyMaterial, err := certProvider.KeyMaterial(info.Context()) - if err != nil { - log.Errorf("failed to get keyMaterial :%s", err.Error()) - return nil, err - } - return &keyMaterial.Certs[0], nil - }, - }) - return cred, nil - - //agent not exist, get cert from istio citadel - } else { - tokenProvider, err := NewSaTokenProvider() - if err != nil { - return nil, err - } - citadelClient, err := remote.NewCitadelClient(&remote.Options{ - CAEndpoint: envconfig.IstioCAEndpoint, - TokenProvider: tokenProvider, - }) - - //use default - options := certgenerate.CertOptions{ - Host: "dubbo-go", - RSAKeySize: 2048, - PKCS8Key: true, - ECSigAlg: certgenerate.EcdsaSigAlg, - } - - // Generate the cert/key, send CSR to CA. - csrPEM, keyPEM, err := certgenerate.GenCSR(options) - if err != nil { - log.Errorf("failed to generate key and certificate for CSR: %v", err) - return nil, err - } - sign, err := citadelClient.CSRSign(csrPEM, int64(0)) - - if err != nil { - return nil, err - } - - cert, err := ParseCert(concatCerts(sign), keyPEM) - if err != nil { - return nil, err - } - - cred := credentials.NewTLS(&tls.Config{ - RootCAs: nil, //todo fetch rootCA - Certificates: []tls.Certificate{*cert}, - GetClientCertificate: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - //todo check cert expired - return cert, nil - }, - }) - return cred, nil - } - -} - -//provide k8s service account -type saTokenProvider struct { - token string -} - -func NewSaTokenProvider() (*saTokenProvider, error) { - sa, err := ioutil.ReadFile(envconfig.KubernetesServiceAccountPath) - if err != nil { - return nil, err - } - return &saTokenProvider{ - token: string(sa), - }, nil -} - -func (s *saTokenProvider) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - - meta := make(map[string]string) - - meta["authorization"] = "Bearer " + s.token - return meta, nil -} - -func (s *saTokenProvider) RequireTransportSecurity() bool { - return false -} - -func buildProviderFunc(configs map[string]*certprovider2.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider2.Provider, error) { - cfg, ok := configs[instanceName] - if !ok { - return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) - } - provider, err := cfg.Build(certprovider2.BuildOptions{ - CertName: certName, - WantIdentity: wantIdentity, - WantRoot: wantRoot, - }) - if err != nil { - // This error is not expected since the bootstrap process parses the - // config and makes sure that it is acceptable to the plugin. Still, it - // is possible that the plugin parses the config successfully, but its - // Build() method errors out. - return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) - } - return provider, nil -} -func concatCerts(certsPEM []string) []byte { - if len(certsPEM) == 0 { - return []byte{} - } - var certChain bytes.Buffer - for i, c := range certsPEM { - certChain.WriteString(c) - if i < len(certsPEM)-1 && !strings.HasSuffix(c, "\n") { - certChain.WriteString("\n") - } - } - return certChain.Bytes() -} - -func ParseCert(certByte []byte, keyByte []byte) (*tls.Certificate, error) { - block, _ := pem.Decode(certByte) - if block == nil { - return nil, fmt.Errorf("failed to decode certificate") - } - cert, err := x509.ParseCertificate(block.Bytes) - - expired := cert.NotAfter - log.Infof("cert expired after:" + expired.String()) - - pair, err := tls.X509KeyPair(certByte, keyByte) - if err != nil { - return nil, fmt.Errorf("failed to parse certificate: %v", err) - } - return &pair, nil -} diff --git a/remoting/xds/xds_client_factory_test.go b/remoting/xds/xds_client_factory_test.go deleted file mode 100644 index bf6a0d8492..0000000000 --- a/remoting/xds/xds_client_factory_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package xds - -import ( - "fmt" - "testing" -) - -func TestBuildCertificate(t *testing.T) { - fmt.Println("test begin ") - certificate, err := buildCertificate() - - fmt.Println(err) - fmt.Println(certificate) - -} diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go new file mode 100644 index 0000000000..6b72aa5ea5 --- /dev/null +++ b/xds/credentials/cert_manager.go @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + log "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "strings" + "time" +) + +//CertManager manage agent or no agent cert +type CertManager interface { + GetCertificate() ([]tls.Certificate, error) + GetRootCertificate() (*x509.CertPool, error) +} + +// NewCertManager return a manager +func NewCertManager() (CertManager, error) { + bootstrapPath := os.Getenv(envconfig.XDSBootstrapFileNameEnv) + if bootstrapPath != "" { + manager := &AgentCertManager{} + config, err := bootstrap.NewConfig() + if err != nil { + log.Errorf("build bootstrap config error :%s", err.Error()) + return nil, err + } + certProvider, err := buildProviderFunc(config.CertProviderConfigs, "default", "rootCA", false, true) + + if err != nil { + log.Errorf("get cert provider error :%s", err.Error()) + return nil, err + } + manager.provider = certProvider + return manager, nil + } else { + manager := &CACertManager{} + manager.rootPath = RootCertPath + return manager, nil + } + +} + +// AgentCertManager work in istio agent mode +type AgentCertManager struct { + provider certprovider.Provider +} + +//GetRootCertificate return certificate of ca +func (c *AgentCertManager) GetRootCertificate() (*x509.CertPool, error) { + material, err := c.provider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return material.Roots, nil +} + +//GetCertificate return certificate of application +func (c *AgentCertManager) GetCertificate() ([]tls.Certificate, error) { + material, err := c.provider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return material.Certs, nil +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} + +// CACertManager work in no agent mode, fetch cert form CA +type CACertManager struct { + // Certs contains a slice of cert/key pairs used to prove local identity. + Certs []tls.Certificate + // Roots contains the set of trusted roots to validate the peer's identity. + Roots *x509.CertPool + + NoAfter time.Time + + RootNoAfter time.Time + + rootPath string +} + +//GetCertificate return certificate of application +func (c *CACertManager) GetCertificate() ([]tls.Certificate, error) { + //cert expired + if time.Now().After(c.NoAfter) { + if err := c.UpdateCert(); err != nil { + return nil, err + } + } + return c.Certs, nil +} + +//GetRootCertificate return certificate of ca +func (c *CACertManager) GetRootCertificate() (*x509.CertPool, error) { + //root expired + if time.Now().After(c.RootNoAfter) { + if err := c.UpdateRoot(); err != nil { + return nil, err + } + } + return c.Roots, nil +} + +func (c *CACertManager) UpdateRoot() error { + rootFileContents, err := ioutil.ReadFile(c.rootPath) + if err != nil { + return err + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + log.Warn("failed to parse root certificate") + } + c.Roots = trustPool + block, _ := pem.Decode(rootFileContents) + if block == nil { + return fmt.Errorf("failed to decode certificate") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil + } + c.RootNoAfter = cert.NotAfter + return nil +} + +func (c *CACertManager) UpdateCert() error { + tokenProvider, err := NewSaTokenProvider(IsitoCaServiceAccountPath) + if err != nil { + return err + } + + trustRoot, err := c.GetRootCertificate() + if err != nil { + return nil + } + citadelClient, err := remote.NewCitadelClient(&remote.Options{ + CAEndpoint: IstioCAEndpoint, + TrustedRoots: trustRoot, + TokenProvider: tokenProvider, + CertSigner: "kubernetes.default.svc", + ClusterID: "Kubernetes", + }) + host := "spiffe://" + "cluster.local" + "/ns/" + "echo-grpc" + "/sa/" + tokenProvider.Token + + //use default + options := certgenerate.CertOptions{ + Host: host, + RSAKeySize: 2048, + PKCS8Key: true, + ECSigAlg: certgenerate.EcdsaSigAlg, + } + + // Generate the cert/key, send CSR to CA. + csrPEM, keyPEM, err := certgenerate.GenCSR(options) + if err != nil { + log.Errorf("failed to generate key and certificate for CSR: %v", err) + return err + } + sign, err := citadelClient.CSRSign(csrPEM, int64(200000)) + + if err != nil { + return err + } + + cert, _, err := c.ParseCert(concatCerts(sign), keyPEM) + if err != nil { + return err + } + c.Certs = []tls.Certificate{*cert} + return nil +} + +func (c *CACertManager) ParseCert(certByte []byte, keyByte []byte) (*tls.Certificate, time.Time, error) { + block, _ := pem.Decode(certByte) + if block == nil { + return nil, time.Now(), fmt.Errorf("failed to decode certificate") + } + cert, err := x509.ParseCertificate(block.Bytes) + + expired := cert.NotAfter + log.Infof("cert expired after:" + expired.String()) + c.NoAfter = expired + pair, err := tls.X509KeyPair(certByte, keyByte) + if err != nil { + return nil, time.Now(), fmt.Errorf("failed to parse certificate: %v", err) + } + return &pair, expired, nil +} + +func concatCerts(certsPEM []string) []byte { + if len(certsPEM) == 0 { + return []byte{} + } + var certChain bytes.Buffer + for i, c := range certsPEM { + certChain.WriteString(c) + if i < len(certsPEM)-1 && !strings.HasSuffix(c, "\n") { + certChain.WriteString("\n") + } + } + return certChain.Bytes() +} diff --git a/xds/credentials/cert_manager_test.go b/xds/credentials/cert_manager_test.go new file mode 100644 index 0000000000..5a1e1730da --- /dev/null +++ b/xds/credentials/cert_manager_test.go @@ -0,0 +1,14 @@ +package credentials + +import ( + "fmt" + "testing" +) + +func TestNewCertManager(t *testing.T) { + manager, err := NewCertManager() + fmt.Println(err) + fmt.Println(manager.GetCertificate()) + fmt.Println(manager.GetRootCertificate()) + fmt.Println() +} diff --git a/xds/credentials/certprovider/pemfile/builder.go b/xds/credentials/certprovider/pemfile/builder.go index 932ab186f4..7860d5efa7 100644 --- a/xds/credentials/certprovider/pemfile/builder.go +++ b/xds/credentials/certprovider/pemfile/builder.go @@ -55,8 +55,6 @@ func (p *PluginBuilder) ParseConfig(c interface{}) (*certprovider.BuildableConfi if err != nil { return nil, err } - fmt.Println("option") - fmt.Println(opts) return certprovider.NewBuildableConfig(pluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider { return newProvider(opts) }), nil diff --git a/xds/credentials/env.go b/xds/credentials/env.go new file mode 100644 index 0000000000..050c668077 --- /dev/null +++ b/xds/credentials/env.go @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +const ( + KubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + IsitoCaServiceAccountPath = "/var/run/secrets/tokens/istio-token" + RootCertPath = "/var/run/secrets/istio/root-cert.pem" + IstioCAEndpoint = "istiod.istio-system.svc:15012" +) diff --git a/xds/credentials/token_provider.go b/xds/credentials/token_provider.go new file mode 100644 index 0000000000..473524938d --- /dev/null +++ b/xds/credentials/token_provider.go @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "context" + "io/ioutil" +) + +//provide k8s service account +type saTokenProvider struct { + Token string + tokenPath string +} + +func NewSaTokenProvider(tokenPath string) (*saTokenProvider, error) { + sa, err := ioutil.ReadFile(tokenPath) + if err != nil { + return nil, err + } + return &saTokenProvider{ + tokenPath: tokenPath, + Token: string(sa), + }, nil +} + +func (s *saTokenProvider) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + + meta := make(map[string]string) + + meta["authorization"] = "Bearer " + s.Token + return meta, nil +} + +func (s *saTokenProvider) RequireTransportSecurity() bool { + return false +} diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index 868a980ce5..90ff78f083 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -50,10 +50,6 @@ const ( rlsInXDSEnv = "GRPC_EXPERIMENTAL_XDS_RLS_LB" c2pResolverTestOnlyTrafficDirectorURIEnv = "GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI" - - KubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" - - IstioCAEndpoint = "istiod.istio-system.svc:15012" ) var ( From ccb8ca9dc527df7a6c5ab5550ac87838af3206cd Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Tue, 12 Jul 2022 23:56:46 +0800 Subject: [PATCH 03/18] adjust code --- remoting/xds/xds_client_factory.go | 4 +++- xds/credentials/cert_manager.go | 15 +++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go index 4c04242d0c..f038e6467a 100644 --- a/remoting/xds/xds_client_factory.go +++ b/remoting/xds/xds_client_factory.go @@ -19,7 +19,9 @@ package xds import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) import ( @@ -45,7 +47,7 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ ServerURI: istioAddr.String(), - Creds: grpc.WithInsecure(), + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), TransportAPI: version.TransportV3, NodeProto: v3NodeProto, }, diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 6b72aa5ea5..76203dfa7d 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -22,12 +22,6 @@ import ( "context" "crypto/tls" "crypto/x509" - log "dubbo.apache.org/dubbo-go/v3/common/logger" - "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "encoding/pem" "fmt" "io/ioutil" @@ -36,6 +30,15 @@ import ( "time" ) +import ( + log "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" +) + //CertManager manage agent or no agent cert type CertManager interface { GetCertificate() ([]tls.Certificate, error) From 1b281737f98e6c4b2cfec757f99f3c02fa484874 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Wed, 13 Jul 2022 00:07:52 +0800 Subject: [PATCH 04/18] add comment --- xds/credentials/cert_manager.go | 19 +++++++++++-------- xds/credentials/env.go | 6 ++++++ xds/credentials/token_provider.go | 3 +++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 76203dfa7d..8e84cf33f4 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -55,7 +55,7 @@ func NewCertManager() (CertManager, error) { log.Errorf("build bootstrap config error :%s", err.Error()) return nil, err } - certProvider, err := buildProviderFunc(config.CertProviderConfigs, "default", "rootCA", false, true) + certProvider, err := buildProvider(config.CertProviderConfigs, "default") if err != nil { log.Errorf("get cert provider error :%s", err.Error()) @@ -94,15 +94,16 @@ func (c *AgentCertManager) GetCertificate() ([]tls.Certificate, error) { return material.Certs, nil } -func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { +// buildProvider build cert provider from config +func buildProvider(configs map[string]*certprovider.BuildableConfig, instanceName string) (certprovider.Provider, error) { cfg, ok := configs[instanceName] if !ok { return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) } provider, err := cfg.Build(certprovider.BuildOptions{ - CertName: certName, - WantIdentity: wantIdentity, - WantRoot: wantRoot, + CertName: "root-ca", + WantIdentity: true, + WantRoot: true, }) if err != nil { return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) @@ -146,6 +147,7 @@ func (c *CACertManager) GetRootCertificate() (*x509.CertPool, error) { return c.Roots, nil } +// UpdateRoot update root cert func (c *CACertManager) UpdateRoot() error { rootFileContents, err := ioutil.ReadFile(c.rootPath) if err != nil { @@ -168,6 +170,7 @@ func (c *CACertManager) UpdateRoot() error { return nil } +// UpdateCert update cert func (c *CACertManager) UpdateCert() error { tokenProvider, err := NewSaTokenProvider(IsitoCaServiceAccountPath) if err != nil { @@ -185,7 +188,7 @@ func (c *CACertManager) UpdateCert() error { CertSigner: "kubernetes.default.svc", ClusterID: "Kubernetes", }) - host := "spiffe://" + "cluster.local" + "/ns/" + "echo-grpc" + "/sa/" + tokenProvider.Token + host := "spiffe://" + "cluster.local" + "/ns/" + POD_NAMESPACE + "/sa/" + tokenProvider.Token //use default options := certgenerate.CertOptions{ @@ -207,7 +210,7 @@ func (c *CACertManager) UpdateCert() error { return err } - cert, _, err := c.ParseCert(concatCerts(sign), keyPEM) + cert, _, err := c.parseCert(concatCerts(sign), keyPEM) if err != nil { return err } @@ -215,7 +218,7 @@ func (c *CACertManager) UpdateCert() error { return nil } -func (c *CACertManager) ParseCert(certByte []byte, keyByte []byte) (*tls.Certificate, time.Time, error) { +func (c *CACertManager) parseCert(certByte []byte, keyByte []byte) (*tls.Certificate, time.Time, error) { block, _ := pem.Decode(certByte) if block == nil { return nil, time.Now(), fmt.Errorf("failed to decode certificate") diff --git a/xds/credentials/env.go b/xds/credentials/env.go index 050c668077..f95318d680 100644 --- a/xds/credentials/env.go +++ b/xds/credentials/env.go @@ -17,9 +17,15 @@ package credentials +import "os" + const ( KubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" IsitoCaServiceAccountPath = "/var/run/secrets/tokens/istio-token" RootCertPath = "/var/run/secrets/istio/root-cert.pem" IstioCAEndpoint = "istiod.istio-system.svc:15012" ) + +var ( + POD_NAMESPACE = os.Getenv("POD_NAMESPACE") +) diff --git a/xds/credentials/token_provider.go b/xds/credentials/token_provider.go index 473524938d..0faadf4b15 100644 --- a/xds/credentials/token_provider.go +++ b/xds/credentials/token_provider.go @@ -28,6 +28,7 @@ type saTokenProvider struct { tokenPath string } +// NewSaTokenProvider return a provider func NewSaTokenProvider(tokenPath string) (*saTokenProvider, error) { sa, err := ioutil.ReadFile(tokenPath) if err != nil { @@ -39,6 +40,7 @@ func NewSaTokenProvider(tokenPath string) (*saTokenProvider, error) { }, nil } +// GetRequestMetadata return meta of authorization func (s *saTokenProvider) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { meta := make(map[string]string) @@ -47,6 +49,7 @@ func (s *saTokenProvider) GetRequestMetadata(ctx context.Context, uri ...string) return meta, nil } +// RequireTransportSecurity always false func (s *saTokenProvider) RequireTransportSecurity() bool { return false } From 03e1a143963b22debd4cd3da8f968e02c92367cc Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Mon, 18 Jul 2022 23:17:12 +0800 Subject: [PATCH 05/18] add env param --- xds/credentials/cert_manager.go | 20 ++++++++++++-------- xds/credentials/cert_manager_test.go | 14 -------------- xds/credentials/env.go | 25 ++++++++++++++++++++----- 3 files changed, 32 insertions(+), 27 deletions(-) delete mode 100644 xds/credentials/cert_manager_test.go diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 8e84cf33f4..16e3f444ae 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -26,6 +26,7 @@ import ( "fmt" "io/ioutil" "os" + "strconv" "strings" "time" ) @@ -101,7 +102,7 @@ func buildProvider(configs map[string]*certprovider.BuildableConfig, instanceNam return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) } provider, err := cfg.Build(certprovider.BuildOptions{ - CertName: "root-ca", + CertName: "ca", WantIdentity: true, WantRoot: true, }) @@ -172,7 +173,7 @@ func (c *CACertManager) UpdateRoot() error { // UpdateCert update cert func (c *CACertManager) UpdateCert() error { - tokenProvider, err := NewSaTokenProvider(IsitoCaServiceAccountPath) + tokenProvider, err := NewSaTokenProvider(ServiceAccountPath) if err != nil { return err } @@ -185,17 +186,20 @@ func (c *CACertManager) UpdateCert() error { CAEndpoint: IstioCAEndpoint, TrustedRoots: trustRoot, TokenProvider: tokenProvider, - CertSigner: "kubernetes.default.svc", - ClusterID: "Kubernetes", + CertSigner: certSigner, + ClusterID: clusterID, }) - host := "spiffe://" + "cluster.local" + "/ns/" + POD_NAMESPACE + "/sa/" + tokenProvider.Token + host := "spiffe://" + "cluster.local" + "/ns/" + PodNamespace + "/sa/" + "default" + ttl, err := strconv.ParseInt(CertTTL, 10, 64) + if err != nil { + return err + } - //use default options := certgenerate.CertOptions{ Host: host, RSAKeySize: 2048, PKCS8Key: true, - ECSigAlg: certgenerate.EcdsaSigAlg, + TTL: time.Duration(ttl), } // Generate the cert/key, send CSR to CA. @@ -204,7 +208,7 @@ func (c *CACertManager) UpdateCert() error { log.Errorf("failed to generate key and certificate for CSR: %v", err) return err } - sign, err := citadelClient.CSRSign(csrPEM, int64(200000)) + sign, err := citadelClient.CSRSign(csrPEM, ttl) if err != nil { return err diff --git a/xds/credentials/cert_manager_test.go b/xds/credentials/cert_manager_test.go deleted file mode 100644 index 5a1e1730da..0000000000 --- a/xds/credentials/cert_manager_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package credentials - -import ( - "fmt" - "testing" -) - -func TestNewCertManager(t *testing.T) { - manager, err := NewCertManager() - fmt.Println(err) - fmt.Println(manager.GetCertificate()) - fmt.Println(manager.GetRootCertificate()) - fmt.Println() -} diff --git a/xds/credentials/env.go b/xds/credentials/env.go index f95318d680..23f984b462 100644 --- a/xds/credentials/env.go +++ b/xds/credentials/env.go @@ -20,12 +20,27 @@ package credentials import "os" const ( - KubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" - IsitoCaServiceAccountPath = "/var/run/secrets/tokens/istio-token" - RootCertPath = "/var/run/secrets/istio/root-cert.pem" - IstioCAEndpoint = "istiod.istio-system.svc:15012" + DefaultKubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + DefaultIsitoCaServiceAccountPath = "/var/run/secrets/tokens/istio-token" + DefaultRootCertPath = "/var/run/secrets/istio/root-cert.pem" + DefaultIstioCAEndpoint = "istiod.istio-system.svc:15012" + DefaultCertSigner = "kubernetes.default.svc" + DefaultClusterID = "Kubernetes" ) var ( - POD_NAMESPACE = os.Getenv("POD_NAMESPACE") + PodNamespace = f(os.Getenv("POD_NAMESPACE"), "default") + CertTTL = f(os.Getenv("CERT_TTL"), "31536000") + RootCertPath = f(os.Getenv("ROOT_CERT_PATH"), DefaultRootCertPath) + IstioCAEndpoint = f(os.Getenv("ISTIO_CA_ENDPOINT"), DefaultIstioCAEndpoint) + ServiceAccountPath = f(os.Getenv("SERVICE_ACCOUNT_PATH"), DefaultIsitoCaServiceAccountPath) + certSigner = f(os.Getenv("CERT_SIGNER"), DefaultCertSigner) + clusterID = f(os.Getenv("CLUSTER_ID"), DefaultClusterID) ) + +var f = func(a, b string) string { + if "" == a { + return b + } + return a +} From f92459aff540b9c53bd10eec552adece6b89d4b1 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Mon, 18 Jul 2022 23:27:05 +0800 Subject: [PATCH 06/18] add env param --- xds/client/bootstrap/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index e6729298db..c15e3c4039 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -50,7 +50,7 @@ import ( import ( dubboLogger "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - internal "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) From 98b736288bd642a1fe14c2f0a6f56412a9cf89f9 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Mon, 18 Jul 2022 23:35:51 +0800 Subject: [PATCH 07/18] fix golint --- xds/credentials/cert_manager.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 16e3f444ae..b51c4b265a 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -189,6 +189,9 @@ func (c *CACertManager) UpdateCert() error { CertSigner: certSigner, ClusterID: clusterID, }) + if err != nil { + return err + } host := "spiffe://" + "cluster.local" + "/ns/" + PodNamespace + "/sa/" + "default" ttl, err := strconv.ParseInt(CertTTL, 10, 64) if err != nil { @@ -214,7 +217,7 @@ func (c *CACertManager) UpdateCert() error { return err } - cert, _, err := c.parseCert(concatCerts(sign), keyPEM) + cert, err := c.parseCert(concatCerts(sign), keyPEM) if err != nil { return err } @@ -222,21 +225,23 @@ func (c *CACertManager) UpdateCert() error { return nil } -func (c *CACertManager) parseCert(certByte []byte, keyByte []byte) (*tls.Certificate, time.Time, error) { +func (c *CACertManager) parseCert(certByte []byte, keyByte []byte) (*tls.Certificate, error) { block, _ := pem.Decode(certByte) if block == nil { - return nil, time.Now(), fmt.Errorf("failed to decode certificate") + return nil, fmt.Errorf("failed to decode certificate") } cert, err := x509.ParseCertificate(block.Bytes) - + if err != nil { + return nil, err + } expired := cert.NotAfter log.Infof("cert expired after:" + expired.String()) c.NoAfter = expired pair, err := tls.X509KeyPair(certByte, keyByte) if err != nil { - return nil, time.Now(), fmt.Errorf("failed to parse certificate: %v", err) + return nil, fmt.Errorf("failed to parse certificate: %v", err) } - return &pair, expired, nil + return &pair, nil } func concatCerts(certsPEM []string) []byte { From ff7df6a44d5781266360cdcead0de27ea034fe6f Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Tue, 19 Jul 2022 23:53:40 +0800 Subject: [PATCH 08/18] fix golint --- xds/client/bootstrap/bootstrap.go | 10 +--------- xds/credentials/cert_manager.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index c15e3c4039..5e15fc1645 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -27,8 +27,6 @@ package bootstrap import ( "bytes" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/pemfile" "encoding/json" "fmt" "io/ioutil" @@ -50,6 +48,7 @@ import ( import ( dubboLogger "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" @@ -68,13 +67,6 @@ const ( clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) -func init() { - //init file_watcher builder - certprovider.Register(&pemfile.PluginBuilder{}) - //init builder func - internal.GetCertificateProviderBuilder = certprovider.GetBuilder -} - var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version) // For overriding in unit tests. diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index b51c4b265a..2c6d8fc28b 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -38,8 +38,18 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/pemfile" + "dubbo.apache.org/dubbo-go/v3/xds/internal" ) +func init() { + //init file_watcher builder + certprovider.Register(&pemfile.PluginBuilder{}) + //init builder func + internal.GetCertificateProviderBuilder = certprovider.GetBuilder +} + //CertManager manage agent or no agent cert type CertManager interface { GetCertificate() ([]tls.Certificate, error) From 2b4aec2bc1fffdf8d44971719f1ae916081da9c8 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sat, 30 Jul 2022 17:42:14 +0800 Subject: [PATCH 09/18] fix type compatible --- xds/balancer/cdsbalancer/cdsbalancer.go | 2 +- xds/server/conn_wrapper.go | 5 +---- xds/utils/credentials/xds/handshake_info.go | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index 3907a795ed..0e344b397e 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -37,7 +37,6 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/resolver" @@ -50,6 +49,7 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index 421a1bdd2f..7f8d42e0a7 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -31,12 +31,9 @@ import ( "time" ) -import ( - "google.golang.org/grpc/credentials/tls/certprovider" -) - import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" ) diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index a7d62ab5fd..a7213a5678 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -37,12 +37,11 @@ import ( import ( "google.golang.org/grpc/attributes" - "google.golang.org/grpc/credentials/tls/certprovider" - "google.golang.org/grpc/resolver" ) import ( + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" ) From 6bca631c97c287da01caababfc64ccc80f3387a1 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sun, 31 Jul 2022 15:34:51 +0800 Subject: [PATCH 10/18] fix imports-formatter --- xds/credentials/cert_manager.go | 5 ++--- xds/credentials/certgenerate/generate_cert.go | 5 ++++- xds/credentials/certgenerate/generate_csr.go | 5 ++++- xds/credentials/certprovider/distributor.go | 5 ++++- xds/credentials/certprovider/pemfile/watcher.go | 5 ++++- xds/credentials/certprovider/remote/istioca_client.go | 10 ++++++++-- xds/credentials/certprovider/remote/v1alpha1/ca.pb.go | 9 +++++++-- .../certprovider/remote/v1alpha1/ca_grpc.pb.go | 3 +++ xds/credentials/env.go | 4 +++- 9 files changed, 39 insertions(+), 12 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 2c6d8fc28b..4de4662c2e 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -36,11 +36,10 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/pemfile" + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote" "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) func init() { diff --git a/xds/credentials/certgenerate/generate_cert.go b/xds/credentials/certgenerate/generate_cert.go index e9b264cecb..9b99c8b2f6 100644 --- a/xds/credentials/certgenerate/generate_cert.go +++ b/xds/credentials/certgenerate/generate_cert.go @@ -32,7 +32,6 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" - log "dubbo.apache.org/dubbo-go/v3/common/logger" "encoding/pem" "errors" "fmt" @@ -42,6 +41,10 @@ import ( "time" ) +import ( + log "dubbo.apache.org/dubbo-go/v3/common/logger" +) + // SupportedECSignatureAlgorithms are the types of EC Signature Algorithms // to be used in key generation (e.g. ECDSA or ED2551) type SupportedECSignatureAlgorithms string diff --git a/xds/credentials/certgenerate/generate_csr.go b/xds/credentials/certgenerate/generate_csr.go index 43eec682f0..76a455cc72 100644 --- a/xds/credentials/certgenerate/generate_csr.go +++ b/xds/credentials/certgenerate/generate_csr.go @@ -31,13 +31,16 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" - log "dubbo.apache.org/dubbo-go/v3/common/logger" "errors" "fmt" "os" "strings" ) +import ( + log "dubbo.apache.org/dubbo-go/v3/common/logger" +) + // minimumRsaKeySize is the minimum RSA key size to generate certificates // to ensure proper security const minimumRsaKeySize = 2048 diff --git a/xds/credentials/certprovider/distributor.go b/xds/credentials/certprovider/distributor.go index 88e7a81f4b..0393d9a0b9 100644 --- a/xds/credentials/certprovider/distributor.go +++ b/xds/credentials/certprovider/distributor.go @@ -25,10 +25,13 @@ package certprovider import ( "context" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "sync" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" +) + // Distributor makes it easy for provider implementations to furnish new key // materials by handling synchronization between the producer and consumers of // the key material. diff --git a/xds/credentials/certprovider/pemfile/watcher.go b/xds/credentials/certprovider/pemfile/watcher.go index 1599275481..0b1540a8ed 100644 --- a/xds/credentials/certprovider/pemfile/watcher.go +++ b/xds/credentials/certprovider/pemfile/watcher.go @@ -38,10 +38,13 @@ import ( ) import ( - "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" "github.com/dubbogo/grpc-go/grpclog" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" +) + const defaultCertRefreshDuration = 1 * time.Hour var ( diff --git a/xds/credentials/certprovider/remote/istioca_client.go b/xds/credentials/certprovider/remote/istioca_client.go index ee9e8761e2..3b93211d81 100644 --- a/xds/credentials/certprovider/remote/istioca_client.go +++ b/xds/credentials/certprovider/remote/istioca_client.go @@ -18,20 +18,26 @@ import ( "context" "crypto/tls" "crypto/x509" - v1alpha1 "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote/v1alpha1" "errors" "fmt" - structpb "github.com/golang/protobuf/ptypes/struct" "log" "path/filepath" "strings" "time" +) + +import ( + structpb "github.com/golang/protobuf/ptypes/struct" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" ) +import ( + v1alpha1 "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider/remote/v1alpha1" +) + const ( // CertSigner info CertSigner = "CertSigner" diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go index 8be6b75a54..77ec526568 100644 --- a/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go +++ b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go @@ -22,12 +22,17 @@ package v1alpha1 +import ( + reflect "reflect" + sync "sync" +) + import ( _struct "github.com/golang/protobuf/ptypes/struct" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go index 9ae2cb9e7e..89e6424fa2 100644 --- a/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go +++ b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go @@ -8,6 +8,9 @@ package v1alpha1 import ( context "context" +) + +import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/xds/credentials/env.go b/xds/credentials/env.go index 23f984b462..c9e98d09b8 100644 --- a/xds/credentials/env.go +++ b/xds/credentials/env.go @@ -17,7 +17,9 @@ package credentials -import "os" +import ( + "os" +) const ( DefaultKubernetesServiceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" From 10065f3ac2ea0a3f5c9114fac74ae89e291876d1 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sun, 31 Jul 2022 15:44:59 +0800 Subject: [PATCH 11/18] fix imports-formatter --- xds/credentials/certgenerate/generate_cert.go | 6 +++--- xds/credentials/certgenerate/generate_csr.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xds/credentials/certgenerate/generate_cert.go b/xds/credentials/certgenerate/generate_cert.go index 9b99c8b2f6..1ebc19e45c 100644 --- a/xds/credentials/certgenerate/generate_cert.go +++ b/xds/credentials/certgenerate/generate_cert.go @@ -35,8 +35,8 @@ import ( "encoding/pem" "errors" "fmt" + "io/ioutil" "math/big" - "os" "strings" "time" ) @@ -242,12 +242,12 @@ func GenCertFromCSR(csr *x509.CertificateRequest, signingCert *x509.Certificate, // signerCertFile: cert file name // signerPrivFile: private key file name func LoadSignerCredsFromFiles(signerCertFile string, signerPrivFile string) (*x509.Certificate, crypto.PrivateKey, error) { - signerCertBytes, err := os.ReadFile(signerCertFile) + signerCertBytes, err := ioutil.ReadFile(signerCertFile) if err != nil { return nil, nil, fmt.Errorf("certificate file reading failure (%v)", err) } - signerPrivBytes, err := os.ReadFile(signerPrivFile) + signerPrivBytes, err := ioutil.ReadFile(signerPrivFile) if err != nil { return nil, nil, fmt.Errorf("private key file reading failure (%v)", err) } diff --git a/xds/credentials/certgenerate/generate_csr.go b/xds/credentials/certgenerate/generate_csr.go index 76a455cc72..d01f89ca67 100644 --- a/xds/credentials/certgenerate/generate_csr.go +++ b/xds/credentials/certgenerate/generate_csr.go @@ -33,7 +33,7 @@ import ( "crypto/x509/pkix" "errors" "fmt" - "os" + "io/ioutil" "strings" ) @@ -116,7 +116,7 @@ func AppendRootCerts(pemCert []byte, rootCertFile string) ([]byte, error) { rootCerts := pemCert if len(rootCertFile) > 0 { log.Debugf("append root certificates from %v", rootCertFile) - certBytes, err := os.ReadFile(rootCertFile) + certBytes, err := ioutil.ReadFile(rootCertFile) if err != nil { return rootCerts, fmt.Errorf("failed to read root certificates (%v)", err) } From 885686c1bd0b965a6543eb30a61a3f82ae42b986 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sun, 31 Jul 2022 22:13:22 +0800 Subject: [PATCH 12/18] fix go-lint --- .../certprovider/remote/istioca_client.go | 29 +++++++++++-------- .../certprovider/remote/v1alpha1/ca.pb.go | 28 ++++++++++-------- .../remote/v1alpha1/ca_grpc.pb.go | 17 +++++++++++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/xds/credentials/certprovider/remote/istioca_client.go b/xds/credentials/certprovider/remote/istioca_client.go index 3b93211d81..bcc75cf214 100644 --- a/xds/credentials/certprovider/remote/istioca_client.go +++ b/xds/credentials/certprovider/remote/istioca_client.go @@ -1,16 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Copyright Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. package remote diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go index 77ec526568..54ce78e974 100644 --- a/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go +++ b/xds/credentials/certprovider/remote/v1alpha1/ca.pb.go @@ -1,16 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ // Copyright Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. // Code generated by protoc-gen-go. DO NOT EDIT. // versions: diff --git a/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go index 89e6424fa2..22a6334b59 100644 --- a/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go +++ b/xds/credentials/certprovider/remote/v1alpha1/ca_grpc.pb.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.2.0 From 4fdd0a2f995066c4e7a83b33cabea76689a299b6 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sun, 31 Jul 2022 22:27:53 +0800 Subject: [PATCH 13/18] fix go-lint --- xds/credentials/cert_manager.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 4de4662c2e..ea596484c8 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -32,7 +32,7 @@ import ( ) import ( - log "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" @@ -62,13 +62,13 @@ func NewCertManager() (CertManager, error) { manager := &AgentCertManager{} config, err := bootstrap.NewConfig() if err != nil { - log.Errorf("build bootstrap config error :%s", err.Error()) + logger.Errorf("build bootstrap config error :%s", err.Error()) return nil, err } certProvider, err := buildProvider(config.CertProviderConfigs, "default") if err != nil { - log.Errorf("get cert provider error :%s", err.Error()) + logger.Errorf("get cert provider error :%s", err.Error()) return nil, err } manager.provider = certProvider @@ -165,7 +165,7 @@ func (c *CACertManager) UpdateRoot() error { } trustPool := x509.NewCertPool() if !trustPool.AppendCertsFromPEM(rootFileContents) { - log.Warn("failed to parse root certificate") + logger.Warn("failed to parse root certificate") } c.Roots = trustPool block, _ := pem.Decode(rootFileContents) @@ -217,7 +217,7 @@ func (c *CACertManager) UpdateCert() error { // Generate the cert/key, send CSR to CA. csrPEM, keyPEM, err := certgenerate.GenCSR(options) if err != nil { - log.Errorf("failed to generate key and certificate for CSR: %v", err) + logger.Errorf("failed to generate key and certificate for CSR: %v", err) return err } sign, err := citadelClient.CSRSign(csrPEM, ttl) @@ -244,7 +244,7 @@ func (c *CACertManager) parseCert(certByte []byte, keyByte []byte) (*tls.Certifi return nil, err } expired := cert.NotAfter - log.Infof("cert expired after:" + expired.String()) + logger.Infof("cert expired after:" + expired.String()) c.NoAfter = expired pair, err := tls.X509KeyPair(certByte, keyByte) if err != nil { From 0ec5761d1cb743a536613f633dfd000e3f1648b0 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Sun, 31 Jul 2022 22:39:41 +0800 Subject: [PATCH 14/18] fix go-lint --- xds/credentials/cert_manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index ea596484c8..3f788ca7bb 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -32,7 +32,10 @@ import ( ) import ( - "dubbo.apache.org/dubbo-go/v3/common/logger" + "github.com/dubbogo/gost/log/logger" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certgenerate" "dubbo.apache.org/dubbo-go/v3/xds/credentials/certprovider" From 38c195b62a6027466027049c5dfb88a17fd0afb4 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Mon, 1 Aug 2022 22:39:53 +0800 Subject: [PATCH 15/18] fix go-lint --- xds/credentials/certgenerate/generate_cert.go | 10 +++++----- xds/credentials/certgenerate/generate_csr.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/xds/credentials/certgenerate/generate_cert.go b/xds/credentials/certgenerate/generate_cert.go index 1ebc19e45c..d0313b6f47 100644 --- a/xds/credentials/certgenerate/generate_cert.go +++ b/xds/credentials/certgenerate/generate_cert.go @@ -42,7 +42,7 @@ import ( ) import ( - log "dubbo.apache.org/dubbo-go/v3/common/logger" + "github.com/dubbogo/gost/log/logger" ) // SupportedECSignatureAlgorithms are the types of EC Signature Algorithms @@ -299,8 +299,8 @@ func genCertTemplateFromCSR(csr *x509.CertificateRequest, subjectIDs []string, t // In this case, set CN as determined by DualUseCommonName(subjectIDsInString). if len(csr.Subject.CommonName) != 0 { if cn, err := DualUseCommonName(subjectIDsInString); err != nil { - // log and continue - log.Errorf("dual-use failed for cert template - omitting CN (%v)", err) + // logger and continue + logger.Errorf("dual-use failed for cert template - omitting CN (%v)", err) } else { subject.CommonName = cn } @@ -369,8 +369,8 @@ func genCertTemplateFromOptions(options CertOptions) (*x509.Certificate, error) if options.IsDualUse { cn, err := DualUseCommonName(h) if err != nil { - // log and continue - log.Errorf("dual-use failed for cert template - omitting CN (%v)", err) + // logger and continue + logger.Errorf("dual-use failed for cert template - omitting CN (%v)", err) } else { subject.CommonName = cn } diff --git a/xds/credentials/certgenerate/generate_csr.go b/xds/credentials/certgenerate/generate_csr.go index d01f89ca67..7e1edf0afc 100644 --- a/xds/credentials/certgenerate/generate_csr.go +++ b/xds/credentials/certgenerate/generate_csr.go @@ -38,7 +38,7 @@ import ( ) import ( - log "dubbo.apache.org/dubbo-go/v3/common/logger" + "github.com/dubbogo/gost/log/logger" ) // minimumRsaKeySize is the minimum RSA key size to generate certificates @@ -99,8 +99,8 @@ func GenCSRTemplate(options CertOptions) (*x509.CertificateRequest, error) { if options.IsDualUse { cn, err := DualUseCommonName(h) if err != nil { - // log and continue - //log.Errorf("dual-use failed for CSR template - omitting CN (%v)", err) + // logger and continue + //logger.Errorf("dual-use failed for CSR template - omitting CN (%v)", err) } else { template.Subject.CommonName = cn } @@ -115,7 +115,7 @@ func GenCSRTemplate(options CertOptions) (*x509.CertificateRequest, error) { func AppendRootCerts(pemCert []byte, rootCertFile string) ([]byte, error) { rootCerts := pemCert if len(rootCertFile) > 0 { - log.Debugf("append root certificates from %v", rootCertFile) + logger.Debugf("append root certificates from %v", rootCertFile) certBytes, err := ioutil.ReadFile(rootCertFile) if err != nil { return rootCerts, fmt.Errorf("failed to read root certificates (%v)", err) From f139e9bd1b1a731f9ecfe4571ba3e4f36a1a7271 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Tue, 2 Aug 2022 22:21:38 +0800 Subject: [PATCH 16/18] fix go-lint --- xds/credentials/certgenerate/generate_cert.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xds/credentials/certgenerate/generate_cert.go b/xds/credentials/certgenerate/generate_cert.go index d0313b6f47..8f590be33b 100644 --- a/xds/credentials/certgenerate/generate_cert.go +++ b/xds/credentials/certgenerate/generate_cert.go @@ -298,7 +298,8 @@ func genCertTemplateFromCSR(csr *x509.CertificateRequest, subjectIDs []string, t // Dual use mode if common name in CSR is not empty. // In this case, set CN as determined by DualUseCommonName(subjectIDsInString). if len(csr.Subject.CommonName) != 0 { - if cn, err := DualUseCommonName(subjectIDsInString); err != nil { + var cn string + if cn, err = DualUseCommonName(subjectIDsInString); err != nil { // logger and continue logger.Errorf("dual-use failed for cert template - omitting CN (%v)", err) } else { From 9e2fb083f459a11624d93fac4f338b4d501e63ff Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Thu, 8 Sep 2022 23:08:21 +0800 Subject: [PATCH 17/18] make host configurable --- xds/credentials/cert_manager.go | 2 +- xds/credentials/certprovider/remote/istioca_client.go | 1 - xds/credentials/env.go | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index 3f788ca7bb..ac6326b5ca 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -204,7 +204,7 @@ func (c *CACertManager) UpdateCert() error { if err != nil { return err } - host := "spiffe://" + "cluster.local" + "/ns/" + PodNamespace + "/sa/" + "default" + host := URIPrefix + Domain + "/ns/" + PodNamespace + "/sa/" + ServiceAccountName ttl, err := strconv.ParseInt(CertTTL, 10, 64) if err != nil { return err diff --git a/xds/credentials/certprovider/remote/istioca_client.go b/xds/credentials/certprovider/remote/istioca_client.go index bcc75cf214..8bd18fb8b6 100644 --- a/xds/credentials/certprovider/remote/istioca_client.go +++ b/xds/credentials/certprovider/remote/istioca_client.go @@ -194,7 +194,6 @@ func (c *CitadelClient) buildConnection() (*grpc.ClientConn, error) { } ol = append(ol, grpc.WithPerRPCCredentials(c.opts.TokenProvider)) ol = append(ol, c.opts.GRPCOptions...) - //security.CARetryInterceptor()) conn, err := grpc.Dial(c.opts.CAEndpoint, ol...) if err != nil { diff --git a/xds/credentials/env.go b/xds/credentials/env.go index c9e98d09b8..f7c86b5973 100644 --- a/xds/credentials/env.go +++ b/xds/credentials/env.go @@ -38,6 +38,9 @@ var ( ServiceAccountPath = f(os.Getenv("SERVICE_ACCOUNT_PATH"), DefaultIsitoCaServiceAccountPath) certSigner = f(os.Getenv("CERT_SIGNER"), DefaultCertSigner) clusterID = f(os.Getenv("CLUSTER_ID"), DefaultClusterID) + URIPrefix = f(os.Getenv("URI_PREFIX"), "spiffe://") + Domain = f(os.Getenv("DOMAIN"), "cluster.local") + ServiceAccountName = f(os.Getenv("SERVICE_ACCOUNT_NAME"), "default") ) var f = func(a, b string) string { From 9eae05eaf8ac0c68ea7c861807d147b5a2b32119 Mon Sep 17 00:00:00 2001 From: zlb <1098294815@qq.com> Date: Wed, 21 Sep 2022 23:47:57 +0800 Subject: [PATCH 18/18] fix --- xds/credentials/cert_manager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xds/credentials/cert_manager.go b/xds/credentials/cert_manager.go index ac6326b5ca..0e50d96b3b 100644 --- a/xds/credentials/cert_manager.go +++ b/xds/credentials/cert_manager.go @@ -107,7 +107,7 @@ func (c *AgentCertManager) GetCertificate() ([]tls.Certificate, error) { return material.Certs, nil } -// buildProvider build cert provider from config +// buildProvider build cert provider from config func buildProvider(configs map[string]*certprovider.BuildableConfig, instanceName string) (certprovider.Provider, error) { cfg, ok := configs[instanceName] if !ok { @@ -177,13 +177,13 @@ func (c *CACertManager) UpdateRoot() error { } cert, err := x509.ParseCertificate(block.Bytes) if err != nil { - return nil + return err } c.RootNoAfter = cert.NotAfter return nil } -// UpdateCert update cert +// UpdateCert update cert func (c *CACertManager) UpdateCert() error { tokenProvider, err := NewSaTokenProvider(ServiceAccountPath) if err != nil { @@ -192,7 +192,7 @@ func (c *CACertManager) UpdateCert() error { trustRoot, err := c.GetRootCertificate() if err != nil { - return nil + return err } citadelClient, err := remote.NewCitadelClient(&remote.Options{ CAEndpoint: IstioCAEndpoint,