From fc8196438aadb5e5da7ff33e0bfd31772a8edec1 Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Wed, 15 Jan 2020 10:45:47 +0100 Subject: [PATCH] NSX-T loadbalancer support --- cmd/vsphere-cloud-controller-manager/main.go | 14 +- go.mod | 12 +- go.sum | 45 +- hack/release.sh | 8 +- pkg/cloudprovider/vsphere/cloud.go | 55 +- pkg/cloudprovider/vsphere/config.go | 4 + .../vsphere/loadbalancer/README.md | 222 ++++++++ .../vsphere/loadbalancer/access.go | 507 ++++++++++++++++ .../vsphere/loadbalancer/class.go | 177 ++++++ .../vsphere/loadbalancer/cleanup.go | 168 ++++++ .../vsphere/loadbalancer/config/config.go | 261 +++++++++ .../loadbalancer/config/config_test.go | 115 ++++ .../vsphere/loadbalancer/helpers.go | 72 +++ .../vsphere/loadbalancer/interface.go | 120 ++++ .../vsphere/loadbalancer/lbprovider.go | 183 ++++++ .../vsphere/loadbalancer/lbservice.go | 83 +++ .../vsphere/loadbalancer/lock.go | 56 ++ .../vsphere/loadbalancer/mapping.go | 73 +++ .../vsphere/loadbalancer/nsxt_broker.go | 539 ++++++++++++++++++ .../loadbalancer/nsxt_type_converter.go | 70 +++ .../vsphere/loadbalancer/state.go | 455 +++++++++++++++ pkg/cloudprovider/vsphere/loadbalancer/tag.go | 102 ++++ .../vsphere/loadbalancer/tag_test.go | 85 +++ pkg/cloudprovider/vsphere/types.go | 4 + pkg/cloudprovider/vsphere/vapilogger.go | 58 ++ 25 files changed, 3459 insertions(+), 29 deletions(-) create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/README.md create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/access.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/class.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/cleanup.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/config/config.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/helpers.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/interface.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/lbservice.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/lock.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/mapping.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/nsxt_broker.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/nsxt_type_converter.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/state.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/tag.go create mode 100644 pkg/cloudprovider/vsphere/loadbalancer/tag_test.go create mode 100644 pkg/cloudprovider/vsphere/vapilogger.go diff --git a/cmd/vsphere-cloud-controller-manager/main.go b/cmd/vsphere-cloud-controller-manager/main.go index ba3e8ec5a4..4b25e81519 100644 --- a/cmd/vsphere-cloud-controller-manager/main.go +++ b/cmd/vsphere-cloud-controller-manager/main.go @@ -33,6 +33,7 @@ import ( _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere" + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -45,6 +46,9 @@ const AppName string = "vsphere-cloud-controller-manager" var version string func main() { + loadbalancer.Version = version + loadbalancer.AppName = AppName + rand.Seed(time.Now().UTC().UnixNano()) command := app.NewCloudControllerManagerCommand() @@ -69,10 +73,13 @@ func main() { } }) - // var versionFlag *pflag.Value + var clusterNameFlag *pflag.Value pflag.CommandLine.VisitAll(func(flag *pflag.Flag) { - if flag.Name == "version" { + switch flag.Name { + case "cluster-name": + clusterNameFlag = &flag.Value + case "version": versionFlag = &flag.Value } }) @@ -84,6 +91,9 @@ func main() { fmt.Printf("%s %s\n", AppName, version) os.Exit(0) } + if clusterNameFlag != nil { + loadbalancer.ClusterName = (*clusterNameFlag).String() + } innerRun(cmd, args) } diff --git a/go.mod b/go.mod index 385f96368f..671c815087 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 // indirect + github.com/google/uuid v1.1.1 github.com/imdario/mergo v0.3.7 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/pkg/errors v0.8.0 @@ -16,15 +17,18 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/vmware/govmomi v0.21.0 - golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect - golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 + github.com/vmware/vsphere-automation-sdk-go/lib v0.1.1 + github.com/vmware/vsphere-automation-sdk-go/runtime v0.1.1 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.1.1 + golang.org/x/lint v0.0.0-20200130185559-910be7a94367 // indirect + golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect - golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a // indirect + golang.org/x/tools v0.0.0-20200301222351-066e0c02454c // indirect google.golang.org/grpc v1.22.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/square/go-jose.v2 v2.3.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - honnef.co/go/tools v0.0.1-2019.2.2 // indirect + honnef.co/go/tools v0.0.1-2020.1.3 // indirect k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 k8s.io/client-go v0.0.0 diff --git a/go.sum b/go.sum index c715f51aa9..31e266da46 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1: github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1:uHBSeeATKpVazAACZBDPL/Nk/UhQDDsJWDlqYJo8/Us= github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= +github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= +github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -93,6 +95,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 h1:bRzFpEzvausOAt4va+I/22BZ1vXDtERngp0BNYDKej0= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI= +github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -154,12 +158,16 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -319,6 +327,12 @@ github.com/vmware/govmomi v0.21.0 h1:jc8uMuxpcV2xMAA/cnEDlnsIjvqcMra5Y8onh/U3VuY github.com/vmware/govmomi v0.21.0/go.mod h1:zbnFoBQ9GIjs2RVETy8CNEpb+L+Lwkjs3XZUL0B3/m0= github.com/vmware/photon-controller-go-sdk v0.0.0-20170310013346-4a435daef6cc/go.mod h1:e6humHha1ekIwTCm+A5Qed5mG8V4JL+ChHcUOJ+L/8U= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vsphere-automation-sdk-go/lib v0.1.1 h1:PmDaeuToX1QKKe9VWRJztAp2/IyjbbGZp6fEiff4Dr8= +github.com/vmware/vsphere-automation-sdk-go/lib v0.1.1/go.mod h1:BkjnHZykqeKKYDZEhyT4pxrEWprYqp4yC0xoCky6wjA= +github.com/vmware/vsphere-automation-sdk-go/runtime v0.1.1 h1:gsbyhqLBiYZQRs0EBPNijKkJNSGcvs1IlRmRi790o84= +github.com/vmware/vsphere-automation-sdk-go/runtime v0.1.1/go.mod h1:SRcvjNB5LycQEoeuwdwf9tSZ/glCmIBPfclZyk/+GLc= +github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.1.1 h1:F4t3V0fe3EZcomNHNZdGIusnAGvcTBkl+eDmyikAg2o= +github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.1.1/go.mod h1:5zUcOhT4aCd6V3Xs6mqXVxVGHoKQ/zr1iwCvbSZtUfo= github.com/xanzy/go-cloudstack v0.0.0-20160728180336-1e2cbf647e57/go.mod h1:s3eL3z5pNXF5FVybcT+LIVdId8pYn709yv6v5mrkrQE= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -333,18 +347,21 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -356,8 +373,8 @@ golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -377,6 +394,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -388,11 +407,13 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a h1:0JEq5ZQ3TgsRlFmz4BcD+E6U6cOk4pOImCQSyIG59ZM= -golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200301222351-066e0c02454c h1:FD7jysxM+EJqg5UYYy3XYDsAiUickFsn4UiaanJkf8c= +golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -410,6 +431,7 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -435,9 +457,10 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.2 h1:TEgegKbBqByGUb1Coo1pc2qIdf2xw6v0mYyLSYtyopE= -honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= diff --git a/hack/release.sh b/hack/release.sh index b7277ec8f9..3180a1c1c9 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -23,16 +23,16 @@ set -o nounset set -o pipefail # BASE_REPO is the root path of the image repository -readonly BASE_IMAGE_REPO=gcr.io/cloud-provider-vsphere +readonly BASE_IMAGE_REPO=${BASE_IMAGE_REPO:-gcr.io/cloud-provider-vsphere} # Release images -readonly CPI_IMAGE_RELEASE=${BASE_IMAGE_REPO}/cpi/release/manager +readonly CPI_IMAGE_RELEASE=${CPI_IMAGE_RELEASE:-${BASE_IMAGE_REPO}/cpi/release/manager} # PR images -readonly CPI_IMAGE_PR=${BASE_IMAGE_REPO}/cpi/pr/manager +readonly CPI_IMAGE_PR=${CPI_IMAGE_PR:-${BASE_IMAGE_REPO}/cpi/pr/manager} # CI images -readonly CPI_IMAGE_CI=${BASE_IMAGE_REPO}/cpi/ci/manager +readonly CPI_IMAGE_CI=${CPI_IMAGE_CI:-${BASE_IMAGE_REPO}/cpi/ci/manager} AUTH= PUSH= diff --git a/pkg/cloudprovider/vsphere/cloud.go b/pkg/cloudprovider/vsphere/cloud.go index de74a49d80..6afd7f0cca 100644 --- a/pkg/cloudprovider/vsphere/cloud.go +++ b/pkg/cloudprovider/vsphere/cloud.go @@ -17,14 +17,18 @@ limitations under the License. package vsphere import ( + "fmt" "io" + "os" "runtime" v1 "k8s.io/api/core/v1" + cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" - cloudprovider "k8s.io/cloud-provider" + "github.com/vmware/vsphere-automation-sdk-go/runtime/log" + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer" "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/server" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" k8s "k8s.io/cloud-provider-vsphere/pkg/common/kubernetes" @@ -77,7 +81,7 @@ func (vs *VSphere) Initialize(clientBuilder cloudprovider.ControllerClientBuilde vs.informMgr.Listen() - //if running secrets, init them + // if running secrets, init them connMgr.InitializeSecretLister() if !vs.cfg.Global.APIDisable { @@ -89,11 +93,25 @@ func (vs *VSphere) Initialize(clientBuilder cloudprovider.ControllerClientBuilde } else { klog.Errorf("Kubernetes Client Init Failed: %v", err) } + if vs.isLoadBalancerSupportEnabled() { + klog.Info("initializing load balancer support") + if loadbalancer.ClusterName == "" { + klog.Warning("Missing cluster id, no periodical cleanup possible") + } + vs.loadbalancer.Initialize(loadbalancer.ClusterName, client, stop) + } +} + +func (vs *VSphere) isLoadBalancerSupportEnabled() bool { + return vs.loadbalancer != nil } // LoadBalancer returns a balancer interface. Also returns true if the // interface is supported, false otherwise. func (vs *VSphere) LoadBalancer() (cloudprovider.LoadBalancer, bool) { + if vs.isLoadBalancerSupportEnabled() { + return vs.loadbalancer, true + } klog.Warning("The vSphere cloud provider does not support load balancers") return nil, false } @@ -145,13 +163,34 @@ func (vs *VSphere) HasClusterID() bool { // Initializes vSphere from vSphere CloudProvider Configuration func buildVSphereFromConfig(cfg *CPIConfig) (*VSphere, error) { nm := newNodeManager(cfg, nil) - + lb, err := loadbalancer.NewLBProvider(&cfg.LBConfig) + if err != nil { + return nil, err + } + if _, ok := os.LookupEnv("ENABLE_ALPHA_NSXT_LB"); !ok { + if lb != nil { + klog.Infof("To enable NSX-T load balancer support you need to set the env variable ENABLE_ALPHA_NSXT_LB") + lb = nil + } + } else { + if lb == nil { + return nil, fmt.Errorf("To enable NSX-T load balancer support you need to configure section LoadBalancer") + } + } + if lb == nil { + klog.Infof("NSX-T load balancer support disabled") + } else { + klog.Infof("NSX-T load balancer support enabled. This feature is alpha, use in production at your own risk.") + // redirect vapi logging from the NSX-T GO SDK to klog + log.SetLogger(NewKlogBridge()) + } vs := VSphere{ - cfg: cfg, - nodeManager: nm, - instances: newInstances(nm), - zones: newZones(nm, cfg.Labels.Zone, cfg.Labels.Region), - server: server.NewServer(cfg.Global.APIBinding, nm), + cfg: cfg, + nodeManager: nm, + loadbalancer: lb, + instances: newInstances(nm), + zones: newZones(nm, cfg.Labels.Zone, cfg.Labels.Region), + server: server.NewServer(cfg.Global.APIBinding, nm), } return &vs, nil } diff --git a/pkg/cloudprovider/vsphere/config.go b/pkg/cloudprovider/vsphere/config.go index 3caf8b5b7d..cafb4286f6 100644 --- a/pkg/cloudprovider/vsphere/config.go +++ b/pkg/cloudprovider/vsphere/config.go @@ -68,5 +68,9 @@ func ReadCPIConfig(config io.Reader) (*CPIConfig, error) { return nil, err } + if err := cfg.LBConfig.CompleteAndValidate(); err != nil { + return nil, err + } + return cfg, nil } diff --git a/pkg/cloudprovider/vsphere/loadbalancer/README.md b/pkg/cloudprovider/vsphere/loadbalancer/README.md new file mode 100644 index 0000000000..7557aaf3f0 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/README.md @@ -0,0 +1,222 @@ +# NSX-T Load Balancer Controller + +*Kubernetes load balancer support using NSX-T for the `vSphere` cloud controller +manager.* + +This package enriches the cloud provider interface by implementing the load +balancing API of the cloud controller for an NSX-T environment. + +**To activate the load balancer support, the environment variable +`ENABLE_ALPHA_NSXT_LB` must be set**. +Since this is an alpha feature, this implementation is a work in progress and the underlying implementation details can change. + +If this feature gate is enabled the load balancer service must be configured +properly, also. + +The basic assumption is that all nodes are bound to a logical tier1 router. +Here the load balancer service is attached to. Because there may be only +one such service here, the configuration of the service must be done +during the creation of the service. Here the selection of the (t-shirt) size is +required (S, M, L or XL). +For every service port a dedicated virtual server is managed which is +connected to this load balancer service. + +## Features + +The load balancer controller part of the vsphere cloud controller manager +is optional. If no `LoadBalancer` or `LoadBalancerClass` section is given +in the controller configuration loadbalancing support is disabled. +If enabled the follwing feature are supported: + +### Tagging + +All generated NSX-T elements are tagged with the app name of the controller, +the cluster name and the information from the service. Using this tagging it is +able to handle recovery of lost or dangling elements and garbage collection of +unused elements previously generated by the controller, even if the kubernetes +service object is already (accidentally) gone. + +### Load Balancer Classes + +This load balancer controller supports the usage of multiple load balancer +classes. Classes are preconfigured in the configuration file of the cloud +controller manager. There may be an arbitrary set of such classes in a dedicated +setup. Every class may use another `IPPool` configured in NSX-T. +This supports the creation of load balancers in different visibility realms, +for example an `*internet facing* or a *private* load balancer. The IPPools must +be preconfigured in NSX-T. +Additionally dedicated TCP and/or UDP profiles can be selected differing from +the default ones. + +The class used to create a Kubernetes load balancer can then be selected on +the level of the Kubernetes service object. +To select a dedicated load balancer class different from the default one, the +Kubernetes service object must be annotated with the annotation: + +```yaml +loadbalancer.vmware.io/class: +``` + +If no such annotation is given the default class will be used. This gives +the adminstrator of the cluster a chance to restrict the usage of the +NSXT-T resources for cluster users. They can determine which elements should +be used for a dedicated purpose. The cluster user just needs to know and select +the purpose by annotating the appropriate load balancer class. + +### Health Checks + +For TCP load balancers a health check will be generated. + +## Configuration File + +The controller manager requires dedicated entries in the cloud controller's +configuration file: + +```ini +[LoadBalancer] +ipPoolName = pool1 +lbServiceId = 4711 +size = SMALL +tcpAppProfileName = default-tcp-lb-app-profile +udpAppProfileName = default-udp-lb-app-profile +tags = {\"tag1\": \"value1\", \"tag2\": \"value 2\"} + +[LoadBalancerClass "public"] +ipPoolName = poolPublic + +[LoadBalancerClass "private"] +ipPoolName = poolPrivate + +[NSX-T] +user = admin +password = secret +host = nsxt-server +insecure-flag = false +``` + +If the `LoadBalancer` section or at least one `LoadBalancerClass` section is +given, the load balancer support of the vSphere cloud controller manager is +enabled, otherwise it is disabled. + +Only one of `ipPoolId` or `ipPoolName` may be given. +As the `lbServiceId` is given the controller is running in the *unmanaged* +mode. + +The ``tcpAppProfileName`` and `udpAppProfileName` are used on creating +virtual servers. Alternatively `tcpAppProfilePath` and `udpAppProfilePath` +can be specified. + +The `tags` field allows to specify additional tags which will be added +to all generated elements in NSX-T. The value must be a JSON object containing +the tags and string values. +The tag scope `owner` can be used to overwrite the owner name using the +controller's app name by default. + +The `LoadBalancer` section defines an implicit default load balancer class. This +load balancer class is used if the service does not specify a dedicated +load balancer class via annotation. Its values are also used as defaults +for all explicitly specified load balancer classes. + +Additionaly classes may be configured by the `LoadBalancerClass` +subsections. + +### Managing Modes + +There are two different modes the load balancer support can be used with: + +- the *unmanaged* mode is used if the configuration specifies a load balancer + service id. Here only the virtual servers are managed for the specified + loadbalancer service. + +- the *managed* mode manages the load balancer service, also. Here the tier1 + gateway must be specified, which is used for the segments the cluster + nodes are connected to. The NSX-T load balancer service is only created if it + is required. This saves resources if no kubernetes service of type + `LoadBalancer` is actually used. + +Exactly one of the properties `lbServiceId` or `tier1GatewayPath` must be +specified if the load balancer support for the vSphere cloud controller +manager is enabled. + +If the load balancer service should be managed by the controller (*managed* mode), +the `tier1GatewayPath` must be set (`lbServiceId` must not be set in this case): + +```ini +[LoadBalancer] +ipPoolName = pool1 +tier1GatewayPath = /infra/tier-1s/12345 +size = SMALL +tcpAppProfileName = default-tcp-lb-app-profile +udpAppProfileName = default-udp-lb-app-profile +... +``` + +### Configuraton Option Reference + +The load balancer configuration uses the sections `[NSX-T]`, `[LoadBalancer]` and +the subsections `[LoadBalancerClass ""]` + +#### Section NSX-T + +The section NSX-T specifies the access to the NSX-T environment used to +provision the load balancers. +The following attributes are supported: + +|Attribute|Meaning| +|---------|-------| +|`host`|NSXT-T host| +|`insecure-flag`|to be set to true if NSX-T uses locally signed cert without specifying a ca| +|`ca-file`|certificate authority for the server certificate for locally signed certificates | +|`user`|user name (either password, access token or certificate based authentification must be specified)| +|`password`|password in clear text for password based authentification| +|`vmcAccessToken`|access token for token based authentification| +|`vmcAuthHost`|verification host for token based authentification| +|`client-auth-cert-file`|client certificate for the certificate based authorization| +|`client-auth-key-file`|private key for the client certificate| + +#### Section LoadBalancer + +The load balancer section contains general settings and default settings for +the load balancer classes. The following attributes are supported: + +|Attribute|Meaning| +|---------|-------| +|`size`|Size of load balancer service (`SMALL`,`MEDIUM`,`LARGE`,`XLARGE`)| +|`lbServiceId`|service id of the load balancer service to use (for unmanaged mode)| +|`tier1GatewayPath`|policy path for the tier1 gateway| +|`tags`|JSON map with name/value pairs used for creating additional tags for the generated NSX-T elements| + +If the tag key `owner` is given it overwrites the default owner +(application name of the cloud controller manager). The owner is used together +with the cluster name (specified with the option `--cluster-name`) to identify +dangling elements in the infrastructure originating from this controller manager. +If the cluster name option is not given, there will be no automated cleanup of +dangling elements. + +Additionally the attributes of a `LoadBalancerClass` can be specified here. These +values are used as defaults for configured load balancer classes. +If no explicit default load balancer class (with name `default`) is configured, +these settings are used for the default load balancer class. + +The default load balancer class settings are always used if the kubernetes +service object does not explicitly specify a load balancer class by using the +annotation `loadbalancer.vmware.io/class`. + +#### Subsections LoadBalancerClass + +The name of the subsection is used as name for the load balancer class to configure. +A load balancer class configuration uses the following attributes: + +|Attribute|Meaning| +|---------|-------| +|`ipPoolName`| name of the ip pool used for the virtual servers (either `ipPoolName` or `ipPoolID` must be specified)| +|`ipPoolID`| id of the ip pool | +|`tcpAppProfileName`| name of application profile used for TCP connections (either `tcpAppProfileName` or `tcpAppProfileID` must be specified)| +|`tcpAppProfileID`| id of application profile used for TCP connections| +|`udpAppProfileName`| name of application profile used for UDP connections (either `udpAppProfileName` or `udpAppProfileID` must be specified)| +|`udpAppProfileID`| id of application profile used for UDP connections| + +If a name/id pair is missing completely it will be defaulted by the settings from the `LoadBalancer` section. +If there no value is specified, also, the configuration is invalid. +If a name is specified instead of an id there *MUST* not be multiple such elements with the same name, even this +is possible in NSX-T. diff --git a/pkg/cloudprovider/vsphere/loadbalancer/access.go b/pkg/cloudprovider/vsphere/loadbalancer/access.go new file mode 100644 index 0000000000..af4f13efd5 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/access.go @@ -0,0 +1,507 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" +) + +const ( + // ScopeOwner is the owner scope + ScopeOwner = "owner" + // ScopeCluster is the cluster scope + ScopeCluster = "cluster" + // ScopeService is the service scope + ScopeService = "service" + // ScopePort is the port scope + ScopePort = "port" + // ScopeIPPoolID is the IP pool id scope + ScopeIPPoolID = "ippoolid" + // ScopeLBClass is the load balancer class scope + ScopeLBClass = "lbclass" +) + +type access struct { + broker NsxtBroker + config *config.LBConfig + ownerTag model.Tag + standardTags Tags +} + +var _ NSXTAccess = &access{} + +// NewNSXTAccess creates a new NSXTAccess instance +func NewNSXTAccess(broker NsxtBroker, config *config.LBConfig) (NSXTAccess, error) { + standardTags := Tags{ + ScopeOwner: newTag(ScopeOwner, AppName), + } + for k, v := range config.LoadBalancer.AdditionalTags { + standardTags[k] = newTag(k, v) + } + return &access{ + broker: broker, + config: config, + ownerTag: standardTags[ScopeOwner], + standardTags: standardTags, + }, nil +} + +func (a *access) FindIPPoolByName(poolName string) (string, error) { + list, err := a.broker.ListIPPools() + if err != nil { + return "", errors.Wrap(err, "listing IP pools failed") + } + for _, item := range list { + if item.DisplayName != nil && *item.DisplayName == poolName { + return *item.Id, nil + } + } + return "", fmt.Errorf("load balancer IP pool named %s not found", poolName) +} + +func (a *access) CreateLoadBalancerService(clusterName string) (*model.LBService, error) { + lbService := model.LBService{ + Description: strptr(fmt.Sprintf("virtual server pool for cluster %s created by %s", clusterName, AppName)), + DisplayName: displayName(clusterName), + Tags: a.standardTags.Append(clusterTag(clusterName)).Normalize(), + Size: strptr(a.config.LoadBalancer.Size), + Enabled: boolptr(true), + ConnectivityPath: strptr(a.config.LoadBalancer.Tier1GatewayPath), + } + result, err := a.broker.CreateLoadBalancerService(lbService) + if err != nil { + return nil, errors.Wrapf(err, "creating load balancer service failed for cluster %s", clusterName) + } + return &result, nil +} + +func (a *access) FindLoadBalancerService(clusterName string, id string) (*model.LBService, error) { + if id == "" { + return a.findLoadBalancerService(a.ownerTag, clusterTag(clusterName)) + } + + result, err := a.broker.ReadLoadBalancerService(id) + if err != nil { + return nil, err + } + if a.config.LoadBalancer.Tier1GatewayPath != "" && (result.ConnectivityPath == nil || *result.ConnectivityPath != a.config.LoadBalancer.Tier1GatewayPath) { + connectivityPath := "nil" + if result.ConnectivityPath != nil { + connectivityPath = *result.ConnectivityPath + } + return nil, fmt.Errorf("load balancer service %q is configured for router %q not %q", + *result.Id, + connectivityPath, + a.config.LoadBalancer.Tier1GatewayPath, + ) + } + return &result, nil +} + +func (a *access) findLoadBalancerService(tags ...model.Tag) (*model.LBService, error) { + list, err := a.broker.ListLoadBalancerServices() + if err != nil { + return nil, errors.Wrapf(err, "listing load balancer services failed") + } + for _, item := range list { + if a.config.LoadBalancer.Tier1GatewayPath != "" && item.ConnectivityPath != nil && *item.ConnectivityPath == a.config.LoadBalancer.Tier1GatewayPath { + return &item, nil + } + if checkTags(item.Tags, tags...) { + return &item, nil + } + } + return nil, nil +} + +func (a *access) UpdateLoadBalancerService(lbService *model.LBService) error { + _, err := a.broker.UpdateLoadBalancerService(*lbService) + if err != nil { + return errors.Wrapf(err, "updating load balancer service %s (%s) failed", *lbService.DisplayName, *lbService.Id) + } + return nil +} + +func (a *access) DeleteLoadBalancerService(id string) error { + err := a.broker.DeleteLoadBalancerService(id) + if isNotFoundError(err) { + return nil + } + if err != nil { + return errors.Wrapf(err, "deleting load balancer service %s failed", id) + } + return nil +} + +func (a *access) findAppProfilePathByName(profileName string, resourceType string) (string, error) { + list, err := a.broker.ListAppProfiles() + if err != nil { + return "", err + } + path := "" + for _, item := range list { + itemResourceType, err := item.String("resource_type") + if err != nil { + return "", errors.Wrapf(err, "findAppProfilePathByName cannot find field resource_type") + } + itemName, err := item.String("display_name") + if err != nil { + return "", errors.Wrapf(err, "findAppProfilePathByName cannot find field name") + } + if itemResourceType == resourceType && itemName == profileName { + if path != "" { + return "", fmt.Errorf("profile name %s for resource type %s is not unique", profileName, resourceType) + } + path, err = item.String("path") + if err != nil { + return "", errors.Wrapf(err, "findAppProfilePathByName cannot find field path") + } + } + } + if path == "" { + return "", fmt.Errorf("application profile named %s of type %s not found", profileName, resourceType) + } + return path, nil +} + +func (a *access) GetAppProfilePath(class LBClass, protocol corev1.Protocol) (string, error) { + profileReference, err := class.AppProfile(protocol) + if err != nil { + return "", err + } + if profileReference.Identifier != "" { + return profileReference.Identifier, nil + } + resourceType := "" + switch protocol { + case corev1.ProtocolTCP: + resourceType = model.LBAppProfile_RESOURCE_TYPE_LBFASTTCPPROFILE + case corev1.ProtocolUDP: + resourceType = model.LBAppProfile_RESOURCE_TYPE_LBFASTUDPPROFILE + default: + return "", fmt.Errorf("Unsupported protocol %s", protocol) + } + return a.findAppProfilePathByName(profileReference.Name, resourceType) +} + +func (a *access) CreateVirtualServer(clusterName string, objectName types.NamespacedName, class LBClass, ipAddress string, + mapping Mapping, lbServicePath, applicationProfilePath string, poolPath *string) (*model.LBVirtualServer, error) { + allTags := append(class.Tags(), clusterTag(clusterName), serviceTag(objectName), portTag(mapping)) + virtualServer := model.LBVirtualServer{ + Description: strptr(fmt.Sprintf("virtual server for cluster %s, service %s created by %s", + clusterName, objectName, AppName)), + DisplayName: displayNameObject(clusterName, objectName), + Tags: a.standardTags.Append(allTags...).Normalize(), + DefaultPoolMemberPorts: []string{fmt.Sprintf("%d", mapping.NodePort)}, + Enabled: boolptr(true), + IpAddress: ipAddress, + ApplicationProfilePath: applicationProfilePath, + PoolPath: poolPath, + Ports: []string{fmt.Sprintf("%d", mapping.SourcePort)}, + LbServicePath: strptr(lbServicePath), + } + result, err := a.broker.CreateLoadBalancerVirtualServer(virtualServer) + if err != nil { + return nil, errors.Wrapf(err, "creating virtual server failed for %s:%s with IP address %s", clusterName, objectName, ipAddress) + } + return &result, nil +} + +func (a *access) FindVirtualServers(clusterName string, objectName types.NamespacedName) ([]*model.LBVirtualServer, error) { + return a.listVirtualServers(a.ownerTag, clusterTag(clusterName), serviceTag(objectName)) +} + +func (a *access) ListVirtualServers(clusterName string) ([]*model.LBVirtualServer, error) { + return a.listVirtualServers(a.ownerTag, clusterTag(clusterName)) +} + +func (a *access) listVirtualServers(tags ...model.Tag) ([]*model.LBVirtualServer, error) { + list, err := a.broker.ListLoadBalancerVirtualServers() + if err != nil { + return nil, errors.Wrapf(err, "listing virtual servers failed") + } + var result []*model.LBVirtualServer + for _, item := range list { + if checkTags(item.Tags, tags...) { + itemCopy := item + result = append(result, &itemCopy) + } + } + return result, nil +} + +func (a *access) UpdateVirtualServer(server *model.LBVirtualServer) error { + _, err := a.broker.UpdateLoadBalancerVirtualServer(*server) + if err != nil { + return errors.Wrapf(err, "updating load balancer virtual server %s (%s) failed", *server.DisplayName, *server.Id) + } + return nil +} + +func (a *access) DeleteVirtualServer(id string) error { + err := a.broker.DeleteLoadBalancerVirtualServer(id) + if isNotFoundError(err) { + return nil + } + if err != nil { + return errors.Wrapf(err, "deleting virtual server %s failed", id) + } + return nil +} + +func (a *access) CreatePool(clusterName string, objectName types.NamespacedName, mapping Mapping, members []model.LBPoolMember, activeMonitorPaths []string) (*model.LBPool, error) { + snatTranslation, err := newNsxtTypeConverter().createLBSnatAutoMap() + if err != nil { + return nil, errors.Wrapf(err, "creating pool failed on preparing LBSnatAutoMap failed") + } + pool := model.LBPool{ + Description: strptr(fmt.Sprintf("pool for cluster %s, service %s created by %s", clusterName, objectName, AppName)), + DisplayName: displayNameObject(clusterName, objectName), + Tags: a.standardTags.Append(clusterTag(clusterName), serviceTag(objectName), portTag(mapping)).Normalize(), + SnatTranslation: snatTranslation, + Members: members, + ActiveMonitorPaths: activeMonitorPaths, + } + result, err := a.broker.CreateLoadBalancerPool(pool) + if err != nil { + return nil, errors.Wrapf(err, "creating pool failed for %s:%s", clusterName, objectName) + } + return &result, nil +} + +func (a *access) GetPool(id string) (*model.LBPool, error) { + pool, err := a.broker.ReadLoadBalancerPool(id) + if err != nil { + return nil, err + } + return &pool, nil +} + +func (a *access) FindPool(clusterName string, objectName types.NamespacedName, mapping Mapping) (*model.LBPool, error) { + list, err := a.broker.ListLoadBalancerPools() + if err != nil { + return nil, errors.Wrapf(err, "listing load balancer pools failed") + } + tags := []model.Tag{a.ownerTag, clusterTag(clusterName), serviceTag(objectName), portTag(mapping)} + for _, item := range list { + if checkTags(item.Tags, tags...) { + return &item, nil + } + } + return nil, nil +} + +func (a *access) FindPools(clusterName string, objectName types.NamespacedName) ([]*model.LBPool, error) { + return a.listPools(a.ownerTag, clusterTag(clusterName), serviceTag(objectName)) +} + +func (a *access) ListPools(clusterName string) ([]*model.LBPool, error) { + return a.listPools(a.ownerTag, clusterTag(clusterName)) +} + +func (a *access) listPools(tags ...model.Tag) ([]*model.LBPool, error) { + list, err := a.broker.ListLoadBalancerPools() + if err != nil { + return nil, errors.Wrapf(err, "listing pools failed") + } + var result []*model.LBPool + for _, item := range list { + if checkTags(item.Tags, tags...) { + itemCopy := item + result = append(result, &itemCopy) + } + } + return result, nil +} + +func (a *access) UpdatePool(pool *model.LBPool) error { + _, err := a.broker.UpdateLoadBalancerPool(*pool) + if err != nil { + return errors.Wrapf(err, "updating load balancer pool %s (%s) failed", *pool.DisplayName, *pool.Id) + } + return nil +} + +func (a *access) DeletePool(id string) error { + err := a.broker.DeleteLoadBalancerPool(id) + if isNotFoundError(err) { + return nil + } + if err != nil { + return errors.Wrapf(err, "deleting load balancer pool %s failed", id) + } + return nil +} + +func (a *access) CreateTCPMonitorProfile(clusterName string, objectName types.NamespacedName, mapping Mapping) (*model.LBTcpMonitorProfile, error) { + profile := model.LBTcpMonitorProfile{ + Description: strptr(fmt.Sprintf("tcp monitor for cluster %s, service %s, port %d created by %s", + clusterName, objectName, mapping.NodePort, AppName)), + DisplayName: displayNameMapping(clusterName, objectName, mapping), + Tags: a.standardTags.Append(clusterTag(clusterName), serviceTag(objectName), portTag(mapping)).Normalize(), + MonitorPort: int64ptr(int64(mapping.NodePort)), + } + monitor, err := a.broker.CreateLoadBalancerTCPMonitorProfile(profile) + if err != nil { + return nil, errors.Wrapf(err, "creating tcp monitor failed for %s:%s:%d", clusterName, objectName, mapping.NodePort) + } + return &monitor, nil +} + +func (a *access) GetTCPMonitorProfile(id string) (*model.LBTcpMonitorProfile, error) { + monitor, err := a.broker.ReadLoadBalancerTCPMonitorProfile(id) + if err != nil { + return nil, errors.Wrapf(err, "reading tcp monitor %s failed", id) + } + return &monitor, nil +} + +func (a *access) FindTCPMonitorProfiles(clusterName string, objectName types.NamespacedName) ([]*model.LBTcpMonitorProfile, error) { + return a.listTCPMonitorProfiles(a.ownerTag, clusterTag(clusterName), serviceTag(objectName)) +} + +func (a *access) ListTCPMonitorProfiles(clusterName string) ([]*model.LBTcpMonitorProfile, error) { + return a.listTCPMonitorProfiles(a.ownerTag, clusterTag(clusterName)) +} + +func (a *access) listTCPMonitorProfiles(tags ...model.Tag) ([]*model.LBTcpMonitorProfile, error) { + list, err := a.broker.ListLoadBalancerMonitorProfiles() + if err != nil { + return nil, errors.Wrapf(err, "listing load balancer monitors failed") + } + result := []*model.LBTcpMonitorProfile{} + converter := newNsxtTypeConverter() + for _, item := range list { + resourceType, err := item.String("resource_type") + if err != nil || resourceType != model.LBMonitorProfile_RESOURCE_TYPE_LBTCPMONITORPROFILE { + continue + } + profile, err := converter.convertStructValueToLBTCPMonitorProfile(item) + if err != nil { + return nil, err + } + if checkTags(profile.Tags, tags...) { + result = append(result, &profile) + } + } + return result, nil +} + +func (a *access) UpdateTCPMonitorProfile(monitor *model.LBTcpMonitorProfile) error { + _, err := a.broker.UpdateLoadBalancerTCPMonitorProfile(*monitor) + if err != nil { + return errors.Wrapf(err, "updating load balancer TCP monitor %s (%s) failed", *monitor.DisplayName, *monitor.Id) + } + return nil +} + +func (a *access) DeleteTCPMonitorProfile(id string) error { + err := a.broker.DeleteLoadBalancerMonitorProfile(id) + if isNotFoundError(err) { + return nil + } + if err != nil { + return errors.Wrapf(err, "deleting monitor %s failed", id) + } + return nil +} + +func (a *access) AllocateExternalIPAddress(ipPoolID string, clusterName string, objectName types.NamespacedName) (*model.IpAddressAllocation, *string, error) { + allocation := model.IpAddressAllocation{ + Tags: a.standardTags.Append(clusterTag(clusterName), serviceTag(objectName)).Normalize(), + } + allocated, ipAdress, err := a.broker.AllocateFromIPPool(ipPoolID, allocation) + if err != nil { + return nil, nil, errors.Wrapf(err, "allocating external IP address failed") + } + return &allocated, &ipAdress, nil +} + +func (a *access) FindExternalIPAddressForObject(ipPoolID string, clusterName string, objectName types.NamespacedName) (*model.IpAddressAllocation, *string, error) { + results, err := a.findExternalIPAddresses(ipPoolID, a.ownerTag, clusterTag(clusterName), serviceTag(objectName)) + if err != nil { + return nil, nil, err + } + if len(results) == 0 { + return nil, nil, nil + } + if len(results) > 1 { + return nil, nil, fmt.Errorf("Multiple IP address allocations") + } + + item := results[0] + ipAddress := item.AllocationIp + if ipAddress == nil { + ipAddress, err = a.broker.GetRealizedExternalIPAddress(*item.Path, 5*time.Second) + if err != nil { + return nil, nil, errors.Wrapf(err, "GetReleaziedExternalIPAddress failed for allocation %s IP pool %s failed", *item.Path, ipPoolID) + } + } + + return item, ipAddress, nil +} + +func (a *access) ListExternalIPAddresses(ipPoolID string, clusterName string) ([]*model.IpAddressAllocation, error) { + return a.findExternalIPAddresses(ipPoolID, a.ownerTag, clusterTag(clusterName)) +} + +func (a *access) findExternalIPAddresses(ipPoolID string, tags ...model.Tag) ([]*model.IpAddressAllocation, error) { + list, err := a.broker.ListIPPoolAllocations(ipPoolID) + if err != nil { + return nil, errors.Wrapf(err, "listing IP address allocations from IP pool %s failed", ipPoolID) + } + results := []*model.IpAddressAllocation{} + for _, item := range list { + if checkTags(item.Tags, tags...) { + itemCopy := item + results = append(results, &itemCopy) + } + } + return results, nil +} + +func (a *access) ReleaseExternalIPAddress(ipPoolID string, id string) error { + err := a.broker.ReleaseFromIPPool(ipPoolID, id) + if isNotFoundError(err) { + return nil + } + if err != nil { + return errors.Wrapf(err, "releasing external IP address allocation id=%s failed", id) + } + return nil +} + +func displayName(clusterName string) *string { + return strptr(fmt.Sprintf("cluster:%s", clusterName)) +} + +func displayNameObject(clusterName string, objectName types.NamespacedName) *string { + return strptr(fmt.Sprintf("cluster:%s:%s", clusterName, objectName)) +} + +func displayNameMapping(clusterName string, objectName types.NamespacedName, mapping Mapping) *string { + return strptr(fmt.Sprintf("cluster:%s:%s:%d", clusterName, objectName, mapping.NodePort)) +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/class.go b/pkg/cloudprovider/vsphere/loadbalancer/class.go new file mode 100644 index 0000000000..bbd9876a73 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/class.go @@ -0,0 +1,177 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" +) + +type loadBalancerClasses struct { + size string + classes map[string]*loadBalancerClass +} + +type loadBalancerClass struct { + className string + ipPool Reference + tcpAppProfile Reference + udpAppProfile Reference + + tags []model.Tag +} + +func setupClasses(access NSXTAccess, cfg *config.LBConfig) (*loadBalancerClasses, error) { + if !config.LoadBalancerSizes.Has(cfg.LoadBalancer.Size) { + return nil, fmt.Errorf("invalid load balancer size %s", cfg.LoadBalancer.Size) + } + + lbClasses := &loadBalancerClasses{ + size: cfg.LoadBalancer.Size, + classes: map[string]*loadBalancerClass{}, + } + + resolver := &ipPoolResolver{access: access, knownIPPools: map[string]string{}} + defaultClass, err := newLBClass(config.DefaultLoadBalancerClass, &cfg.LoadBalancer.LoadBalancerClassConfig, nil, resolver) + if err != nil { + return nil, errors.Wrapf(err, "invalid LoadBalancerClass %s", config.DefaultLoadBalancerClass) + } + if defCfg, ok := cfg.LoadBalancerClasses[defaultClass.className]; ok { + defaultClass, err = newLBClass(config.DefaultLoadBalancerClass, defCfg, defaultClass, resolver) + if err != nil { + return nil, errors.Wrapf(err, "invalid LoadBalancerClass %s", config.DefaultLoadBalancerClass) + } + } else { + lbClasses.add(defaultClass) + } + + for name, classConfig := range cfg.LoadBalancerClasses { + if _, ok := lbClasses.classes[name]; ok { + return nil, fmt.Errorf("duplicate LoadBalancerClass %s", name) + } + class, err := newLBClass(name, classConfig, defaultClass, resolver) + if err != nil { + return nil, errors.Wrapf(err, "invalid LoadBalancerClass %s", name) + } + lbClasses.add(class) + } + + return lbClasses, nil +} + +func (c *loadBalancerClasses) GetClassNames() []string { + names := make([]string, 0, len(c.classes)) + for name := range c.classes { + names = append(names, name) + } + return names +} + +func (c *loadBalancerClasses) GetClass(name string) *loadBalancerClass { + return c.classes[name] +} + +func (c *loadBalancerClasses) add(class *loadBalancerClass) { + c.classes[class.className] = class +} + +type ipPoolResolver struct { + access NSXTAccess + knownIPPools map[string]string +} + +func (r *ipPoolResolver) resolve(ipPool *Reference) error { + var err error + ipPoolID := ipPool.Identifier + if ipPoolID == "" { + var ok bool + ipPoolID, ok = r.knownIPPools[ipPool.Name] + if !ok { + ipPoolID, err = r.access.FindIPPoolByName(ipPool.Name) + if err != nil { + return err + } + r.knownIPPools[ipPool.Name] = ipPoolID + } + ipPool.Identifier = ipPoolID + } + return nil +} + +func newLBClass(name string, classConfig *config.LoadBalancerClassConfig, defaults *loadBalancerClass, resolver *ipPoolResolver) (*loadBalancerClass, error) { + class := loadBalancerClass{ + className: name, + ipPool: Reference{ + Identifier: classConfig.IPPoolID, + Name: classConfig.IPPoolName, + }, + tcpAppProfile: Reference{ + Identifier: classConfig.TCPAppProfilePath, + Name: classConfig.TCPAppProfileName, + }, + udpAppProfile: Reference{ + Identifier: classConfig.UDPAppProfilePath, + Name: classConfig.UDPAppProfileName, + }, + } + if defaults != nil { + if class.ipPool.IsEmpty() { + class.ipPool = defaults.ipPool + } + if class.tcpAppProfile.IsEmpty() { + class.tcpAppProfile = defaults.tcpAppProfile + } + if class.udpAppProfile.IsEmpty() { + class.udpAppProfile = defaults.udpAppProfile + } + } + if resolver != nil { + err := resolver.resolve(&class.ipPool) + if err != nil { + return nil, err + } + } else if class.ipPool.Identifier == "" { + return nil, fmt.Errorf("ipPoolResolver needed if IP pool ID not provided") + } + class.tags = []model.Tag{ + newTag(ScopeIPPoolID, class.ipPool.Identifier), + newTag(ScopeLBClass, class.className), + } + + return &class, nil +} + +func (c *loadBalancerClass) Tags() []model.Tag { + return c.tags +} + +func (c *loadBalancerClass) AppProfile(protocol corev1.Protocol) (Reference, error) { + switch protocol { + case corev1.ProtocolTCP: + return c.tcpAppProfile, nil + case corev1.ProtocolUDP: + return c.udpAppProfile, nil + default: + return Reference{}, fmt.Errorf("unexpected protocol: %s", protocol) + } +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/cleanup.go b/pkg/cloudprovider/vsphere/loadbalancer/cleanup.go new file mode 100644 index 0000000000..c176ff7f5c --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/cleanup.go @@ -0,0 +1,168 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog" +) + +const maxPeriod = 30 * time.Minute + +// cleanup is used to cleanup obsolete and potentially forgotten objects +// created by the loadbalancer controller in NSX-T. This should not +// happen, but if users play with finalizers or some error condition +// appears in the controller there might be orphaned objects in the +// infrastructure. This is important for higher level automations like +// cluster fleet managements that manage the infrastructure of a cluster, +// because various elements cannot be deleted if they are still in use, +// after the cluster has been deleted. +// The controller tags all elements it creates with the cluster name and +// its identity (the app name of the controller, or a dedicated name chosen +// by the config file in the tags section). This tagging can then be used +// to identify all elements originally created by this controller. By +// comparing this set with the actually required objects it is possible +// to identify those that are orphaned and safely delete them. +func (p *lbProvider) cleanup(clusterName string, client clientcorev1.ServiceInterface, stop <-chan struct{}) { + timer := time.NewTimer(1 * time.Second) + lastErrNext := 0 * time.Second + for { + select { + case <-stop: + return + case <-timer.C: + var next time.Duration + err := p.doCleanupStep(clusterName, client) + if err == nil { + next = maxPeriod + lastErrNext = 0 + } else { + klog.Warningf("cleanup failed with %s", err) + if lastErrNext == 0 { + lastErrNext = 500 * time.Millisecond + } else { + lastErrNext = 5 * lastErrNext / 4 + if lastErrNext > maxPeriod { + lastErrNext = maxPeriod + } + } + next = lastErrNext + } + timer.Reset(next) + } + } +} + +func (p *lbProvider) doCleanupStep(clusterName string, client clientcorev1.ServiceInterface) error { + klog.Infof("starting cleanup...") + list, err := client.List(metav1.ListOptions{}) + if err != nil { + return err + } + + services := map[types.NamespacedName]corev1.Service{} + for _, item := range list.Items { + if item.Spec.Type == corev1.ServiceTypeLoadBalancer { + services[namespacedNameFromService(&item)] = item + } + } + + return p.CleanupServices(clusterName, services) +} + +func (p *lbProvider) CleanupServices(clusterName string, validServices map[types.NamespacedName]corev1.Service) error { + ipPoolIds := sets.NewString() + for _, name := range p.classes.GetClassNames() { + class := p.classes.GetClass(name) + ipPoolIds.Insert(class.ipPool.Identifier) + } + + lbs := map[types.NamespacedName]struct{}{} + servers, err := p.access.ListVirtualServers(ClusterName) + if err != nil { + return err + } + for _, server := range servers { + tag := getTag(server.Tags, ScopeService) + if tag != "" { + lbs[parseNamespacedName(tag)] = struct{}{} + } + ipPoolID := getTag(server.Tags, ScopeIPPoolID) + ipPoolIds.Insert(ipPoolID) + } + ipPoolIds.Delete("") + + pools, err := p.access.ListPools(clusterName) + if err != nil { + return err + } + for _, pool := range pools { + tag := getTag(pool.Tags, ScopeService) + if tag != "" { + lbs[parseNamespacedName(tag)] = struct{}{} + } + } + + monitors, err := p.access.ListTCPMonitorProfiles(clusterName) + if err != nil { + return err + } + for _, pool := range monitors { + tag := getTag(pool.Tags, ScopeService) + if tag != "" { + lbs[parseNamespacedName(tag)] = struct{}{} + } + } + + for ipPoolID := range ipPoolIds { + ipAddressAllocs, err := p.access.ListExternalIPAddresses(ipPoolID, clusterName) + if err != nil { + return err + } + for _, ipAddressAlloc := range ipAddressAllocs { + tag := getTag(ipAddressAlloc.Tags, ScopeService) + if tag != "" { + lbs[parseNamespacedName(tag)] = struct{}{} + } + } + } + + klog.Infof("cleanup: %d existing services, artefacts for %d services", len(validServices), len(lbs)) + for lb := range lbs { + if svc, ok := validServices[lb]; !ok || svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: lb.Namespace, + Name: lb.Name, + }, + } + klog.Infof("deleting artefacts for non-existing service %s/%s", lb.Namespace, lb.Name) + err = p.EnsureLoadBalancerDeleted(context.TODO(), clusterName, service) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config.go new file mode 100644 index 0000000000..68b26b348b --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config.go @@ -0,0 +1,261 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + + "gopkg.in/gcfg.v1" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +const ( + // DefaultLoadBalancerClass is the default load balancer class + DefaultLoadBalancerClass = "default" +) + +// LoadBalancerSizes contains the valid size names +var LoadBalancerSizes = sets.NewString( + model.LBService_SIZE_SMALL, + model.LBService_SIZE_MEDIUM, + model.LBService_SIZE_LARGE, + model.LBService_SIZE_XLARGE, + model.LBService_SIZE_DLB, +) + +// LBConfig is used to read and store information from the cloud configuration file +type LBConfig struct { + LoadBalancer LoadBalancerConfig `gcfg:"LoadBalancer"` + LoadBalancerClasses map[string]*LoadBalancerClassConfig `gcfg:"LoadBalancerClass"` + NSXT NsxtConfig `gcfg:"NSX-T"` +} + +// LoadBalancerConfig contains the configuration for the load balancer itself +type LoadBalancerConfig struct { + LoadBalancerClassConfig + Size string `gcfg:"size"` + LBServiceID string `gcfg:"lbServiceId"` + Tier1GatewayPath string `gcfg:"tier1GatewayPath"` + RawTags string `gcfg:"tags"` + AdditionalTags map[string]string +} + +// LoadBalancerClassConfig contains the configuration for a load balancer class +type LoadBalancerClassConfig struct { + IPPoolName string `gcfg:"ipPoolName"` + IPPoolID string `gcfg:"ipPoolID"` + TCPAppProfileName string `gcfg:"tcpAppProfileName"` + TCPAppProfilePath string `gcfg:"tcpAppProfilePath"` + UDPAppProfileName string `gcfg:"udpAppProfileName"` + UDPAppProfilePath string `gcfg:"udpAppProfilePath"` +} + +// NsxtConfig contains the NSX-T specific configuration +type NsxtConfig struct { + // NSX-T username. + User string `gcfg:"user"` + // NSX-T password in clear text. + Password string `gcfg:"password"` + // NSX-T host. + Host string `gcfg:"host"` + // InsecureFlag is to be set to true if NSX-T uses self-signed cert. + InsecureFlag bool `gcfg:"insecure-flag"` + + VMCAccessToken string `gcfg:"vmcAccessToken"` + VMCAuthHost string `gcfg:"vmcAuthHost"` + ClientAuthCertFile string `gcfg:"client-auth-cert-file"` + ClientAuthKeyFile string `gcfg:"client-auth-key-file"` + CAFile string `gcfg:"ca-file"` +} + +// IsEnabled checks whether the load balancer feature is enabled +// It is enabled if any flavor of the load balancer configuration is given. +func (cfg *LBConfig) IsEnabled() bool { + return len(cfg.LoadBalancerClasses) > 0 || !cfg.LoadBalancer.IsEmpty() +} + +func (cfg *LBConfig) validateConfig() error { + if cfg.LoadBalancer.LBServiceID == "" && cfg.LoadBalancer.Tier1GatewayPath == "" { + msg := "either load balancer service id or T1 gateway path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if cfg.LoadBalancer.TCPAppProfileName == "" && cfg.LoadBalancer.TCPAppProfilePath == "" { + msg := "either load balancer TCP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if cfg.LoadBalancer.UDPAppProfileName == "" && cfg.LoadBalancer.UDPAppProfilePath == "" { + msg := "either load balancer UDP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if !LoadBalancerSizes.Has(cfg.LoadBalancer.Size) { + msg := fmt.Sprintf("load balancer size is invalid. Valid values are: %s", strings.Join(LoadBalancerSizes.List(), ",")) + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if cfg.LoadBalancer.IPPoolID == "" && cfg.LoadBalancer.IPPoolName == "" { + class, ok := cfg.LoadBalancerClasses[DefaultLoadBalancerClass] + if !ok { + msg := "no default load balancer class defined" + klog.Errorf(msg) + return fmt.Errorf(msg) + } else if class.IPPoolName == "" && class.IPPoolID == "" { + msg := "default load balancer class: ipPoolName and ipPoolID is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + if cfg.LoadBalancer.IPPoolName != "" && cfg.LoadBalancer.IPPoolID != "" { + msg := "either load balancer ipPoolName or ipPoolID can be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } + return cfg.NSXT.validateConfig() +} + +// IsEmpty checks whether the load balancer config is empty (no values specified) +func (cfg *LoadBalancerConfig) IsEmpty() bool { + return cfg.Size == "" && cfg.LBServiceID == "" && + cfg.IPPoolID == "" && cfg.IPPoolName == "" && + cfg.Tier1GatewayPath == "" +} + +func (cfg *NsxtConfig) validateConfig() error { + if cfg.VMCAccessToken != "" { + if cfg.VMCAuthHost == "" { + msg := "vmc auth host must be provided if auth token is provided" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else if cfg.User != "" { + if cfg.Password == "" { + msg := "password is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + msg := "either user or vmc access token must be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if cfg.Host == "" { + msg := "host is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + return nil +} + +// FromEnv initializes the provided configuration object with values +// obtained from environment variables. If an environment variable is set +// for a property that's already initialized, the environment variable's value +// takes precedence. +func (cfg *NsxtConfig) FromEnv() error { + if v := os.Getenv("NSXT_MANAGER_HOST"); v != "" { + cfg.Host = v + } + if v := os.Getenv("NSXT_USERNAME"); v != "" { + cfg.User = v + } + if v := os.Getenv("NSXT_PASSWORD"); v != "" { + cfg.Password = v + } + if v := os.Getenv("NSXT_ALLOW_UNVERIFIED_SSL"); v != "" { + InsecureFlag, err := strconv.ParseBool(v) + if err != nil { + klog.Errorf("Failed to parse NSXT_ALLOW_UNVERIFIED_SSL: %s", err) + return fmt.Errorf("Failed to parse NSXT_ALLOW_UNVERIFIED_SSL: %s", err) + } + cfg.InsecureFlag = InsecureFlag + } + if v := os.Getenv("NSXT_CLIENT_AUTH_CERT_FILE"); v != "" { + cfg.ClientAuthCertFile = v + } + if v := os.Getenv("NSXT_CLIENT_AUTH_KEY_FILE"); v != "" { + cfg.ClientAuthKeyFile = v + } + if v := os.Getenv("NSXT_CA_FILE"); v != "" { + cfg.CAFile = v + } + + return nil +} + +// ReadConfig parses vSphere cloud config file and stores it into LBConfig. +// Environment variables are also checked +func ReadConfig(config io.Reader) (*LBConfig, error) { + if config == nil { + return nil, fmt.Errorf("no vSphere cloud provider config file given") + } + + cfg := &LBConfig{} + + if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, config)); err != nil { + return nil, err + } + + err := cfg.CompleteAndValidate() + if err != nil { + return nil, err + } + return cfg, nil +} + +// CompleteAndValidate sets default values, overrides by env and validates the resulting config +func (cfg *LBConfig) CompleteAndValidate() error { + if !cfg.IsEnabled() { + return nil + } + + cfg.LoadBalancer.AdditionalTags = map[string]string{} + if cfg.LoadBalancer.RawTags != "" { + err := json.Unmarshal([]byte(cfg.LoadBalancer.RawTags), &cfg.LoadBalancer.AdditionalTags) + if err != nil { + return fmt.Errorf("unmarshalling load balancer tags failed: %s", err) + } + } + if cfg.LoadBalancerClasses == nil { + cfg.LoadBalancerClasses = map[string]*LoadBalancerClassConfig{} + } + for _, class := range cfg.LoadBalancerClasses { + if class.IPPoolName == "" { + if class.IPPoolID == "" { + class.IPPoolID = cfg.LoadBalancer.IPPoolID + class.IPPoolName = cfg.LoadBalancer.IPPoolName + } + } + } + + // Env Vars should override config file entries if present + if err := cfg.NSXT.FromEnv(); err != nil { + return err + } + + return cfg.validateConfig() +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go new file mode 100644 index 0000000000..0fd9dbd2da --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go @@ -0,0 +1,115 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "strings" + "testing" +) + +func TestReadConfig(t *testing.T) { + contents := ` +[LoadBalancer] +ipPoolName = pool1 +size = MEDIUM +lbServiceId = 4711 +tier1GatewayPath = 1234 +tcpAppProfileName = default-tcp-lb-app-profile +udpAppProfileName = default-udp-lb-app-profile +tags = {\"tag1\": \"value1\", \"tag2\": \"value 2\"} + +[LoadBalancerClass "public"] +ipPoolName = poolPublic + +[LoadBalancerClass "private"] +ipPoolName = poolPrivate +tcpAppProfileName = tcp2 +udpAppProfileName = udp2 + +[NSX-T] +user = admin +password = secret +host = nsxt-server +` + config, err := ReadConfig(strings.NewReader(contents)) + if err != nil { + t.Error(err) + return + } + + assertEquals := func(name, left, right string) { + if left != right { + t.Errorf("%s %s != %s", name, left, right) + } + } + assertEquals("LoadBalancer.ipPoolName", config.LoadBalancer.IPPoolName, "pool1") + assertEquals("LoadBalancer.lbServiceId", config.LoadBalancer.LBServiceID, "4711") + assertEquals("LoadBalancer.tier1GatewayPath", config.LoadBalancer.Tier1GatewayPath, "1234") + assertEquals("LoadBalancer.tcpAppProfileName", config.LoadBalancer.TCPAppProfileName, "default-tcp-lb-app-profile") + assertEquals("LoadBalancer.udpAppProfileName", config.LoadBalancer.UDPAppProfileName, "default-udp-lb-app-profile") + assertEquals("LoadBalancer.size", config.LoadBalancer.Size, "MEDIUM") + if len(config.LoadBalancerClasses) != 2 { + t.Errorf("expected two LoadBalancerClass subsections, but got %d", len(config.LoadBalancerClasses)) + } + assertEquals("LoadBalancerClass.public.ipPoolName", config.LoadBalancerClasses["public"].IPPoolName, "poolPublic") + assertEquals("LoadBalancerClass.private.tcpAppProfileName", config.LoadBalancerClasses["private"].TCPAppProfileName, "tcp2") + assertEquals("LoadBalancerClass.private.udpAppProfileName", config.LoadBalancerClasses["private"].UDPAppProfileName, "udp2") + if len(config.LoadBalancer.AdditionalTags) != 2 || config.LoadBalancer.AdditionalTags["tag1"] != "value1" || config.LoadBalancer.AdditionalTags["tag2"] != "value 2" { + t.Errorf("unexpected additionalTags %v", config.LoadBalancer.AdditionalTags) + } + assertEquals("NSX-T.user", config.NSXT.User, "admin") + assertEquals("NSX-T.password", config.NSXT.Password, "secret") + assertEquals("NSX-T.host", config.NSXT.Host, "nsxt-server") +} + +func TestReadConfigOnVMC(t *testing.T) { + contents := ` +[LoadBalancer] +ipPoolID = 123-456 +size = MEDIUM +tier1GatewayPath = 1234 +tcpAppProfilePath = infra/xxx/tcp1234 +udpAppProfilePath = infra/xxx/udp1234 + +[NSX-T] +vmcAccessToken = token123 +vmcAuthHost = authHost +host = nsxt-server +insecure-flag = true +` + config, err := ReadConfig(strings.NewReader(contents)) + if err != nil { + t.Error(err) + return + } + assertEquals := func(name, left, right string) { + if left != right { + t.Errorf("%s %s != %s", name, left, right) + } + } + assertEquals("LoadBalancer.ipPoolID", config.LoadBalancer.IPPoolID, "123-456") + assertEquals("LoadBalancer.size", config.LoadBalancer.Size, "MEDIUM") + assertEquals("LoadBalancer.tier1GatewayPath", config.LoadBalancer.Tier1GatewayPath, "1234") + assertEquals("LoadBalancer.tcpAppProfilePath", config.LoadBalancer.TCPAppProfilePath, "infra/xxx/tcp1234") + assertEquals("LoadBalancer.udpAppProfilePath", config.LoadBalancer.UDPAppProfilePath, "infra/xxx/udp1234") + assertEquals("NSX-T.vmcAccessToken", config.NSXT.VMCAccessToken, "token123") + assertEquals("NSX-T.vmcAuthHost", config.NSXT.VMCAuthHost, "authHost") + assertEquals("NSX-T.host", config.NSXT.Host, "nsxt-server") + if !config.NSXT.InsecureFlag { + t.Errorf("NSX-T.insecure-flag != true") + } +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/helpers.go b/pkg/cloudprovider/vsphere/loadbalancer/helpers.go new file mode 100644 index 0000000000..eca9dbf61c --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/helpers.go @@ -0,0 +1,72 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + vapi_errors "github.com/vmware/vsphere-automation-sdk-go/lib/vapi/std/errors" +) + +func namespacedNameFromService(service *corev1.Service) types.NamespacedName { + return types.NamespacedName{Namespace: service.Namespace, Name: service.Name} +} + +func parseNamespacedName(name string) types.NamespacedName { + parts := strings.Split(name, "/") + return types.NamespacedName{Namespace: parts[0], Name: parts[1]} +} + +func collectNodeInternalAddresses(nodes []*corev1.Node) map[string]string { + set := map[string]string{} + for _, node := range nodes { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + set[addr.Address] = node.Name + break + } + } + } + return set +} + +func strptr(s string) *string { + return &s +} + +func isNotFoundError(err error) bool { + _, ok := err.(vapi_errors.NotFound) + return ok +} + +func boolptr(b bool) *bool { + return &b +} + +func int64ptr(i int64) *int64 { + return &i +} + +func safeEquals(a, b *string) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/interface.go b/pkg/cloudprovider/vsphere/loadbalancer/interface.go new file mode 100644 index 0000000000..73c237250b --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/interface.go @@ -0,0 +1,120 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + cloudprovider "k8s.io/cloud-provider" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// LBProvider is the interface used call the load balancer functionality +// It extends the cloud controller manager LoadBalancer interface by an +// initialization function +type LBProvider interface { + cloudprovider.LoadBalancer + Initialize(clusterName string, client clientset.Interface, stop <-chan struct{}) + CleanupServices(clusterName string, services map[types.NamespacedName]corev1.Service) error +} + +// NSXTAccess provides methods for dealing with NSX-T objects +type NSXTAccess interface { + // CreateLoadBalancerService creates a LbService + CreateLoadBalancerService(clusterName string) (*model.LBService, error) + // FindLoadBalancerService finds a LbService by cluster name and LB service id + FindLoadBalancerService(clusterName string, lbServiceID string) (lbService *model.LBService, err error) + // UpdateLoadBalancerService updates a LbService + UpdateLoadBalancerService(lbService *model.LBService) error + // DeleteLoadBalancerService deletes a LbService by id + DeleteLoadBalancerService(id string) error + + // CreateVirtualServer creates a virtual server + CreateVirtualServer(clusterName string, objectName types.NamespacedName, class LBClass, ipAddress string, mapping Mapping, + lbServicePath, applicationProfilePath string, poolPath *string) (*model.LBVirtualServer, error) + // FindVirtualServers finds a virtual server by cluster and object name + FindVirtualServers(clusterName string, objectName types.NamespacedName) ([]*model.LBVirtualServer, error) + // ListVirtualServers finds all virtual servers for a cluster + ListVirtualServers(clusterName string) ([]*model.LBVirtualServer, error) + // UpdateVirtualServer updates a virtual server + UpdateVirtualServer(server *model.LBVirtualServer) error + // DeleteVirtualServer deletes a virtual server by id + DeleteVirtualServer(id string) error + + // CreatePool creates a LbPool + CreatePool(clusterName string, objectName types.NamespacedName, mapping Mapping, members []model.LBPoolMember, + activeMonitorPaths []string) (*model.LBPool, error) + // GetPool gets a LbPool by id + GetPool(id string) (*model.LBPool, error) + // FindPool finds a LbPool for a mapping + FindPool(clusterName string, objectName types.NamespacedName, mapping Mapping) (*model.LBPool, error) + // FindPools finds a LbPool by cluster and object name + FindPools(clusterName string, objectName types.NamespacedName) ([]*model.LBPool, error) + // ListPools lists all LbPool for a cluster + ListPools(clusterName string) ([]*model.LBPool, error) + // UpdatePool updates a LbPool + UpdatePool(*model.LBPool) error + // DeletePool deletes a LbPool by id + DeletePool(id string) error + + // FindIPPoolByName finds an IP pool by name + FindIPPoolByName(poolName string) (string, error) + + // GetAppProfilePath gets the application profile for given loadbalancer class and protocol + GetAppProfilePath(class LBClass, protocol corev1.Protocol) (string, error) + + // AllocateExternalIPAddress allocates an IP address from the given IP pool + AllocateExternalIPAddress(ipPoolID string, clusterName string, objectName types.NamespacedName) (allocation *model.IpAddressAllocation, ipAddress *string, err error) + // ListExternalIPAddresses finds all IP addresses belonging to a clusterName from the given IP pool + ListExternalIPAddresses(ipPoolID string, clusterName string) ([]*model.IpAddressAllocation, error) + // FindExternalIPAddressForObject finds an IP address belonging to an object + FindExternalIPAddressForObject(ipPoolID string, clusterName string, objectName types.NamespacedName) (allocation *model.IpAddressAllocation, ipAddress *string, err error) + // ReleaseExternalIPAddress releases an allocated IP address + ReleaseExternalIPAddress(ipPoolID string, id string) error + + // CreateTCPMonitorProfile creates a LBTcpMonitorProfile + CreateTCPMonitorProfile(clusterName string, objectName types.NamespacedName, mapping Mapping) (*model.LBTcpMonitorProfile, error) + // FindTCPMonitors finds a LBTcpMonitorProfile by cluster and object name + FindTCPMonitorProfiles(clusterName string, objectName types.NamespacedName) ([]*model.LBTcpMonitorProfile, error) + // ListTCPMonitorProfile lists LBTcpMonitorProfile by cluster + ListTCPMonitorProfiles(clusterName string) ([]*model.LBTcpMonitorProfile, error) + // UpdateTCPMonitorProfile updates a LBTcpMonitorProfile + UpdateTCPMonitorProfile(monitor *model.LBTcpMonitorProfile) error + // DeleteTCPMonitorProfile deletes a LBTcpMonitorProfile by id + DeleteTCPMonitorProfile(id string) error +} + +// Reference references an object either by identifier or name +type Reference struct { + Identifier string + Name string +} + +// IsEmpty returns true if neither identifier and name is set. +func (r *Reference) IsEmpty() bool { + return r.Identifier == "" && r.Name == "" +} + +// LBClass is an interface to retrieve settings of load balancer class. +type LBClass interface { + // Tags retrieves tags of an object + Tags() []model.Tag + // AppProfile retrieves application profile either by path (stored in Reference.Identifier) or by name + AppProfile(protocol corev1.Protocol) (Reference, error) +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go b/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go new file mode 100644 index 0000000000..184e4f4aab --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go @@ -0,0 +1,183 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" +) + +const ( + // LoadBalancerClassAnnotation is the optional class annotation at the service + LoadBalancerClassAnnotation = "loadbalancer.vmware.io/class" +) + +var ( + // AppName is set by the main program to the name of the application + AppName string + // Version is set by the main program to the version of the application + Version string +) + +type lbProvider struct { + *lbService + classes *loadBalancerClasses + keyLock *keyLock +} + +// ClusterName contains the cluster-name flag injected from main, needed for cleanup +var ClusterName string + +var _ LBProvider = &lbProvider{} + +// NewLBProvider creates a new LBProvider +func NewLBProvider(cfg *config.LBConfig) (LBProvider, error) { + if !cfg.IsEnabled() { + return nil, nil + } + + broker, err := NewNsxtBroker(&cfg.NSXT) + if err != nil { + return nil, err + } + access, err := NewNSXTAccess(broker, cfg) + if err != nil { + return nil, errors.Wrap(err, "creating access handler failed") + } + classes, err := setupClasses(access, cfg) + if err != nil { + return nil, errors.Wrap(err, "creating load balancer classes failed") + } + return &lbProvider{ + lbService: newLbService(access, cfg.LoadBalancer.LBServiceID), + classes: classes, + keyLock: newKeyLock(), + }, nil +} + +func (p *lbProvider) Initialize(clusterName string, client clientset.Interface, stop <-chan struct{}) { + if clusterName != "" { + go p.cleanup(clusterName, client.CoreV1().Services(""), stop) + } +} + +// GetLoadBalancer returns the LoadBalancerStatus +// Implementations must treat the *corev1.Service parameter as read-only and not modify it. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +func (p *lbProvider) GetLoadBalancer(_ context.Context, clusterName string, service *corev1.Service) (status *corev1.LoadBalancerStatus, exists bool, err error) { + servers, err := p.access.FindVirtualServers(clusterName, namespacedNameFromService(service)) + if err != nil { + return nil, false, err + } + if len(servers) == 0 { + return nil, false, nil + } + return newLoadBalancerStatus(&servers[0].IpAddress), true, nil +} + +func newLoadBalancerStatus(ipAddress *string) *corev1.LoadBalancerStatus { + status := &corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{}, + } + if ipAddress != nil { + status.Ingress = append(status.Ingress, corev1.LoadBalancerIngress{IP: *ipAddress}) + } + return status +} + +// GetLoadBalancerName returns the name of the load balancer. Implementations must treat the +// *corev1.Service parameter as read-only and not modify it. +func (p *lbProvider) GetLoadBalancerName(_ context.Context, clusterName string, service *corev1.Service) string { + return *displayNameObject(clusterName, namespacedNameFromService(service)) +} + +// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer +// Implementations must treat the *corev1.Service and *corev1.Node +// parameters as read-only and not modify them. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +func (p *lbProvider) EnsureLoadBalancer(_ context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) (*corev1.LoadBalancerStatus, error) { + key := namespacedNameFromService(service).String() + p.keyLock.Lock(key) + defer p.keyLock.Unlock(key) + + class, err := p.classFromService(service) + if err != nil { + return nil, err + } + + state := newState(p.lbService, clusterName, service, nodes) + err = state.Process(class) + status, err2 := state.Finish() + if err != nil { + return status, err + } + return status, err2 +} + +func (p *lbProvider) classFromService(service *corev1.Service) (*loadBalancerClass, error) { + annos := service.GetAnnotations() + if annos == nil { + annos = map[string]string{} + } + name, ok := annos[LoadBalancerClassAnnotation] + name = strings.TrimSpace(name) + if !ok || name == "" { + name = config.DefaultLoadBalancerClass + } + + class := p.classes.GetClass(name) + if class == nil { + return nil, fmt.Errorf("invalid load balancer class %s", name) + } + return class, nil +} + +// UpdateLoadBalancer updates hosts under the specified load balancer. +// Implementations must treat the *corev1.Service and *corev1.Node +// parameters as read-only and not modify them. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +func (p *lbProvider) UpdateLoadBalancer(_ context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { + key := namespacedNameFromService(service).String() + p.keyLock.Lock(key) + defer p.keyLock.Unlock(key) + + state := newState(p.lbService, clusterName, service, nodes) + + return state.UpdatePoolMembers() +} + +// EnsureLoadBalancerDeleted deletes the specified load balancer if it +// exists, returning nil if the load balancer specified either didn't exist or +// was successfully deleted. +// This construction is useful because many cloud providers' load balancers +// have multiple underlying components, meaning a Get could say that the LB +// doesn't exist even if some part of it is still laying around. +// Implementations must treat the *corev1.Service parameter as read-only and not modify it. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +func (p *lbProvider) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error { + emptyService := service.DeepCopy() + emptyService.Spec.Ports = nil + _, err := p.EnsureLoadBalancer(ctx, clusterName, emptyService, nil) + return err +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/lbservice.go b/pkg/cloudprovider/vsphere/loadbalancer/lbservice.go new file mode 100644 index 0000000000..a6dedebc8a --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/lbservice.go @@ -0,0 +1,83 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "sync" +) + +type lbService struct { + access NSXTAccess + lbServiceID string + managed bool + lbLock sync.Mutex +} + +func newLbService(access NSXTAccess, lbServiceID string) *lbService { + return &lbService{access: access, lbServiceID: lbServiceID, managed: lbServiceID == ""} +} + +func (s *lbService) getOrCreateLoadBalancerService(clusterName string) (string, error) { + s.lbLock.Lock() + defer s.lbLock.Unlock() + + lbService, err := s.access.FindLoadBalancerService(clusterName, s.lbServiceID) + if err != nil { + return "", err + } + if lbService != nil { + return *lbService.Path, nil + } + if s.managed { + lbService, err = s.access.CreateLoadBalancerService(clusterName) + if err != nil { + return "", err + } + s.lbServiceID = *lbService.Id + return *lbService.Path, nil + } + return "", fmt.Errorf("no load balancer service found with id %s", s.lbServiceID) +} + +func (s *lbService) removeLoadBalancerServiceIfUnused(clusterName string) error { + s.lbLock.Lock() + defer s.lbLock.Unlock() + + if !s.managed { + return nil + } + + lbService, err := s.access.FindLoadBalancerService(clusterName, s.lbServiceID) + if err != nil { + return err + } + if lbService == nil { + return nil + } + virtualServers, err := s.access.ListVirtualServers(clusterName) + if err != nil { + return err + } + if len(virtualServers) == 0 { + err := s.access.DeleteLoadBalancerService(*lbService.Id) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/lock.go b/pkg/cloudprovider/vsphere/loadbalancer/lock.go new file mode 100644 index 0000000000..211155fd62 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/lock.go @@ -0,0 +1,56 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "sync" +) + +type keyLock struct { + lock sync.Mutex + keys map[string]*sync.Mutex +} + +func newKeyLock() *keyLock { + return &keyLock{keys: map[string]*sync.Mutex{}} +} + +// Lock locks the key +func (l *keyLock) Lock(key string) { + l.lock.Lock() + lock := l.keys[key] + if lock == nil { + lock = &sync.Mutex{} + l.keys[key] = lock + } + l.lock.Unlock() + + lock.Lock() +} + +// Unlock unlocks the key +func (l *keyLock) Unlock(key string) { + l.lock.Lock() + defer l.lock.Unlock() + + lock := l.keys[key] + if lock == nil { + panic(fmt.Sprintf("unlock of unknown keyLock %s", key)) + } + lock.Unlock() +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/mapping.go b/pkg/cloudprovider/vsphere/loadbalancer/mapping.go new file mode 100644 index 0000000000..5a04dc214d --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/mapping.go @@ -0,0 +1,73 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// Mapping defines the port mapping and protocol +type Mapping struct { + // SourcePort is the service source port + SourcePort int + // NodePort is the service node port + NodePort int + // Protoocl is the protocol on the service port + Protocol corev1.Protocol +} + +// NewMapping creates a new Mapping for the given service port +func NewMapping(servicePort corev1.ServicePort) Mapping { + return Mapping{ + SourcePort: int(servicePort.Port), + NodePort: int(servicePort.NodePort), + Protocol: servicePort.Protocol, + } +} + +func (m Mapping) String() string { + return fmt.Sprintf("%s/%d->%d", m.Protocol, m.SourcePort, m.NodePort) +} + +// MatchVirtualServer returns true if source port is matching +func (m Mapping) MatchVirtualServer(server *model.LBVirtualServer) bool { + return len(server.Ports) == 1 && server.Ports[0] == formatPort(m.SourcePort) && checkTags(server.Tags, portTag(m)) +} + +// MatchPool returns true if the pool has the correct port tag +func (m Mapping) MatchPool(pool *model.LBPool) bool { + return checkTags(pool.Tags, portTag(m)) +} + +// MatchTCPMonitor returns true if the monitor has the correct port tag +func (m Mapping) MatchTCPMonitor(monitor *model.LBTcpMonitorProfile) bool { + return checkTags(monitor.Tags, portTag(m)) +} + +// MatchNodePort returns true if the server pool member port is equal to the mapping's node port +func (m Mapping) MatchNodePort(server *model.LBVirtualServer) bool { + return len(server.DefaultPoolMemberPorts) == 1 && server.DefaultPoolMemberPorts[0] == formatPort(m.NodePort) +} + +func formatPort(port int) string { + return strconv.FormatInt(int64(port), 10) +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/nsxt_broker.go b/pkg/cloudprovider/vsphere/loadbalancer/nsxt_broker.go new file mode 100644 index 0000000000..6c6feb4ade --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/nsxt_broker.go @@ -0,0 +1,539 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/vmware/vsphere-automation-sdk-go/lib/vapi/std" + vapi_errors "github.com/vmware/vsphere-automation-sdk-go/lib/vapi/std/errors" + "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" + "github.com/vmware/vsphere-automation-sdk-go/runtime/core" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/runtime/protocol/client" + "github.com/vmware/vsphere-automation-sdk-go/runtime/security" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/ip_pools" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/realized_state" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" +) + +// NsxtBroker is an internal interface to enable mocking the nsxt backend +type NsxtBroker interface { + ReadLoadBalancerService(id string) (model.LBService, error) + CreateLoadBalancerService(service model.LBService) (model.LBService, error) + ListLoadBalancerServices() ([]model.LBService, error) + UpdateLoadBalancerService(service model.LBService) (model.LBService, error) + DeleteLoadBalancerService(id string) error + CreateLoadBalancerVirtualServer(server model.LBVirtualServer) (model.LBVirtualServer, error) + ListLoadBalancerVirtualServers() ([]model.LBVirtualServer, error) + UpdateLoadBalancerVirtualServer(server model.LBVirtualServer) (model.LBVirtualServer, error) + DeleteLoadBalancerVirtualServer(id string) error + CreateLoadBalancerPool(pool model.LBPool) (model.LBPool, error) + ReadLoadBalancerPool(id string) (model.LBPool, error) + ListLoadBalancerPools() ([]model.LBPool, error) + UpdateLoadBalancerPool(pool model.LBPool) (model.LBPool, error) + DeleteLoadBalancerPool(id string) error + ListIPPools() ([]model.IpAddressPool, error) + AllocateFromIPPool(ipPoolID string, allocation model.IpAddressAllocation) (model.IpAddressAllocation, string, error) + ListIPPoolAllocations(ipPoolID string) ([]model.IpAddressAllocation, error) + ReleaseFromIPPool(ipPoolID, ipAllocationID string) error + GetRealizedExternalIPAddress(ipAllocationPath string, timeout time.Duration) (*string, error) + ListAppProfiles() ([]*data.StructValue, error) + + CreateLoadBalancerTCPMonitorProfile(monitor model.LBTcpMonitorProfile) (model.LBTcpMonitorProfile, error) + ListLoadBalancerMonitorProfiles() ([]*data.StructValue, error) + ReadLoadBalancerTCPMonitorProfile(id string) (model.LBTcpMonitorProfile, error) + UpdateLoadBalancerTCPMonitorProfile(monitor model.LBTcpMonitorProfile) (model.LBTcpMonitorProfile, error) + DeleteLoadBalancerMonitorProfile(id string) error +} + +type nsxtBroker struct { + lbServicesClient infra.LbServicesClient + lbVirtServersClient infra.LbVirtualServersClient + lbPoolsClient infra.LbPoolsClient + ipPoolsClient infra.IpPoolsClient + ipAllocationsClient ip_pools.IpAllocationsClient + lbAppProfilesClient infra.LbAppProfilesClient + lbMonitorProfilesClient infra.LbMonitorProfilesClient + realizedEntitiesClient realized_state.RealizedEntitiesClient +} + +// NewNsxtBroker creates a new NsxtBroker using the configuration +func NewNsxtBroker(nsxtConfig *config.NsxtConfig) (NsxtBroker, error) { + url := fmt.Sprintf("https://%s", nsxtConfig.Host) + securityCtx := core.NewSecurityContextImpl() + securityContextNeeded := true + if len(nsxtConfig.ClientAuthCertFile) > 0 { + securityContextNeeded = false + } + + if securityContextNeeded { + if len(nsxtConfig.VMCAccessToken) > 0 { + if nsxtConfig.VMCAuthHost == "" { + return nil, fmt.Errorf("vmc auth host must be provided if auth token is provided") + } + + apiToken, err := getAPIToken(nsxtConfig.VMCAuthHost, nsxtConfig.VMCAccessToken) + if err != nil { + return nil, err + } + + securityCtx.SetProperty(security.AUTHENTICATION_SCHEME_ID, security.OAUTH_SCHEME_ID) + securityCtx.SetProperty(security.ACCESS_TOKEN, apiToken) + } else { + if nsxtConfig.User == "" { + return nil, fmt.Errorf("username must be provided") + } + + if nsxtConfig.Password == "" { + return nil, fmt.Errorf("password must be provided") + } + + securityCtx.SetProperty(security.AUTHENTICATION_SCHEME_ID, security.USER_PASSWORD_SCHEME_ID) + securityCtx.SetProperty(security.USER_KEY, nsxtConfig.User) + securityCtx.SetProperty(security.PASSWORD_KEY, nsxtConfig.Password) + } + } + + tlsConfig, err := getConnectorTLSConfig(nsxtConfig.InsecureFlag, nsxtConfig.ClientAuthCertFile, nsxtConfig.ClientAuthKeyFile, nsxtConfig.CAFile) + if err != nil { + return nil, err + } + httpClient := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + connector := client.NewRestConnector(url, httpClient) + connector.SetSecurityContext(securityCtx) + + // perform API call to check connector + _, err = infra.NewDefaultLbMonitorProfilesClient(connector).List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, errors.Wrapf(err, "Connection to NSX-T API failed. Please check your connection settings.") + } + return NewNsxtBrokerFromConnector(connector), nil +} + +func getConnectorTLSConfig(insecure bool, clientCertFile string, clientKeyFile string, caFile string) (*tls.Config, error) { + tlsConfig := tls.Config{InsecureSkipVerify: insecure} + + if len(clientCertFile) > 0 { + if len(clientKeyFile) == 0 { + return nil, fmt.Errorf("Please provide key file for client certificate") + } + + cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) + if err != nil { + return nil, fmt.Errorf("Failed to load client cert/key pair: %v", err) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + + if len(caFile) > 0 { + caCert, err := ioutil.ReadFile(caFile) + if err != nil { + return nil, err + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + tlsConfig.RootCAs = caCertPool + } + + tlsConfig.BuildNameToCertificate() + + return &tlsConfig, nil +} + +type jwtToken struct { + IDToken string `json:"id_token"` + TokenType string `json:"token_type"` + ExpiresIn string `json:"expires_in"` + Scope string `json:"scope"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` +} + +func getAPIToken(vmcAuthHost string, vmcAccessToken string) (string, error) { + + payload := strings.NewReader("refresh_token=" + vmcAccessToken) + req, _ := http.NewRequest("POST", "https://"+vmcAuthHost, payload) + + req.Header.Add("content-type", "application/x-www-form-urlencoded") + res, err := http.DefaultClient.Do(req) + + if err != nil { + return "", err + } + + if res.StatusCode != 200 { + b, _ := ioutil.ReadAll(res.Body) + return "", fmt.Errorf("Unexpected status code %d trying to get auth token. %s", res.StatusCode, string(b)) + } + + defer res.Body.Close() + token := jwtToken{} + err = json.NewDecoder(res.Body).Decode(&token) + if err != nil { + return "", errors.Wrapf(err, "Decoding token failed with") + } + + return token.AccessToken, nil +} + +// NewNsxtBrokerFromConnector creates a new NsxtBroker to the real API +func NewNsxtBrokerFromConnector(connector client.Connector) NsxtBroker { + return &nsxtBroker{ + lbServicesClient: infra.NewDefaultLbServicesClient(connector), + lbVirtServersClient: infra.NewDefaultLbVirtualServersClient(connector), + lbPoolsClient: infra.NewDefaultLbPoolsClient(connector), + ipPoolsClient: infra.NewDefaultIpPoolsClient(connector), + ipAllocationsClient: ip_pools.NewDefaultIpAllocationsClient(connector), + lbAppProfilesClient: infra.NewDefaultLbAppProfilesClient(connector), + lbMonitorProfilesClient: infra.NewDefaultLbMonitorProfilesClient(connector), + realizedEntitiesClient: realized_state.NewDefaultRealizedEntitiesClient(connector), + } +} + +func (b *nsxtBroker) ReadLoadBalancerService(id string) (model.LBService, error) { + return b.lbServicesClient.Get(id) +} + +func (b *nsxtBroker) CreateLoadBalancerService(service model.LBService) (model.LBService, error) { + id := uuid.New().String() + result, err := b.lbServicesClient.Update(id, service) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) ListLoadBalancerServices() ([]model.LBService, error) { + result, err := b.lbServicesClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.lbServicesClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) UpdateLoadBalancerService(service model.LBService) (model.LBService, error) { + result, err := b.lbServicesClient.Update(*service.Id, service) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) DeleteLoadBalancerService(id string) error { + err := b.lbServicesClient.Delete(id, nil) + return nicerVAPIError(err) +} + +func (b *nsxtBroker) CreateLoadBalancerVirtualServer(server model.LBVirtualServer) (model.LBVirtualServer, error) { + id := uuid.New().String() + result, err := b.lbVirtServersClient.Update(id, server) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) ListLoadBalancerVirtualServers() ([]model.LBVirtualServer, error) { + result, err := b.lbVirtServersClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.lbVirtServersClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) UpdateLoadBalancerVirtualServer(server model.LBVirtualServer) (model.LBVirtualServer, error) { + result, err := b.lbVirtServersClient.Update(*server.Id, server) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) DeleteLoadBalancerVirtualServer(id string) error { + err := b.lbVirtServersClient.Delete(id, nil) + return nicerVAPIError(err) +} + +func (b *nsxtBroker) CreateLoadBalancerPool(pool model.LBPool) (model.LBPool, error) { + id := uuid.New().String() + result, err := b.lbPoolsClient.Update(id, pool) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) ReadLoadBalancerPool(id string) (model.LBPool, error) { + result, err := b.lbPoolsClient.Get(id) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) ListLoadBalancerPools() ([]model.LBPool, error) { + result, err := b.lbPoolsClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.lbPoolsClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) UpdateLoadBalancerPool(pool model.LBPool) (model.LBPool, error) { + result, err := b.lbPoolsClient.Update(*pool.Id, pool) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) DeleteLoadBalancerPool(id string) error { + err := b.lbPoolsClient.Delete(id, nil) + return nicerVAPIError(err) +} + +func (b *nsxtBroker) ListAppProfiles() ([]*data.StructValue, error) { + result, err := b.lbAppProfilesClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.lbAppProfilesClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) CreateLoadBalancerTCPMonitorProfile(monitor model.LBTcpMonitorProfile) (model.LBTcpMonitorProfile, error) { + id := uuid.New().String() + result, err := b.createOrUpdateLoadBalancerTCPMonitorProfile(id, monitor) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) createOrUpdateLoadBalancerTCPMonitorProfile(id string, monitor model.LBTcpMonitorProfile) (model.LBTcpMonitorProfile, error) { + monitor.ResourceType = model.LBMonitorProfile_RESOURCE_TYPE_LBTCPMONITORPROFILE + converter := newNsxtTypeConverter() + value, err := converter.convertLBTCPMonitorProfileToStructValue(monitor) + if err != nil { + return model.LBTcpMonitorProfile{}, errors.Wrapf(err, "converting LBTcpMonitorProfile failed") + } + result, err := b.lbMonitorProfilesClient.Update(id, value) + if err != nil { + return model.LBTcpMonitorProfile{}, nicerVAPIError(err) + } + return converter.convertStructValueToLBTCPMonitorProfile(result) +} + +func (b *nsxtBroker) ListLoadBalancerMonitorProfiles() ([]*data.StructValue, error) { + result, err := b.lbMonitorProfilesClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.lbMonitorProfilesClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) ReadLoadBalancerTCPMonitorProfile(id string) (model.LBTcpMonitorProfile, error) { + itf, err := b.lbMonitorProfilesClient.Get(id) + if err != nil { + return model.LBTcpMonitorProfile{}, errors.Wrapf(nicerVAPIError(err), "getting LBTcpMonitorProfile %s failed", id) + } + return newNsxtTypeConverter().convertStructValueToLBTCPMonitorProfile(itf) +} + +func (b *nsxtBroker) UpdateLoadBalancerTCPMonitorProfile(monitor model.LBTcpMonitorProfile) (model.LBTcpMonitorProfile, error) { + result, err := b.createOrUpdateLoadBalancerTCPMonitorProfile(*monitor.Id, monitor) + return result, nicerVAPIError(err) +} + +func (b *nsxtBroker) DeleteLoadBalancerMonitorProfile(id string) error { + err := b.lbMonitorProfilesClient.Delete(id, nil) + return nicerVAPIError(err) +} + +func (b *nsxtBroker) ListIPPools() ([]model.IpAddressPool, error) { + result, err := b.ipPoolsClient.List(nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.ipPoolsClient.List(result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) AllocateFromIPPool(ipPoolID string, allocation model.IpAddressAllocation) (model.IpAddressAllocation, string, error) { + id := uuid.New().String() + err := b.ipAllocationsClient.Patch(ipPoolID, id, allocation) + if err != nil { + return allocation, "", nicerVAPIError(err) + } + allocated, err := b.ipAllocationsClient.Get(ipPoolID, id) + if err != nil { + return allocation, "", nicerVAPIError(err) + } + ipAddress, err := b.GetRealizedExternalIPAddress(*allocated.Path, 15*time.Second) + if err != nil { + return allocated, "", nicerVAPIError(err) + } + if ipAddress == nil { + return allocated, "", fmt.Errorf("no IP address allocated for %s", *allocated.Path) + } + return allocated, *ipAddress, nil +} + +func (b *nsxtBroker) ListIPPoolAllocations(ipPoolID string) ([]model.IpAddressAllocation, error) { + result, err := b.ipAllocationsClient.List(ipPoolID, nil, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list := result.Results + count := int(*result.ResultCount) + for len(list) < count { + result, err = b.ipAllocationsClient.List(ipPoolID, result.Cursor, nil, nil, nil, nil, nil) + if err != nil { + return nil, nicerVAPIError(err) + } + list = append(list, result.Results...) + } + return list, nil +} + +func (b *nsxtBroker) ReleaseFromIPPool(ipPoolID, ipAllocationID string) error { + err := b.ipAllocationsClient.Delete(ipPoolID, ipAllocationID) + return nicerVAPIError(err) +} + +func (b *nsxtBroker) GetRealizedExternalIPAddress(ipAllocationPath string, timeout time.Duration) (*string, error) { + // wait for realized state + limit := time.Now().Add(timeout) + sleepIncr := 100 * time.Millisecond + sleepMax := 1000 * time.Millisecond + sleep := sleepIncr + for time.Now().Before(limit) { + time.Sleep(sleep) + sleep += sleepIncr + if sleep > sleepMax { + sleep = sleepMax + } + list, err := b.realizedEntitiesClient.List(ipAllocationPath) + if err != nil { + return nil, nicerVAPIError(err) + } + for _, realizedResource := range list.Results { + for _, attr := range realizedResource.ExtendedAttributes { + if *attr.Key == "allocation_ip" { + return &attr.Values[0], nil + } + } + } + } + return nil, fmt.Errorf("Timeout of wait for realized state of IP allocation") +} + +func nicerVAPIError(err error) error { + switch vapiError := err.(type) { + case vapi_errors.InvalidRequest: + // Connection errors end up here + return nicerVapiErrorData("InvalidRequest", vapiError.Data, vapiError.Messages) + case vapi_errors.NotFound: + return nicerVapiErrorData("NotFound", vapiError.Data, vapiError.Messages) + case vapi_errors.Unauthorized: + return nicerVapiErrorData("Unauthorized", vapiError.Data, vapiError.Messages) + case vapi_errors.Unauthenticated: + return nicerVapiErrorData("Unauthenticated", vapiError.Data, vapiError.Messages) + case vapi_errors.InternalServerError: + return nicerVapiErrorData("InternalServerError", vapiError.Data, vapiError.Messages) + case vapi_errors.ServiceUnavailable: + return nicerVapiErrorData("ServiceUnavailable", vapiError.Data, vapiError.Messages) + } + + return err +} + +func nicerVapiErrorData(errorMsg string, apiErrorDataValue *data.StructValue, messages []std.LocalizableMessage) error { + if apiErrorDataValue == nil { + if len(messages) > 0 { + return fmt.Errorf("%s (%s)", errorMsg, messages[0].DefaultMessage) + } + return fmt.Errorf("%s (no additional details provided)", errorMsg) + } + + var typeConverter = bindings.NewTypeConverter() + typeConverter.SetMode(bindings.REST) + rawData, err := typeConverter.ConvertToGolang(apiErrorDataValue, model.ApiErrorBindingType()) + + if err != nil { + return fmt.Errorf("%s (failed to extract additional details due to %s)", errorMsg, err) + } + apiError := rawData.(model.ApiError) + details := fmt.Sprintf(" %s: %s (code %v)", errorMsg, *apiError.ErrorMessage, *apiError.ErrorCode) + + if len(apiError.RelatedErrors) > 0 { + details += "\nRelated errors:\n" + for _, relatedErr := range apiError.RelatedErrors { + details += fmt.Sprintf("%s (code %v)", *relatedErr.ErrorMessage, relatedErr.ErrorCode) + } + } + return fmt.Errorf(details) +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/nsxt_type_converter.go b/pkg/cloudprovider/vsphere/loadbalancer/nsxt_type_converter.go new file mode 100644 index 0000000000..1daf8ccbb2 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/nsxt_type_converter.go @@ -0,0 +1,70 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + + "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +type nsxtTypeConverter struct { + bindings.TypeConverter +} + +func newNsxtTypeConverter() *nsxtTypeConverter { + converter := bindings.NewTypeConverter() + converter.SetMode(bindings.REST) + return &nsxtTypeConverter{TypeConverter: *converter} +} + +func (c *nsxtTypeConverter) createLBSnatAutoMap() (*data.StructValue, error) { + entry := model.LBSnatAutoMap{ + Type_: model.LBSnatAutoMap__TYPE_IDENTIFIER, + } + + dataValue, errs := c.ConvertToVapi(entry, model.LBSnatAutoMapBindingType()) + if errs != nil { + return nil, errs[0] + } + + return dataValue.(*data.StructValue), nil +} + +func (c *nsxtTypeConverter) convertLBTCPMonitorProfileToStructValue(monitor model.LBTcpMonitorProfile) (*data.StructValue, error) { + dataValue, errs := c.ConvertToVapi(monitor, model.LBTcpMonitorProfileBindingType()) + if errs != nil { + return nil, errs[0] + } + + return dataValue.(*data.StructValue), nil +} + +func (c *nsxtTypeConverter) convertStructValueToLBTCPMonitorProfile(dataValue *data.StructValue) (model.LBTcpMonitorProfile, error) { + itf, errs := c.ConvertToGolang(dataValue, model.LBTcpMonitorProfileBindingType()) + if errs != nil { + return model.LBTcpMonitorProfile{}, errs[0] + } + + profile, ok := itf.(model.LBTcpMonitorProfile) + if !ok { + return model.LBTcpMonitorProfile{}, fmt.Errorf("converting struct value to LBTcpMonitorProfile failed") + } + return profile, nil +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/state.go b/pkg/cloudprovider/vsphere/loadbalancer/state.go new file mode 100644 index 0000000000..ebe35a6c40 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/state.go @@ -0,0 +1,455 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "reflect" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" +) + +type state struct { + *lbService + klog.Verbose + clusterName string + objectName types.NamespacedName + service *corev1.Service + nodes []*corev1.Node + servers []*model.LBVirtualServer + pools []*model.LBPool + tcpMonitors []*model.LBTcpMonitorProfile + ipAddressAlloc *model.IpAddressAllocation + ipAddress *string + class *loadBalancerClass +} + +func newState(lbService *lbService, clusterName string, service *corev1.Service, nodes []*corev1.Node) *state { + return &state{ + lbService: lbService, + clusterName: clusterName, + service: service, + nodes: nodes, + objectName: namespacedNameFromService(service), + Verbose: klog.V(klog.Level(2)), + } +} + +// CxtInfof logs with object name context +func (s *state) CtxInfof(format string, args ...interface{}) { + if s.Verbose { + s.Infof("%s: %s", s.objectName, fmt.Sprintf(format, args...)) + } +} + +// Process processes a load balancer and ensures that all needed objects are existing +func (s *state) Process(class *loadBalancerClass) error { + var err error + s.ipAddressAlloc, s.ipAddress, err = s.access.FindExternalIPAddressForObject(class.ipPool.Identifier, s.clusterName, s.objectName) + if err != nil { + return err + } + s.servers, err = s.access.FindVirtualServers(s.clusterName, s.objectName) + if err != nil { + return err + } + s.pools, err = s.access.FindPools(s.clusterName, s.objectName) + if err != nil { + return err + } + s.tcpMonitors, err = s.access.FindTCPMonitorProfiles(s.clusterName, s.objectName) + if err != nil { + return err + } + if len(s.servers) > 0 { + className := getTag(s.servers[0].Tags, ScopeLBClass) + ipPoolID := getTag(s.servers[0].Tags, ScopeIPPoolID) + if class.className != className || class.ipPool.Identifier != ipPoolID { + classConfig := &config.LoadBalancerClassConfig{ + IPPoolID: ipPoolID, + } + class, err = newLBClass(className, classConfig, class, nil) + if err != nil { + return err + } + } + } + s.class = class + + for _, servicePort := range s.service.Spec.Ports { + mapping := NewMapping(servicePort) + + monitor, err := s.getTCPMonitor(mapping) + if err != nil { + return err + } + pool, err := s.getPool(mapping, monitor) + if err != nil { + return err + } + _, err = s.getVirtualServer(mapping, pool.Path) + if err != nil { + return err + } + } + validPoolPaths, err := s.deleteOrphanVirtualServers() + if err != nil { + return err + } + s.CtxInfof("validPoolPaths: %v", validPoolPaths.List()) + validTCPMonitorPaths, err := s.deleteOrphanPools(validPoolPaths) + if err != nil { + return err + } + s.CtxInfof("validTCPMonitorPaths: %v", validTCPMonitorPaths.List()) + err = s.deleteOrphanTCPMonitors(validTCPMonitorPaths) + if err != nil { + return err + } + return nil +} + +func (s *state) deleteOrphanVirtualServers() (sets.String, error) { + validPoolPaths := sets.String{} + for _, server := range s.servers { + found := false + for _, servicePort := range s.service.Spec.Ports { + mapping := NewMapping(servicePort) + if mapping.MatchVirtualServer(server) { + if server.PoolPath != nil { + validPoolPaths.Insert(*server.PoolPath) + } + found = true + break + } + } + if !found { + err := s.deleteVirtualServer(server) + if err != nil { + return nil, err + } + } + } + return validPoolPaths, nil +} + +func (s *state) deleteOrphanPools(validPoolPaths sets.String) (sets.String, error) { + validTCPMonitorPaths := sets.String{} + for _, pool := range s.pools { + found := false + for _, servicePort := range s.service.Spec.Ports { + mapping := NewMapping(servicePort) + if mapping.MatchPool(pool) && validPoolPaths.Has(*pool.Path) { + if len(pool.ActiveMonitorPaths) > 0 { + validTCPMonitorPaths.Insert(pool.ActiveMonitorPaths...) + } + found = true + break + } + } + if !found { + err := s.deletePool(pool) + if err != nil { + return nil, err + } + } + } + return validTCPMonitorPaths, nil +} + +func (s *state) deleteOrphanTCPMonitors(validTCPMonitorPaths sets.String) error { + for _, monitor := range s.tcpMonitors { + found := false + for _, servicePort := range s.service.Spec.Ports { + mapping := NewMapping(servicePort) + if mapping.MatchTCPMonitor(monitor) && monitor.Path != nil && validTCPMonitorPaths.Has(*monitor.Path) { + found = true + break + } + } + if !found { + err := s.deleteTCPMonitor(monitor) + if err != nil { + return err + } + } + } + return nil +} + +func (s *state) allocateResources() (allocated bool, err error) { + if s.ipAddressAlloc == nil { + ipPoolID := s.class.ipPool.Identifier + s.ipAddressAlloc, s.ipAddress, err = s.access.AllocateExternalIPAddress(ipPoolID, s.clusterName, s.objectName) + if err != nil { + return + } + allocated = true + s.CtxInfof("allocated IP address %s from pool %s", *s.ipAddress, ipPoolID) + } + return +} + +func (s *state) releaseResources() error { + if s.ipAddressAlloc != nil { + ipPoolID := s.class.ipPool.Identifier + err := s.access.ReleaseExternalIPAddress(ipPoolID, *s.ipAddressAlloc.Id) + if err != nil { + return err + } + s.ipAddressAlloc = nil + s.ipAddress = nil + } + return nil +} + +func (s *state) loggedReleaseResources() { + ipAddress := s.ipAddress + err := s.releaseResources() + if err != nil { + s.CtxInfof("failed to release IP address %s to pool %s", *ipAddress, s.class.ipPool.Identifier) + } +} + +// Finish performs cleanup after Process +func (s *state) Finish() (*corev1.LoadBalancerStatus, error) { + if len(s.service.Spec.Ports) == 0 { + err := s.releaseResources() + if err != nil { + return nil, err + } + return nil, nil + } + return newLoadBalancerStatus(s.ipAddress), nil +} + +func (s *state) getTCPMonitor(mapping Mapping) (*model.LBTcpMonitorProfile, error) { + if mapping.Protocol == corev1.ProtocolTCP { + for _, m := range s.tcpMonitors { + if mapping.MatchTCPMonitor(m) { + err := s.updateTCPMonitor(m, mapping) + if err != nil { + return nil, err + } + return m, nil + } + } + return s.createTCPMonitor(mapping) + } + return nil, nil +} + +func (s *state) createTCPMonitor(mapping Mapping) (*model.LBTcpMonitorProfile, error) { + monitor, err := s.access.CreateTCPMonitorProfile(s.clusterName, s.objectName, mapping) + if err == nil { + s.CtxInfof("created LbTcpMonitor %s for %s", *monitor.Id, mapping) + s.tcpMonitors = append(s.tcpMonitors, monitor) + } + return monitor, err +} + +func (s *state) updateTCPMonitor(monitor *model.LBTcpMonitorProfile, mapping Mapping) error { + if monitor.MonitorPort != nil && *monitor.MonitorPort == int64(mapping.NodePort) { + return nil + } + monitor.MonitorPort = int64ptr(int64(mapping.NodePort)) + s.CtxInfof("updating LbTcpMonitor %s for %s", *monitor.Id, mapping) + return s.access.UpdateTCPMonitorProfile(monitor) +} + +func (s *state) deleteTCPMonitor(monitor *model.LBTcpMonitorProfile) error { + s.CtxInfof("deleting LbTcpMonitor %s for %s", *monitor.Id, getTag(monitor.Tags, ScopePort)) + return s.access.DeleteTCPMonitorProfile(*monitor.Id) +} + +func (s *state) getPool(mapping Mapping, monitor *model.LBTcpMonitorProfile) (*model.LBPool, error) { + var activeMonitorPaths []string + if monitor != nil { + activeMonitorPaths = []string{*monitor.Path} + } + for _, pool := range s.pools { + if mapping.MatchPool(pool) { + err := s.updatePool(pool, mapping, activeMonitorPaths) + return pool, err + } + } + return s.createPool(mapping, activeMonitorPaths) +} + +func (s *state) createPool(mapping Mapping, activeMonitorIds []string) (*model.LBPool, error) { + members, _ := s.updatedPoolMembers(nil) + pool, err := s.access.CreatePool(s.clusterName, s.objectName, mapping, members, activeMonitorIds) + if err == nil { + s.CtxInfof("created LbPool %s for %s", *pool.Id, mapping) + s.pools = append(s.pools, pool) + } + return pool, err +} + +func (s *state) UpdatePoolMembers() error { + pools, err := s.access.FindPools(s.clusterName, s.objectName) + if err != nil { + return err + } + for _, servicePort := range s.service.Spec.Ports { + mapping := NewMapping(servicePort) + for _, pool := range pools { + if mapping.MatchPool(pool) { + err = s.updatePool(pool, mapping, pool.ActiveMonitorPaths) + if err != nil { + return err + } + } + } + } + return nil +} + +func (s *state) updatePool(pool *model.LBPool, mapping Mapping, activeMonitorPaths []string) error { + newMembers, modified := s.updatedPoolMembers(pool.Members) + if modified || !reflect.DeepEqual(activeMonitorPaths, pool.ActiveMonitorPaths) { + pool.Members = newMembers + pool.ActiveMonitorPaths = activeMonitorPaths + s.CtxInfof("updating LbPool %s for %s, #members=%d", *pool.Id, mapping, len(pool.Members)) + err := s.access.UpdatePool(pool) + if err != nil { + return err + } + } + return nil +} + +func (s *state) updatedPoolMembers(oldMembers []model.LBPoolMember) ([]model.LBPoolMember, bool) { + modified := false + nodeIPAddresses := collectNodeInternalAddresses(s.nodes) + newMembers := []model.LBPoolMember{} + for _, member := range oldMembers { + if _, ok := nodeIPAddresses[member.IpAddress]; ok { + newMembers = append(newMembers, member) + } else { + modified = true + } + } + if len(nodeIPAddresses) > len(newMembers) { + for nodeIPAddress, nodeName := range nodeIPAddresses { + found := false + for _, member := range oldMembers { + if member.IpAddress == nodeIPAddress { + found = true + break + } + } + if !found { + member := model.LBPoolMember{ + AdminState: strptr("ENABLED"), + DisplayName: strptr(fmt.Sprintf("%s:%s", s.clusterName, nodeName)), + IpAddress: nodeIPAddress, + } + newMembers = append(newMembers, member) + modified = true + } + } + } + return newMembers, modified +} + +func (s *state) deletePool(pool *model.LBPool) error { + s.CtxInfof("deleting LbPool %s for %s", *pool.Id, getTag(pool.Tags, ScopePort)) + return s.access.DeletePool(*pool.Id) +} + +func (s *state) getVirtualServer(mapping Mapping, poolPath *string) (*model.LBVirtualServer, error) { + for _, server := range s.servers { + if mapping.MatchVirtualServer(server) { + err := s.updateVirtualServer(server, mapping, poolPath) + if err != nil { + return nil, err + } + return server, nil + } + } + + return s.createVirtualServer(mapping, poolPath) +} + +func (s *state) createVirtualServer(mapping Mapping, poolPath *string) (*model.LBVirtualServer, error) { + allocated, err := s.allocateResources() + if err != nil { + return nil, err + } + + lbServicePath, err := s.lbService.getOrCreateLoadBalancerService(s.clusterName) + if err != nil { + return nil, errors.Wrapf(err, "get or create LBService failed") + } + + applicationProfilePath, err := s.access.GetAppProfilePath(s.class, mapping.Protocol) + if err != nil { + return nil, errors.Wrapf(err, "Lookup of application profile failed for %s", mapping.Protocol) + } + + server, err := s.access.CreateVirtualServer(s.clusterName, s.objectName, s.class, *s.ipAddress, mapping, + lbServicePath, applicationProfilePath, poolPath) + if err != nil { + if allocated { + s.loggedReleaseResources() + } + return nil, err + } + s.CtxInfof("created LBVirtualServer %s for %s", *server.Id, mapping) + s.servers = append(s.servers, server) + return server, nil +} + +func (s *state) updateVirtualServer(server *model.LBVirtualServer, mapping Mapping, poolPath *string) error { + applicationProfilePath, err := s.access.GetAppProfilePath(s.class, mapping.Protocol) + if err != nil { + return errors.Wrapf(err, "Lookup of application profile failed for %s", mapping.Protocol) + } + if !mapping.MatchNodePort(server) || !safeEquals(server.PoolPath, poolPath) || server.ApplicationProfilePath != applicationProfilePath { + server.ApplicationProfilePath = applicationProfilePath + server.DefaultPoolMemberPorts = []string{formatPort(mapping.NodePort)} + server.PoolPath = poolPath + s.CtxInfof("updating LbVirtualServer %s for %s", *server.Id, mapping) + err = s.access.UpdateVirtualServer(server) + if err != nil { + return err + } + } + return nil +} + +func (s *state) deleteVirtualServer(server *model.LBVirtualServer) error { + port := "?" + if len(server.DefaultPoolMemberPorts) > 0 { + port = server.DefaultPoolMemberPorts[0] + } + s.CtxInfof("deleting LbVirtualServer %s for %s->%s", *server.Id, getTag(server.Tags, ScopePort), port) + err := s.access.DeleteVirtualServer(*server.Id) + if err != nil { + return err + } + return s.lbService.removeLoadBalancerServiceIfUnused(s.clusterName) +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/tag.go b/pkg/cloudprovider/vsphere/loadbalancer/tag.go new file mode 100644 index 0000000000..3bc21d78d2 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/tag.go @@ -0,0 +1,102 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "fmt" + "sort" + "strings" + + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// Tags is a map of NSXT-T tags indexed by the tag scope +type Tags map[string]model.Tag + +// ByScope is an array of sags sortable by tag scope +type ByScope []model.Tag + +func (a ByScope) Len() int { return len(a) } +func (a ByScope) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByScope) Less(i, j int) bool { return strings.Compare(*a[i].Scope, *a[j].Scope) < 0 } + +// Append clones the Tags and optionally adds additional tags +func (m Tags) Append(tags ...model.Tag) Tags { + result := Tags{} + for n, t := range m { + result[n] = t + } + for _, t := range tags { + result[*t.Scope] = t + } + return result +} + +// Normalize returns a tag array sorted by scopes +func (m Tags) Normalize() []model.Tag { + result := make(ByScope, len(m)) + cnt := 0 + for _, t := range m { + result[cnt] = t + cnt++ + } + sort.Sort(result) + return result +} + +func newTag(scope, tag string) model.Tag { + return model.Tag{Scope: &scope, Tag: &tag} +} + +func clusterTag(clusterName string) model.Tag { + return newTag(ScopeCluster, clusterName) +} + +func serviceTag(objectName types.NamespacedName) model.Tag { + return newTag(ScopeService, objectName.String()) +} + +func portTag(mapping Mapping) model.Tag { + return newTag(ScopePort, fmt.Sprintf("%s/%d", mapping.Protocol, mapping.SourcePort)) +} + +func checkTags(tags []model.Tag, required ...model.Tag) bool { +outer: + for _, req := range required { + for _, tag := range tags { + if *tag.Scope == *req.Scope { + if *tag.Tag != *req.Tag { + return false + } + continue outer + } + } + return false + } + return true +} + +func getTag(tags []model.Tag, scope string) string { + for _, tag := range tags { + if *tag.Scope == scope { + return *tag.Tag + } + } + return "" +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/tag_test.go b/pkg/cloudprovider/vsphere/loadbalancer/tag_test.go new file mode 100644 index 0000000000..8019a94cb0 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/tag_test.go @@ -0,0 +1,85 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loadbalancer + +import ( + "testing" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func _checkTags(t *testing.T, msg string, tags Tags, tag ...model.Tag) { + if len(tags) != len(tag) { + t.Errorf("%s: length mismatch: expected %d entries, but found %d", msg, len(tag), len(tags)) + } + for _, _t := range tag { + scope := *_t.Scope + if f, ok := tags[scope]; ok { + if !_equalTag(f, _t) { + t.Errorf("%s: tag %q mismatch: expected %s, but found %s", msg, scope, *_t.Tag, *f.Tag) + } + } else { + t.Errorf("%s: tag with scope %q missing", msg, scope) + } + } +} + +func _equalTag(a, b model.Tag) bool { + return *a.Scope == *b.Scope && *a.Tag == *b.Tag +} + +func _checkNormTags(t *testing.T, msg string, tags []model.Tag, tag ...model.Tag) { + if len(tags) != len(tag) { + t.Errorf("%s: length mismatch: expected %d entries, but found %d", msg, len(tag), len(tags)) + return + } + for i, _t := range tag { + if !_equalTag(tags[i], _t) { + t.Errorf("%s: entry %d: tag %q mismatch: expected %v, but found %v", msg, i, *_t.Scope, *_t.Tag, *tags[i].Tag) + } + } +} + +func TestTagAdd(t *testing.T) { + tags := Tags{} + + t1 := newTag("t1", "v1") + t1a := newTag("t1", "v1a") + t2 := newTag("t2", "v2") + t3 := newTag("t3", "v3") + + n := tags.Append(t1, t2) + + _checkTags(t, "original tags still empty after add", tags) + _checkTags(t, "simple add", n, t1, t2) + + tags = n + + n = tags.Append(t1a) + _checkTags(t, "replacing keeps original unchanged", tags, t1, t2) + _checkTags(t, "replace tag", n, t1a, t2) + + n = tags.Append(t3) + _checkTags(t, "adding keeps original unchanged", tags, t1, t2) + _checkTags(t, "add tag", n, t1, t2, t3) + + norm := n.Normalize() + _checkNormTags(t, "Normalize tags", norm, t1, t2, t3) + + norm = Tags{}.Append(t3).Append(t2, t1).Normalize() + _checkNormTags(t, "Normalize tags with other add order", norm, t1, t2, t3) +} diff --git a/pkg/cloudprovider/vsphere/types.go b/pkg/cloudprovider/vsphere/types.go index 70e135da84..778c26c52c 100644 --- a/pkg/cloudprovider/vsphere/types.go +++ b/pkg/cloudprovider/vsphere/types.go @@ -22,6 +22,8 @@ import ( v1 "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer" + lbcfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" k8s "k8s.io/cloud-provider-vsphere/pkg/common/kubernetes" @@ -36,6 +38,7 @@ type GRPCServer interface { // CPIConfig is used to read and store information (related only to the CPI) from the cloud configuration file type CPIConfig struct { vcfg.Config + lbcfg.LBConfig Nodes struct { // IP address on VirtualMachine's network interfaces included in the fields' CIDRs @@ -57,6 +60,7 @@ type VSphere struct { connectionManager *cm.ConnectionManager nodeManager *NodeManager informMgr *k8s.InformerManager + loadbalancer loadbalancer.LBProvider instances cloudprovider.Instances zones cloudprovider.Zones server GRPCServer diff --git a/pkg/cloudprovider/vsphere/vapilogger.go b/pkg/cloudprovider/vsphere/vapilogger.go new file mode 100644 index 0000000000..17347ffcd2 --- /dev/null +++ b/pkg/cloudprovider/vsphere/vapilogger.go @@ -0,0 +1,58 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vsphere + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/log" + "k8s.io/klog" +) + +// klogBridge is a connector for the vapi logger to klog +// the github.com/vmware/vsphere-automation-sdk-go SDK used for the NSX-T +// load balancer support logs a lot of stuff on its own logger defaulted +// to standard output. This bridge redirects the SDK log to the +// logging environment used by the controller manager (klog). +type klogBridge struct{} + +// NewKlogBridge provides a vapi logger with klog backend +func NewKlogBridge() log.Logger { + return klogBridge{} +} + +func (d klogBridge) Error(args ...interface{}) { + klog.Error(args...) +} + +func (d klogBridge) Errorf(a string, args ...interface{}) { + klog.Errorf(a, args...) +} + +func (d klogBridge) Info(args ...interface{}) { + klog.Info(args...) +} + +func (d klogBridge) Infof(a string, args ...interface{}) { + klog.Infof(a, args...) +} + +func (d klogBridge) Debug(args ...interface{}) { + klog.V(4).Info(args...) +} + +func (d klogBridge) Debugf(a string, args ...interface{}) { + klog.V(4).Infof(a, args...) +}