From ac3712ab4d0f1b6b5e3f8dcc0cd69e5293603bfc Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Fri, 3 Dec 2021 12:05:46 +0000 Subject: [PATCH 01/23] Implement first interface methods for the kubernetes kv --- go.mod | 7 +- go.sum | 252 ++++++++++++++++++- kv/client.go | 4 + kv/kubernetes/controller.go | 107 ++++++++ kv/kubernetes/kubernetes.go | 246 ++++++++++++++++++ kv/kubernetes/kubernetes_integration_test.go | 92 +++++++ 6 files changed, 700 insertions(+), 8 deletions(-) create mode 100644 kv/kubernetes/controller.go create mode 100644 kv/kubernetes/kubernetes.go create mode 100644 kv/kubernetes/kubernetes_integration_test.go diff --git a/go.mod b/go.mod index 9671a2349..78669adfa 100644 --- a/go.mod +++ b/go.mod @@ -34,10 +34,11 @@ require ( golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 google.golang.org/grpc v1.38.0 gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.21.7 + k8s.io/apimachinery v0.21.7 + k8s.io/client-go v0.21.7 ) -replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.21.4 +replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.21.7 -// Replace memberlist with our fork which includes some fixes that haven't been -// merged upstream yet. replace github.com/hashicorp/memberlist v0.2.3 => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b diff --git a/go.sum b/go.sum index d43f910c3..2ec59f3d1 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,40 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -23,6 +55,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= @@ -37,6 +70,9 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -60,32 +96,41 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBt github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -96,6 +141,16 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -116,11 +171,19 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -147,9 +210,23 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -157,8 +234,10 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b h1:UlCBLaqvS4wVYNrMKSfqTBVsed/EOY9dnenhYZMUnrA= github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= @@ -214,6 +293,9 @@ github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKEN github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -227,7 +309,10 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -239,15 +324,18 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= @@ -286,12 +374,17 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -299,12 +392,18 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg= @@ -327,6 +426,7 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -398,18 +498,24 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFCbQYJ3KJlPs/FvPz2dy1tkpxyeNESVyCNNzRXFR0= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -440,7 +546,10 @@ go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -462,22 +571,45 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b h1:wSOdpTq0/eI46Ez/LkDwIsAKA71YP2SRKBODiRWM0as= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -493,24 +625,36 @@ golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -537,21 +681,31 @@ golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -560,17 +714,27 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -580,17 +744,39 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -600,15 +786,43 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= @@ -619,12 +833,14 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= @@ -643,12 +859,15 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -667,8 +886,31 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.21.7 h1:h+uKRQM59vNKUiv8jo5/h9w9zjk7etiBBfaTZ2Q1nLs= +k8s.io/api v0.21.7/go.mod h1:9Z7hGak48detDeDBCo3Db9N/EqdFSTOEJ9BpIRC3Cms= +k8s.io/apimachinery v0.21.7 h1:7Aeg8xTM367t1J/6x8WPZeHe0MSzL3PVMw8gDaBR4A0= +k8s.io/apimachinery v0.21.7/go.mod h1:Ee84YWaZJo/QdW7/nsjTQCSaCJEJ/CyHkdWbdiBZ3Ns= +k8s.io/client-go v0.21.7 h1:YzpdGcyT0dXWo2af4DpxLsHSBxvQqOYQTnF/RHbVcyg= +k8s.io/client-go v0.21.7/go.mod h1:IdmcpVUFBlFrzDtr58R5o/q3OaA8AJ+FF6LyE9Fpr0w= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/utils v0.0.0-20210521133846-da695404a2bc h1:dx6VGe+PnOW/kD/2UV4aUSsRfJGd7+lcqgJ6Xg0HwUs= +k8s.io/utils v0.0.0-20210521133846-da695404a2bc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/kv/client.go b/kv/client.go index b73620dfd..052ecc3d2 100644 --- a/kv/client.go +++ b/kv/client.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/dskit/kv/codec" "github.com/grafana/dskit/kv/consul" "github.com/grafana/dskit/kv/etcd" + "github.com/grafana/dskit/kv/kubernetes" "github.com/grafana/dskit/kv/memberlist" ) @@ -142,6 +143,9 @@ func createClient(backend string, prefix string, cfg StoreConfig, codec codec.Co }) client = inmemoryStore + case "kubernetes": + client, err = kubernetes.NewClient(&kubernetes.Config{}, codec, logger, reg) + case "memberlist": kv, err := cfg.MemberlistKV() if err != nil { diff --git a/kv/kubernetes/controller.go b/kv/kubernetes/controller.go new file mode 100644 index 000000000..629e235f6 --- /dev/null +++ b/kv/kubernetes/controller.go @@ -0,0 +1,107 @@ +package kubernetes + +import ( + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" +) + +func (c *Client) startController() error { + // create the pod watcher + configMapWatcher := cache.NewFilteredListWatchFromClient(c.client.CoreV1().RESTClient(), "configMaps", c.namespace, func(options *metav1.ListOptions) { + options.FieldSelector = "metadata.name=" + c.name + }) + + // create the workqueue + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + + indexer, informer := cache.NewIndexerInformer(configMapWatcher, &v1.ConfigMap{}, 0, cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err == nil { + queue.Add(key) + } + }, + UpdateFunc: func(old interface{}, new interface{}) { + key, err := cache.MetaNamespaceKeyFunc(new) + if err == nil { + queue.Add(key) + } + }, + DeleteFunc: func(obj interface{}) { + // IndexerInformer uses a delta queue, therefore for deletes we have to use this + // key function. + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err == nil { + queue.Add(key) + } + }, + }, cache.Indexers{}) + + c.queue = queue + c.indexer = indexer + c.informer = informer + + go c.runController() + + return nil +} + +func (c *Client) runController() { + defer runtime.HandleCrash() + + // Let the workers stop when we are done + defer c.queue.ShutDown() + c.logger.Log("msg", "starting configmap controller") + + go c.informer.Run(c.stopCh) + + // Wait for all involved caches to be synced, before processing items from the queue is started + if !cache.WaitForCacheSync(c.stopCh, c.informer.HasSynced) { + runtime.HandleError(fmt.Errorf("Timed out waiting for caches to sync")) + return + } + + go wait.Until(func() { + for c.process() { + } + }, time.Second, c.stopCh) + + <-c.stopCh + c.logger.Log("msg", "stopping configmap controller") +} + +func (c *Client) process() bool { + // Wait until there is a new item in the working queue + key, quit := c.queue.Get() + if quit { + return false + } + // Tell the queue that we are done with processing this key. This unblocks the key for other workers + // This allows safe parallel processing because two pods with the same key are never processed in + // parallel. + defer c.queue.Done(key) + + obj, exists, err := c.indexer.GetByKey(key.(string)) + if err != nil { + c.logger.Log("Fetching object with key %s from store failed with %v", key.(string), err) + return false + } + + if !exists { + // Below we will warm up our cache with a Pod, so that we will see a delete for one pod + fmt.Printf("configMap %s does not exist anymore\n", key) + } else { + // Note that you also have to check the uid if you have a local controlled resource, which + // is dependent on the actual instance, to detect that a Pod was recreated with the same name + fmt.Printf("Sync/Add/Update for configMap %s\n", obj.(*v1.Pod).GetName()) + } + return true + +} diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go new file mode 100644 index 000000000..aebc114be --- /dev/null +++ b/kv/kubernetes/kubernetes.go @@ -0,0 +1,246 @@ +package kubernetes + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "sort" + "strings" + "sync" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/workqueue" + + "github.com/grafana/dskit/kv/codec" +) + +type Config struct { + Name string // name of the config map +} + +type Client struct { + logger log.Logger + name string // config map name + namespace string + client *kubernetes.Clientset + codec codec.Codec + + indexer cache.Indexer + queue workqueue.RateLimitingInterface + informer cache.Controller + stopCh chan struct{} + + configMapMtx sync.RWMutex + configMap *v1.ConfigMap +} + +func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { + var config *rest.Config + var err error + + client := &Client{ + logger: logger, + codec: cod, + name: "dskit-ring", + stopCh: make(chan struct{}), + } + + if cfg != nil { + if cfg.Name != "" { + client.name = cfg.Name + } + } + + // if environment variable is set use local kubeconfig otherwise fall back to in-cluster client + if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + kubeconfigCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + &clientcmd.ConfigOverrides{}, + ) + config, err = kubeconfigCfg.ClientConfig() + if err != nil { + return nil, err + } + client.namespace, _, err = kubeconfigCfg.Namespace() + if err != nil { + return nil, err + } + } else { + config, err = rest.InClusterConfig() + if err != nil { + return nil, err + } + } + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + client.client = clientset + + // check if config already exits + client.configMapMtx.Lock() + defer client.configMapMtx.Unlock() + client.configMap, err = clientset.CoreV1().ConfigMaps(client.namespace).Get(context.Background(), client.name, metav1.GetOptions{}) + if err == nil { + if err := client.startController(); err != nil { + return nil, err + } + return client, nil + } else if !errors.IsNotFound(err) { + return nil, err + } + + // create a new config map + client.configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: client.name, + }, + } + client.configMap, err = clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), client.configMap, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + // start controller to watch for changes to the config map + if err := client.startController(); err != nil { + return nil, err + } + + return client, nil + +} + +func convertKeyToStore(in string) string { + return base64.RawURLEncoding.EncodeToString([]byte(in)) +} + +func convertKeyFromStore(in string) (string, error) { + body, err := base64.RawURLEncoding.DecodeString(in) + if err != nil { + return "", err + } + return string(body), nil +} + +// List returns a list of keys under the given prefix. Returned keys will +// include the prefix. +func (c *Client) List(ctx context.Context, prefix string) ([]string, error) { + c.configMapMtx.RLock() + cm := c.configMap + c.configMapMtx.RUnlock() + + var keys []string + + for keyStore := range cm.BinaryData { + key, err := convertKeyFromStore(keyStore) + if err != nil { + c.logger.Log(fmt.Sprintf("unable to decode key '%s'", keyStore)) + continue + } + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys, nil +} + +// Get a specific key. Will use a codec to deserialise key to appropriate type. +// If the key does not exist, Get will return nil and no error. +func (c *Client) Get(ctx context.Context, key string) (interface{}, error) { + c.configMapMtx.RLock() + cm := c.configMap + c.configMapMtx.RUnlock() + + value, ok := cm.BinaryData[convertKeyToStore(key)] + if !ok { + return nil, nil + } + return value, nil +} + +// Delete a specific key. Deletions are best-effort and no error will +// be returned if the key does not exist. +func (c *Client) Delete(ctx context.Context, key string) error { + return fmt.Errorf("unimplemented") +} + +// CAS stands for Compare-And-Swap. Will call provided callback f with the +// current value of the key and allow callback to return a different value. +// Will then attempt to atomically swap the current value for the new value. +// If that doesn't succeed will try again - callback will be called again +// with new value etc. Guarantees that only a single concurrent CAS +// succeeds. Callback can return nil to indicate it is happy with existing +// value. +func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (out interface{}, retry bool, err error)) error { + var ( + intermediate interface{} + err error + + cm *v1.ConfigMap + ) + + c.configMapMtx.RLock() + cm = c.configMap + c.configMapMtx.RUnlock() + + storedValue, ok := cm.BinaryData[convertKeyToStore(key)] + if ok && storedValue != nil { + intermediate, err = c.codec.Decode(storedValue) + if err != nil { + return err + } + } + + intermediate, _, err = f(intermediate) + if err != nil { + return err + } + + if intermediate == nil { + return nil + } + + encoded, err := c.codec.Encode(intermediate) + if err != nil { + return err + } + + newCM := cm.DeepCopy() + if newCM.BinaryData == nil { + newCM.BinaryData = make(map[string][]byte) + } + newCM.BinaryData[convertKeyToStore(key)] = encoded + + updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Update(ctx, newCM, metav1.UpdateOptions{}) + if err != nil { + return err + } + + c.configMapMtx.Lock() + c.configMap = updatedCM + c.configMapMtx.Unlock() + + return nil +} + +// WatchKey calls f whenever the value stored under key changes. +func (c *Client) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { + panic("implement me") +} + +// WatchPrefix calls f whenever any value stored under prefix changes. +func (c *Client) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { + panic("implement me") +} diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go new file mode 100644 index 000000000..96e3bf002 --- /dev/null +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -0,0 +1,92 @@ +package kubernetes + +import ( + "context" + "math/rand" + "os" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +type testCodec struct { +} + +func (testCodec) Decode(in []byte) (interface{}, error) { + return string(in), nil +} + +func (testCodec) Encode(in interface{}) ([]byte, error) { + return []byte(in.(string)), nil +} + +func (testCodec) CodecID() string { + return "test" +} + +func newClient(t testing.TB) *Client { + logger := log.NewNopLogger() + if testing.Verbose() { + logger = log.NewLogfmtLogger(os.Stderr) + } + c, err := NewClient( + &Config{Name: "test-integration-" + randStringRunes(8)}, + testCodec{}, + logger, + nil, + ) + require.NoError(t, err) + + t.Cleanup(func() { + // TODO: Implement cleanup of config map + if err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(context.Background(), c.name, metav1.DeleteOptions{}); err != nil { + t.Logf("unable to delete config map: %v", err) + } + }) + + return c +} + +func Test_Integration(t *testing.T) { + c := newClient(t) + + keys, err := c.List(context.Background(), "") + require.NoError(t, err) + assert.ElementsMatch(t, []string{}, keys) + + value, err := c.Get(context.Background(), "not-exists") + require.NoError(t, err) + assert.Nil(t, value) + + require.NoError(t, c.CAS(context.Background(), "/test", func(_ interface{}) (out interface{}, retry bool, err error) { + out = "test" + retry = false + return + })) + + keys, err = c.List(context.TODO(), "/test") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"/test"}, keys) + + value, err = c.Get(context.TODO(), "/test") + require.NoError(t, err) + assert.NotNil(t, value) +} From 68962f1a0c4a554874700af0ad3d2d8759095696 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 3 Dec 2021 17:40:47 +0100 Subject: [PATCH 02/23] Add k8s kv CAS implementation --- kv/kubernetes/controller.go | 2 +- kv/kubernetes/kubernetes.go | 82 ++++++++++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/kv/kubernetes/controller.go b/kv/kubernetes/controller.go index 629e235f6..e9a019f1a 100644 --- a/kv/kubernetes/controller.go +++ b/kv/kubernetes/controller.go @@ -100,7 +100,7 @@ func (c *Client) process() bool { } else { // Note that you also have to check the uid if you have a local controlled resource, which // is dependent on the actual instance, to detect that a Pod was recreated with the same name - fmt.Printf("Sync/Add/Update for configMap %s\n", obj.(*v1.Pod).GetName()) + fmt.Printf("Sync/Add/Update for configMap %s\n", obj.(*v1.ConfigMap).GetName()) } return true diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index aebc114be..31beca9c2 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -2,7 +2,9 @@ package kubernetes import ( "context" + "crypto/sha1" "encoding/base64" + "encoding/json" "fmt" "os" "sort" @@ -14,6 +16,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -106,6 +109,8 @@ func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prome ObjectMeta: metav1.ObjectMeta{ Name: client.name, }, + // We want non-empty .data and .binaryData; otherwise CAS will fail because it cannot find the parent key + BinaryData: map[string][]byte{convertKeyToStore("_"): []byte("_")}, } client.configMap, err = clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), client.configMap, metav1.CreateOptions{}) if err != nil { @@ -125,6 +130,10 @@ func convertKeyToStore(in string) string { return base64.RawURLEncoding.EncodeToString([]byte(in)) } +func convertKeyToStoreHash(in string) string { + return "__hash_" + base64.RawURLEncoding.EncodeToString([]byte(in)) +} + func convertKeyFromStore(in string) (string, error) { body, err := base64.RawURLEncoding.DecodeString(in) if err != nil { @@ -148,6 +157,9 @@ func (c *Client) List(ctx context.Context, prefix string) ([]string, error) { c.logger.Log(fmt.Sprintf("unable to decode key '%s'", keyStore)) continue } + if key == "_" { + continue + } if strings.HasPrefix(key, prefix) { keys = append(keys, key) } @@ -163,11 +175,16 @@ func (c *Client) Get(ctx context.Context, key string) (interface{}, error) { cm := c.configMap c.configMapMtx.RUnlock() + if key == "_" { + return nil, nil + } + value, ok := cm.BinaryData[convertKeyToStore(key)] if !ok { return nil, nil } - return value, nil + + return c.codec.Decode(value) } // Delete a specific key. Deletions are best-effort and no error will @@ -217,13 +234,15 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou return err } - newCM := cm.DeepCopy() - if newCM.BinaryData == nil { - newCM.BinaryData = make(map[string][]byte) + oldEncodedHash := cm.BinaryData[convertKeyToStoreHash(key)] + newHash := hash(encoded) + + patch, err := preparePatch(key, oldEncodedHash, encoded, newHash) + if err != nil { + return err } - newCM.BinaryData[convertKeyToStore(key)] = encoded - updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Update(ctx, newCM, metav1.UpdateOptions{}) + updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) if err != nil { return err } @@ -235,6 +254,57 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou return nil } +func preparePatch(key string, oldHash, newVal, newHash []byte) ([]byte, error) { + hashKey := "/binaryData/" + convertKeyToStoreHash(key) + + b64 := func(b []byte) *string { + str := base64.StdEncoding.EncodeToString(b) + return &str + } + + type operation struct { + Op string `json:"op"` + Path string `json:"path"` + Value *string `json:"value"` + } + + var expectedHash *string + if len(oldHash) > 0 { + expectedHash = b64(oldHash) + } + + testHashOp := operation{ + Op: "test", + Path: hashKey, + Value: expectedHash, + } + + setHashOp := operation{ + Op: "replace", + Path: hashKey, + Value: b64(newHash), + } + + setDataOp := operation{ + Op: "replace", + Path: "/binaryData/" + convertKeyToStore(key), + Value: b64(newVal), + } + + patch := []operation{testHashOp, setHashOp, setDataOp} + + return json.Marshal(patch) +} + +func hash(b []byte) []byte { + hasher := sha1.New() + _, err := hasher.Write(b) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + // WatchKey calls f whenever the value stored under key changes. func (c *Client) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { panic("implement me") From dc77595ad6fc1244f4ca77354476c24874a0939c Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 3 Dec 2021 17:44:11 +0100 Subject: [PATCH 03/23] Strengthen assertions in kubernetes_integration_test.go --- kv/kubernetes/kubernetes_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 96e3bf002..7652ae18c 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -88,5 +88,5 @@ func Test_Integration(t *testing.T) { value, err = c.Get(context.TODO(), "/test") require.NoError(t, err) - assert.NotNil(t, value) + assert.Equal(t, "test", value) } From fabeae0a62a63667b8fb2e1b51714033f00673d9 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 3 Dec 2021 17:46:13 +0100 Subject: [PATCH 04/23] Add some more assertions in kubernetes_integration_test.go --- kv/kubernetes/kubernetes_integration_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 7652ae18c..30aba7472 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -82,6 +82,13 @@ func Test_Integration(t *testing.T) { return })) + require.NoError(t, c.CAS(context.Background(), "/test", func(old interface{}) (out interface{}, retry bool, err error) { + assert.Equal(t, "test", old) + out = nil + retry = false + return + })) + keys, err = c.List(context.TODO(), "/test") require.NoError(t, err) assert.ElementsMatch(t, []string{"/test"}, keys) From 5c384408d75865a983d1e323244e02969f356880 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 3 Dec 2021 18:29:03 +0100 Subject: [PATCH 05/23] Implement k8s kv Delete --- kv/kubernetes/jsonpatch.go | 70 ++++++++++++++++++++ kv/kubernetes/kubernetes.go | 69 +++++++------------ kv/kubernetes/kubernetes_integration_test.go | 38 ++++++++++- 3 files changed, 132 insertions(+), 45 deletions(-) create mode 100644 kv/kubernetes/jsonpatch.go diff --git a/kv/kubernetes/jsonpatch.go b/kv/kubernetes/jsonpatch.go new file mode 100644 index 000000000..b10bf6e20 --- /dev/null +++ b/kv/kubernetes/jsonpatch.go @@ -0,0 +1,70 @@ +package kubernetes + +import ( + "encoding/base64" + "encoding/json" +) + +type operation struct { + Op string `json:"op"` + Path string `json:"path"` + // Value should be a base64-encoded value or nil. + Value *string `json:"value"` +} + +// preparePatch prepares a JSON patch (RFC 6902) to update a configmap key. If oldHash is empty or nil +// this is equivalent to adding a key. +func preparePatch(key string, oldHash, newVal, newHash []byte) ([]byte, error) { + hashKey := "/binaryData/" + convertKeyToStoreHash(key) + + b64 := func(b []byte) *string { + str := base64.StdEncoding.EncodeToString(b) + return &str + } + + // testing with nil is equivalent to testing that the key doesn't exist + var expectedHash *string + if len(oldHash) > 0 { + expectedHash = b64(oldHash) + } + + testHashOp := operation{ + Op: "test", + Path: hashKey, + Value: expectedHash, + } + + setHashOp := operation{ + Op: "replace", + Path: hashKey, + Value: b64(newHash), + } + + setDataOp := operation{ + Op: "replace", + Path: "/binaryData/" + convertKeyToStore(key), + Value: b64(newVal), + } + + patch := []operation{testHashOp, setHashOp, setDataOp} + + return json.Marshal(patch) +} + +func prepareDeletePatch(key string) ([]byte, error) { + removeHashOp := operation{ + Op: "remove", + Path: "/binaryData/" + convertKeyToStoreHash(key), + Value: nil, + } + + removeObjectOp := operation{ + Op: "remove", + Path: "/binaryData/" + convertKeyToStore(key), + Value: nil, + } + + patch := []operation{removeHashOp, removeObjectOp} + + return json.Marshal(patch) +} diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index 31beca9c2..3ed18ebd8 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha1" "encoding/base64" - "encoding/json" "fmt" "os" "sort" @@ -190,7 +189,31 @@ func (c *Client) Get(ctx context.Context, key string) (interface{}, error) { // Delete a specific key. Deletions are best-effort and no error will // be returned if the key does not exist. func (c *Client) Delete(ctx context.Context, key string) error { - return fmt.Errorf("unimplemented") + c.configMapMtx.RLock() + cm := c.configMap + c.configMapMtx.RUnlock() + + _, ok := cm.BinaryData[convertKeyToStore(key)] + if !ok { + // Object is already deleted or never existed + return nil + } + + patch, err := prepareDeletePatch(key) + if err != nil { + return err + } + + updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + + c.configMapMtx.Lock() + c.configMap = updatedCM + c.configMapMtx.Unlock() + + return nil } // CAS stands for Compare-And-Swap. Will call provided callback f with the @@ -254,48 +277,6 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou return nil } -func preparePatch(key string, oldHash, newVal, newHash []byte) ([]byte, error) { - hashKey := "/binaryData/" + convertKeyToStoreHash(key) - - b64 := func(b []byte) *string { - str := base64.StdEncoding.EncodeToString(b) - return &str - } - - type operation struct { - Op string `json:"op"` - Path string `json:"path"` - Value *string `json:"value"` - } - - var expectedHash *string - if len(oldHash) > 0 { - expectedHash = b64(oldHash) - } - - testHashOp := operation{ - Op: "test", - Path: hashKey, - Value: expectedHash, - } - - setHashOp := operation{ - Op: "replace", - Path: hashKey, - Value: b64(newHash), - } - - setDataOp := operation{ - Op: "replace", - Path: "/binaryData/" + convertKeyToStore(key), - Value: b64(newVal), - } - - patch := []operation{testHashOp, setHashOp, setDataOp} - - return json.Marshal(patch) -} - func hash(b []byte) []byte { hasher := sha1.New() _, err := hasher.Write(b) diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 30aba7472..304c9e415 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -70,7 +70,7 @@ func Test_Integration(t *testing.T) { keys, err := c.List(context.Background(), "") require.NoError(t, err) - assert.ElementsMatch(t, []string{}, keys) + assert.Empty(t, keys) value, err := c.Get(context.Background(), "not-exists") require.NoError(t, err) @@ -97,3 +97,39 @@ func Test_Integration(t *testing.T) { require.NoError(t, err) assert.Equal(t, "test", value) } + +func Test_Delete(t *testing.T) { + t.Run("happy flow", func(t *testing.T) { + c := newClient(t) + + require.NoError(t, c.CAS(context.Background(), "/test", func(_ interface{}) (out interface{}, retry bool, err error) { + out = "test" + retry = false + return + })) + + require.NoError(t, c.Delete(context.Background(), "/test")) + + keys, err := c.List(context.TODO(), "/test") + require.NoError(t, err) + assert.Empty(t, keys) + + value, err := c.Get(context.TODO(), "/test") + assert.NoError(t, err) + assert.Nil(t, value) + }) + + t.Run("deleting non-existent key also works", func(t *testing.T) { + c := newClient(t) + + require.NoError(t, c.Delete(context.Background(), "/test")) + + keys, err := c.List(context.TODO(), "/test") + require.NoError(t, err) + assert.Empty(t, keys) + + value, err := c.Get(context.TODO(), "/test") + assert.NoError(t, err) + assert.Nil(t, value) + }) +} From fe2ae24d70d30f1a9898ad99a3bf96201f41e051 Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Fri, 3 Dec 2021 18:00:03 +0000 Subject: [PATCH 06/23] Implement in memory client Note: This client currently crashes, to be investigated --- go.sum | 7 ++ kv/kubernetes/controller.go | 2 +- kv/kubernetes/kubernetes.go | 70 ++++++++++++-------- kv/kubernetes/kubernetes_integration_test.go | 49 +++++++++----- kv/kubernetes/mock.go | 25 +++++++ kv/kv_test.go | 5 ++ 6 files changed, 115 insertions(+), 43 deletions(-) create mode 100644 kv/kubernetes/mock.go diff --git a/go.sum b/go.sum index 2ec59f3d1..1ba50cffe 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= @@ -291,6 +292,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -400,9 +402,11 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= @@ -864,11 +868,13 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -901,6 +907,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909 h1:s77MRc/+/eQjsF89MB12JssAlsoi9mnNoaacRqibeAU= k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20210521133846-da695404a2bc h1:dx6VGe+PnOW/kD/2UV4aUSsRfJGd7+lcqgJ6Xg0HwUs= k8s.io/utils v0.0.0-20210521133846-da695404a2bc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/kv/kubernetes/controller.go b/kv/kubernetes/controller.go index e9a019f1a..9d85c1620 100644 --- a/kv/kubernetes/controller.go +++ b/kv/kubernetes/controller.go @@ -14,7 +14,7 @@ import ( func (c *Client) startController() error { // create the pod watcher - configMapWatcher := cache.NewFilteredListWatchFromClient(c.client.CoreV1().RESTClient(), "configMaps", c.namespace, func(options *metav1.ListOptions) { + configMapWatcher := cache.NewFilteredListWatchFromClient(c.clientset.CoreV1().RESTClient(), "configMaps", c.namespace, func(options *metav1.ListOptions) { options.FieldSelector = "metadata.name=" + c.name }) diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index 3ed18ebd8..63784b93a 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -33,7 +33,7 @@ type Client struct { logger log.Logger name string // config map name namespace string - client *kubernetes.Clientset + clientset kubernetes.Interface codec codec.Codec indexer cache.Indexer @@ -45,22 +45,11 @@ type Client struct { configMap *v1.ConfigMap } -func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { - var config *rest.Config - var err error - - client := &Client{ - logger: logger, - codec: cod, - name: "dskit-ring", - stopCh: make(chan struct{}), - } - - if cfg != nil { - if cfg.Name != "" { - client.name = cfg.Name - } - } +func realClientGenerator(c *Client) error { + var ( + config *rest.Config + err error + ) // if environment variable is set use local kubeconfig otherwise fall back to in-cluster client if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { @@ -70,30 +59,59 @@ func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prome ) config, err = kubeconfigCfg.ClientConfig() if err != nil { - return nil, err + return err } - client.namespace, _, err = kubeconfigCfg.Namespace() + c.namespace, _, err = kubeconfigCfg.Namespace() if err != nil { - return nil, err + return err } } else { config, err = rest.InClusterConfig() if err != nil { - return nil, err + return err } + // TODO: detect namespace } // creates the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { + return err + } + c.clientset = clientset + return nil +} + +func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { + return newClient(cfg, cod, logger, registerer, realClientGenerator) +} + +func newClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { + var err error + + client := &Client{ + logger: logger, + codec: cod, + name: "dskit-ring", + stopCh: make(chan struct{}), + } + + // configure configuration options on the client struct + if cfg != nil { + if cfg.Name != "" { + client.name = cfg.Name + } + } + + // creates the clientset + if err := clientGenerator(client); err != nil { return nil, err } - client.client = clientset // check if config already exits client.configMapMtx.Lock() defer client.configMapMtx.Unlock() - client.configMap, err = clientset.CoreV1().ConfigMaps(client.namespace).Get(context.Background(), client.name, metav1.GetOptions{}) + client.configMap, err = client.clientset.CoreV1().ConfigMaps(client.namespace).Get(context.Background(), client.name, metav1.GetOptions{}) if err == nil { if err := client.startController(); err != nil { return nil, err @@ -111,7 +129,7 @@ func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prome // We want non-empty .data and .binaryData; otherwise CAS will fail because it cannot find the parent key BinaryData: map[string][]byte{convertKeyToStore("_"): []byte("_")}, } - client.configMap, err = clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), client.configMap, metav1.CreateOptions{}) + client.configMap, err = client.clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), client.configMap, metav1.CreateOptions{}) if err != nil { return nil, err } @@ -204,7 +222,7 @@ func (c *Client) Delete(ctx context.Context, key string) error { return err } - updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) + updatedCM, err := c.clientset.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) if err != nil { return err } @@ -265,7 +283,7 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou return err } - updatedCM, err := c.client.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) + updatedCM, err := c.clientset.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) if err != nil { return err } diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 304c9e415..19077111e 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -42,31 +42,48 @@ func (testCodec) CodecID() string { return "test" } -func newClient(t testing.TB) *Client { - logger := log.NewNopLogger() +func newTestClient(t testing.TB) *Client { + var ( + logger = log.NewNopLogger() + ) + if testing.Verbose() { logger = log.NewLogfmtLogger(os.Stderr) } - c, err := NewClient( - &Config{Name: "test-integration-" + randStringRunes(8)}, + + // use a real Kubernetes client if both environment DSKIT_TEST_KUBERNETES and KUBECONFIG are set + if os.Getenv("DSKIT_TEST_KUBERNETES") != "" && os.Getenv("KUBECONFIG") != "" { + t.Logf("connecting to real Kubernetes cluster") + client, err := NewClient( + &Config{Name: "test-integration-" + randStringRunes(8)}, + testCodec{}, + logger, + nil, + ) + require.NoError(t, err) + t.Cleanup(func() { + if err := client.clientset.CoreV1().ConfigMaps(client.namespace).Delete(context.Background(), client.name, metav1.DeleteOptions{}); err != nil { + t.Logf("unable to delete config map: %v", err) + } + }) + return client + } + + // otherwise use in memory client + client, closer := NewInMemoryClient( testCodec{}, logger, - nil, ) - require.NoError(t, err) - t.Cleanup(func() { - // TODO: Implement cleanup of config map - if err := c.client.CoreV1().ConfigMaps(c.namespace).Delete(context.Background(), c.name, metav1.DeleteOptions{}); err != nil { - t.Logf("unable to delete config map: %v", err) - } + closer.Close() + close(client.stopCh) }) - return c + return client } -func Test_Integration(t *testing.T) { - c := newClient(t) +func Test_Integration_Simple(t *testing.T) { + c := newTestClient(t) keys, err := c.List(context.Background(), "") require.NoError(t, err) @@ -100,7 +117,7 @@ func Test_Integration(t *testing.T) { func Test_Delete(t *testing.T) { t.Run("happy flow", func(t *testing.T) { - c := newClient(t) + c := newTestClient(t) require.NoError(t, c.CAS(context.Background(), "/test", func(_ interface{}) (out interface{}, retry bool, err error) { out = "test" @@ -120,7 +137,7 @@ func Test_Delete(t *testing.T) { }) t.Run("deleting non-existent key also works", func(t *testing.T) { - c := newClient(t) + c := newTestClient(t) require.NoError(t, c.Delete(context.Background(), "/test")) diff --git a/kv/kubernetes/mock.go b/kv/kubernetes/mock.go new file mode 100644 index 000000000..5d45289fc --- /dev/null +++ b/kv/kubernetes/mock.go @@ -0,0 +1,25 @@ +package kubernetes + +import ( + "io" + + "github.com/go-kit/log" + + "github.com/grafana/dskit/kv/codec" + + "k8s.io/client-go/kubernetes/fake" +) + +func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) { + fakeClientset := fake.NewSimpleClientset() + + client, err := newClient(&Config{}, codec, logger, nil, func(c *Client) error { + c.clientset = fakeClientset + return nil + }) + if err != nil { + panic("error generating in memory client: " + err.Error()) + } + + return client, io.NopCloser(nil) +} diff --git a/kv/kv_test.go b/kv/kv_test.go index f5ac6207f..6070a131a 100644 --- a/kv/kv_test.go +++ b/kv/kv_test.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/dskit/kv/codec" "github.com/grafana/dskit/kv/consul" "github.com/grafana/dskit/kv/etcd" + "github.com/grafana/dskit/kv/kubernetes" ) func withFixtures(t *testing.T, f func(*testing.T, Client)) { @@ -32,6 +33,10 @@ func withFixtures(t *testing.T, f func(*testing.T, Client)) { client, closer := etcd.NewInMemoryClient(codec.String{}, testLogger{}) return client, closer, nil }}, + {"kubernetes", func() (Client, io.Closer, error) { + client, closer := kubernetes.NewInMemoryClient(codec.String{}, testLogger{}) + return client, closer, nil + }}, } { t.Run(fixture.name, func(t *testing.T) { client, closer, err := fixture.factory() From 24d680237744956706389f495ebc28dbb883a0e0 Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Sat, 4 Dec 2021 09:54:29 +0000 Subject: [PATCH 07/23] Ensure restclient is populated --- kv/kubernetes/mock.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/kv/kubernetes/mock.go b/kv/kubernetes/mock.go index 5d45289fc..64690ab5d 100644 --- a/kv/kubernetes/mock.go +++ b/kv/kubernetes/mock.go @@ -8,10 +8,29 @@ import ( "github.com/grafana/dskit/kv/codec" "k8s.io/client-go/kubernetes/fake" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + fake_rest "k8s.io/client-go/rest/fake" ) +type fakeClientset struct { + *fake.Clientset +} + +func (c fakeClientset) CoreV1() corev1.CoreV1Interface { + return fakeCoreV1{c.Clientset.CoreV1()} +} + +type fakeCoreV1 struct { + corev1.CoreV1Interface +} + +func (c fakeCoreV1) RESTClient() rest.Interface { + return &fake_rest.RESTClient{} +} + func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) { - fakeClientset := fake.NewSimpleClientset() + fakeClientset := fakeClientset{fake.NewSimpleClientset()} client, err := newClient(&Config{}, codec, logger, nil, func(c *Client) error { c.clientset = fakeClientset From edd830f34727812e15f1afa0909880785b3ac372 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Mon, 6 Dec 2021 14:15:13 +0100 Subject: [PATCH 08/23] Register config flags --- kv/client.go | 10 ++++--- kv/kubernetes/kubernetes.go | 30 ++++++++++++-------- kv/kubernetes/kubernetes_integration_test.go | 2 +- kv/kubernetes/mock.go | 2 +- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/kv/client.go b/kv/client.go index 052ecc3d2..f299c67cc 100644 --- a/kv/client.go +++ b/kv/client.go @@ -41,9 +41,10 @@ var inmemoryStore Client // Consul, Etcd, Memberlist or MultiClient. It was extracted from Config to keep // single-client config separate from final client-config (with all the wrappers) type StoreConfig struct { - Consul consul.Config `yaml:"consul"` - Etcd etcd.Config `yaml:"etcd"` - Multi MultiConfig `yaml:"multi"` + Consul consul.Config `yaml:"consul"` + Etcd etcd.Config `yaml:"etcd"` + Multi MultiConfig `yaml:"multi"` + Kubernetes kubernetes.Config `yaml:"kubernetes"` // Function that returns memberlist.KV store to use. By using a function, we can delay // initialization of memberlist.KV until it is actually required. @@ -73,6 +74,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix, defaultPrefix string, f cfg.Consul.RegisterFlags(f, flagsPrefix) cfg.Etcd.RegisterFlagsWithPrefix(f, flagsPrefix) cfg.Multi.RegisterFlagsWithPrefix(f, flagsPrefix) + cfg.Kubernetes.RegisterFlags(f, flagsPrefix) if flagsPrefix == "" { flagsPrefix = "ring." @@ -144,7 +146,7 @@ func createClient(backend string, prefix string, cfg StoreConfig, codec codec.Co client = inmemoryStore case "kubernetes": - client, err = kubernetes.NewClient(&kubernetes.Config{}, codec, logger, reg) + client, err = kubernetes.NewClient(cfg.Kubernetes, codec, logger, reg) case "memberlist": kv, err := cfg.MemberlistKV() diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index 63784b93a..ff1b04c5e 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -4,7 +4,9 @@ import ( "context" "crypto/sha1" "encoding/base64" + "flag" "fmt" + "io/ioutil" "os" "sort" "strings" @@ -26,7 +28,13 @@ import ( ) type Config struct { - Name string // name of the config map + ConfigMapName string // name of the config map +} + +// RegisterFlags adds the flags required to config this to the given FlagSet +// If prefix is not an empty string it should end with a period. +func (cfg *Config) RegisterFlags(f *flag.FlagSet, prefix string) { + f.StringVar(&cfg.ConfigMapName, prefix+"kubernetes.config_map_name", "dskit-ring", "Name of kubernetes configmap to use for KV store.") } type Client struct { @@ -70,7 +78,12 @@ func realClientGenerator(c *Client) error { if err != nil { return err } - // TODO: detect namespace + + nsBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return err + } + c.namespace = string(nsBytes) } // creates the clientset @@ -82,27 +95,20 @@ func realClientGenerator(c *Client) error { return nil } -func NewClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { +func NewClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { return newClient(cfg, cod, logger, registerer, realClientGenerator) } -func newClient(cfg *Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { +func newClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { var err error client := &Client{ logger: logger, codec: cod, - name: "dskit-ring", + name: cfg.ConfigMapName, stopCh: make(chan struct{}), } - // configure configuration options on the client struct - if cfg != nil { - if cfg.Name != "" { - client.name = cfg.Name - } - } - // creates the clientset if err := clientGenerator(client); err != nil { return nil, err diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 19077111e..f95b75365 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -55,7 +55,7 @@ func newTestClient(t testing.TB) *Client { if os.Getenv("DSKIT_TEST_KUBERNETES") != "" && os.Getenv("KUBECONFIG") != "" { t.Logf("connecting to real Kubernetes cluster") client, err := NewClient( - &Config{Name: "test-integration-" + randStringRunes(8)}, + Config{ConfigMapName: "test-integration-" + randStringRunes(8)}, testCodec{}, logger, nil, diff --git a/kv/kubernetes/mock.go b/kv/kubernetes/mock.go index 64690ab5d..b135404e0 100644 --- a/kv/kubernetes/mock.go +++ b/kv/kubernetes/mock.go @@ -32,7 +32,7 @@ func (c fakeCoreV1) RESTClient() rest.Interface { func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) { fakeClientset := fakeClientset{fake.NewSimpleClientset()} - client, err := newClient(&Config{}, codec, logger, nil, func(c *Client) error { + client, err := newClient(Config{}, codec, logger, nil, func(c *Client) error { c.clientset = fakeClientset return nil }) From 8d3b45e695de431361e732bd52594cb0121ebc12 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Mon, 6 Dec 2021 18:20:16 +0100 Subject: [PATCH 09/23] Implement k8s CAS watch --- kv/internal/watcher/watcher.go | 171 ++++++++++++++++++++++++++++ kv/internal/watcher/watcher_test.go | 83 ++++++++++++++ kv/kubernetes/controller.go | 30 ++++- kv/kubernetes/kubernetes.go | 60 ++++++---- 4 files changed, 317 insertions(+), 27 deletions(-) create mode 100644 kv/internal/watcher/watcher.go create mode 100644 kv/internal/watcher/watcher_test.go diff --git a/kv/internal/watcher/watcher.go b/kv/internal/watcher/watcher.go new file mode 100644 index 000000000..43b990cc3 --- /dev/null +++ b/kv/internal/watcher/watcher.go @@ -0,0 +1,171 @@ +package watch + +import ( + "context" + "strings" + "sync" + + "github.com/go-kit/log" + + "github.com/go-kit/log/level" +) + +type Watcher struct { + logger log.Logger + + // closed on shutdown + shutdown chan struct{} + + // Key watchers + watchersMu sync.Mutex + watchers map[string][]chan update + prefixWatchers map[string][]chan update +} + +type update struct { + key string + value interface{} +} + +func NewWatcher(l log.Logger) *Watcher { + return &Watcher{ + logger: l, + shutdown: make(chan struct{}), + watchers: make(map[string][]chan update), + prefixWatchers: make(map[string][]chan update), + } +} + +// WatchKey watches for value changes for given key. When value changes, 'f' function is called with the +// latest value. Notifications that arrive while 'f' is running are coalesced into one subsequent 'f' call. +// +// Watching ends when 'f' returns false, context is done, or this client is shut down. +func (w *Watcher) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { + // keep one extra notification, to avoid missing notification if we're busy running the function + watchChan := make(chan update, 1) + + // register watcher + w.watchersMu.Lock() + w.watchers[key] = append(w.watchers[key], watchChan) + w.watchersMu.Unlock() + + defer func() { + // unregister watcher on exit + w.watchersMu.Lock() + defer w.watchersMu.Unlock() + + removeWatcherChannel(key, watchChan, w.watchers) + }() + + for { + select { + case newValue := <-watchChan: + // value changed + if !f(newValue.value) { + return + } + + case <-w.shutdown: + // stop watching on shutdown + return + + case <-ctx.Done(): + return + } + } +} + +// WatchPrefix watches for any change of values stored under keys with given prefix. When change occurs, +// function 'f' is called with key and current value. +// Each change of the key results in one notification. If there are too many pending notifications ('f' is slow), +// some notifications may be lost. +// +// Watching ends when 'f' returns false, context is done, or this client is shut down. +func (w *Watcher) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { + // we use bigger buffer here, since keys are interesting and we don't want to lose them. + watchChan := make(chan update, 16) + + // register watcher + w.watchersMu.Lock() + w.prefixWatchers[prefix] = append(w.prefixWatchers[prefix], watchChan) + w.watchersMu.Unlock() + + defer func() { + // unregister watcher on exit + w.watchersMu.Lock() + defer w.watchersMu.Unlock() + + removeWatcherChannel(prefix, watchChan, w.prefixWatchers) + }() + + for { + select { + case newValue := <-watchChan: + if !f(newValue.key, newValue.value) { + return + } + + case <-w.shutdown: + // stop watching on shutdown + return + + case <-ctx.Done(): + return + } + } +} + +func removeWatcherChannel(k string, w chan update, watchers map[string][]chan update) { + ws := watchers[k] + for ix, kw := range ws { + if kw == w { + ws = append(ws[:ix], ws[ix+1:]...) + break + } + } + + if len(ws) > 0 { + watchers[k] = ws + } else { + delete(watchers, k) + } +} + +func (w *Watcher) Notify(key string, value interface{}) { + w.watchersMu.Lock() + defer w.watchersMu.Unlock() + + watchKey := update{ + key: key, + value: value, + } + + for _, kw := range w.watchers[key] { + select { + case kw <- watchKey: + // notification sent. + default: + // cannot send notification to this watcher at the moment + // but since this is a buffered channel, it means that + // there is already a pending notification anyway + } + } + + for p, ws := range w.prefixWatchers { + if strings.HasPrefix(key, p) { + for _, pw := range ws { + select { + case pw <- watchKey: + // notification sent. + default: + level.Warn(w.logger).Log("msg", "failed to send notification to prefix watcher", "prefix", p) + } + } + } + } +} + +func (w *Watcher) Stop(_ error) error { + close(w.shutdown) + return nil +} diff --git a/kv/internal/watcher/watcher_test.go b/kv/internal/watcher/watcher_test.go new file mode 100644 index 000000000..66fc8819b --- /dev/null +++ b/kv/internal/watcher/watcher_test.go @@ -0,0 +1,83 @@ +package watch + +import ( + "context" + "math/rand" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" +) + +func TestWatcher(t *testing.T) { + t.Run("Notify notifies watchers that are watching a key", func(t *testing.T) { + w := newTestWatcher() + const key = "/key" + notifyVal := rand.Int() + notified := make(chan struct{}) + + go w.WatchKey(context.Background(), key, func(val interface{}) bool { + assert.Equal(t, notifyVal, val) + close(notified) + return false + }) + + // Wait until the watcher has been registered + require.NoError(t, wait.Poll(time.Millisecond*10, time.Second, func() (bool, error) { + return len(w.watchers) > 0, nil + })) + + w.Notify(key, notifyVal) + + select { + case <-time.NewTimer(time.Second).C: + assert.FailNow(t, "notifier didn't get called within a second") + case <-notified: + } + }) + + t.Run("Notify notifies watchers that are watching a prefix", func(t *testing.T) { + w := newTestWatcher() + + const prefix = "/prefix" + const key = prefix + "/key" + notifyVal := rand.Int() + notified := make(chan struct{}) + + go w.WatchPrefix(context.Background(), prefix, func(k string, val interface{}) bool { + assert.Equal(t, notifyVal, val) + assert.Equal(t, key, k) + close(notified) + return false + }) + + // Wait until the watcher has been registered + require.NoError(t, wait.Poll(time.Millisecond*10, time.Second, func() (bool, error) { + return len(w.prefixWatchers) > 0, nil + })) + + w.Notify(key, notifyVal) + + select { + case <-time.NewTimer(time.Second).C: + assert.FailNow(t, "notifier didn't get called within a second") + case <-notified: + } + }) +} + +func newTestWatcher() *Watcher { + var logger log.Logger + if testing.Verbose() { + logger = log.NewLogfmtLogger(os.Stdout) + } else { + logger = log.NewNopLogger() + } + + return NewWatcher(logger) +} diff --git a/kv/kubernetes/controller.go b/kv/kubernetes/controller.go index 9d85c1620..3d2eed2d1 100644 --- a/kv/kubernetes/controller.go +++ b/kv/kubernetes/controller.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/go-kit/log/level" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" @@ -95,13 +96,32 @@ func (c *Client) process() bool { } if !exists { - // Below we will warm up our cache with a Pod, so that we will see a delete for one pod - fmt.Printf("configMap %s does not exist anymore\n", key) + level.Warn(c.logger).Log("msg", "configMap does not exist anymore", "name", key) } else { - // Note that you also have to check the uid if you have a local controlled resource, which - // is dependent on the actual instance, to detect that a Pod was recreated with the same name - fmt.Printf("Sync/Add/Update for configMap %s\n", obj.(*v1.ConfigMap).GetName()) + c.processCMUpdate(obj.(*v1.ConfigMap)) } return true +} + +func (c *Client) processCMUpdate(newValue *v1.ConfigMap) { + c.configMapMtx.Lock() + oldValue := c.configMap + c.configMap = newValue + c.configMapMtx.Unlock() + for key, value := range newValue.BinaryData { + decodedKey, err := convertKeyFromStoreHash(key) + if err != nil { + continue + } + if string(oldValue.BinaryData[key]) == string(value) { + continue + } + decoded, err := c.codec.Decode(value) + if err != nil { + level.Warn(c.logger).Log("msg", "couldn't deserialize key contents") + continue + } + c.watcher.Notify(decodedKey, decoded) + } } diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index ff1b04c5e..bce91e6fc 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -12,6 +12,8 @@ import ( "strings" "sync" + watch "github.com/grafana/dskit/kv/internal/watcher" + "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" @@ -27,6 +29,8 @@ import ( "github.com/grafana/dskit/kv/codec" ) +const hashPrefix = "__hash_" + type Config struct { ConfigMapName string // name of the config map } @@ -51,6 +55,8 @@ type Client struct { configMapMtx sync.RWMutex configMap *v1.ConfigMap + + watcher *watch.Watcher } func realClientGenerator(c *Client) error { @@ -99,14 +105,15 @@ func NewClient(cfg Config, cod codec.Codec, logger log.Logger, registerer promet return newClient(cfg, cod, logger, registerer, realClientGenerator) } -func newClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { +func newClient(cfg Config, cod codec.Codec, logger log.Logger, _ prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { var err error client := &Client{ - logger: logger, - codec: cod, - name: cfg.ConfigMapName, - stopCh: make(chan struct{}), + logger: logger, + codec: cod, + name: cfg.ConfigMapName, + stopCh: make(chan struct{}), + watcher: watch.NewWatcher(logger), } // creates the clientset @@ -128,14 +135,14 @@ func newClient(cfg Config, cod codec.Codec, logger log.Logger, registerer promet } // create a new config map - client.configMap = &v1.ConfigMap{ + configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: client.name, }, // We want non-empty .data and .binaryData; otherwise CAS will fail because it cannot find the parent key BinaryData: map[string][]byte{convertKeyToStore("_"): []byte("_")}, } - client.configMap, err = client.clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), client.configMap, metav1.CreateOptions{}) + client.configMap, err = client.clientset.CoreV1().ConfigMaps(client.namespace).Create(context.Background(), configMap, metav1.CreateOptions{}) if err != nil { return nil, err } @@ -154,7 +161,7 @@ func convertKeyToStore(in string) string { } func convertKeyToStoreHash(in string) string { - return "__hash_" + base64.RawURLEncoding.EncodeToString([]byte(in)) + return hashPrefix + base64.RawURLEncoding.EncodeToString([]byte(in)) } func convertKeyFromStore(in string) (string, error) { @@ -165,9 +172,19 @@ func convertKeyFromStore(in string) (string, error) { return string(body), nil } +func convertKeyFromStoreHash(in string) (string, error) { + withoutPrefix := strings.TrimPrefix(in, hashPrefix) + + if withoutPrefix == in { + return "", fmt.Errorf("key is not prefixed as a hash key") + } + + return convertKeyFromStore(withoutPrefix) +} + // List returns a list of keys under the given prefix. Returned keys will // include the prefix. -func (c *Client) List(ctx context.Context, prefix string) ([]string, error) { +func (c *Client) List(_ context.Context, prefix string) ([]string, error) { c.configMapMtx.RLock() cm := c.configMap c.configMapMtx.RUnlock() @@ -180,7 +197,7 @@ func (c *Client) List(ctx context.Context, prefix string) ([]string, error) { c.logger.Log(fmt.Sprintf("unable to decode key '%s'", keyStore)) continue } - if key == "_" { + if key == "_" { // the value we pre-populate the map with on creation continue } if strings.HasPrefix(key, prefix) { @@ -193,12 +210,12 @@ func (c *Client) List(ctx context.Context, prefix string) ([]string, error) { // Get a specific key. Will use a codec to deserialise key to appropriate type. // If the key does not exist, Get will return nil and no error. -func (c *Client) Get(ctx context.Context, key string) (interface{}, error) { +func (c *Client) Get(_ context.Context, key string) (interface{}, error) { c.configMapMtx.RLock() cm := c.configMap c.configMapMtx.RUnlock() - if key == "_" { + if key == "_" { // the value we pre-populate the map with on creation return nil, nil } @@ -233,9 +250,7 @@ func (c *Client) Delete(ctx context.Context, key string) error { return err } - c.configMapMtx.Lock() - c.configMap = updatedCM - c.configMapMtx.Unlock() + c.processCMUpdate(updatedCM) return nil } @@ -294,9 +309,7 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou return err } - c.configMapMtx.Lock() - c.configMap = updatedCM - c.configMapMtx.Unlock() + c.processCMUpdate(updatedCM) return nil } @@ -310,12 +323,15 @@ func hash(b []byte) []byte { return hasher.Sum(nil) } -// WatchKey calls f whenever the value stored under key changes. func (c *Client) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { - panic("implement me") + c.watcher.WatchKey(ctx, key, f) } -// WatchPrefix calls f whenever any value stored under prefix changes. func (c *Client) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { - panic("implement me") + c.watcher.WatchPrefix(ctx, prefix, f) +} + +func (c *Client) Stop(err error) error { + close(c.stopCh) + return c.watcher.Stop(err) } From 5aad4786c2baeca726fb350673d6953bdc25218a Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Mon, 6 Dec 2021 18:44:20 +0100 Subject: [PATCH 10/23] Remove usage of go-kit/kit --- kv/internal/watcher/watcher_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/internal/watcher/watcher_test.go b/kv/internal/watcher/watcher_test.go index 66fc8819b..9dce3c8f2 100644 --- a/kv/internal/watcher/watcher_test.go +++ b/kv/internal/watcher/watcher_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/wait" - "github.com/go-kit/kit/log" + "github.com/go-kit/log" "github.com/stretchr/testify/assert" ) From 86fd0f98fd2ad4724d5819367f878f8e5a6f4ac5 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Mon, 6 Dec 2021 18:57:45 +0100 Subject: [PATCH 11/23] Fix data race in watcher_test.go --- kv/internal/watcher/watcher.go | 12 ++++++++++++ kv/internal/watcher/watcher_test.go | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/kv/internal/watcher/watcher.go b/kv/internal/watcher/watcher.go index 43b990cc3..aef536353 100644 --- a/kv/internal/watcher/watcher.go +++ b/kv/internal/watcher/watcher.go @@ -165,6 +165,18 @@ func (w *Watcher) Notify(key string, value interface{}) { } } +func (w *Watcher) prefixWatchersCount() int { + w.watchersMu.Lock() + defer w.watchersMu.Unlock() + return len(w.prefixWatchers) +} + +func (w *Watcher) keyWatchersCount() int { + w.watchersMu.Lock() + defer w.watchersMu.Unlock() + return len(w.watchers) +} + func (w *Watcher) Stop(_ error) error { close(w.shutdown) return nil diff --git a/kv/internal/watcher/watcher_test.go b/kv/internal/watcher/watcher_test.go index 9dce3c8f2..698161902 100644 --- a/kv/internal/watcher/watcher_test.go +++ b/kv/internal/watcher/watcher_test.go @@ -29,7 +29,7 @@ func TestWatcher(t *testing.T) { // Wait until the watcher has been registered require.NoError(t, wait.Poll(time.Millisecond*10, time.Second, func() (bool, error) { - return len(w.watchers) > 0, nil + return w.keyWatchersCount() > 0, nil })) w.Notify(key, notifyVal) @@ -58,7 +58,7 @@ func TestWatcher(t *testing.T) { // Wait until the watcher has been registered require.NoError(t, wait.Poll(time.Millisecond*10, time.Second, func() (bool, error) { - return len(w.prefixWatchers) > 0, nil + return w.prefixWatchersCount() > 0, nil })) w.Notify(key, notifyVal) From 2e255e96877fb3ec1067b9d6dced2b0f38a60f36 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Mon, 6 Dec 2021 19:03:47 +0100 Subject: [PATCH 12/23] Add a TODO --- kv/internal/watcher/watcher.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kv/internal/watcher/watcher.go b/kv/internal/watcher/watcher.go index aef536353..1df856ac8 100644 --- a/kv/internal/watcher/watcher.go +++ b/kv/internal/watcher/watcher.go @@ -10,6 +10,8 @@ import ( "github.com/go-kit/log/level" ) +// TODO dimitarvdimitrov this is code copied form memberlist; make sure to refactor memberlist to use this implementation before merging + type Watcher struct { logger log.Logger From 2eed82237e3440aba54c47ea8789e1f3f52d40d2 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 8 Dec 2021 11:29:48 +0100 Subject: [PATCH 13/23] Fix encoding during watch --- go.sum | 9 +++++++++ kv/kubernetes/controller.go | 19 +++++++++++-------- kv/kubernetes/kubernetes_integration_test.go | 1 + 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/go.sum b/go.sum index 1ba50cffe..589749e32 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,7 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -21,12 +22,19 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -124,6 +132,7 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= diff --git a/kv/kubernetes/controller.go b/kv/kubernetes/controller.go index 3d2eed2d1..7753e4597 100644 --- a/kv/kubernetes/controller.go +++ b/kv/kubernetes/controller.go @@ -103,23 +103,26 @@ func (c *Client) process() bool { return true } -func (c *Client) processCMUpdate(newValue *v1.ConfigMap) { +func (c *Client) processCMUpdate(newMap *v1.ConfigMap) { c.configMapMtx.Lock() - oldValue := c.configMap - c.configMap = newValue + oldMap := c.configMap + c.configMap = newMap c.configMapMtx.Unlock() - for key, value := range newValue.BinaryData { + for key, keyHash := range newMap.BinaryData { decodedKey, err := convertKeyFromStoreHash(key) if err != nil { - continue + continue // skip non-hash keys } - if string(oldValue.BinaryData[key]) == string(value) { + if string(oldMap.BinaryData[key]) == string(keyHash) { continue } - decoded, err := c.codec.Decode(value) + + keyContents := newMap.BinaryData[convertKeyToStore(decodedKey)] + + decoded, err := c.codec.Decode(keyContents) if err != nil { - level.Warn(c.logger).Log("msg", "couldn't deserialize key contents") + level.Warn(c.logger).Log("msg", "couldn't deserialize key contents", "error", err, "key", decodedKey) continue } c.watcher.Notify(decodedKey, decoded) diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index f95b75365..9f5147213 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + _ "k8s.io/client-go/plugin/pkg/client/auth" ) func init() { From 900b0a8a0d7f981c440980a23c32096398511e13 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 8 Dec 2021 17:54:53 +0100 Subject: [PATCH 14/23] Add retries and metrics to k8s KV --- kv/kubernetes/kubernetes.go | 103 ++++++++++++------- kv/kubernetes/kubernetes_integration_test.go | 42 +++++++- kv/kubernetes/metrics.go | 40 +++++++ 3 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 kv/kubernetes/metrics.go diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index bce91e6fc..6fc9aa34e 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -7,14 +7,17 @@ import ( "flag" "fmt" "io/ioutil" + "net/http" "os" "sort" "strings" "sync" + "github.com/grafana/dskit/kv/codec" watch "github.com/grafana/dskit/kv/internal/watcher" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -25,20 +28,22 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/workqueue" - - "github.com/grafana/dskit/kv/codec" ) -const hashPrefix = "__hash_" +const ( + hashPrefix = "__hash_" + maxCASRetries = 100 +) type Config struct { - ConfigMapName string // name of the config map + ConfigMapName string `yaml:"config_map_name"` + MetricsNamespace string `yaml:"-"` } // RegisterFlags adds the flags required to config this to the given FlagSet // If prefix is not an empty string it should end with a period. func (cfg *Config) RegisterFlags(f *flag.FlagSet, prefix string) { - f.StringVar(&cfg.ConfigMapName, prefix+"kubernetes.config_map_name", "dskit-ring", "Name of kubernetes configmap to use for KV store.") + f.StringVar(&cfg.ConfigMapName, prefix+"kubernetes.config-map-name", "dskit-ring", "Name of kubernetes configmap to use for KV store.") } type Client struct { @@ -57,6 +62,8 @@ type Client struct { configMap *v1.ConfigMap watcher *watch.Watcher + + metrics *metrics } func realClientGenerator(c *Client) error { @@ -105,7 +112,7 @@ func NewClient(cfg Config, cod codec.Codec, logger log.Logger, registerer promet return newClient(cfg, cod, logger, registerer, realClientGenerator) } -func newClient(cfg Config, cod codec.Codec, logger log.Logger, _ prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { +func newClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { var err error client := &Client{ @@ -114,6 +121,7 @@ func newClient(cfg Config, cod codec.Codec, logger log.Logger, _ prometheus.Regi name: cfg.ConfigMapName, stopCh: make(chan struct{}), watcher: watch.NewWatcher(logger), + metrics: newMetrics(cfg.MetricsNamespace, registerer), } // creates the clientset @@ -266,52 +274,75 @@ func (c *Client) CAS(ctx context.Context, key string, f func(in interface{}) (ou var ( intermediate interface{} err error + patch []byte + encoded []byte + retry = true cm *v1.ConfigMap ) - c.configMapMtx.RLock() - cm = c.configMap - c.configMapMtx.RUnlock() + refreshIntermediate := func() error { + c.configMapMtx.RLock() + cm = c.configMap + c.configMapMtx.RUnlock() + + storedValue, ok := cm.BinaryData[convertKeyToStore(key)] + if ok && storedValue != nil { + intermediate, err = c.codec.Decode(storedValue) + if err != nil { + return err + } + } + return nil + } + + for tries := 0; tries < maxCASRetries && retry; tries++ { + c.metrics.casAttempts.Inc() - storedValue, ok := cm.BinaryData[convertKeyToStore(key)] - if ok && storedValue != nil { - intermediate, err = c.codec.Decode(storedValue) + err = refreshIntermediate() if err != nil { return err } - } - intermediate, _, err = f(intermediate) - if err != nil { - return err - } + intermediate, retry, err = f(intermediate) + if err != nil { + return err + } - if intermediate == nil { - return nil - } + if intermediate == nil { + return nil + } - encoded, err := c.codec.Encode(intermediate) - if err != nil { - return err - } + encoded, err = c.codec.Encode(intermediate) + if err != nil { + return err + } - oldEncodedHash := cm.BinaryData[convertKeyToStoreHash(key)] - newHash := hash(encoded) + oldEncodedHash := cm.BinaryData[convertKeyToStoreHash(key)] + newHash := hash(encoded) - patch, err := preparePatch(key, oldEncodedHash, encoded, newHash) - if err != nil { - return err - } + patch, err = preparePatch(key, oldEncodedHash, encoded, newHash) + if err != nil { + return err + } - updatedCM, err := c.clientset.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) - if err != nil { - return err - } + updatedCM, casErr := c.clientset.CoreV1().ConfigMaps(c.namespace).Patch(ctx, c.name, types.JSONPatchType, patch, metav1.PatchOptions{}) - c.processCMUpdate(updatedCM) + if statusErr, ok := casErr.(*errors.StatusError); ok && statusErr.ErrStatus.Code == http.StatusUnprocessableEntity { + err = casErr + continue + } else if casErr != nil { + c.metrics.casFailures.Inc() + return casErr + } - return nil + c.processCMUpdate(updatedCM) + c.metrics.casSuccesses.Inc() + return nil + } + + c.metrics.casFailures.Inc() + return fmt.Errorf("exceeded maximum CAS retries (%d)", maxCASRetries) } func hash(b []byte) []byte { diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 9f5147213..590643b6a 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -3,6 +3,7 @@ package kubernetes import ( "context" "math/rand" + "net/http" "os" "testing" "time" @@ -10,8 +11,11 @@ import ( "github.com/go-kit/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" _ "k8s.io/client-go/plugin/pkg/client/auth" + k8s_testing "k8s.io/client-go/testing" ) func init() { @@ -76,7 +80,7 @@ func newTestClient(t testing.TB) *Client { logger, ) t.Cleanup(func() { - closer.Close() + _ = closer.Close() close(client.stopCh) }) @@ -151,3 +155,39 @@ func Test_Delete(t *testing.T) { assert.Nil(t, value) }) } + +func Test_CAS(t *testing.T) { + t.Run("retry=true is respected", func(t *testing.T) { + c := newTestClient(t) + mockK8sResponse(c, http.StatusUnprocessableEntity) + + var counter int + + require.Error(t, c.CAS(context.Background(), "/test", func(interface{}) (interface{}, bool, error) { + counter++ + return "something", true, nil + })) + + assert.Equal(t, maxCASRetries, counter) + }) + + t.Run("retry=false is also respected", func(t *testing.T) { + c := newTestClient(t) + mockK8sResponse(c, http.StatusUnprocessableEntity) + + var counter int + + require.Error(t, c.CAS(context.Background(), "/test", func(interface{}) (interface{}, bool, error) { + counter++ + return "something", false, nil + })) + + assert.Equal(t, 1, counter) + }) +} + +func mockK8sResponse(c *Client, status int) { + c.clientset.(fakeClientset).PrependReactor("*", "*", func(action k8s_testing.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, &errors.StatusError{ErrStatus: metav1.Status{Code: int32(status)}} + }) +} diff --git a/kv/kubernetes/metrics.go b/kv/kubernetes/metrics.go new file mode 100644 index 000000000..55859f46b --- /dev/null +++ b/kv/kubernetes/metrics.go @@ -0,0 +1,40 @@ +package kubernetes + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type metrics struct { + casAttempts prometheus.Counter + casSuccesses prometheus.Counter + casFailures prometheus.Counter +} + +func newMetrics(namespace string, registerer prometheus.Registerer) *metrics { + const subsystem = "kubernetes_ring" + m := metrics{} + + m.casAttempts = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "cas_attempt_total", + Help: "Attempted CAS operations", + }) + + m.casSuccesses = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "cas_success_total", + Help: "Successful CAS operations", + }) + + m.casFailures = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "cas_failure_total", + Help: "Failed CAS operations", + }) + + return &m +} From 73cdf22f7e95b02c10e6ba1505509c786e9e9e76 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 9 Dec 2021 12:00:20 +0100 Subject: [PATCH 15/23] Add tests for k8s CAS --- kv/kubernetes/kubernetes.go | 1 - kv/kubernetes/kubernetes_integration_test.go | 63 +++++++++++++------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index 6fc9aa34e..56109172d 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -17,7 +17,6 @@ import ( watch "github.com/grafana/dskit/kv/internal/watcher" "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 590643b6a..42ae90b48 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" _ "k8s.io/client-go/plugin/pkg/client/auth" k8s_testing "k8s.io/client-go/testing" ) @@ -104,13 +105,6 @@ func Test_Integration_Simple(t *testing.T) { return })) - require.NoError(t, c.CAS(context.Background(), "/test", func(old interface{}) (out interface{}, retry bool, err error) { - assert.Equal(t, "test", old) - out = nil - retry = false - return - })) - keys, err = c.List(context.TODO(), "/test") require.NoError(t, err) assert.ElementsMatch(t, []string{"/test"}, keys) @@ -157,36 +151,61 @@ func Test_Delete(t *testing.T) { } func Test_CAS(t *testing.T) { - t.Run("retry=true is respected", func(t *testing.T) { + t.Run("CAS is retried is retry=true", func(t *testing.T) { + const key = "/test" + var casAttempts int c := newTestClient(t) - mockK8sResponse(c, http.StatusUnprocessableEntity) - - var counter int + // API server returns 422 (Unprocessable Entity) when the "test" JSON patch operation failed + mockK8sResponseStatusCode(t, c, http.StatusUnprocessableEntity) - require.Error(t, c.CAS(context.Background(), "/test", func(interface{}) (interface{}, bool, error) { - counter++ + err := c.CAS(context.Background(), key, func(interface{}) (interface{}, bool, error) { + casAttempts++ return "something", true, nil - })) + }) - assert.Equal(t, maxCASRetries, counter) + assert.Error(t, err) + + assert.Equal(t, maxCASRetries, casAttempts) }) - t.Run("retry=false is also respected", func(t *testing.T) { + t.Run("CAS is not retried if retry=false", func(t *testing.T) { + const key = "/test" + var casAttempts int c := newTestClient(t) - mockK8sResponse(c, http.StatusUnprocessableEntity) - var counter int + // API server returns 422 (Unprocessable Entity) when the "test" JSON patch operation failed + mockK8sResponseStatusCode(t, c, http.StatusUnprocessableEntity) - require.Error(t, c.CAS(context.Background(), "/test", func(interface{}) (interface{}, bool, error) { - counter++ + assert.Error(t, c.CAS(context.Background(), key, func(interface{}) (interface{}, bool, error) { + casAttempts++ return "something", false, nil })) - assert.Equal(t, 1, counter) + assert.Equal(t, 1, casAttempts) + }) + + t.Run("non-compare errors are not retried", func(t *testing.T) { + const key = "/test" + var casAttempts int + c := newTestClient(t) + mockK8sResponseStatusCode(t, c, http.StatusBadRequest) + + assert.Error(t, c.CAS(context.Background(), key, func(interface{}) (interface{}, bool, error) { + casAttempts++ + return "something", true, nil + })) + + assert.Equal(t, 1, casAttempts) }) } -func mockK8sResponse(c *Client, status int) { +func updateK8sConfigMap(t *testing.T, c *Client, object runtime.Object) { + var configmapsResource = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} + + require.NoError(t, c.clientset.(fakeClientset).Tracker().Update(configmapsResource, object, "")) +} + +func mockK8sResponseStatusCode(_ *testing.T, c *Client, status int) { c.clientset.(fakeClientset).PrependReactor("*", "*", func(action k8s_testing.Action) (handled bool, ret runtime.Object, err error) { return true, nil, &errors.StatusError{ErrStatus: metav1.Status{Code: int32(status)}} }) From 230c1dfe9b970a78452d81827306a1f1fc0ec293 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 9 Dec 2021 16:28:02 +0100 Subject: [PATCH 16/23] Remove deadcode from kubernetes_integration_test.go --- kv/kubernetes/kubernetes_integration_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kv/kubernetes/kubernetes_integration_test.go b/kv/kubernetes/kubernetes_integration_test.go index 42ae90b48..dfdfe63e5 100644 --- a/kv/kubernetes/kubernetes_integration_test.go +++ b/kv/kubernetes/kubernetes_integration_test.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" _ "k8s.io/client-go/plugin/pkg/client/auth" k8s_testing "k8s.io/client-go/testing" ) @@ -199,12 +198,6 @@ func Test_CAS(t *testing.T) { }) } -func updateK8sConfigMap(t *testing.T, c *Client, object runtime.Object) { - var configmapsResource = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} - - require.NoError(t, c.clientset.(fakeClientset).Tracker().Update(configmapsResource, object, "")) -} - func mockK8sResponseStatusCode(_ *testing.T, c *Client, status int) { c.clientset.(fakeClientset).PrependReactor("*", "*", func(action k8s_testing.Action) (handled bool, ret runtime.Object, err error) { return true, nil, &errors.StatusError{ErrStatus: metav1.Status{Code: int32(status)}} From ef54cbf89809a1d22e4b8284b0ba851bbddf1e2a Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 10:25:22 +0100 Subject: [PATCH 17/23] Add docs to kv/kubernetes --- kv/kubernetes/kubernetes.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index 56109172d..f9fbffb32 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -1,3 +1,5 @@ +// Package kubernetes provides a KV store implementation based on a kubernetes configmap. +// The entrypoint to the package is NewClient. package kubernetes import ( @@ -107,6 +109,13 @@ func realClientGenerator(c *Client) error { return nil } +// NewClient returns a new KV store client. It will default to reading the KUBECONFIG environment variable. +// If KUBECONFIG is not set, it will use the rest.InClusterConfig to connect to the kubernetes cluster. +// +// The client will try to get the configmap specified in cfg. If the configmap doesn't exist, NewClient will create it +// before returning. +// +// The Client needs a role with CREATE, LIST, GET, PATCH and WATCH verbs on configmaps in the namespace. func NewClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer) (*Client, error) { return newClient(cfg, cod, logger, registerer, realClientGenerator) } From a08e42a1c99b26b027df1700824d6b8e81967204 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 11:20:46 +0100 Subject: [PATCH 18/23] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f50d722e..607f58ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,5 +22,6 @@ * [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #76 #77 #84 #91 * [ENHANCEMENT] Memberlist: prepare the data to send on the write before starting counting the elapsed time for `-memberlist.packet-write-timeout`, in order to reduce chances we hit the timeout when sending a packet to other node. #89 * [ENHANCEMENT] flagext: for cases such as `DeprecatedFlag()` that need a logger, add RegisterFlagsWithLogger. #80 +* [ENHANCEMENT] Implement kubernetes KV store client. #94 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59 * [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85 From ab789cf841a0e49e779b3e0366f873865d92fcb4 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 11:35:10 +0100 Subject: [PATCH 19/23] Update config flags --- kv/client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kv/client.go b/kv/client.go index f299c67cc..55e8bc457 100644 --- a/kv/client.go +++ b/kv/client.go @@ -74,13 +74,15 @@ func (cfg *Config) RegisterFlagsWithPrefix(flagsPrefix, defaultPrefix string, f cfg.Consul.RegisterFlags(f, flagsPrefix) cfg.Etcd.RegisterFlagsWithPrefix(f, flagsPrefix) cfg.Multi.RegisterFlagsWithPrefix(f, flagsPrefix) - cfg.Kubernetes.RegisterFlags(f, flagsPrefix) if flagsPrefix == "" { flagsPrefix = "ring." } + + cfg.Kubernetes.RegisterFlags(f, flagsPrefix) + f.StringVar(&cfg.Prefix, flagsPrefix+"prefix", defaultPrefix, "The prefix for the keys in the store. Should end with a /.") - f.StringVar(&cfg.Store, flagsPrefix+"store", "consul", "Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi.") + f.StringVar(&cfg.Store, flagsPrefix+"store", "consul", "Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, kubernetes, multi.") } // Client is a high-level client for key-value stores (such as Etcd and From 337a5ff728a0594065285ee4879aff68f20f8078 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 14:06:45 +0100 Subject: [PATCH 20/23] Refactor memberlist to use shared watcher implementation --- .../{watcher/watcher.go => watch/watch.go} | 19 +- .../watcher_test.go => watch/watch_test.go} | 2 +- kv/kubernetes/kubernetes.go | 11 +- kv/kubernetes/metrics.go | 29 ++-- kv/memberlist/memberlist_client.go | 164 +++--------------- kv/memberlist/memberlist_client_test.go | 4 +- kv/memberlist/metrics.go | 41 ++--- 7 files changed, 79 insertions(+), 191 deletions(-) rename kv/internal/{watcher/watcher.go => watch/watch.go} (87%) rename kv/internal/{watcher/watcher_test.go => watch/watch_test.go} (98%) diff --git a/kv/internal/watcher/watcher.go b/kv/internal/watch/watch.go similarity index 87% rename from kv/internal/watcher/watcher.go rename to kv/internal/watch/watch.go index 1df856ac8..9b8c80009 100644 --- a/kv/internal/watcher/watcher.go +++ b/kv/internal/watch/watch.go @@ -6,12 +6,11 @@ import ( "sync" "github.com/go-kit/log" - "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) -// TODO dimitarvdimitrov this is code copied form memberlist; make sure to refactor memberlist to use this implementation before merging - type Watcher struct { logger log.Logger @@ -22,6 +21,8 @@ type Watcher struct { watchersMu sync.Mutex watchers map[string][]chan update prefixWatchers map[string][]chan update + + watchPrefixDroppedNotifications *prometheus.CounterVec } type update struct { @@ -29,12 +30,17 @@ type update struct { value interface{} } -func NewWatcher(l log.Logger) *Watcher { +func NewWatcher(l log.Logger, r prometheus.Registerer) *Watcher { return &Watcher{ logger: l, shutdown: make(chan struct{}), watchers: make(map[string][]chan update), prefixWatchers: make(map[string][]chan update), + + watchPrefixDroppedNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "watch_prefix_dropped_notifications", + Help: "Number of dropped notifications in WatchPrefix function", + }, []string{"prefix"}), } } @@ -160,6 +166,11 @@ func (w *Watcher) Notify(key string, value interface{}) { case pw <- watchKey: // notification sent. default: + c, _ := w.watchPrefixDroppedNotifications.GetMetricWithLabelValues(p) + if c != nil { + c.Inc() + } + level.Warn(w.logger).Log("msg", "failed to send notification to prefix watcher", "prefix", p) } } diff --git a/kv/internal/watcher/watcher_test.go b/kv/internal/watch/watch_test.go similarity index 98% rename from kv/internal/watcher/watcher_test.go rename to kv/internal/watch/watch_test.go index 698161902..ae7c63849 100644 --- a/kv/internal/watcher/watcher_test.go +++ b/kv/internal/watch/watch_test.go @@ -79,5 +79,5 @@ func newTestWatcher() *Watcher { logger = log.NewNopLogger() } - return NewWatcher(logger) + return NewWatcher(logger, nil) } diff --git a/kv/kubernetes/kubernetes.go b/kv/kubernetes/kubernetes.go index f9fbffb32..514977e84 100644 --- a/kv/kubernetes/kubernetes.go +++ b/kv/kubernetes/kubernetes.go @@ -16,7 +16,7 @@ import ( "sync" "github.com/grafana/dskit/kv/codec" - watch "github.com/grafana/dskit/kv/internal/watcher" + "github.com/grafana/dskit/kv/internal/watch" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -37,8 +37,7 @@ const ( ) type Config struct { - ConfigMapName string `yaml:"config_map_name"` - MetricsNamespace string `yaml:"-"` + ConfigMapName string `yaml:"config_map_name"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -123,13 +122,15 @@ func NewClient(cfg Config, cod codec.Codec, logger log.Logger, registerer promet func newClient(cfg Config, cod codec.Codec, logger log.Logger, registerer prometheus.Registerer, clientGenerator func(*Client) error) (*Client, error) { var err error + m := newMetrics(registerer) + client := &Client{ logger: logger, codec: cod, name: cfg.ConfigMapName, stopCh: make(chan struct{}), - watcher: watch.NewWatcher(logger), - metrics: newMetrics(cfg.MetricsNamespace, registerer), + watcher: watch.NewWatcher(logger, prometheus.WrapRegistererWithPrefix(m.prefix, registerer)), + metrics: m, } // creates the clientset diff --git a/kv/kubernetes/metrics.go b/kv/kubernetes/metrics.go index 55859f46b..4c3bfe047 100644 --- a/kv/kubernetes/metrics.go +++ b/kv/kubernetes/metrics.go @@ -6,34 +6,33 @@ import ( ) type metrics struct { + prefix string + casAttempts prometheus.Counter casSuccesses prometheus.Counter casFailures prometheus.Counter } -func newMetrics(namespace string, registerer prometheus.Registerer) *metrics { - const subsystem = "kubernetes_ring" - m := metrics{} +func newMetrics(registerer prometheus.Registerer) *metrics { + m := metrics{ + prefix: "kubernetes_ring_", + } + + registerer = prometheus.WrapRegistererWithPrefix(m.prefix, registerer) m.casAttempts = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "cas_attempt_total", - Help: "Attempted CAS operations", + Name: "cas_attempt_total", + Help: "Attempted CAS operations", }) m.casSuccesses = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "cas_success_total", - Help: "Successful CAS operations", + Name: "cas_success_total", + Help: "Successful CAS operations", }) m.casFailures = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "cas_failure_total", - Help: "Failed CAS operations", + Name: "cas_failure_total", + Help: "Failed CAS operations", }) return &m diff --git a/kv/memberlist/memberlist_client.go b/kv/memberlist/memberlist_client.go index d7ad176d0..0283c172c 100644 --- a/kv/memberlist/memberlist_client.go +++ b/kv/memberlist/memberlist_client.go @@ -21,6 +21,7 @@ import ( "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/kv/codec" + "github.com/grafana/dskit/kv/internal/watch" "github.com/grafana/dskit/services" ) @@ -65,7 +66,7 @@ func (c *Client) Get(ctx context.Context, key string) (interface{}, error) { return nil, err } - return c.kv.Get(key, c.codec) + return c.kv.Get(key) } // Delete is part of kv.Client interface. @@ -237,10 +238,7 @@ type KV struct { // Codec registry codecs map[string]codec.Codec - // Key watchers - watchersMu sync.Mutex - watchers map[string][]chan string - prefixWatchers map[string][]chan string + watcher *watch.Watcher // Buffers with sent and received messages. Used for troubleshooting only. // New messages are appended, old messages (based on configured size limit) removed from the front. @@ -343,12 +341,11 @@ func NewKV(cfg KVConfig, logger log.Logger, dnsProvider DNSProvider, registerer registerer: registerer, provider: dnsProvider, - store: make(map[string]valueDesc), - codecs: make(map[string]codec.Codec), - watchers: make(map[string][]chan string), - prefixWatchers: make(map[string][]chan string), - shutdown: make(chan struct{}), - maxCasRetries: maxCasRetries, + store: make(map[string]valueDesc), + codecs: make(map[string]codec.Codec), + shutdown: make(chan struct{}), + maxCasRetries: maxCasRetries, + watcher: watch.NewWatcher(logger, prometheus.WrapRegistererWithPrefix(fmt.Sprintf("%s_%s_", cfg.MetricsNamespace, metricsSubsystem), registerer)), } mlkv.createAndRegisterMetrics() @@ -625,13 +622,13 @@ func (m *KV) List(prefix string) []string { // Get returns current value associated with given key. // No communication with other nodes in the cluster is done here. -func (m *KV) Get(key string, codec codec.Codec) (interface{}, error) { - val, _, err := m.get(key, codec) - return val, err +func (m *KV) Get(key string) (interface{}, error) { + val, _ := m.get(key) + return val, nil } // Returns current value with removed tombstones. -func (m *KV) get(key string, codec codec.Codec) (out interface{}, version uint, err error) { +func (m *KV) get(key string) (out interface{}, version uint) { m.storeMu.Lock() v := m.store[key].Clone() m.storeMu.Unlock() @@ -642,52 +639,15 @@ func (m *KV) get(key string, codec codec.Codec) (out interface{}, version uint, _, _ = v.value.RemoveTombstones(time.Time{}) } - return v.value, v.version, nil + return v.value, v.version } // WatchKey watches for value changes for given key. When value changes, 'f' function is called with the // latest value. Notifications that arrive while 'f' is running are coalesced into one subsequent 'f' call. // // Watching ends when 'f' returns false, context is done, or this client is shut down. -func (m *KV) WatchKey(ctx context.Context, key string, codec codec.Codec, f func(interface{}) bool) { - // keep one extra notification, to avoid missing notification if we're busy running the function - w := make(chan string, 1) - - // register watcher - m.watchersMu.Lock() - m.watchers[key] = append(m.watchers[key], w) - m.watchersMu.Unlock() - - defer func() { - // unregister watcher on exit - m.watchersMu.Lock() - defer m.watchersMu.Unlock() - - removeWatcherChannel(key, w, m.watchers) - }() - - for { - select { - case <-w: - // value changed - val, _, err := m.get(key, codec) - if err != nil { - level.Warn(m.logger).Log("msg", "failed to decode value while watching for changes", "key", key, "err", err) - continue - } - - if !f(val) { - return - } - - case <-m.shutdown: - // stop watching on shutdown - return - - case <-ctx.Done(): - return - } - } +func (m *KV) WatchKey(ctx context.Context, key string, _ codec.Codec, f func(interface{}) bool) { + m.watcher.WatchKey(ctx, key, f) } // WatchPrefix watches for any change of values stored under keys with given prefix. When change occurs, @@ -696,94 +656,13 @@ func (m *KV) WatchKey(ctx context.Context, key string, codec codec.Codec, f func // some notifications may be lost. // // Watching ends when 'f' returns false, context is done, or this client is shut down. -func (m *KV) WatchPrefix(ctx context.Context, prefix string, codec codec.Codec, f func(string, interface{}) bool) { - // we use bigger buffer here, since keys are interesting and we don't want to lose them. - w := make(chan string, 16) - - // register watcher - m.watchersMu.Lock() - m.prefixWatchers[prefix] = append(m.prefixWatchers[prefix], w) - m.watchersMu.Unlock() - - defer func() { - // unregister watcher on exit - m.watchersMu.Lock() - defer m.watchersMu.Unlock() - - removeWatcherChannel(prefix, w, m.prefixWatchers) - }() - - for { - select { - case key := <-w: - val, _, err := m.get(key, codec) - if err != nil { - level.Warn(m.logger).Log("msg", "failed to decode value while watching for changes", "key", key, "err", err) - continue - } - - if !f(key, val) { - return - } - - case <-m.shutdown: - // stop watching on shutdown - return - - case <-ctx.Done(): - return - } - } -} - -func removeWatcherChannel(k string, w chan string, watchers map[string][]chan string) { - ws := watchers[k] - for ix, kw := range ws { - if kw == w { - ws = append(ws[:ix], ws[ix+1:]...) - break - } - } - - if len(ws) > 0 { - watchers[k] = ws - } else { - delete(watchers, k) - } +func (m *KV) WatchPrefix(ctx context.Context, prefix string, _ codec.Codec, f func(string, interface{}) bool) { + m.watcher.WatchPrefix(ctx, prefix, f) } func (m *KV) notifyWatchers(key string) { - m.watchersMu.Lock() - defer m.watchersMu.Unlock() - - for _, kw := range m.watchers[key] { - select { - case kw <- key: - // notification sent. - default: - // cannot send notification to this watcher at the moment - // but since this is a buffered channel, it means that - // there is already a pending notification anyway - } - } - - for p, ws := range m.prefixWatchers { - if strings.HasPrefix(key, p) { - for _, pw := range ws { - select { - case pw <- key: - // notification sent. - default: - c, _ := m.watchPrefixDroppedNotifications.GetMetricWithLabelValues(p) - if c != nil { - c.Inc() - } - - level.Warn(m.logger).Log("msg", "failed to send notification to prefix watcher", "prefix", p) - } - } - } - } + val, _ := m.get(key) + m.watcher.Notify(key, val) } // CAS implements Compare-And-Set/Swap operation. @@ -853,10 +732,7 @@ outer: // returns change, error (or nil, if CAS succeeded), and whether to retry or not. // returns errNoChangeDetected if merge failed to detect change in f's output. func (m *KV) trySingleCas(key string, codec codec.Codec, f func(in interface{}) (out interface{}, retry bool, err error)) (Mergeable, uint, bool, error) { - val, ver, err := m.get(key, codec) - if err != nil { - return nil, 0, false, fmt.Errorf("failed to get value: %v", err) - } + val, ver := m.get(key) out, retry, err := f(val) if err != nil { diff --git a/kv/memberlist/memberlist_client_test.go b/kv/memberlist/memberlist_client_test.go index a8c6ab590..b5bf6b14a 100644 --- a/kv/memberlist/memberlist_client_test.go +++ b/kv/memberlist/memberlist_client_test.go @@ -1001,13 +1001,13 @@ func TestMultipleCodecs(t *testing.T) { // Now read both values from second KV. It should have both values by now. // fetch directly from single KV, to see that both are stored in the same one - val, err := mkv2.Get("data", dataCodec{}) + val, err := mkv2.Get("data") require.NoError(t, err) require.NotNil(t, val) require.NotZero(t, val.(*data).Members["test"].Timestamp) require.Equal(t, ACTIVE, val.(*data).Members["test"].State) - val, err = mkv2.Get("counter", distributedCounterCodec{}) + val, err = mkv2.Get("counter") require.NoError(t, err) require.NotNil(t, val) require.Equal(t, 5, val.(distributedCounter)["test"]) diff --git a/kv/memberlist/metrics.go b/kv/memberlist/metrics.go index c7d3f01c2..c20a5379e 100644 --- a/kv/memberlist/metrics.go +++ b/kv/memberlist/metrics.go @@ -12,61 +12,62 @@ import ( "github.com/grafana/dskit/services" ) +const metricsSubsystem = "memberlist_client" + func (m *KV) createAndRegisterMetrics() { - const subsystem = "memberlist_client" m.numberOfReceivedMessages = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "received_broadcasts_total", Help: "Number of received broadcast user messages", }) m.totalSizeOfReceivedMessages = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "received_broadcasts_bytes_total", Help: "Total size of received broadcast user messages", }) m.numberOfInvalidReceivedMessages = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "received_broadcasts_invalid_total", Help: "Number of received broadcast user messages that were invalid. Hopefully 0.", }) m.numberOfPushes = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "state_pushes_total", Help: "How many times did this node push its full state to another node", }) m.totalSizeOfPushes = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "state_pushes_bytes_total", Help: "Total size of pushed state", }) m.numberOfPulls = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "state_pulls_total", Help: "How many times did this node pull full state from another node", }) m.totalSizeOfPulls = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "state_pulls_bytes_total", Help: "Total size of pulled state", }) m.numberOfBroadcastMessagesInQueue = promauto.With(m.registerer).NewGaugeFunc(prometheus.GaugeOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "messages_in_broadcast_queue", Help: "Number of user messages in the broadcast queue", }, func() float64 { @@ -79,61 +80,61 @@ func (m *KV) createAndRegisterMetrics() { m.totalSizeOfBroadcastMessagesInQueue = promauto.With(m.registerer).NewGauge(prometheus.GaugeOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "messages_in_broadcast_queue_bytes", Help: "Total size of messages waiting in the broadcast queue", }) m.numberOfBroadcastMessagesDropped = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "messages_to_broadcast_dropped_total", Help: "Number of broadcast messages intended to be sent but were dropped due to encoding errors or for being too big", }) m.casAttempts = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "cas_attempt_total", Help: "Attempted CAS operations", }) m.casSuccesses = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "cas_success_total", Help: "Successful CAS operations", }) m.casFailures = promauto.With(m.registerer).NewCounter(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "cas_failure_total", Help: "Failed CAS operations", }) m.storeValuesDesc = prometheus.NewDesc( - prometheus.BuildFQName(m.cfg.MetricsNamespace, subsystem, "kv_store_count"), // gauge + prometheus.BuildFQName(m.cfg.MetricsNamespace, metricsSubsystem, "kv_store_count"), // gauge "Number of values in KV Store", nil, nil) m.storeTombstones = promauto.With(m.registerer).NewGaugeVec(prometheus.GaugeOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "kv_store_value_tombstones", Help: "Number of tombstones currently present in KV store values", }, []string{"key"}) m.storeRemovedTombstones = promauto.With(m.registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "kv_store_value_tombstones_removed_total", Help: "Total number of tombstones which have been removed from KV store values", }, []string{"key"}) m.memberlistMembersCount = promauto.With(m.registerer).NewGaugeFunc(prometheus.GaugeOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "cluster_members_count", Help: "Number of members in memberlist cluster", }, func() float64 { @@ -146,7 +147,7 @@ func (m *KV) createAndRegisterMetrics() { m.memberlistHealthScore = promauto.With(m.registerer).NewGaugeFunc(prometheus.GaugeOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "cluster_node_health_score", Help: "Health score of this cluster. Lower value is better. 0 = healthy", }, func() float64 { @@ -159,7 +160,7 @@ func (m *KV) createAndRegisterMetrics() { m.watchPrefixDroppedNotifications = promauto.With(m.registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: m.cfg.MetricsNamespace, - Subsystem: subsystem, + Subsystem: metricsSubsystem, Name: "watch_prefix_dropped_notifications", Help: "Number of dropped notifications in WatchPrefix function", }, []string{"prefix"}) From 0e9f3be98dad5d75e61bc6a39fafb4f79daa4d04 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 14:11:23 +0100 Subject: [PATCH 21/23] Further simplify signatures in memberlist --- kv/memberlist/memberlist_client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kv/memberlist/memberlist_client.go b/kv/memberlist/memberlist_client.go index 0283c172c..2e26c89ea 100644 --- a/kv/memberlist/memberlist_client.go +++ b/kv/memberlist/memberlist_client.go @@ -91,7 +91,7 @@ func (c *Client) WatchKey(ctx context.Context, key string, f func(interface{}) b return } - c.kv.WatchKey(ctx, key, c.codec, f) + c.kv.WatchKey(ctx, key, f) } // WatchPrefix calls f whenever any value stored under prefix changes. @@ -102,7 +102,7 @@ func (c *Client) WatchPrefix(ctx context.Context, prefix string, f func(string, return } - c.kv.WatchPrefix(ctx, prefix, c.codec, f) + c.kv.WatchPrefix(ctx, prefix, f) } // We want to use KV in Running and Stopping states. @@ -646,7 +646,7 @@ func (m *KV) get(key string) (out interface{}, version uint) { // latest value. Notifications that arrive while 'f' is running are coalesced into one subsequent 'f' call. // // Watching ends when 'f' returns false, context is done, or this client is shut down. -func (m *KV) WatchKey(ctx context.Context, key string, _ codec.Codec, f func(interface{}) bool) { +func (m *KV) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { m.watcher.WatchKey(ctx, key, f) } @@ -656,7 +656,7 @@ func (m *KV) WatchKey(ctx context.Context, key string, _ codec.Codec, f func(int // some notifications may be lost. // // Watching ends when 'f' returns false, context is done, or this client is shut down. -func (m *KV) WatchPrefix(ctx context.Context, prefix string, _ codec.Codec, f func(string, interface{}) bool) { +func (m *KV) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { m.watcher.WatchPrefix(ctx, prefix, f) } From bf386c85dbf48c5debf8fa8ca65f82788b10a151 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 14:14:24 +0100 Subject: [PATCH 22/23] Restore comment in go.mod --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 78669adfa..b6a9d86dd 100644 --- a/go.mod +++ b/go.mod @@ -41,4 +41,6 @@ require ( replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.21.7 +// Replace memberlist with our fork which includes some fixes that haven't been +// merged upstream yet. replace github.com/hashicorp/memberlist v0.2.3 => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b From 1a392e737e1cc5f04a83043e427ef525257ec1c3 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Fri, 10 Dec 2021 14:18:42 +0100 Subject: [PATCH 23/23] Propagate stops to watcher in memberlist --- kv/memberlist/memberlist_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kv/memberlist/memberlist_client.go b/kv/memberlist/memberlist_client.go index 2e26c89ea..8aa18b431 100644 --- a/kv/memberlist/memberlist_client.go +++ b/kv/memberlist/memberlist_client.go @@ -597,6 +597,7 @@ func (m *KV) stopping(_ error) error { } close(m.shutdown) + _ = m.watcher.Stop(nil) err = m.memberlist.Shutdown() if err != nil {