diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml index 5ec1f7a31e..1c18d7a9c9 100644 --- a/.github/workflows/test-scripts.yml +++ b/.github/workflows/test-scripts.yml @@ -23,6 +23,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: '~1.16' - run: | sudo apt-get install conntrack curl -sLo minikube "$(curl -sL https://api.github.com/repos/kubernetes/minikube/releases/latest | jq -r '[.assets[] | select(.name == "minikube-linux-amd64")] | first | .browser_download_url')" @@ -34,6 +37,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: '~1.16' - run: | curl -sLo kind "$(curl -sL https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | jq -r '[.assets[] | select(.name == "kind-linux-amd64")] | first | .browser_download_url')" chmod +x kind diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go index 805334efbb..f8128a4c2c 100644 --- a/cmd/catalog/main.go +++ b/cmd/catalog/main.go @@ -28,7 +28,8 @@ const ( catalogNamespaceEnvVarName = "GLOBAL_CATALOG_NAMESPACE" defaultWakeupInterval = 15 * time.Minute defaultCatalogNamespace = "openshift-operator-lifecycle-manager" - defaultConfigMapServerImage = "quay.io/operatorframework/configmap-operator-registry:latest" + defaultConfigMapServerImage = "quay.io/operator-framework/configmap-operator-registry:latest" + defaultOPMImage = "quay.io/operator-framework/upstream-opm-builder:latest" defaultUtilImage = "quay.io/operator-framework/olm:latest" defaultOperatorName = "" ) @@ -47,6 +48,9 @@ var ( configmapServerImage = flag.String( "configmapServerImage", defaultConfigMapServerImage, "the image to use for serving the operator registry api for a configmap") + opmImage = flag.String( + "opmImage", defaultOPMImage, "the image to use for unpacking bundle content with opm") + utilImage = flag.String( "util-image", defaultUtilImage, "an image containing custom olm utilities") @@ -131,7 +135,7 @@ func main() { } // Create a new instance of the operator. - op, err := catalog.NewOperator(ctx, *kubeConfigPath, utilclock.RealClock{}, logger, *wakeupInterval, *configmapServerImage, *utilImage, *catalogNamespace, k8sscheme.Scheme, *installPlanTimeout, *bundleUnpackTimeout) + op, err := catalog.NewOperator(ctx, *kubeConfigPath, utilclock.RealClock{}, logger, *wakeupInterval, *configmapServerImage, *opmImage, *utilImage, *catalogNamespace, k8sscheme.Scheme, *installPlanTimeout, *bundleUnpackTimeout) if err != nil { log.Panicf("error configuring operator: %s", err.Error()) } diff --git a/go.mod b/go.mod index 3bccaa8fd8..8a02b55ae5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/openshift/api v0.0.0-20200331152225-585af27e34fd github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 github.com/operator-framework/api v0.10.1 - github.com/operator-framework/operator-registry v1.13.6 + github.com/operator-framework/operator-registry v1.17.5 github.com/otiai10/copy v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 diff --git a/go.sum b/go.sum index 0f485b6228..6a43a12331 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,7 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Masterminds/vcs v1.13.1 h1:NL3G1X7/7xduQtA2sJLpVpfHTNBALVNSjob6KEjPXNQ= github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= @@ -103,6 +104,7 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -111,7 +113,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -119,6 +121,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -294,6 +297,7 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -319,6 +323,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -336,12 +341,21 @@ github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49P github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-bindata/go-bindata/v3 v3.1.3 h1:F0nVttLC3ws0ojc7p60veTurcOm//D4QBODNM7EGrCI= github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.1.0 h1:4pl5BV4o7ZG/lterP4S6WzJ6xr49Ba5ET9ygheTYahk= +github.com/go-git/go-billy/v5 v5.1.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= +github.com/go-git/go-git/v5 v5.3.0 h1:8WKMtJR2j8RntEXR/uvTKagfEt4GYlwQ7mntE4+0GWc= +github.com/go-git/go-git/v5 v5.3.0/go.mod h1:xdX4bWJ48aOrdhnl2XqHYstHbbp6+LFS4r4X+lNVprw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -355,9 +369,10 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -533,7 +548,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= @@ -568,6 +582,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-health-probe v0.3.2/go.mod h1:izVOQ4RWbjUR6lm4nn+VLJyQ+FyaiGmprEYgI04Gs7U= +github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= +github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= +github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -601,8 +619,7 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -618,9 +635,12 @@ github.com/itchyny/gojq v0.11.0 h1:z7HnaKZ6erVzxPo3BkhJBG7jHmzKdAW8Hcse5N6YAic= github.com/itchyny/gojq v0.11.0/go.mod h1:my6D2qN2Sm6qa+/5GsPDUZlCWGR+U8Qsa9he76sudv0= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 h1:IPJ3dvxmJ4uczJe5YQdrYB16oTJlGSC/OyZDqUk9xX4= github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -628,6 +648,8 @@ github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d h1:A2/B900ip/Z20TzkLeGRNy1s6J2HmH9AmGt+dHyqb4I= +github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d/go.mod h1:7HQupe4vyNxMKXmM5DFuwXHsqwMyglcYmZBtlDPIcZ8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -649,6 +671,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= @@ -660,6 +683,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -731,8 +756,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 h1:g+4J5sZg6osfvEfkRZxJ1em0VT95/UOZgi/l7zi1/oE= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU= -github.com/mikefarah/yq/v2 v2.4.1/go.mod h1:i8SYf1XdgUvY2OFwSqGAtWOOgimD2McJ6iutoxRm4k0= github.com/mikefarah/yq/v3 v3.0.0-20201202084205-8846255d1c37 h1:lPmsut5Sk7eK2BmDXuvNEvMbT7MkAJBu64Yxr7iJ6nk= github.com/mikefarah/yq/v3 v3.0.0-20201202084205-8846255d1c37/go.mod h1:dYWq+UWoFCDY1TndvFUQuhBbIYmZpjreC8adEAx93zE= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -787,7 +810,6 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -806,9 +828,9 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -818,8 +840,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= @@ -851,11 +873,11 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/operator-framework/api v0.3.7-0.20200602203552-431198de9fc2/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q= +github.com/operator-framework/api v0.7.1/go.mod h1:L7IvLd/ckxJEJg/t4oTTlnHKAJIP/p51AvEslW3wYdY= github.com/operator-framework/api v0.10.1 h1:2tBjIr4hRZ0iaJ4UuenVjocbo9xrJi1O409K/tlEddo= github.com/operator-framework/api v0.10.1/go.mod h1:tV0BUNvly7szq28ZPBXhjp1Sqg5yHCOeX19ui9K4vjI= -github.com/operator-framework/operator-registry v1.13.6 h1:h/dIjQQS7uneQNRifrSz7h0xg4Xyjg6C9f6XZofbMPg= -github.com/operator-framework/operator-registry v1.13.6/go.mod h1:YhnIzOVjRU2ZwZtzt+fjcjW8ujJaSFynBEu7QVKaSdU= +github.com/operator-framework/operator-registry v1.17.5 h1:LR8m1rFz5Gcyje8WK6iYt+gIhtzqo52zMRALdmTYHT0= +github.com/operator-framework/operator-registry v1.17.5/go.mod h1:sRQIgDMZZdUcmHltzyCnM6RUoDF+WS8Arj1BQIARDS8= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -923,7 +945,6 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -983,7 +1004,6 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= @@ -1031,6 +1051,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -1041,7 +1062,6 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -1062,7 +1082,6 @@ github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wK gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -1123,17 +1142,21 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1141,6 +1164,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1157,8 +1181,9 @@ golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1224,7 +1249,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1239,6 +1263,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -1329,6 +1355,8 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1389,7 +1417,6 @@ golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1411,6 +1438,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1423,7 +1451,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1446,6 +1474,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= @@ -1474,7 +1503,6 @@ google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200701001935-0939c5918c31/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= @@ -1521,8 +1549,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1532,7 +1561,6 @@ gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8 gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= -gopkg.in/imdario/mergo.v0 v0.3.7/go.mod h1:9qPP6AGrlC1G2PTNXko614FwGZvorN7MiBU0Eppok+U= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1544,6 +1572,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1578,13 +1607,17 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= k8s.io/api v0.22.0-beta.0 h1:Icf4R7+a0zV6GexTqOXlcLw4OojdQVZE263bd4ui8ms= k8s.io/api v0.22.0-beta.0/go.mod h1:qORVa/3lcMBQXyTmjzpk8IN3VRg619iFNr8d16VXkUQ= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= +k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= +k8s.io/apiextensions-apiserver v0.20.6/go.mod h1:qO8YMqeMmZH+lV21LUNzV41vfpoE9QVAJRA+MNqj0mo= k8s.io/apiextensions-apiserver v0.21.0/go.mod h1:gsQGNtGkc/YoDG9loKI0V+oLZM4ljRPjc/sql5tmvzc= k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= @@ -1592,26 +1625,32 @@ k8s.io/apiextensions-apiserver v0.22.0-beta.0 h1:0w1Z8vtPSH8j0S1ryaztOhwJ1C/4RWT k8s.io/apiextensions-apiserver v0.22.0-beta.0/go.mod h1:Gvf2TX9jNuFdzd+b39kVksy9pEq9rVexezn3SeJoLeI= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= k8s.io/apimachinery v0.22.0-beta.0 h1:y7EnOS9n+ZhCrk9nlwW7kwIr67syKFkGWj/HPAiE+XI= k8s.io/apimachinery v0.22.0-beta.0/go.mod h1:AZCsxGyiXb/4yTvUIiY+4LESjQ12B77LFrbW2xd61is= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.21.0/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg= k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= k8s.io/apiserver v0.22.0-beta.0 h1:pSXFpHIw3PBA4XNPiwkajD+XyOjJwqSGDCNR8d3orTA= k8s.io/apiserver v0.22.0-beta.0/go.mod h1:8nCnNsRYExj/mPHowit5UvsjbExm1YWojDs4ziem9q4= -k8s.io/cli-runtime v0.18.0/go.mod h1:1eXfmBsIJosjn9LjEBUd2WVPoPAY9XGTqTFcPMIBsUQ= +k8s.io/cli-runtime v0.20.6/go.mod h1:JVERW478qcxWrUjJuWQSqyJeiz9QC4T6jmBznHFBC8w= k8s.io/cli-runtime v0.21.0 h1:/V2Kkxtf6x5NI2z+Sd/mIrq4FQyQ8jzZAUD6N5RnN7Y= k8s.io/cli-runtime v0.21.0/go.mod h1:XoaHP93mGPF37MkLbjGVYqg3S1MnsFdKtiA/RZzzxOo= k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= @@ -1619,20 +1658,24 @@ k8s.io/client-go v0.22.0-beta.0 h1:CMKtGoQPNjt70Ii94mvuRO/eJM1iXuRh+u3V9Fb1ZvM= k8s.io/client-go v0.22.0-beta.0/go.mod h1:h2hyZ6clrIgKFF4Upjz2Epnvo9cq8S7rZfWegGQkNb0= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.20.6/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= k8s.io/code-generator v0.21.0/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= k8s.io/code-generator v0.22.0-beta.0 h1:fwlAIV7nZmcjHkj4NmIbZCpJyUn2AQGXArvWYVxaND0= k8s.io/code-generator v0.22.0-beta.0/go.mod h1:lBZLjrQ5bAwlmQalYYgV0gH3G7SS+Z2d33ijEte85FY= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw= k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= k8s.io/component-base v0.22.0-beta.0 h1:9FPWLb4nWVyKQDtj7NUEdm+nTSlEdKvLseLAeo10wXg= k8s.io/component-base v0.22.0-beta.0/go.mod h1:/sa2flwcRBe8XTwdrueKBHBrR+J+ptB669U5R4ytRsU= +k8s.io/component-helpers v0.20.6/go.mod h1:d4rFhZS/wxrZCxRiJJiWf1mVGVeMB5/ey3Yv8/rOp78= k8s.io/component-helpers v0.21.0/go.mod h1:tezqefP7lxfvJyR+0a+6QtVrkZ/wIkyMLK4WcQ3Cj8U= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -1658,10 +1701,10 @@ k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iL k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d h1:lUK8GPtuJy8ClWZhuvKoaLdKGPLq9H1PxWp7VPBZBkU= k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kubectl v0.18.0/go.mod h1:LOkWx9Z5DXMEg5KtOjHhRiC1fqJPLyCr3KtQgEolCkU= +k8s.io/kubectl v0.20.6/go.mod h1:yTCGVrlkBuQhFbKA1R65+lQ9hH7XeyOqUd0FUPFicPg= k8s.io/kubectl v0.21.0 h1:WZXlnG/yjcE4LWO2g6ULjFxtzK6H1TKzsfaBFuVIhNg= k8s.io/kubectl v0.21.0/go.mod h1:EU37NukZRXn1TpAkMUoy8Z/B2u6wjHDS4aInsDzVvks= -k8s.io/metrics v0.18.0/go.mod h1:8aYTW18koXqjLVKL7Ds05RPMX9ipJZI3mywYvBOxXd4= +k8s.io/metrics v0.20.6/go.mod h1:d+OAIaXutom9kGWcBit/M8OkDpIzBKTsm47+KcUt7VI= k8s.io/metrics v0.21.0/go.mod h1:L3Ji9EGPP1YBbfm9sPfEXSpnj8i24bfQbAFAsW0NueQ= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1679,11 +1722,11 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyz sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.20 h1:jLWvYcn/9pCws8p54so9QpxkkL0SWc2fTZTg77NRyhg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.20/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo= +sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q= sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= -sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= +sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/controller-tools v0.6.0/go.mod h1:baRMVPrctU77F+rfAuH2uPqW93k6yQnZA2dhUOr7ihc= sigs.k8s.io/controller-tools v0.6.1 h1:nODRx2YrSNcaGd+90+CVC9SGEG6ygHlz3nSJmweR5as= sigs.k8s.io/controller-tools v0.6.1/go.mod h1:U6O1RF5w17iX2d+teSXELpJsdexmrTb126DMeJM8r+U= @@ -1700,6 +1743,7 @@ sigs.k8s.io/kustomize/kyaml v0.10.15/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.1 h1:nYqY2A6oy37sKLYuSBXuQhbj4JVclzJK13BOIvJG5XU= sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= @@ -1707,4 +1751,3 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/pkg/controller/bundle/bundle_unpacker.go b/pkg/controller/bundle/bundle_unpacker.go index 67e86de300..962e8066dd 100644 --- a/pkg/controller/bundle/bundle_unpacker.go +++ b/pkg/controller/bundle/bundle_unpacker.go @@ -100,9 +100,14 @@ func (c *ConfigMapUnpacker) job(cmRef *corev1.ObjectReference, bundlePath string ImagePullSecrets: secrets, Containers: []corev1.Container{ { - Name: "extract", - Image: c.opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", cmRef.Namespace, "-c", cmRef.Name}, + Name: "extract", + Image: c.opmImage, + Command: []string{"opm", "alpha", "bundle", "extract", + "-m", "/bundle/", + "-n", cmRef.Namespace, + "-c", cmRef.Name, + "-z", + }, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, diff --git a/pkg/controller/bundle/bundle_unpacker_test.go b/pkg/controller/bundle/bundle_unpacker_test.go index 0149606048..5d8ff740ab 100644 --- a/pkg/controller/bundle/bundle_unpacker_test.go +++ b/pkg/controller/bundle/bundle_unpacker_test.go @@ -222,7 +222,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, @@ -399,7 +399,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, @@ -616,7 +616,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, @@ -828,7 +828,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, @@ -1010,7 +1010,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, @@ -1203,7 +1203,7 @@ func TestConfigMapUnpacker(t *testing.T) { { Name: "extract", Image: opmImage, - Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash}, + Command: []string{"opm", "alpha", "bundle", "extract", "-m", "/bundle/", "-n", "ns-a", "-c", pathHash, "-z"}, Env: []corev1.EnvVar{ { Name: configmap.EnvContainerImage, diff --git a/pkg/controller/operators/catalog/operator.go b/pkg/controller/operators/catalog/operator.go index efd1685b90..6aec1d1ffe 100644 --- a/pkg/controller/operators/catalog/operator.go +++ b/pkg/controller/operators/catalog/operator.go @@ -115,7 +115,7 @@ type Operator struct { type CatalogSourceSyncFunc func(logger *logrus.Entry, in *v1alpha1.CatalogSource) (out *v1alpha1.CatalogSource, continueSync bool, syncError error) // NewOperator creates a new Catalog Operator. -func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clock, logger *logrus.Logger, resync time.Duration, configmapRegistryImage, utilImage string, operatorNamespace string, scheme *runtime.Scheme, installPlanTimeout time.Duration, bundleUnpackTimeout time.Duration) (*Operator, error) { +func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clock, logger *logrus.Logger, resync time.Duration, configmapRegistryImage, opmImage, utilImage string, operatorNamespace string, scheme *runtime.Scheme, installPlanTimeout time.Duration, bundleUnpackTimeout time.Duration) (*Operator, error) { resyncPeriod := queueinformer.ResyncWithJitter(resync, 0.2) config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) if err != nil { @@ -351,7 +351,7 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo bundle.WithPodLister(podInformer.Lister()), bundle.WithRoleLister(roleInformer.Lister()), bundle.WithRoleBindingLister(roleBindingInformer.Lister()), - bundle.WithOPMImage(configmapRegistryImage), + bundle.WithOPMImage(opmImage), bundle.WithUtilImage(utilImage), bundle.WithNow(op.now), bundle.WithUnpackTimeout(op.bundleUnpackTimeout), diff --git a/pkg/fakes/fake_registry_store.go b/pkg/fakes/fake_registry_store.go index 8d27b36632..5894a426ae 100644 --- a/pkg/fakes/fake_registry_store.go +++ b/pkg/fakes/fake_registry_store.go @@ -57,6 +57,20 @@ type FakeQuery struct { result1 *api.Bundle result2 error } + GetBundlePathIfExistsStub func(context.Context, string) (string, error) + getBundlePathIfExistsMutex sync.RWMutex + getBundlePathIfExistsArgsForCall []struct { + arg1 context.Context + arg2 string + } + getBundlePathIfExistsReturns struct { + result1 string + result2 error + } + getBundlePathIfExistsReturnsOnCall map[int]struct { + result1 string + result2 error + } GetBundlePathsForPackageStub func(context.Context, string) ([]string, error) getBundlePathsForPackageMutex sync.RWMutex getBundlePathsForPackageArgsForCall []struct { @@ -331,6 +345,19 @@ type FakeQuery struct { result1 []string result2 error } + ListRegistryBundlesStub func(context.Context) ([]*registry.Bundle, error) + listRegistryBundlesMutex sync.RWMutex + listRegistryBundlesArgsForCall []struct { + arg1 context.Context + } + listRegistryBundlesReturns struct { + result1 []*registry.Bundle + result2 error + } + listRegistryBundlesReturnsOnCall map[int]struct { + result1 []*registry.Bundle + result2 error + } ListTablesStub func(context.Context) ([]string, error) listTablesMutex sync.RWMutex listTablesArgsForCall []struct { @@ -546,6 +573,70 @@ func (fake *FakeQuery) GetBundleForChannelReturnsOnCall(i int, result1 *api.Bund }{result1, result2} } +func (fake *FakeQuery) GetBundlePathIfExists(arg1 context.Context, arg2 string) (string, error) { + fake.getBundlePathIfExistsMutex.Lock() + ret, specificReturn := fake.getBundlePathIfExistsReturnsOnCall[len(fake.getBundlePathIfExistsArgsForCall)] + fake.getBundlePathIfExistsArgsForCall = append(fake.getBundlePathIfExistsArgsForCall, struct { + arg1 context.Context + arg2 string + }{arg1, arg2}) + fake.recordInvocation("GetBundlePathIfExists", []interface{}{arg1, arg2}) + fake.getBundlePathIfExistsMutex.Unlock() + if fake.GetBundlePathIfExistsStub != nil { + return fake.GetBundlePathIfExistsStub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.getBundlePathIfExistsReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeQuery) GetBundlePathIfExistsCallCount() int { + fake.getBundlePathIfExistsMutex.RLock() + defer fake.getBundlePathIfExistsMutex.RUnlock() + return len(fake.getBundlePathIfExistsArgsForCall) +} + +func (fake *FakeQuery) GetBundlePathIfExistsCalls(stub func(context.Context, string) (string, error)) { + fake.getBundlePathIfExistsMutex.Lock() + defer fake.getBundlePathIfExistsMutex.Unlock() + fake.GetBundlePathIfExistsStub = stub +} + +func (fake *FakeQuery) GetBundlePathIfExistsArgsForCall(i int) (context.Context, string) { + fake.getBundlePathIfExistsMutex.RLock() + defer fake.getBundlePathIfExistsMutex.RUnlock() + argsForCall := fake.getBundlePathIfExistsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeQuery) GetBundlePathIfExistsReturns(result1 string, result2 error) { + fake.getBundlePathIfExistsMutex.Lock() + defer fake.getBundlePathIfExistsMutex.Unlock() + fake.GetBundlePathIfExistsStub = nil + fake.getBundlePathIfExistsReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeQuery) GetBundlePathIfExistsReturnsOnCall(i int, result1 string, result2 error) { + fake.getBundlePathIfExistsMutex.Lock() + defer fake.getBundlePathIfExistsMutex.Unlock() + fake.GetBundlePathIfExistsStub = nil + if fake.getBundlePathIfExistsReturnsOnCall == nil { + fake.getBundlePathIfExistsReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.getBundlePathIfExistsReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + func (fake *FakeQuery) GetBundlePathsForPackage(arg1 context.Context, arg2 string) ([]string, error) { fake.getBundlePathsForPackageMutex.Lock() ret, specificReturn := fake.getBundlePathsForPackageReturnsOnCall[len(fake.getBundlePathsForPackageArgsForCall)] @@ -1770,6 +1861,69 @@ func (fake *FakeQuery) ListPackagesReturnsOnCall(i int, result1 []string, result }{result1, result2} } +func (fake *FakeQuery) ListRegistryBundles(arg1 context.Context) ([]*registry.Bundle, error) { + fake.listRegistryBundlesMutex.Lock() + ret, specificReturn := fake.listRegistryBundlesReturnsOnCall[len(fake.listRegistryBundlesArgsForCall)] + fake.listRegistryBundlesArgsForCall = append(fake.listRegistryBundlesArgsForCall, struct { + arg1 context.Context + }{arg1}) + fake.recordInvocation("ListRegistryBundles", []interface{}{arg1}) + fake.listRegistryBundlesMutex.Unlock() + if fake.ListRegistryBundlesStub != nil { + return fake.ListRegistryBundlesStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.listRegistryBundlesReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeQuery) ListRegistryBundlesCallCount() int { + fake.listRegistryBundlesMutex.RLock() + defer fake.listRegistryBundlesMutex.RUnlock() + return len(fake.listRegistryBundlesArgsForCall) +} + +func (fake *FakeQuery) ListRegistryBundlesCalls(stub func(context.Context) ([]*registry.Bundle, error)) { + fake.listRegistryBundlesMutex.Lock() + defer fake.listRegistryBundlesMutex.Unlock() + fake.ListRegistryBundlesStub = stub +} + +func (fake *FakeQuery) ListRegistryBundlesArgsForCall(i int) context.Context { + fake.listRegistryBundlesMutex.RLock() + defer fake.listRegistryBundlesMutex.RUnlock() + argsForCall := fake.listRegistryBundlesArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeQuery) ListRegistryBundlesReturns(result1 []*registry.Bundle, result2 error) { + fake.listRegistryBundlesMutex.Lock() + defer fake.listRegistryBundlesMutex.Unlock() + fake.ListRegistryBundlesStub = nil + fake.listRegistryBundlesReturns = struct { + result1 []*registry.Bundle + result2 error + }{result1, result2} +} + +func (fake *FakeQuery) ListRegistryBundlesReturnsOnCall(i int, result1 []*registry.Bundle, result2 error) { + fake.listRegistryBundlesMutex.Lock() + defer fake.listRegistryBundlesMutex.Unlock() + fake.ListRegistryBundlesStub = nil + if fake.listRegistryBundlesReturnsOnCall == nil { + fake.listRegistryBundlesReturnsOnCall = make(map[int]struct { + result1 []*registry.Bundle + result2 error + }) + } + fake.listRegistryBundlesReturnsOnCall[i] = struct { + result1 []*registry.Bundle + result2 error + }{result1, result2} +} + func (fake *FakeQuery) ListTables(arg1 context.Context) ([]string, error) { fake.listTablesMutex.Lock() ret, specificReturn := fake.listTablesReturnsOnCall[len(fake.listTablesArgsForCall)] @@ -1842,6 +1996,8 @@ func (fake *FakeQuery) Invocations() map[string][][]interface{} { defer fake.getBundleMutex.RUnlock() fake.getBundleForChannelMutex.RLock() defer fake.getBundleForChannelMutex.RUnlock() + fake.getBundlePathIfExistsMutex.RLock() + defer fake.getBundlePathIfExistsMutex.RUnlock() fake.getBundlePathsForPackageMutex.RLock() defer fake.getBundlePathsForPackageMutex.RUnlock() fake.getBundleThatProvidesMutex.RLock() @@ -1880,6 +2036,8 @@ func (fake *FakeQuery) Invocations() map[string][][]interface{} { defer fake.listImagesMutex.RUnlock() fake.listPackagesMutex.RLock() defer fake.listPackagesMutex.RUnlock() + fake.listRegistryBundlesMutex.RLock() + defer fake.listRegistryBundlesMutex.RUnlock() fake.listTablesMutex.RLock() defer fake.listTablesMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/vendor/github.com/h2non/filetype/.editorconfig b/vendor/github.com/h2non/filetype/.editorconfig new file mode 100644 index 0000000000..000dc0a7aa --- /dev/null +++ b/vendor/github.com/h2non/filetype/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +indent_style = tabs +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false diff --git a/vendor/github.com/h2non/filetype/.gitignore b/vendor/github.com/h2non/filetype/.gitignore new file mode 100644 index 0000000000..6fefe6cce2 --- /dev/null +++ b/vendor/github.com/h2non/filetype/.gitignore @@ -0,0 +1,2 @@ +bin +.DS_Store diff --git a/vendor/github.com/h2non/filetype/.travis.yml b/vendor/github.com/h2non/filetype/.travis.yml new file mode 100644 index 0000000000..c9cdbc8daf --- /dev/null +++ b/vendor/github.com/h2non/filetype/.travis.yml @@ -0,0 +1,16 @@ +language: go +arch: + - AMD64 + - ppc64le +go: + - "1.13" + - "1.14" + +before_install: + - go get -u -v golang.org/x/lint/golint + +script: + - diff -u <(echo -n) <(gofmt -s -d ./) + - diff -u <(echo -n) <(go vet ./...) + - diff -u <(echo -n) <(golint) + - go test -v -race ./... diff --git a/vendor/github.com/h2non/filetype/History.md b/vendor/github.com/h2non/filetype/History.md new file mode 100644 index 0000000000..ecace553ef --- /dev/null +++ b/vendor/github.com/h2non/filetype/History.md @@ -0,0 +1,148 @@ + +v1.1.0 / 2020-06-06 +=================== + + * feat: version bump v1.10 + * feat(ci): add go 1.14 + * Merge pull request #82 from andrewstucki/sqlite-update + * Merge pull request #84 from evanoberholster/master + * Better differentiation: between image/x-canon-cr2 and image/tiff + * Merge pull request #1 from h2non/master + * Update ico filetype per https://www.iana.org/assignments/media-types/image/vnd.microsoft.icon + * Update rar filetype per https://www.iana.org/assignments/media-types/application/vnd.rar + * Update exe filetype per https://www.iana.org/assignments/media-types/application/vnd.microsoft.portable-executable + * Update deb filetype per https://www.iana.org/assignments/media-types/application/vnd.debian.binary-package + * Update sqlite filetype per https://www.iana.org/assignments/media-types/application/vnd.sqlite3 + * Merge pull request #72 from turn88/master + * Update document.go + * Update document.go + * Update document.go + * add matchers for office 2003 + +v1.0.10 / 2019-08-06 +==================== + + * Merge pull request #76 from lex-r/fix-matroska-detection + * fix: mkv and webm types detection + +v1.0.9 / 2019-07-25 +=================== + + * Merge pull request #75 from Trane9991/master + * add video/3gpp support + * fix: use proper iso file mime type + * feat: add iso image format + * Merge pull request #65 from Fentonz/master + * Merge pull request #70 from fanpei91/master + * add image/vnd.dwg to README + * add image/vnd.dwg support + * Added support for .iso files + +v1.0.8 / 2019-02-10 +=================== + + * refactor(images): heic -> heif + * feat(docs): add heif format + * Merge pull request #60 from rikonor/master + * add heif/heic support + * fix(docs): dicom -> dcm + * feat: add dicom type + * Merge pull request #58 from Fentonz/master + * Merge pull request #59 from kmanley/master + * fix example; related to h2non/filetype#43 + * Added DICOM type to archive + + +v1.0.7 / 2019-02-09 +=================== + + * Merge pull request #56 from akupila/wasm + * add wasm to readme + * detect wasm file type + +v1.0.6 / 2019-01-22 +=================== + + * Merge pull request #55 from ivanlemeshev/master + * Added ftypmp4v to MP4 matcher + * Merge pull request #54 from aofei/master + * chore: add support for Go modules + * feat: add support for AAC (audio/aac) + * Merge pull request #53 from lynxbyorion/check-for-docoments + * Added checks for documents. + * Merge pull request #51 from eriken/master + * fixed bad mime and import paths + * Merge pull request #50 from eriken/jpeg2000_support + * fix import paths + * jpeg2000 support + * Merge pull request #47 from Ma124/master + * Merge pull request #49 from amoore614/master + * more robust check for .mov files + * bugfix: reverse order of matcher key list so user registered matchers appear first + * bugfix: store ptr to MatcherKeys in case user registered matchers are used. + * update comment + * Bump buffer size to 8K to allow for more custom file matching + * refactor(readme): update package import path + * Merge pull request #48 from kumakichi/support_msooxml + * do not use v1 + * ok, master already changed travis + * add fixtures, but MatchReader may not work for some msooxml files, 4096 bytes maybe not enough + * support ms ooxml, #40 + * Fixed misspells + * fix(travis): use string notation for matrix items + * Merge pull request #42 from bruth/patch-2 + * refactor(travis): remove Go 1.6, add Go 1.10 + * Change maximum bytes required for detection + * Merge pull request #36 from yiiTT/patch-1 + * Add MP4 dash and additional ISO formats + * Merge pull request #34 from RangelReale/fix-mp4-case + * Merge pull request #32 from yiiTT/fix-m4v + * Fixed mp4 detection case-sensitivity according to http://www.ftyps.com/ + * Fix M4v matcher + +v1.0.5 / 2017-12-12 +=================== + + * Merge pull request #30 from RangelReale/fix_mp4 + * Fix duplicated item in mp4 fix + * Fix MP4 matcher, with information from http://www.file-recovery.com/mp4-signature-format.htm + * Merge pull request #28 from ikovic/master + * Updated file header example. + +v1.0.4 / 2017-11-29 +=================== + + * fix: tests and document types matchers + * refactor(docs): remove codesponsor + * Merge pull request #26 from bienkma/master + * Add support check file type: .doc, .docx, .pptx, .ppt, .xls, .xlsx + * feat(docs): add code sponsor banner + * feat(travis): add go 1.9 + * Merge pull request #24 from strazzere/patch-1 + * Fix typo in unknown + +v1.0.3 / 2017-08-03 +=================== + + * Merge pull request #21 from elemeta/master + * Add Elf file as supported matcher archive type + +v1.0.2 / 2017-07-26 +=================== + + * Merge pull request #20 from marshyski/master + * Added RedHat RPM as supported matcher archive type + * Merge pull request #19 from nlamirault/patch-1 + * Fix typo in documentation + +v1.0.1 / 2017-02-24 +=================== + + * Merge pull request #18 from Impyy/enable-webm + * Enable the webm matcher + * feat(docs): add Go version badge + +1.0.0 / 2016-12-11 +================== + +- Initial stable version (v1.0.0). diff --git a/vendor/github.com/h2non/filetype/LICENSE b/vendor/github.com/h2non/filetype/LICENSE new file mode 100644 index 0000000000..30ede59b65 --- /dev/null +++ b/vendor/github.com/h2non/filetype/LICENSE @@ -0,0 +1,24 @@ +The MIT License + +Copyright (c) Tomas Aparicio + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/h2non/filetype/README.md b/vendor/github.com/h2non/filetype/README.md new file mode 100644 index 0000000000..3ba06cb4e5 --- /dev/null +++ b/vendor/github.com/h2non/filetype/README.md @@ -0,0 +1,292 @@ +# filetype [![Build Status](https://travis-ci.org/h2non/filetype.svg)](https://travis-ci.org/h2non/filetype) [![GoDoc](https://godoc.org/github.com/h2non/filetype?status.svg)](https://godoc.org/github.com/h2non/filetype) [![Go Report Card](http://goreportcard.com/badge/h2non/filetype)](http://goreportcard.com/report/h2non/filetype) [![Go Version](https://img.shields.io/badge/go-v1.0+-green.svg?style=flat)](https://github.com/h2non/gentleman) + +Small and dependency free [Go](https://golang.org) package to infer file and MIME type checking the [magic numbers]() signature. + +For SVG file type checking, see [go-is-svg](https://github.com/h2non/go-is-svg) package. Python port: [filetype.py](https://github.com/h2non/filetype.py). + +## Features + +- Supports a [wide range](#supported-types) of file types +- Provides file extension and proper MIME type +- File discovery by extension or MIME type +- File discovery by class (image, video, audio...) +- Provides a bunch of helpers and file matching shortcuts +- [Pluggable](#add-additional-file-type-matchers): add custom new types and matchers +- Simple and semantic API +- [Blazing fast](#benchmarks), even processing large files +- Only first 262 bytes representing the max file header is required, so you can just [pass a slice](#file-header) +- Dependency free (just Go code, no C compilation needed) +- Cross-platform file recognition + +## Installation + +```bash +go get github.com/h2non/filetype +``` + +## API + +See [Godoc](https://godoc.org/github.com/h2non/filetype) reference. + +### Subpackages + +- [`github.com/h2non/filetype/types`](https://godoc.org/github.com/h2non/filetype/types) +- [`github.com/h2non/filetype/matchers`](https://godoc.org/github.com/h2non/filetype/matchers) + +## Examples + +#### Simple file type checking + +```go +package main + +import ( + "fmt" + "io/ioutil" + + "github.com/h2non/filetype" +) + +func main() { + buf, _ := ioutil.ReadFile("sample.jpg") + + kind, _ := filetype.Match(buf) + if kind == filetype.Unknown { + fmt.Println("Unknown file type") + return + } + + fmt.Printf("File type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value) +} +``` + +#### Check type class + +```go +package main + +import ( + "fmt" + "io/ioutil" + + "github.com/h2non/filetype" +) + +func main() { + buf, _ := ioutil.ReadFile("sample.jpg") + + if filetype.IsImage(buf) { + fmt.Println("File is an image") + } else { + fmt.Println("Not an image") + } +} +``` + +#### Supported type + +```go +package main + +import ( + "fmt" + + "github.com/h2non/filetype" +) + +func main() { + // Check if file is supported by extension + if filetype.IsSupported("jpg") { + fmt.Println("Extension supported") + } else { + fmt.Println("Extension not supported") + } + + // Check if file is supported by extension + if filetype.IsMIMESupported("image/jpeg") { + fmt.Println("MIME type supported") + } else { + fmt.Println("MIME type not supported") + } +} +``` + +#### File header + +```go +package main + +import ( + "fmt" + "io/ioutil" + + "github.com/h2non/filetype" +) + +func main() { + // Open a file descriptor + file, _ := os.Open("movie.mp4") + + // We only have to pass the file header = first 261 bytes + head := make([]byte, 261) + file.Read(head) + + if filetype.IsImage(head) { + fmt.Println("File is an image") + } else { + fmt.Println("Not an image") + } +} +``` + +#### Add additional file type matchers + +```go +package main + +import ( + "fmt" + + "github.com/h2non/filetype" +) + +var fooType = filetype.NewType("foo", "foo/foo") + +func fooMatcher(buf []byte) bool { + return len(buf) > 1 && buf[0] == 0x01 && buf[1] == 0x02 +} + +func main() { + // Register the new matcher and its type + filetype.AddMatcher(fooType, fooMatcher) + + // Check if the new type is supported by extension + if filetype.IsSupported("foo") { + fmt.Println("New supported type: foo") + } + + // Check if the new type is supported by MIME + if filetype.IsMIMESupported("foo/foo") { + fmt.Println("New supported MIME type: foo/foo") + } + + // Try to match the file + fooFile := []byte{0x01, 0x02} + kind, _ := filetype.Match(fooFile) + if kind == filetype.Unknown { + fmt.Println("Unknown file type") + } else { + fmt.Printf("File type matched: %s\n", kind.Extension) + } +} +``` + +## Supported types + +#### Image + +- **jpg** - `image/jpeg` +- **png** - `image/png` +- **gif** - `image/gif` +- **webp** - `image/webp` +- **cr2** - `image/x-canon-cr2` +- **tif** - `image/tiff` +- **bmp** - `image/bmp` +- **heif** - `image/heif` +- **jxr** - `image/vnd.ms-photo` +- **psd** - `image/vnd.adobe.photoshop` +- **ico** - `image/vnd.microsoft.icon` +- **dwg** - `image/vnd.dwg` + +#### Video + +- **mp4** - `video/mp4` +- **m4v** - `video/x-m4v` +- **mkv** - `video/x-matroska` +- **webm** - `video/webm` +- **mov** - `video/quicktime` +- **avi** - `video/x-msvideo` +- **wmv** - `video/x-ms-wmv` +- **mpg** - `video/mpeg` +- **flv** - `video/x-flv` +- **3gp** - `video/3gpp` + +#### Audio + +- **mid** - `audio/midi` +- **mp3** - `audio/mpeg` +- **m4a** - `audio/m4a` +- **ogg** - `audio/ogg` +- **flac** - `audio/x-flac` +- **wav** - `audio/x-wav` +- **amr** - `audio/amr` +- **aac** - `audio/aac` + +#### Archive + +- **epub** - `application/epub+zip` +- **zip** - `application/zip` +- **tar** - `application/x-tar` +- **rar** - `application/vnd.rar` +- **gz** - `application/gzip` +- **bz2** - `application/x-bzip2` +- **7z** - `application/x-7z-compressed` +- **xz** - `application/x-xz` +- **pdf** - `application/pdf` +- **exe** - `application/vnd.microsoft.portable-executable` +- **swf** - `application/x-shockwave-flash` +- **rtf** - `application/rtf` +- **iso** - `application/x-iso9660-image` +- **eot** - `application/octet-stream` +- **ps** - `application/postscript` +- **sqlite** - `application/vnd.sqlite3` +- **nes** - `application/x-nintendo-nes-rom` +- **crx** - `application/x-google-chrome-extension` +- **cab** - `application/vnd.ms-cab-compressed` +- **deb** - `application/vnd.debian.binary-package` +- **ar** - `application/x-unix-archive` +- **Z** - `application/x-compress` +- **lz** - `application/x-lzip` +- **rpm** - `application/x-rpm` +- **elf** - `application/x-executable` +- **dcm** - `application/dicom` + +#### Documents + +- **doc** - `application/msword` +- **docx** - `application/vnd.openxmlformats-officedocument.wordprocessingml.document` +- **xls** - `application/vnd.ms-excel` +- **xlsx** - `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet` +- **ppt** - `application/vnd.ms-powerpoint` +- **pptx** - `application/vnd.openxmlformats-officedocument.presentationml.presentation` + +#### Font + +- **woff** - `application/font-woff` +- **woff2** - `application/font-woff` +- **ttf** - `application/font-sfnt` +- **otf** - `application/font-sfnt` + +#### Application + +- **wasm** - `application/wasm` +- **dex** - `application/vnd.android.dex` +- **dey** - `application/vnd.android.dey` + +## Benchmarks + +Measured using [real files](https://github.com/h2non/filetype/tree/master/fixtures). + +Environment: OSX x64 i7 2.7 Ghz + +```bash +BenchmarkMatchTar-8 1000000 1083 ns/op +BenchmarkMatchZip-8 1000000 1162 ns/op +BenchmarkMatchJpeg-8 1000000 1280 ns/op +BenchmarkMatchGif-8 1000000 1315 ns/op +BenchmarkMatchPng-8 1000000 1121 ns/op +``` + +## License + +MIT - Tomas Aparicio diff --git a/vendor/github.com/h2non/filetype/filetype.go b/vendor/github.com/h2non/filetype/filetype.go new file mode 100644 index 0000000000..c99691e5fc --- /dev/null +++ b/vendor/github.com/h2non/filetype/filetype.go @@ -0,0 +1,102 @@ +package filetype + +import ( + "errors" + + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" +) + +// Types stores a map of supported types +var Types = types.Types + +// NewType creates and registers a new type +var NewType = types.NewType + +// Unknown represents an unknown file type +var Unknown = types.Unknown + +// ErrEmptyBuffer represents an empty buffer error +var ErrEmptyBuffer = errors.New("Empty buffer") + +// ErrUnknownBuffer represents a unknown buffer error +var ErrUnknownBuffer = errors.New("Unknown buffer type") + +// AddType registers a new file type +func AddType(ext, mime string) types.Type { + return types.NewType(ext, mime) +} + +// Is checks if a given buffer matches with the given file type extension +func Is(buf []byte, ext string) bool { + kind := types.Get(ext) + if kind != types.Unknown { + return IsType(buf, kind) + } + return false +} + +// IsExtension semantic alias to Is() +func IsExtension(buf []byte, ext string) bool { + return Is(buf, ext) +} + +// IsType checks if a given buffer matches with the given file type +func IsType(buf []byte, kind types.Type) bool { + matcher := matchers.Matchers[kind] + if matcher == nil { + return false + } + return matcher(buf) != types.Unknown +} + +// IsMIME checks if a given buffer matches with the given MIME type +func IsMIME(buf []byte, mime string) bool { + result := false + types.Types.Range(func(k, v interface{}) bool { + kind := v.(types.Type) + if kind.MIME.Value == mime { + matcher := matchers.Matchers[kind] + result = matcher(buf) != types.Unknown + return false + } + return true + }) + + return result +} + +// IsSupported checks if a given file extension is supported +func IsSupported(ext string) bool { + result := false + types.Types.Range(func(k, v interface{}) bool { + key := k.(string) + if key == ext { + result = true + return false + } + return true + }) + + return result +} + +// IsMIMESupported checks if a given MIME type is supported +func IsMIMESupported(mime string) bool { + result := false + types.Types.Range(func(k, v interface{}) bool { + kind := v.(types.Type) + if kind.MIME.Value == mime { + result = true + return false + } + return true + }) + + return result +} + +// GetType retrieves a Type by file extension +func GetType(ext string) types.Type { + return types.Get(ext) +} diff --git a/vendor/github.com/h2non/filetype/go.mod b/vendor/github.com/h2non/filetype/go.mod new file mode 100644 index 0000000000..071ea73201 --- /dev/null +++ b/vendor/github.com/h2non/filetype/go.mod @@ -0,0 +1,3 @@ +module github.com/h2non/filetype + +go 1.13 diff --git a/vendor/github.com/h2non/filetype/kind.go b/vendor/github.com/h2non/filetype/kind.go new file mode 100644 index 0000000000..de8473507d --- /dev/null +++ b/vendor/github.com/h2non/filetype/kind.go @@ -0,0 +1,80 @@ +package filetype + +import ( + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" +) + +// Image tries to match a file as image type +func Image(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Image) +} + +// IsImage checks if the given buffer is an image type +func IsImage(buf []byte) bool { + kind, _ := Image(buf) + return kind != types.Unknown +} + +// Audio tries to match a file as audio type +func Audio(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Audio) +} + +// IsAudio checks if the given buffer is an audio type +func IsAudio(buf []byte) bool { + kind, _ := Audio(buf) + return kind != types.Unknown +} + +// Video tries to match a file as video type +func Video(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Video) +} + +// IsVideo checks if the given buffer is a video type +func IsVideo(buf []byte) bool { + kind, _ := Video(buf) + return kind != types.Unknown +} + +// Font tries to match a file as text font type +func Font(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Font) +} + +// IsFont checks if the given buffer is a font type +func IsFont(buf []byte) bool { + kind, _ := Font(buf) + return kind != types.Unknown +} + +// Archive tries to match a file as generic archive type +func Archive(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Archive) +} + +// IsArchive checks if the given buffer is an archive type +func IsArchive(buf []byte) bool { + kind, _ := Archive(buf) + return kind != types.Unknown +} + +// Document tries to match a file as document type +func Document(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Document) +} + +// IsDocument checks if the given buffer is an document type +func IsDocument(buf []byte) bool { + kind, _ := Document(buf) + return kind != types.Unknown +} + +func doMatchMap(buf []byte, machers matchers.Map) (types.Type, error) { + kind := MatchMap(buf, machers) + if kind != types.Unknown { + return kind, nil + } + return kind, ErrUnknownBuffer +} diff --git a/vendor/github.com/h2non/filetype/match.go b/vendor/github.com/h2non/filetype/match.go new file mode 100644 index 0000000000..82cf804684 --- /dev/null +++ b/vendor/github.com/h2non/filetype/match.go @@ -0,0 +1,90 @@ +package filetype + +import ( + "io" + "os" + + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" +) + +// Matchers is an alias to matchers.Matchers +var Matchers = matchers.Matchers + +// MatcherKeys is an alias to matchers.MatcherKeys +var MatcherKeys = &matchers.MatcherKeys + +// NewMatcher is an alias to matchers.NewMatcher +var NewMatcher = matchers.NewMatcher + +// Match infers the file type of a given buffer inspecting its magic numbers signature +func Match(buf []byte) (types.Type, error) { + length := len(buf) + if length == 0 { + return types.Unknown, ErrEmptyBuffer + } + + for _, kind := range *MatcherKeys { + checker := Matchers[kind] + match := checker(buf) + if match != types.Unknown && match.Extension != "" { + return match, nil + } + } + + return types.Unknown, nil +} + +// Get is an alias to Match() +func Get(buf []byte) (types.Type, error) { + return Match(buf) +} + +// MatchFile infers a file type for a file +func MatchFile(filepath string) (types.Type, error) { + file, err := os.Open(filepath) + if err != nil { + return types.Unknown, err + } + defer file.Close() + + return MatchReader(file) +} + +// MatchReader is convenient wrapper to Match() any Reader +func MatchReader(reader io.Reader) (types.Type, error) { + buffer := make([]byte, 8192) // 8K makes msooxml tests happy and allows for expanded custom file checks + + _, err := reader.Read(buffer) + if err != nil && err != io.EOF { + return types.Unknown, err + } + + return Match(buffer) +} + +// AddMatcher registers a new matcher type +func AddMatcher(fileType types.Type, matcher matchers.Matcher) matchers.TypeMatcher { + return matchers.NewMatcher(fileType, matcher) +} + +// Matches checks if the given buffer matches with some supported file type +func Matches(buf []byte) bool { + kind, _ := Match(buf) + return kind != types.Unknown +} + +// MatchMap performs a file matching against a map of match functions +func MatchMap(buf []byte, matchers matchers.Map) types.Type { + for kind, matcher := range matchers { + if matcher(buf) { + return kind + } + } + return types.Unknown +} + +// MatchesMap is an alias to Matches() but using matching against a map of match functions +func MatchesMap(buf []byte, matchers matchers.Map) bool { + return MatchMap(buf, matchers) != types.Unknown +} diff --git a/vendor/github.com/h2non/filetype/matchers/application.go b/vendor/github.com/h2non/filetype/matchers/application.go new file mode 100644 index 0000000000..67fdab3d8d --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/application.go @@ -0,0 +1,43 @@ +package matchers + +var ( + TypeWasm = newType("wasm", "application/wasm") + TypeDex = newType("dex", "application/vnd.android.dex") + TypeDey = newType("dey", "application/vnd.android.dey") +) + +var Application = Map{ + TypeWasm: Wasm, + TypeDex: Dex, + TypeDey: Dey, +} + +// Wasm detects a Web Assembly 1.0 filetype. +func Wasm(buf []byte) bool { + // WASM has starts with `\0asm`, followed by the version. + // http://webassembly.github.io/spec/core/binary/modules.html#binary-magic + return len(buf) >= 8 && + buf[0] == 0x00 && buf[1] == 0x61 && + buf[2] == 0x73 && buf[3] == 0x6D && + buf[4] == 0x01 && buf[5] == 0x00 && + buf[6] == 0x00 && buf[7] == 0x00 +} + +// Dex detects dalvik executable(DEX) +func Dex(buf []byte) bool { + // https://source.android.com/devices/tech/dalvik/dex-format#dex-file-magic + return len(buf) > 36 && + // magic + buf[0] == 0x64 && buf[1] == 0x65 && buf[2] == 0x78 && buf[3] == 0x0A && + // file sise + buf[36] == 0x70 +} + +// Dey Optimized Dalvik Executable(ODEX) +func Dey(buf []byte) bool { + return len(buf) > 100 && + // dey magic + buf[0] == 0x64 && buf[1] == 0x65 && buf[2] == 0x79 && buf[3] == 0x0A && + // dex + Dex(buf[40:100]) +} diff --git a/vendor/github.com/h2non/filetype/matchers/archive.go b/vendor/github.com/h2non/filetype/matchers/archive.go new file mode 100644 index 0000000000..1cef8f5b58 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/archive.go @@ -0,0 +1,246 @@ +package matchers + +var ( + TypeEpub = newType("epub", "application/epub+zip") + TypeZip = newType("zip", "application/zip") + TypeTar = newType("tar", "application/x-tar") + TypeRar = newType("rar", "application/vnd.rar") + TypeGz = newType("gz", "application/gzip") + TypeBz2 = newType("bz2", "application/x-bzip2") + Type7z = newType("7z", "application/x-7z-compressed") + TypeXz = newType("xz", "application/x-xz") + TypePdf = newType("pdf", "application/pdf") + TypeExe = newType("exe", "application/vnd.microsoft.portable-executable") + TypeSwf = newType("swf", "application/x-shockwave-flash") + TypeRtf = newType("rtf", "application/rtf") + TypeEot = newType("eot", "application/octet-stream") + TypePs = newType("ps", "application/postscript") + TypeSqlite = newType("sqlite", "application/vnd.sqlite3") + TypeNes = newType("nes", "application/x-nintendo-nes-rom") + TypeCrx = newType("crx", "application/x-google-chrome-extension") + TypeCab = newType("cab", "application/vnd.ms-cab-compressed") + TypeDeb = newType("deb", "application/vnd.debian.binary-package") + TypeAr = newType("ar", "application/x-unix-archive") + TypeZ = newType("Z", "application/x-compress") + TypeLz = newType("lz", "application/x-lzip") + TypeRpm = newType("rpm", "application/x-rpm") + TypeElf = newType("elf", "application/x-executable") + TypeDcm = newType("dcm", "application/dicom") + TypeIso = newType("iso", "application/x-iso9660-image") + TypeMachO = newType("macho", "application/x-mach-binary") // Mach-O binaries have no common extension. +) + +var Archive = Map{ + TypeEpub: Epub, + TypeZip: Zip, + TypeTar: Tar, + TypeRar: Rar, + TypeGz: Gz, + TypeBz2: Bz2, + Type7z: SevenZ, + TypeXz: Xz, + TypePdf: Pdf, + TypeExe: Exe, + TypeSwf: Swf, + TypeRtf: Rtf, + TypeEot: Eot, + TypePs: Ps, + TypeSqlite: Sqlite, + TypeNes: Nes, + TypeCrx: Crx, + TypeCab: Cab, + TypeDeb: Deb, + TypeAr: Ar, + TypeZ: Z, + TypeLz: Lz, + TypeRpm: Rpm, + TypeElf: Elf, + TypeDcm: Dcm, + TypeIso: Iso, + TypeMachO: MachO, +} + +func Epub(buf []byte) bool { + return len(buf) > 57 && + buf[0] == 0x50 && buf[1] == 0x4B && buf[2] == 0x3 && buf[3] == 0x4 && + buf[30] == 0x6D && buf[31] == 0x69 && buf[32] == 0x6D && buf[33] == 0x65 && + buf[34] == 0x74 && buf[35] == 0x79 && buf[36] == 0x70 && buf[37] == 0x65 && + buf[38] == 0x61 && buf[39] == 0x70 && buf[40] == 0x70 && buf[41] == 0x6C && + buf[42] == 0x69 && buf[43] == 0x63 && buf[44] == 0x61 && buf[45] == 0x74 && + buf[46] == 0x69 && buf[47] == 0x6F && buf[48] == 0x6E && buf[49] == 0x2F && + buf[50] == 0x65 && buf[51] == 0x70 && buf[52] == 0x75 && buf[53] == 0x62 && + buf[54] == 0x2B && buf[55] == 0x7A && buf[56] == 0x69 && buf[57] == 0x70 +} + +func Zip(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x50 && buf[1] == 0x4B && + (buf[2] == 0x3 || buf[2] == 0x5 || buf[2] == 0x7) && + (buf[3] == 0x4 || buf[3] == 0x6 || buf[3] == 0x8) +} + +func Tar(buf []byte) bool { + return len(buf) > 261 && + buf[257] == 0x75 && buf[258] == 0x73 && + buf[259] == 0x74 && buf[260] == 0x61 && + buf[261] == 0x72 +} + +func Rar(buf []byte) bool { + return len(buf) > 6 && + buf[0] == 0x52 && buf[1] == 0x61 && buf[2] == 0x72 && + buf[3] == 0x21 && buf[4] == 0x1A && buf[5] == 0x7 && + (buf[6] == 0x0 || buf[6] == 0x1) +} + +func Gz(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x1F && buf[1] == 0x8B && buf[2] == 0x8 +} + +func Bz2(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x42 && buf[1] == 0x5A && buf[2] == 0x68 +} + +func SevenZ(buf []byte) bool { + return len(buf) > 5 && + buf[0] == 0x37 && buf[1] == 0x7A && buf[2] == 0xBC && + buf[3] == 0xAF && buf[4] == 0x27 && buf[5] == 0x1C +} + +func Pdf(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x25 && buf[1] == 0x50 && + buf[2] == 0x44 && buf[3] == 0x46 +} + +func Exe(buf []byte) bool { + return len(buf) > 1 && + buf[0] == 0x4D && buf[1] == 0x5A +} + +func Swf(buf []byte) bool { + return len(buf) > 2 && + (buf[0] == 0x43 || buf[0] == 0x46) && + buf[1] == 0x57 && buf[2] == 0x53 +} + +func Rtf(buf []byte) bool { + return len(buf) > 4 && + buf[0] == 0x7B && buf[1] == 0x5C && + buf[2] == 0x72 && buf[3] == 0x74 && + buf[4] == 0x66 +} + +func Nes(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x4E && buf[1] == 0x45 && + buf[2] == 0x53 && buf[3] == 0x1A +} + +func Crx(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x43 && buf[1] == 0x72 && + buf[2] == 0x32 && buf[3] == 0x34 +} + +func Cab(buf []byte) bool { + return len(buf) > 3 && + ((buf[0] == 0x4D && buf[1] == 0x53 && buf[2] == 0x43 && buf[3] == 0x46) || + (buf[0] == 0x49 && buf[1] == 0x53 && buf[2] == 0x63 && buf[3] == 0x28)) +} + +func Eot(buf []byte) bool { + return len(buf) > 35 && + buf[34] == 0x4C && buf[35] == 0x50 && + ((buf[8] == 0x02 && buf[9] == 0x00 && + buf[10] == 0x01) || (buf[8] == 0x01 && + buf[9] == 0x00 && buf[10] == 0x00) || + (buf[8] == 0x02 && buf[9] == 0x00 && + buf[10] == 0x02)) +} + +func Ps(buf []byte) bool { + return len(buf) > 1 && + buf[0] == 0x25 && buf[1] == 0x21 +} + +func Xz(buf []byte) bool { + return len(buf) > 5 && + buf[0] == 0xFD && buf[1] == 0x37 && + buf[2] == 0x7A && buf[3] == 0x58 && + buf[4] == 0x5A && buf[5] == 0x00 +} + +func Sqlite(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x53 && buf[1] == 0x51 && + buf[2] == 0x4C && buf[3] == 0x69 +} + +func Deb(buf []byte) bool { + return len(buf) > 20 && + buf[0] == 0x21 && buf[1] == 0x3C && buf[2] == 0x61 && + buf[3] == 0x72 && buf[4] == 0x63 && buf[5] == 0x68 && + buf[6] == 0x3E && buf[7] == 0x0A && buf[8] == 0x64 && + buf[9] == 0x65 && buf[10] == 0x62 && buf[11] == 0x69 && + buf[12] == 0x61 && buf[13] == 0x6E && buf[14] == 0x2D && + buf[15] == 0x62 && buf[16] == 0x69 && buf[17] == 0x6E && + buf[18] == 0x61 && buf[19] == 0x72 && buf[20] == 0x79 +} + +func Ar(buf []byte) bool { + return len(buf) > 6 && + buf[0] == 0x21 && buf[1] == 0x3C && + buf[2] == 0x61 && buf[3] == 0x72 && + buf[4] == 0x63 && buf[5] == 0x68 && + buf[6] == 0x3E +} + +func Z(buf []byte) bool { + return len(buf) > 1 && + ((buf[0] == 0x1F && buf[1] == 0xA0) || + (buf[0] == 0x1F && buf[1] == 0x9D)) +} + +func Lz(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x4C && buf[1] == 0x5A && + buf[2] == 0x49 && buf[3] == 0x50 +} + +func Rpm(buf []byte) bool { + return len(buf) > 96 && + buf[0] == 0xED && buf[1] == 0xAB && + buf[2] == 0xEE && buf[3] == 0xDB +} + +func Elf(buf []byte) bool { + return len(buf) > 52 && + buf[0] == 0x7F && buf[1] == 0x45 && + buf[2] == 0x4C && buf[3] == 0x46 +} + +func Dcm(buf []byte) bool { + return len(buf) > 131 && + buf[128] == 0x44 && buf[129] == 0x49 && + buf[130] == 0x43 && buf[131] == 0x4D +} + +func Iso(buf []byte) bool { + return len(buf) > 32773 && + buf[32769] == 0x43 && buf[32770] == 0x44 && + buf[32771] == 0x30 && buf[32772] == 0x30 && + buf[32773] == 0x31 +} + +func MachO(buf []byte) bool { + return len(buf) > 3 && ((buf[0] == 0xFE && buf[1] == 0xED && buf[2] == 0xFA && buf[3] == 0xCF) || + (buf[0] == 0xFE && buf[1] == 0xED && buf[2] == 0xFA && buf[3] == 0xCE) || + (buf[0] == 0xBE && buf[1] == 0xBA && buf[2] == 0xFE && buf[3] == 0xCA) || + // Big endian versions below here... + (buf[0] == 0xCF && buf[1] == 0xFA && buf[2] == 0xED && buf[3] == 0xFE) || + (buf[0] == 0xCE && buf[1] == 0xFA && buf[2] == 0xED && buf[3] == 0xFE) || + (buf[0] == 0xCA && buf[1] == 0xFE && buf[2] == 0xBA && buf[3] == 0xBE)) +} diff --git a/vendor/github.com/h2non/filetype/matchers/audio.go b/vendor/github.com/h2non/filetype/matchers/audio.go new file mode 100644 index 0000000000..6d532630af --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/audio.go @@ -0,0 +1,75 @@ +package matchers + +var ( + TypeMidi = newType("mid", "audio/midi") + TypeMp3 = newType("mp3", "audio/mpeg") + TypeM4a = newType("m4a", "audio/m4a") + TypeOgg = newType("ogg", "audio/ogg") + TypeFlac = newType("flac", "audio/x-flac") + TypeWav = newType("wav", "audio/x-wav") + TypeAmr = newType("amr", "audio/amr") + TypeAac = newType("aac", "audio/aac") +) + +var Audio = Map{ + TypeMidi: Midi, + TypeMp3: Mp3, + TypeM4a: M4a, + TypeOgg: Ogg, + TypeFlac: Flac, + TypeWav: Wav, + TypeAmr: Amr, + TypeAac: Aac, +} + +func Midi(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x4D && buf[1] == 0x54 && + buf[2] == 0x68 && buf[3] == 0x64 +} + +func Mp3(buf []byte) bool { + return len(buf) > 2 && + ((buf[0] == 0x49 && buf[1] == 0x44 && buf[2] == 0x33) || + (buf[0] == 0xFF && buf[1] == 0xfb)) +} + +func M4a(buf []byte) bool { + return len(buf) > 10 && + ((buf[4] == 0x66 && buf[5] == 0x74 && buf[6] == 0x79 && + buf[7] == 0x70 && buf[8] == 0x4D && buf[9] == 0x34 && buf[10] == 0x41) || + (buf[0] == 0x4D && buf[1] == 0x34 && buf[2] == 0x41 && buf[3] == 0x20)) +} + +func Ogg(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x4F && buf[1] == 0x67 && + buf[2] == 0x67 && buf[3] == 0x53 +} + +func Flac(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x66 && buf[1] == 0x4C && + buf[2] == 0x61 && buf[3] == 0x43 +} + +func Wav(buf []byte) bool { + return len(buf) > 11 && + buf[0] == 0x52 && buf[1] == 0x49 && + buf[2] == 0x46 && buf[3] == 0x46 && + buf[8] == 0x57 && buf[9] == 0x41 && + buf[10] == 0x56 && buf[11] == 0x45 +} + +func Amr(buf []byte) bool { + return len(buf) > 11 && + buf[0] == 0x23 && buf[1] == 0x21 && + buf[2] == 0x41 && buf[3] == 0x4D && + buf[4] == 0x52 && buf[5] == 0x0A +} + +func Aac(buf []byte) bool { + return len(buf) > 1 && + ((buf[0] == 0xFF && buf[1] == 0xF1) || + (buf[0] == 0xFF && buf[1] == 0xF9)) +} diff --git a/vendor/github.com/h2non/filetype/matchers/document.go b/vendor/github.com/h2non/filetype/matchers/document.go new file mode 100644 index 0000000000..b898c0ff75 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/document.go @@ -0,0 +1,197 @@ +package matchers + +import ( + "bytes" + "encoding/binary" +) + +var ( + TypeDoc = newType("doc", "application/msword") + TypeDocx = newType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document") + TypeXls = newType("xls", "application/vnd.ms-excel") + TypeXlsx = newType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + TypePpt = newType("ppt", "application/vnd.ms-powerpoint") + TypePptx = newType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation") +) + +var Document = Map{ + TypeDoc: Doc, + TypeDocx: Docx, + TypeXls: Xls, + TypeXlsx: Xlsx, + TypePpt: Ppt, + TypePptx: Pptx, +} + +type docType int + +const ( + TYPE_DOC docType = iota + TYPE_DOCX + TYPE_XLS + TYPE_XLSX + TYPE_PPT + TYPE_PPTX + TYPE_OOXML +) + +//reference: https://bz.apache.org/ooo/show_bug.cgi?id=111457 +func Doc(buf []byte) bool { + if len(buf) > 513 { + return buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[512] == 0xEC && buf[513] == 0xA5 + } else { + return len(buf) > 3 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 + } +} + +func Docx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_DOCX +} + +func Xls(buf []byte) bool { + if len(buf) > 513 { + return buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[512] == 0x09 && buf[513] == 0x08 + } else { + return len(buf) > 3 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 + } +} + +func Xlsx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_XLSX +} + +func Ppt(buf []byte) bool { + if len(buf) > 513 { + return buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[512] == 0xA0 && buf[513] == 0x46 + } else { + return len(buf) > 3 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 + } +} + +func Pptx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_PPTX +} + +func msooxml(buf []byte) (typ docType, found bool) { + signature := []byte{'P', 'K', 0x03, 0x04} + + // start by checking for ZIP local file header signature + if ok := compareBytes(buf, signature, 0); !ok { + return + } + + // make sure the first file is correct + if v, ok := checkMSOoml(buf, 0x1E); ok { + return v, ok + } + + if !compareBytes(buf, []byte("[Content_Types].xml"), 0x1E) && + !compareBytes(buf, []byte("_rels/.rels"), 0x1E) && + !compareBytes(buf, []byte("docProps"), 0x1E) { + return + } + + // skip to the second local file header + // since some documents include a 520-byte extra field following the file + // header, we need to scan for the next header + startOffset := int(binary.LittleEndian.Uint32(buf[18:22]) + 49) + idx := search(buf, startOffset, 6000) + if idx == -1 { + return + } + + // now skip to the *third* local file header; again, we need to scan due to a + // 520-byte extra field following the file header + startOffset += idx + 4 + 26 + idx = search(buf, startOffset, 6000) + if idx == -1 { + return + } + + // and check the subdirectory name to determine which type of OOXML + // file we have. Correct the mimetype with the registered ones: + // http://technet.microsoft.com/en-us/library/cc179224.aspx + startOffset += idx + 4 + 26 + if typ, ok := checkMSOoml(buf, startOffset); ok { + return typ, ok + } + + // OpenOffice/Libreoffice orders ZIP entry differently, so check the 4th file + startOffset += 26 + idx = search(buf, startOffset, 6000) + if idx == -1 { + return TYPE_OOXML, true + } + + startOffset += idx + 4 + 26 + if typ, ok := checkMSOoml(buf, startOffset); ok { + return typ, ok + } else { + return TYPE_OOXML, true + } +} + +func compareBytes(slice, subSlice []byte, startOffset int) bool { + sl := len(subSlice) + + if startOffset+sl > len(slice) { + return false + } + + s := slice[startOffset : startOffset+sl] + for i := range s { + if subSlice[i] != s[i] { + return false + } + } + + return true +} + +func checkMSOoml(buf []byte, offset int) (typ docType, ok bool) { + ok = true + + switch { + case compareBytes(buf, []byte("word/"), offset): + typ = TYPE_DOCX + case compareBytes(buf, []byte("ppt/"), offset): + typ = TYPE_PPTX + case compareBytes(buf, []byte("xl/"), offset): + typ = TYPE_XLSX + default: + ok = false + } + + return +} + +func search(buf []byte, start, rangeNum int) int { + length := len(buf) + end := start + rangeNum + signature := []byte{'P', 'K', 0x03, 0x04} + + if end > length { + end = length + } + + if start >= end { + return -1 + } + + return bytes.Index(buf[start:end], signature) +} diff --git a/vendor/github.com/h2non/filetype/matchers/font.go b/vendor/github.com/h2non/filetype/matchers/font.go new file mode 100644 index 0000000000..f39171675e --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/font.go @@ -0,0 +1,45 @@ +package matchers + +var ( + TypeWoff = newType("woff", "application/font-woff") + TypeWoff2 = newType("woff2", "application/font-woff") + TypeTtf = newType("ttf", "application/font-sfnt") + TypeOtf = newType("otf", "application/font-sfnt") +) + +var Font = Map{ + TypeWoff: Woff, + TypeWoff2: Woff2, + TypeTtf: Ttf, + TypeOtf: Otf, +} + +func Woff(buf []byte) bool { + return len(buf) > 7 && + buf[0] == 0x77 && buf[1] == 0x4F && + buf[2] == 0x46 && buf[3] == 0x46 && + buf[4] == 0x00 && buf[5] == 0x01 && + buf[6] == 0x00 && buf[7] == 0x00 +} + +func Woff2(buf []byte) bool { + return len(buf) > 7 && + buf[0] == 0x77 && buf[1] == 0x4F && + buf[2] == 0x46 && buf[3] == 0x32 && + buf[4] == 0x00 && buf[5] == 0x01 && + buf[6] == 0x00 && buf[7] == 0x00 +} + +func Ttf(buf []byte) bool { + return len(buf) > 4 && + buf[0] == 0x00 && buf[1] == 0x01 && + buf[2] == 0x00 && buf[3] == 0x00 && + buf[4] == 0x00 +} + +func Otf(buf []byte) bool { + return len(buf) > 4 && + buf[0] == 0x4F && buf[1] == 0x54 && + buf[2] == 0x54 && buf[3] == 0x4F && + buf[4] == 0x00 +} diff --git a/vendor/github.com/h2non/filetype/matchers/image.go b/vendor/github.com/h2non/filetype/matchers/image.go new file mode 100644 index 0000000000..0465d0d681 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/image.go @@ -0,0 +1,143 @@ +package matchers + +import "github.com/h2non/filetype/matchers/isobmff" + +var ( + TypeJpeg = newType("jpg", "image/jpeg") + TypeJpeg2000 = newType("jp2", "image/jp2") + TypePng = newType("png", "image/png") + TypeGif = newType("gif", "image/gif") + TypeWebp = newType("webp", "image/webp") + TypeCR2 = newType("cr2", "image/x-canon-cr2") + TypeTiff = newType("tif", "image/tiff") + TypeBmp = newType("bmp", "image/bmp") + TypeJxr = newType("jxr", "image/vnd.ms-photo") + TypePsd = newType("psd", "image/vnd.adobe.photoshop") + TypeIco = newType("ico", "image/vnd.microsoft.icon") + TypeHeif = newType("heif", "image/heif") + TypeDwg = newType("dwg", "image/vnd.dwg") +) + +var Image = Map{ + TypeJpeg: Jpeg, + TypeJpeg2000: Jpeg2000, + TypePng: Png, + TypeGif: Gif, + TypeWebp: Webp, + TypeCR2: CR2, + TypeTiff: Tiff, + TypeBmp: Bmp, + TypeJxr: Jxr, + TypePsd: Psd, + TypeIco: Ico, + TypeHeif: Heif, + TypeDwg: Dwg, +} + +func Jpeg(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0xFF && + buf[1] == 0xD8 && + buf[2] == 0xFF +} + +func Jpeg2000(buf []byte) bool { + return len(buf) > 12 && + buf[0] == 0x0 && + buf[1] == 0x0 && + buf[2] == 0x0 && + buf[3] == 0xC && + buf[4] == 0x6A && + buf[5] == 0x50 && + buf[6] == 0x20 && + buf[7] == 0x20 && + buf[8] == 0xD && + buf[9] == 0xA && + buf[10] == 0x87 && + buf[11] == 0xA && + buf[12] == 0x0 +} + +func Png(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x89 && buf[1] == 0x50 && + buf[2] == 0x4E && buf[3] == 0x47 +} + +func Gif(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x47 && buf[1] == 0x49 && buf[2] == 0x46 +} + +func Webp(buf []byte) bool { + return len(buf) > 11 && + buf[8] == 0x57 && buf[9] == 0x45 && + buf[10] == 0x42 && buf[11] == 0x50 +} + +func CR2(buf []byte) bool { + return len(buf) > 10 && + ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || // Little Endian + (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) && // Big Endian + buf[8] == 0x43 && buf[9] == 0x52 && // CR2 magic word + buf[10] == 0x02 // CR2 major version +} + +func Tiff(buf []byte) bool { + return len(buf) > 10 && + ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || // Little Endian + (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) && // Big Endian + !CR2(buf) // To avoid conflicts differentiate Tiff from CR2 +} + +func Bmp(buf []byte) bool { + return len(buf) > 1 && + buf[0] == 0x42 && + buf[1] == 0x4D +} + +func Jxr(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x49 && + buf[1] == 0x49 && + buf[2] == 0xBC +} + +func Psd(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x38 && buf[1] == 0x42 && + buf[2] == 0x50 && buf[3] == 0x53 +} + +func Ico(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x00 && buf[1] == 0x00 && + buf[2] == 0x01 && buf[3] == 0x00 +} + +func Heif(buf []byte) bool { + if !isobmff.IsISOBMFF(buf) { + return false + } + + majorBrand, _, compatibleBrands := isobmff.GetFtyp(buf) + if majorBrand == "heic" { + return true + } + + if majorBrand == "mif1" || majorBrand == "msf1" { + for _, compatibleBrand := range compatibleBrands { + if compatibleBrand == "heic" { + return true + } + } + } + + return false +} + +func Dwg(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x41 && buf[1] == 0x43 && + buf[2] == 0x31 && buf[3] == 0x30 +} diff --git a/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go b/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go new file mode 100644 index 0000000000..b3e39bf59a --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go @@ -0,0 +1,37 @@ +package isobmff + +import "encoding/binary" + +// IsISOBMFF checks whether the given buffer represents ISO Base Media File Format data +func IsISOBMFF(buf []byte) bool { + if len(buf) < 16 || string(buf[4:8]) != "ftyp" { + return false + } + + if ftypLength := binary.BigEndian.Uint32(buf[0:4]); len(buf) < int(ftypLength) { + return false + } + + return true +} + +// GetFtyp returns the major brand, minor version and compatible brands of the ISO-BMFF data +func GetFtyp(buf []byte) (string, string, []string) { + if len(buf) < 17 { + return "", "", []string{""} + } + + ftypLength := binary.BigEndian.Uint32(buf[0:4]) + + majorBrand := string(buf[8:12]) + minorVersion := string(buf[12:16]) + + compatibleBrands := []string{} + for i := 16; i < int(ftypLength); i += 4 { + if len(buf) >= (i + 4) { + compatibleBrands = append(compatibleBrands, string(buf[i:i+4])) + } + } + + return majorBrand, minorVersion, compatibleBrands +} diff --git a/vendor/github.com/h2non/filetype/matchers/matchers.go b/vendor/github.com/h2non/filetype/matchers/matchers.go new file mode 100644 index 0000000000..20d74d080d --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/matchers.go @@ -0,0 +1,51 @@ +package matchers + +import ( + "github.com/h2non/filetype/types" +) + +// Internal shortcut to NewType +var newType = types.NewType + +// Matcher function interface as type alias +type Matcher func([]byte) bool + +// Type interface to store pairs of type with its matcher function +type Map map[types.Type]Matcher + +// Type specific matcher function interface +type TypeMatcher func([]byte) types.Type + +// Store registered file type matchers +var Matchers = make(map[types.Type]TypeMatcher) +var MatcherKeys []types.Type + +// Create and register a new type matcher function +func NewMatcher(kind types.Type, fn Matcher) TypeMatcher { + matcher := func(buf []byte) types.Type { + if fn(buf) { + return kind + } + return types.Unknown + } + + Matchers[kind] = matcher + // prepend here so any user defined matchers get added first + MatcherKeys = append([]types.Type{kind}, MatcherKeys...) + return matcher +} + +func register(matchers ...Map) { + MatcherKeys = MatcherKeys[:0] + for _, m := range matchers { + for kind, matcher := range m { + NewMatcher(kind, matcher) + } + } +} + +func init() { + // Arguments order is intentional + // Archive files will be checked last due to prepend above in func NewMatcher + register(Archive, Document, Font, Audio, Video, Image, Application) +} diff --git a/vendor/github.com/h2non/filetype/matchers/video.go b/vendor/github.com/h2non/filetype/matchers/video.go new file mode 100644 index 0000000000..e97cf28a11 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/video.go @@ -0,0 +1,145 @@ +package matchers + +import "bytes" + +var ( + TypeMp4 = newType("mp4", "video/mp4") + TypeM4v = newType("m4v", "video/x-m4v") + TypeMkv = newType("mkv", "video/x-matroska") + TypeWebm = newType("webm", "video/webm") + TypeMov = newType("mov", "video/quicktime") + TypeAvi = newType("avi", "video/x-msvideo") + TypeWmv = newType("wmv", "video/x-ms-wmv") + TypeMpeg = newType("mpg", "video/mpeg") + TypeFlv = newType("flv", "video/x-flv") + Type3gp = newType("3gp", "video/3gpp") +) + +var Video = Map{ + TypeMp4: Mp4, + TypeM4v: M4v, + TypeMkv: Mkv, + TypeWebm: Webm, + TypeMov: Mov, + TypeAvi: Avi, + TypeWmv: Wmv, + TypeMpeg: Mpeg, + TypeFlv: Flv, + Type3gp: Match3gp, +} + +func M4v(buf []byte) bool { + return len(buf) > 10 && + buf[4] == 0x66 && buf[5] == 0x74 && + buf[6] == 0x79 && buf[7] == 0x70 && + buf[8] == 0x4D && buf[9] == 0x34 && + buf[10] == 0x56 +} + +func Mkv(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x1A && buf[1] == 0x45 && + buf[2] == 0xDF && buf[3] == 0xA3 && + containsMatroskaSignature(buf, []byte{'m', 'a', 't', 'r', 'o', 's', 'k', 'a'}) +} + +func Webm(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x1A && buf[1] == 0x45 && + buf[2] == 0xDF && buf[3] == 0xA3 && + containsMatroskaSignature(buf, []byte{'w', 'e', 'b', 'm'}) +} + +func Mov(buf []byte) bool { + return len(buf) > 15 && ((buf[0] == 0x0 && buf[1] == 0x0 && + buf[2] == 0x0 && buf[3] == 0x14 && + buf[4] == 0x66 && buf[5] == 0x74 && + buf[6] == 0x79 && buf[7] == 0x70) || + (buf[4] == 0x6d && buf[5] == 0x6f && buf[6] == 0x6f && buf[7] == 0x76) || + (buf[4] == 0x6d && buf[5] == 0x64 && buf[6] == 0x61 && buf[7] == 0x74) || + (buf[12] == 0x6d && buf[13] == 0x64 && buf[14] == 0x61 && buf[15] == 0x74)) +} + +func Avi(buf []byte) bool { + return len(buf) > 10 && + buf[0] == 0x52 && buf[1] == 0x49 && + buf[2] == 0x46 && buf[3] == 0x46 && + buf[8] == 0x41 && buf[9] == 0x56 && + buf[10] == 0x49 +} + +func Wmv(buf []byte) bool { + return len(buf) > 9 && + buf[0] == 0x30 && buf[1] == 0x26 && + buf[2] == 0xB2 && buf[3] == 0x75 && + buf[4] == 0x8E && buf[5] == 0x66 && + buf[6] == 0xCF && buf[7] == 0x11 && + buf[8] == 0xA6 && buf[9] == 0xD9 +} + +func Mpeg(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x0 && buf[1] == 0x0 && + buf[2] == 0x1 && buf[3] >= 0xb0 && + buf[3] <= 0xbf +} + +func Flv(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x46 && buf[1] == 0x4C && + buf[2] == 0x56 && buf[3] == 0x01 +} + +func Mp4(buf []byte) bool { + return len(buf) > 11 && + (buf[4] == 'f' && buf[5] == 't' && buf[6] == 'y' && buf[7] == 'p') && + ((buf[8] == 'a' && buf[9] == 'v' && buf[10] == 'c' && buf[11] == '1') || + (buf[8] == 'd' && buf[9] == 'a' && buf[10] == 's' && buf[11] == 'h') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '2') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '3') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '4') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '5') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '6') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == 'm') || + (buf[8] == 'm' && buf[9] == 'm' && buf[10] == 'p' && buf[11] == '4') || + (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == '1') || + (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == '2') || + (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == 'v') || + (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '7' && buf[11] == '1') || + (buf[8] == 'M' && buf[9] == 'S' && buf[10] == 'N' && buf[11] == 'V') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'A' && buf[11] == 'S') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'S' && buf[10] == 'D' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'H') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'M') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'P') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'S') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'H') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'M') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'P') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'S') || + (buf[8] == 'F' && buf[9] == '4' && buf[10] == 'V' && buf[11] == ' ') || + (buf[8] == 'F' && buf[9] == '4' && buf[10] == 'P' && buf[11] == ' ')) +} + +func Match3gp(buf []byte) bool { + return len(buf) > 10 && + buf[4] == 0x66 && buf[5] == 0x74 && buf[6] == 0x79 && + buf[7] == 0x70 && buf[8] == 0x33 && buf[9] == 0x67 && + buf[10] == 0x70 +} + +func containsMatroskaSignature(buf, subType []byte) bool { + limit := 4096 + if len(buf) < limit { + limit = len(buf) + } + + index := bytes.Index(buf[:limit], subType) + if index < 3 { + return false + } + + return buf[index-3] == 0x42 && buf[index-2] == 0x82 +} diff --git a/vendor/github.com/h2non/filetype/types/defaults.go b/vendor/github.com/h2non/filetype/types/defaults.go new file mode 100644 index 0000000000..0d985a05d6 --- /dev/null +++ b/vendor/github.com/h2non/filetype/types/defaults.go @@ -0,0 +1,4 @@ +package types + +// Unknown default type +var Unknown = NewType("unknown", "") diff --git a/vendor/github.com/h2non/filetype/types/mime.go b/vendor/github.com/h2non/filetype/types/mime.go new file mode 100644 index 0000000000..fe8ea822e5 --- /dev/null +++ b/vendor/github.com/h2non/filetype/types/mime.go @@ -0,0 +1,14 @@ +package types + +// MIME stores the file MIME type values +type MIME struct { + Type string + Subtype string + Value string +} + +// Creates a new MIME type +func NewMIME(mime string) MIME { + kind, subtype := splitMime(mime) + return MIME{Type: kind, Subtype: subtype, Value: mime} +} diff --git a/vendor/github.com/h2non/filetype/types/split.go b/vendor/github.com/h2non/filetype/types/split.go new file mode 100644 index 0000000000..68a5a8b3b5 --- /dev/null +++ b/vendor/github.com/h2non/filetype/types/split.go @@ -0,0 +1,11 @@ +package types + +import "strings" + +func splitMime(s string) (string, string) { + x := strings.Split(s, "/") + if len(x) > 1 { + return x[0], x[1] + } + return x[0], "" +} diff --git a/vendor/github.com/h2non/filetype/types/type.go b/vendor/github.com/h2non/filetype/types/type.go new file mode 100644 index 0000000000..5cf7dfc4bb --- /dev/null +++ b/vendor/github.com/h2non/filetype/types/type.go @@ -0,0 +1,16 @@ +package types + +// Type represents a file MIME type and its extension +type Type struct { + MIME MIME + Extension string +} + +// NewType creates a new Type +func NewType(ext, mime string) Type { + t := Type{ + MIME: NewMIME(mime), + Extension: ext, + } + return Add(t) +} diff --git a/vendor/github.com/h2non/filetype/types/types.go b/vendor/github.com/h2non/filetype/types/types.go new file mode 100644 index 0000000000..f59e256f0d --- /dev/null +++ b/vendor/github.com/h2non/filetype/types/types.go @@ -0,0 +1,23 @@ +package types + +import "sync" + +// Types Support concurrent map writes +var Types sync.Map + +// Add registers a new type in the package +func Add(t Type) Type { + Types.Store(t.Extension, t) + return t +} + +// Get retrieves a Type by extension +func Get(ext string) Type { + if tmp, ok := Types.Load(ext); ok { + kind := tmp.(Type) + if kind.Extension != "" { + return kind + } + } + return Unknown +} diff --git a/vendor/github.com/h2non/filetype/version.go b/vendor/github.com/h2non/filetype/version.go new file mode 100644 index 0000000000..116e4c78ed --- /dev/null +++ b/vendor/github.com/h2non/filetype/version.go @@ -0,0 +1,4 @@ +package filetype + +// Version exposes the current package version. +const Version = "1.1.1" diff --git a/vendor/github.com/h2non/go-is-svg/.editorconfig b/vendor/github.com/h2non/go-is-svg/.editorconfig new file mode 100644 index 0000000000..000dc0a7aa --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +indent_style = tabs +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false diff --git a/vendor/github.com/h2non/go-is-svg/.gitignore b/vendor/github.com/h2non/go-is-svg/.gitignore new file mode 100644 index 0000000000..3cf2565219 --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/.gitignore @@ -0,0 +1,7 @@ +/bimg +/bundle +bin +/*.jpg +/*.png +/*.webp +/fixtures/*_out.* diff --git a/vendor/github.com/h2non/go-is-svg/.travis.yml b/vendor/github.com/h2non/go-is-svg/.travis.yml new file mode 100644 index 0000000000..d5a81534a0 --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/.travis.yml @@ -0,0 +1,23 @@ +language: go + +go: + - 1.5 + - 1.6 + - 1.7 + - tip + +before_install: + - go get github.com/nbio/st + - go get -u -v github.com/axw/gocov/gocov + - go get -u -v github.com/mattn/goveralls + - go get -u -v github.com/golang/lint/golint + +script: + - diff -u <(echo -n) <(gofmt -s -d ./) + - diff -u <(echo -n) <(go vet ./...) + - diff -u <(echo -n) <(golint ./...) + - go test -v -race ./... + - go test -v -race -covermode=atomic -coverprofile=coverage.out + +after_success: + - goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/h2non/go-is-svg/LICENSE b/vendor/github.com/h2non/go-is-svg/LICENSE new file mode 100644 index 0000000000..f67807d007 --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/LICENSE @@ -0,0 +1,24 @@ +The MIT License + +Copyright (c) 2016 Tomas Aparicio + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/h2non/go-is-svg/README.md b/vendor/github.com/h2non/go-is-svg/README.md new file mode 100644 index 0000000000..d4a0ee6c02 --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/README.md @@ -0,0 +1,47 @@ +# go-is-svg [![Build Status](https://travis-ci.org/h2non/go-is-svg.png)](https://travis-ci.org/h2non/go-is-svg) [![GoDoc](https://godoc.org/github.com/h2non/go-is-svg?status.svg)](https://godoc.org/github.com/h2non/go-is-svg) [![Coverage Status](https://coveralls.io/repos/github/h2non/go-is-svg/badge.svg?branch=master)](https://coveralls.io/github/h2non/go-is-svg?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/h2non/go-is-svg)](https://goreportcard.com/report/github.com/h2non/go-is-svg) + +Tiny package to verify if a given file buffer is an SVG image in Go (golang). + +See also [filetype](https://github.com/h2non/filetype) package for binary files type inference. + +## Installation + +```bash +go get -u github.com/h2non/go-is-svg +``` + +## Example + +```go +package main + +import ( + "fmt" + "io/ioutil" + + svg "github.com/h2non/go-is-svg" +) + +func main() { + buf, err := ioutil.ReadFile("_example/example.svg") + if err != nil { + fmt.Printf("Error: %s\n", err) + return + } + + if svg.Is(buf) { + fmt.Println("File is an SVG") + } else { + fmt.Println("File is NOT an SVG") + } +} +``` + +Run example: +```bash +go run _example/example.go +``` + +## License + +MIT - Tomas Aparicio diff --git a/vendor/github.com/h2non/go-is-svg/svg.go b/vendor/github.com/h2non/go-is-svg/svg.go new file mode 100644 index 0000000000..062f6e1f66 --- /dev/null +++ b/vendor/github.com/h2non/go-is-svg/svg.go @@ -0,0 +1,36 @@ +package issvg + +import ( + "regexp" + "unicode/utf8" +) + +var ( + htmlCommentRegex = regexp.MustCompile("(?i)") + svgRegex = regexp.MustCompile(`(?i)^\s*(?:<\?xml[^>]*>\s*)?(?:]*>\s*)?]*>[^*]*<\/svg>\s*$`) +) + +// isBinary checks if the given buffer is a binary file. +func isBinary(buf []byte) bool { + if len(buf) < 24 { + return false + } + for i := 0; i < 24; i++ { + charCode, _ := utf8.DecodeRuneInString(string(buf[i])) + if charCode == 65533 || charCode <= 8 { + return true + } + } + return false +} + +// Is returns true if the given buffer is a valid SVG image. +func Is(buf []byte) bool { + return !isBinary(buf) && svgRegex.Match(htmlCommentRegex.ReplaceAll(buf, []byte{})) +} + +// IsSVG returns true if the given buffer is a valid SVG image. +// Alias to: Is() +func IsSVG(buf []byte) bool { + return Is(buf) +} diff --git a/vendor/github.com/operator-framework/operator-registry/internal/model/error.go b/vendor/github.com/operator-framework/operator-registry/internal/model/error.go new file mode 100644 index 0000000000..0ad0f7adba --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/internal/model/error.go @@ -0,0 +1,66 @@ +package model + +import ( + "bytes" + "fmt" + "strings" +) + +type validationError struct { + message string + subErrors []error +} + +func newValidationError(message string) *validationError { + return &validationError{message: message} +} + +func (v *validationError) orNil() error { + if len(v.subErrors) == 0 { + return nil + } + return v +} + +func (v *validationError) Error() string { + if v == nil { + return "" + } + return strings.TrimSpace(v.errorPrefix(nil, true, nil)) +} + +func (v *validationError) errorPrefix(prefix []rune, last bool, seen []error) string { + for _, s := range seen { + if v == s { + return "" + } + } + seen = append(seen, v) + sep := ":\n" + if len(v.subErrors) == 0 { + sep = "\n" + } + errMsg := bytes.NewBufferString(fmt.Sprintf("%s%s%s", string(prefix), v.message, sep)) + for i, serr := range v.subErrors { + subPrefix := prefix + if len(subPrefix) >= 4 { + if last { + subPrefix = append(subPrefix[0:len(subPrefix)-4], []rune(" ")...) + } else { + subPrefix = append(subPrefix[0:len(subPrefix)-4], []rune("│ ")...) + } + } + subLast := i == len(v.subErrors)-1 + if subLast { + subPrefix = append(subPrefix, []rune("└── ")...) + } else { + subPrefix = append(subPrefix, []rune("├── ")...) + } + if verr, ok := serr.(*validationError); ok { + errMsg.WriteString(verr.errorPrefix(subPrefix, subLast, seen)) + } else { + errMsg.WriteString(fmt.Sprintf("%s%s\n", string(subPrefix), serr)) + } + } + return errMsg.String() +} diff --git a/vendor/github.com/operator-framework/operator-registry/internal/model/model.go b/vendor/github.com/operator-framework/operator-registry/internal/model/model.go new file mode 100644 index 0000000000..330ea84db1 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/internal/model/model.go @@ -0,0 +1,317 @@ +package model + +import ( + "errors" + "fmt" + "strings" + + "github.com/h2non/filetype" + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" + svg "github.com/h2non/go-is-svg" + + "github.com/operator-framework/operator-registry/internal/property" +) + +func init() { + t := types.NewType("svg", "image/svg+xml") + filetype.AddMatcher(t, svg.Is) + matchers.Image[types.NewType("svg", "image/svg+xml")] = svg.Is +} + +type Model map[string]*Package + +func (m Model) Validate() error { + result := newValidationError("invalid index") + + for name, pkg := range m { + if name != pkg.Name { + result.subErrors = append(result.subErrors, fmt.Errorf("package key %q does not match package name %q", name, pkg.Name)) + } + if err := pkg.Validate(); err != nil { + result.subErrors = append(result.subErrors, err) + } + } + return result.orNil() +} + +type Package struct { + Name string + Description string + Icon *Icon + DefaultChannel *Channel + Channels map[string]*Channel +} + +func (m *Package) Validate() error { + result := newValidationError(fmt.Sprintf("invalid package %q", m.Name)) + + if m.Name == "" { + result.subErrors = append(result.subErrors, errors.New("package name must not be empty")) + } + + if err := m.Icon.Validate(); err != nil { + result.subErrors = append(result.subErrors, err) + } + + if m.DefaultChannel == nil { + result.subErrors = append(result.subErrors, fmt.Errorf("default channel must be set")) + } + + if len(m.Channels) == 0 { + result.subErrors = append(result.subErrors, fmt.Errorf("package must contain at least one channel")) + } + + foundDefault := false + for name, ch := range m.Channels { + if name != ch.Name { + result.subErrors = append(result.subErrors, fmt.Errorf("channel key %q does not match channel name %q", name, ch.Name)) + } + if err := ch.Validate(); err != nil { + result.subErrors = append(result.subErrors, err) + } + if ch == m.DefaultChannel { + foundDefault = true + } + if ch.Package != m { + result.subErrors = append(result.subErrors, fmt.Errorf("channel %q not correctly linked to parent package", ch.Name)) + } + } + + if m.DefaultChannel != nil && !foundDefault { + result.subErrors = append(result.subErrors, fmt.Errorf("default channel %q not found in channels list", m.DefaultChannel.Name)) + } + return result.orNil() +} + +type Icon struct { + Data []byte + MediaType string +} + +func (i *Icon) Validate() error { + if i == nil { + return nil + } + // TODO(joelanford): Should we check that data and mediatype are set, + // and detect the media type of the data and compare it to the + // mediatype listed in the icon field? Currently, some production + // index databases are failing these tests, so leaving this + // commented out for now. + result := newValidationError("invalid icon") + //if len(i.Data) == 0 { + // result.subErrors = append(result.subErrors, errors.New("icon data must be set if icon is defined")) + //} + //if len(i.MediaType) == 0 { + // result.subErrors = append(result.subErrors, errors.New("icon mediatype must be set if icon is defined")) + //} + //if len(i.Data) > 0 { + // if err := i.validateData(); err != nil { + // result.subErrors = append(result.subErrors, err) + // } + //} + return result.orNil() +} + +func (i *Icon) validateData() error { + if !filetype.IsImage(i.Data) { + return errors.New("icon data is not an image") + } + t, err := filetype.Match(i.Data) + if err != nil { + return err + } + if t.MIME.Value != i.MediaType { + return fmt.Errorf("icon media type %q does not match detected media type %q", i.MediaType, t.MIME.Value) + } + return nil +} + +type Channel struct { + Package *Package + Name string + Bundles map[string]*Bundle +} + +// TODO(joelanford): This function determines the channel head by finding the bundle that has 0 +// incoming edges, based on replaces and skips. It also expects to find exactly one such bundle. +// Is this the correct algorithm? +func (c Channel) Head() (*Bundle, error) { + incoming := map[string]int{} + for _, b := range c.Bundles { + if b.Replaces != "" { + incoming[b.Replaces]++ + } + for _, skip := range b.Skips { + incoming[skip]++ + } + } + var heads []*Bundle + for _, b := range c.Bundles { + if _, ok := incoming[b.Name]; !ok { + heads = append(heads, b) + } + } + if len(heads) == 0 { + return nil, fmt.Errorf("no channel head found in graph") + } + if len(heads) > 1 { + var headNames []string + for _, head := range heads { + headNames = append(headNames, head.Name) + } + return nil, fmt.Errorf("multiple channel heads found in graph: %s", strings.Join(headNames, ", ")) + } + return heads[0], nil +} + +func (c *Channel) Validate() error { + result := newValidationError(fmt.Sprintf("invalid channel %q", c.Name)) + + if c.Name == "" { + result.subErrors = append(result.subErrors, errors.New("channel name must not be empty")) + } + + if c.Package == nil { + result.subErrors = append(result.subErrors, errors.New("package must be set")) + } + + if len(c.Bundles) == 0 { + result.subErrors = append(result.subErrors, fmt.Errorf("channel must contain at least one bundle")) + } + + if len(c.Bundles) > 0 { + if _, err := c.Head(); err != nil { + result.subErrors = append(result.subErrors, err) + } + } + + for name, b := range c.Bundles { + if name != b.Name { + result.subErrors = append(result.subErrors, fmt.Errorf("bundle key %q does not match bundle name %q", name, b.Name)) + } + if err := b.Validate(); err != nil { + result.subErrors = append(result.subErrors, err) + } + if b.Channel != c { + result.subErrors = append(result.subErrors, fmt.Errorf("bundle %q not correctly linked to parent channel", b.Name)) + } + } + return result.orNil() +} + +type Bundle struct { + Package *Package + Channel *Channel + Name string + Image string + Replaces string + Skips []string + Properties []property.Property + RelatedImages []RelatedImage + + // These fields are present so that we can continue serving + // the GRPC API the way packageserver expects us to in a + // backwards-compatible way. + Objects []string + CsvJSON string +} + +func (b *Bundle) Validate() error { + result := newValidationError(fmt.Sprintf("invalid bundle %q", b.Name)) + + if b.Name == "" { + result.subErrors = append(result.subErrors, errors.New("name must be set")) + } + if b.Channel == nil { + result.subErrors = append(result.subErrors, errors.New("channel must be set")) + } + if b.Package == nil { + result.subErrors = append(result.subErrors, errors.New("package must be set")) + } + if b.Channel != nil && b.Package != nil && b.Package != b.Channel.Package { + result.subErrors = append(result.subErrors, errors.New("package does not match channel's package")) + } + props, err := property.Parse(b.Properties) + if err != nil { + result.subErrors = append(result.subErrors, err) + } + for i, skip := range b.Skips { + if skip == "" { + result.subErrors = append(result.subErrors, fmt.Errorf("skip[%d] is empty", i)) + } + } + // TODO(joelanford): Validate related images? It looks like some + // CSVs in production databases use incorrect fields ([name,value] + // instead of [name,image]), which results in empty image values. + // Example is in redhat-operators: 3scale-operator.v0.5.5 + //for i, relatedImage := range b.RelatedImages { + // if err := relatedImage.Validate(); err != nil { + // result.subErrors = append(result.subErrors, WithIndex(i, err)) + // } + //} + + if props != nil && len(props.Packages) != 1 { + result.subErrors = append(result.subErrors, fmt.Errorf("must be exactly one property with type %q", property.TypePackage)) + } + + if b.Image == "" && len(b.Objects) == 0 { + result.subErrors = append(result.subErrors, errors.New("bundle image must be set")) + } + + return result.orNil() +} + +type RelatedImage struct { + Name string + Image string +} + +func (i RelatedImage) Validate() error { + result := newValidationError("invalid related image") + if i.Image == "" { + result.subErrors = append(result.subErrors, fmt.Errorf("image must be set")) + } + return result.orNil() +} + +func (m Model) Normalize() { + for _, pkg := range m { + for _, ch := range pkg.Channels { + for _, b := range ch.Bundles { + for i := range b.Properties { + // Ensure property value is encoded in a standard way. + if normalized, err := property.Build(&b.Properties[i]); err == nil { + b.Properties[i] = *normalized + } + } + } + } + } +} + +func (m Model) AddBundle(b Bundle) { + if _, present := m[b.Package.Name]; !present { + m[b.Package.Name] = b.Package + } + p := m[b.Package.Name] + b.Package = p + + if ch, ok := p.Channels[b.Channel.Name]; ok { + b.Channel = ch + ch.Bundles[b.Name] = &b + } else { + newCh := &Channel{ + Name: b.Channel.Name, + Package: p, + Bundles: make(map[string]*Bundle), + } + b.Channel = newCh + newCh.Bundles[b.Name] = &b + p.Channels[newCh.Name] = newCh + } + + if p.DefaultChannel == nil { + p.DefaultChannel = b.Channel + } +} diff --git a/vendor/github.com/operator-framework/operator-registry/internal/property/errors.go b/vendor/github.com/operator-framework/operator-registry/internal/property/errors.go new file mode 100644 index 0000000000..6c3689c5b8 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/internal/property/errors.go @@ -0,0 +1,25 @@ +package property + +import ( + "fmt" +) + +type ParseError struct { + Idx int + Typ string + Err error +} + +func (e ParseError) Error() string { + return fmt.Sprintf("parse property[%d] of type %q: %v", e.Idx, e.Typ, e.Err) +} + +type MatchMissingError struct { + foundType string + foundValue interface{} + expectedType string +} + +func (e MatchMissingError) Error() string { + return fmt.Sprintf("property %q for %+v requires matching %q property", e.foundType, e.foundValue, e.expectedType) +} diff --git a/vendor/github.com/operator-framework/operator-registry/internal/property/property.go b/vendor/github.com/operator-framework/operator-registry/internal/property/property.go new file mode 100644 index 0000000000..5f535eea36 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/internal/property/property.go @@ -0,0 +1,310 @@ +package property + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "path/filepath" + "reflect" +) + +type Property struct { + Type string `json:"type"` + Value json.RawMessage `json:"value"` +} + +func (p Property) Validate() error { + if len(p.Type) == 0 { + return errors.New("type must be set") + } + if len(p.Value) == 0 { + return errors.New("value must be set") + } + var raw json.RawMessage + if err := json.Unmarshal(p.Value, &raw); err != nil { + return fmt.Errorf("value is not valid json: %v", err) + } + return nil +} + +func (p Property) String() string { + return fmt.Sprintf("type: %q, value: %q", p.Type, p.Value) +} + +type Package struct { + PackageName string `json:"packageName"` + Version string `json:"version"` +} + +type PackageRequired struct { + PackageName string `json:"packageName"` + VersionRange string `json:"versionRange"` +} + +type Channel struct { + Name string `json:"name"` + Replaces string `json:"replaces,omitempty"` +} + +type GVK struct { + Group string `json:"group"` + Kind string `json:"kind"` + Version string `json:"version"` +} + +type GVKRequired struct { + Group string `json:"group"` + Kind string `json:"kind"` + Version string `json:"version"` +} + +type Skips string +type SkipRange string + +type BundleObject struct { + File `json:",inline"` +} + +type File struct { + ref string + data []byte +} + +type fileJSON struct { + Ref string `json:"ref,omitempty"` + Data []byte `json:"data,omitempty"` +} + +func (f *File) UnmarshalJSON(data []byte) error { + var t fileJSON + if err := json.Unmarshal(data, &t); err != nil { + return err + } + if len(t.Ref) > 0 && len(t.Data) > 0 { + return errors.New("fields 'ref' and 'data' are mutually exclusive") + } + f.ref = t.Ref + f.data = t.Data + return nil +} + +func (f File) MarshalJSON() ([]byte, error) { + return json.Marshal(fileJSON{ + Ref: f.ref, + Data: f.data, + }) +} + +func (f File) IsRef() bool { + return len(f.ref) > 0 +} + +func (f File) GetRef() string { + return f.ref +} + +func (f File) GetData(root fs.FS, cwd string) ([]byte, error) { + if !f.IsRef() { + return f.data, nil + } + if filepath.IsAbs(f.ref) { + return nil, fmt.Errorf("reference must be a relative path") + } + file, err := root.Open(filepath.Join(cwd, f.ref)) + if err != nil { + return nil, err + } + return ioutil.ReadAll(file) +} + +type Properties struct { + Packages []Package + PackagesRequired []PackageRequired + Channels []Channel + GVKs []GVK + GVKsRequired []GVKRequired + Skips []Skips + SkipRanges []SkipRange + BundleObjects []BundleObject + + Others []Property +} + +const ( + TypePackage = "olm.package" + TypePackageRequired = "olm.package.required" + TypeChannel = "olm.channel" + TypeGVK = "olm.gvk" + TypeGVKRequired = "olm.gvk.required" + TypeSkips = "olm.skips" + TypeSkipRange = "olm.skipRange" + TypeBundleObject = "olm.bundle.object" +) + +func Parse(in []Property) (*Properties, error) { + var out Properties + for i, prop := range in { + switch prop.Type { + case TypePackage: + var p Package + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.Packages = append(out.Packages, p) + case TypePackageRequired: + var p PackageRequired + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.PackagesRequired = append(out.PackagesRequired, p) + case TypeChannel: + var p Channel + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.Channels = append(out.Channels, p) + case TypeGVK: + var p GVK + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.GVKs = append(out.GVKs, p) + case TypeGVKRequired: + var p GVKRequired + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.GVKsRequired = append(out.GVKsRequired, p) + case TypeSkips: + var p Skips + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.Skips = append(out.Skips, p) + case TypeSkipRange: + var p SkipRange + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.SkipRanges = append(out.SkipRanges, p) + case TypeBundleObject: + var p BundleObject + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.BundleObjects = append(out.BundleObjects, p) + default: + var p json.RawMessage + if err := json.Unmarshal(prop.Value, &p); err != nil { + return nil, ParseError{Idx: i, Typ: prop.Type, Err: err} + } + out.Others = append(out.Others, prop) + } + } + return &out, nil +} + +func Deduplicate(in []Property) []Property { + type key struct { + typ string + value string + } + + props := map[key]Property{} + var out []Property + for _, p := range in { + k := key{p.Type, string(p.Value)} + if _, ok := props[k]; ok { + continue + } + props[k] = p + out = append(out, p) + } + return out +} + +func Build(p interface{}) (*Property, error) { + var ( + typ string + val interface{} + ) + if prop, ok := p.(*Property); ok { + typ = prop.Type + val = prop.Value + } else { + t := reflect.TypeOf(p) + if t.Kind() != reflect.Ptr { + return nil, errors.New("input must be a pointer to a type") + } + typ, ok = scheme[t] + if !ok { + return nil, fmt.Errorf("%s not a known property type registered with the scheme", t) + } + val = p + } + d, err := jsonMarshal(val) + if err != nil { + return nil, err + } + + return &Property{ + Type: typ, + Value: d, + }, nil +} + +func MustBuild(p interface{}) Property { + prop, err := Build(p) + if err != nil { + panic(err) + } + return *prop +} + +func jsonMarshal(p interface{}) ([]byte, error) { + buf := &bytes.Buffer{} + dec := json.NewEncoder(buf) + dec.SetEscapeHTML(false) + err := dec.Encode(p) + if err != nil { + return nil, err + } + out := &bytes.Buffer{} + if err := json.Compact(out, buf.Bytes()); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func MustBuildPackage(name, version string) Property { + return MustBuild(&Package{PackageName: name, Version: version}) +} +func MustBuildPackageRequired(name, versionRange string) Property { + return MustBuild(&PackageRequired{name, versionRange}) +} +func MustBuildChannel(name, replaces string) Property { + return MustBuild(&Channel{name, replaces}) +} +func MustBuildGVK(group, version, kind string) Property { + return MustBuild(&GVK{group, kind, version}) +} +func MustBuildGVKRequired(group, version, kind string) Property { + return MustBuild(&GVKRequired{group, kind, version}) +} +func MustBuildSkips(skips string) Property { + s := Skips(skips) + return MustBuild(&s) +} +func MustBuildSkipRange(skipRange string) Property { + s := SkipRange(skipRange) + return MustBuild(&s) +} +func MustBuildBundleObjectRef(ref string) Property { + return MustBuild(&BundleObject{File: File{ref: ref}}) +} +func MustBuildBundleObjectData(data []byte) Property { + return MustBuild(&BundleObject{File: File{data: data}}) +} diff --git a/vendor/github.com/operator-framework/operator-registry/internal/property/scheme.go b/vendor/github.com/operator-framework/operator-registry/internal/property/scheme.go new file mode 100644 index 0000000000..c2fc7d3b9f --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/internal/property/scheme.go @@ -0,0 +1,35 @@ +package property + +import ( + "fmt" + "reflect" +) + +func init() { + skips := Skips("") + skipRange := SkipRange("") + + scheme = map[reflect.Type]string{ + reflect.TypeOf(&Package{}): TypePackage, + reflect.TypeOf(&PackageRequired{}): TypePackageRequired, + reflect.TypeOf(&Channel{}): TypeChannel, + reflect.TypeOf(&GVK{}): TypeGVK, + reflect.TypeOf(&GVKRequired{}): TypeGVKRequired, + reflect.TypeOf(&skips): TypeSkips, + reflect.TypeOf(&skipRange): TypeSkipRange, + reflect.TypeOf(&BundleObject{}): TypeBundleObject, + } +} + +var scheme map[reflect.Type]string + +func AddToScheme(typ string, p interface{}) { + t := reflect.TypeOf(p) + if t.Kind() != reflect.Ptr { + panic("input must be a pointer to a type") + } + if _, ok := scheme[t]; ok { + panic(fmt.Sprintf("scheme already contains registration for type %q", t)) + } + scheme[t] = typ +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/api_to_model.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/api_to_model.go new file mode 100644 index 0000000000..eccbbcf0e5 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/api_to_model.go @@ -0,0 +1,153 @@ +package api + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/operator-framework/operator-registry/internal/model" + "github.com/operator-framework/operator-registry/internal/property" +) + +func ConvertAPIBundleToModelBundle(b *Bundle) (*model.Bundle, error) { + bundleProps, err := convertAPIBundleToModelProperties(b) + if err != nil { + return nil, fmt.Errorf("convert properties: %v", err) + } + + relatedImages, err := getRelatedImages(b.CsvJson) + if err != nil { + return nil, fmt.Errorf("get related iamges: %v", err) + } + + return &model.Bundle{ + Name: b.CsvName, + Image: b.BundlePath, + Replaces: b.Replaces, + Skips: b.Skips, + CsvJSON: b.CsvJson, + Objects: b.Object, + Properties: bundleProps, + RelatedImages: relatedImages, + }, nil +} + +func convertAPIBundleToModelProperties(b *Bundle) ([]property.Property, error) { + var out []property.Property + + for _, skip := range b.Skips { + out = append(out, property.MustBuildSkips(skip)) + } + + if b.SkipRange != "" { + out = append(out, property.MustBuildSkipRange(b.SkipRange)) + } + + out = append(out, property.MustBuildChannel(b.ChannelName, b.Replaces)) + + providedGVKs := map[property.GVK]struct{}{} + requiredGVKs := map[property.GVKRequired]struct{}{} + + foundPackageProperty := false + for i, p := range b.Properties { + switch p.Type { + case property.TypeGVK: + var v GroupVersionKind + if err := json.Unmarshal(json.RawMessage(p.Value), &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + k := property.GVK{Group: v.Group, Kind: v.Kind, Version: v.Version} + providedGVKs[k] = struct{}{} + case property.TypePackage: + foundPackageProperty = true + out = append(out, property.Property{ + Type: property.TypePackage, + Value: json.RawMessage(p.Value), + }) + default: + out = append(out, property.Property{ + Type: p.Type, + Value: json.RawMessage(p.Value), + }) + } + } + + for i, p := range b.Dependencies { + switch p.Type { + case property.TypeGVK: + var v GroupVersionKind + if err := json.Unmarshal(json.RawMessage(p.Value), &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + k := property.GVKRequired{Group: v.Group, Kind: v.Kind, Version: v.Version} + requiredGVKs[k] = struct{}{} + case property.TypePackage: + var v property.Package + if err := json.Unmarshal(json.RawMessage(p.Value), &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + out = append(out, property.MustBuildPackageRequired(v.PackageName, v.Version)) + } + } + + if !foundPackageProperty { + out = append(out, property.MustBuildPackage(b.PackageName, b.Version)) + } + + for _, p := range b.ProvidedApis { + k := property.GVK{Group: p.Group, Kind: p.Kind, Version: p.Version} + if _, ok := providedGVKs[k]; !ok { + providedGVKs[k] = struct{}{} + } + } + for _, p := range b.RequiredApis { + k := property.GVKRequired{Group: p.Group, Kind: p.Kind, Version: p.Version} + if _, ok := requiredGVKs[k]; !ok { + requiredGVKs[k] = struct{}{} + } + } + + for p := range providedGVKs { + out = append(out, property.MustBuildGVK(p.Group, p.Version, p.Kind)) + } + + for p := range requiredGVKs { + out = append(out, property.MustBuildGVKRequired(p.Group, p.Version, p.Kind)) + } + + for _, obj := range b.Object { + out = append(out, property.MustBuildBundleObjectData([]byte(obj))) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].Type != out[j].Type { + return out[i].Type < out[j].Type + } + return string(out[i].Value) < string(out[j].Value) + }) + + return out, nil +} + +func getRelatedImages(csvJSON string) ([]model.RelatedImage, error) { + if len(csvJSON) == 0 { + return nil, nil + } + type csv struct { + Spec struct { + RelatedImages []struct { + Name string `json:"name"` + Image string `json:"image"` + } `json:"relatedImages"` + } `json:"spec"` + } + c := csv{} + if err := json.Unmarshal([]byte(csvJSON), &c); err != nil { + return nil, fmt.Errorf("unmarshal csv: %v", err) + } + relatedImages := []model.RelatedImage{} + for _, ri := range c.Spec.RelatedImages { + relatedImages = append(relatedImages, model.RelatedImage(ri)) + } + return relatedImages, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health.pb.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health.pb.go index 77a709be3f..bbb79b4ae6 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health.pb.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health.pb.go @@ -7,11 +7,12 @@ package grpc_health_v1 import ( + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health_grpc.pb.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health_grpc.pb.go index 1911f0ac32..f24b731121 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health_grpc.pb.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1/health_grpc.pb.go @@ -4,6 +4,7 @@ package grpc_health_v1 import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/model_to_api.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/model_to_api.go new file mode 100644 index 0000000000..ad805a7cfa --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/model_to_api.go @@ -0,0 +1,127 @@ +package api + +import ( + "encoding/json" + "fmt" + + "github.com/operator-framework/operator-registry/internal/model" + "github.com/operator-framework/operator-registry/internal/property" +) + +func ConvertModelBundleToAPIBundle(b model.Bundle) (*Bundle, error) { + props, err := parseProperties(b.Properties) + if err != nil { + return nil, fmt.Errorf("parse properties: %v", err) + } + skipRange := "" + if len(props.SkipRanges) > 0 { + skipRange = string(props.SkipRanges[0]) + } + + apiDeps, err := convertModelPropertiesToAPIDependencies(b.Properties) + if err != nil { + return nil, fmt.Errorf("convert model properties to api dependencies: %v", err) + } + return &Bundle{ + CsvName: b.Name, + PackageName: b.Package.Name, + ChannelName: b.Channel.Name, + BundlePath: b.Image, + ProvidedApis: gvksProvidedtoAPIGVKs(props.GVKs), + RequiredApis: gvksRequirestoAPIGVKs(props.GVKsRequired), + Version: props.Packages[0].Version, + SkipRange: skipRange, + Dependencies: apiDeps, + Properties: convertModelPropertiesToAPIProperties(b.Properties), + Replaces: b.Replaces, + Skips: b.Skips, + CsvJson: b.CsvJSON, + Object: b.Objects, + }, nil +} + +func parseProperties(in []property.Property) (*property.Properties, error) { + props, err := property.Parse(in) + if err != nil { + return nil, err + } + + if len(props.Packages) != 1 { + return nil, fmt.Errorf("expected exactly 1 property of type %q, found %d", property.TypePackage, len(props.Packages)) + } + + if len(props.SkipRanges) > 1 { + return nil, fmt.Errorf("multiple properties of type %q not allowed", property.TypeSkipRange) + } + + return props, nil +} + +func gvksProvidedtoAPIGVKs(in []property.GVK) []*GroupVersionKind { + var out []*GroupVersionKind + for _, gvk := range in { + out = append(out, &GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + }) + } + return out +} +func gvksRequirestoAPIGVKs(in []property.GVKRequired) []*GroupVersionKind { + var out []*GroupVersionKind + for _, gvk := range in { + out = append(out, &GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + }) + } + return out +} + +func convertModelPropertiesToAPIProperties(props []property.Property) []*Property { + var out []*Property + for _, prop := range props { + + // NOTE: This is a special case filter to prevent problems with existing client implementations that + // project bundle properties into CSV annotations and store those CSVs in a size-constrained + // storage backend (e.g. etcd via kube-apiserver). If the bundle object property has data inlined + // in its `Data` field, this CSV annotation projection would cause the size of the on-cluster + // CSV to at least double, which is untenable since CSVs already have known issues running up + // against etcd size constraints. + if prop.Type == property.TypeBundleObject { + continue + } + + out = append(out, &Property{ + Type: prop.Type, + Value: string(prop.Value), + }) + } + return out +} + +func convertModelPropertiesToAPIDependencies(props []property.Property) ([]*Dependency, error) { + var out []*Dependency + for _, prop := range props { + switch prop.Type { + case property.TypeGVKRequired: + out = append(out, &Dependency{ + Type: property.TypeGVK, + Value: string(prop.Value), + }) + case property.TypePackageRequired: + var v property.PackageRequired + if err := json.Unmarshal(prop.Value, &v); err != nil { + return nil, err + } + pkg := property.MustBuildPackage(v.PackageName, v.VersionRange) + out = append(out, &Dependency{ + Type: pkg.Type, + Value: string(pkg.Value), + }) + } + } + return out, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go index 84f3b90916..2654756928 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry.pb.go @@ -7,11 +7,12 @@ package api import ( + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry_grpc.pb.go b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry_grpc.pb.go index dba2e1ee0b..47b680c591 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/api/registry_grpc.pb.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/api/registry_grpc.pb.go @@ -4,6 +4,7 @@ package api import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap.go b/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap.go index 031eabecc1..0c95407e2f 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/operator-framework/operator-registry/pkg/api" + "github.com/operator-framework/operator-registry/pkg/lib/encoding" "github.com/operator-framework/operator-registry/pkg/registry" ) @@ -41,7 +42,7 @@ func (l *BundleLoader) Load(cm *corev1.ConfigMap) (bundle *api.Bundle, err error "configmap": fmt.Sprintf("%s/%s", cm.GetNamespace(), cm.GetName()), }) - bundle, skipped, bundleErr := loadBundle(logger, cm.Data) + bundle, skipped, bundleErr := loadBundle(logger, cm) if bundleErr != nil { err = fmt.Errorf("failed to extract bundle from configmap - %v", bundleErr) return @@ -50,10 +51,21 @@ func (l *BundleLoader) Load(cm *corev1.ConfigMap) (bundle *api.Bundle, err error return } -func loadBundle(entry *logrus.Entry, data map[string]string) (bundle *api.Bundle, skipped map[string]string, err error) { +func loadBundle(entry *logrus.Entry, cm *corev1.ConfigMap) (bundle *api.Bundle, skipped map[string]string, err error) { bundle = &api.Bundle{Object: []string{}} skipped = map[string]string{} + data := cm.Data + if hasGzipEncodingAnnotation(cm) { + entry.Debug("Decoding gzip-encoded bundle data") + + var err error + data, err = decodeGzipBinaryData(cm) + if err != nil { + return nil, nil, err + } + } + // Add kube resources to the bundle. for name, content := range data { reader := strings.NewReader(content) @@ -85,3 +97,18 @@ func loadBundle(entry *logrus.Entry, data map[string]string) (bundle *api.Bundle return } + +func decodeGzipBinaryData(cm *corev1.ConfigMap) (map[string]string, error) { + data := map[string]string{} + + for name, content := range cm.BinaryData { + decoded, err := encoding.GzipBase64Decode(content) + if err != nil { + return nil, fmt.Errorf("error decoding gzip-encoded bundle data: %v", err) + } + + data[name] = string(decoded) + } + + return data, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap_writer.go b/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap_writer.go index 718da72f5a..c2c5e42f29 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap_writer.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/configmap/configmap_writer.go @@ -16,14 +16,17 @@ import ( "github.com/operator-framework/operator-registry/pkg/client" "github.com/operator-framework/operator-registry/pkg/lib/bundle" + "github.com/operator-framework/operator-registry/pkg/lib/encoding" ) // configmap keys can contain underscores, but configmap names can not var unallowedKeyChars = regexp.MustCompile("[^-A-Za-z0-9_.]") const ( - EnvContainerImage = "CONTAINER_IMAGE" - ConfigMapImageAnnotationKey = "olm.sourceImage" + EnvContainerImage = "CONTAINER_IMAGE" + ConfigMapImageAnnotationKey = "olm.sourceImage" + ConfigMapEncodingAnnotationKey = "olm.contentEncoding" + ConfigMapEncodingAnnotationGzip = "gzip+base64" ) type AnnotationsFile struct { @@ -38,23 +41,29 @@ type AnnotationsFile struct { } type ConfigMapWriter struct { + clientset kubernetes.Interface manifestsDir string configMapName string namespace string - clientset *kubernetes.Clientset + gzip bool } -func NewConfigMapLoaderForDirectory(configMapName, namespace, manifestsDir, kubeconfig string) *ConfigMapWriter { +func NewConfigMapLoader(configMapName, namespace, manifestsDir string, gzip bool, kubeconfig string) *ConfigMapWriter { clientset, err := client.NewKubeClient(kubeconfig, logrus.StandardLogger()) if err != nil { logrus.Fatalf("cluster config failed: %v", err) } + return NewConfigMapLoaderWithClient(configMapName, namespace, manifestsDir, gzip, clientset) +} + +func NewConfigMapLoaderWithClient(configMapName, namespace, manifestsDir string, gzip bool, clientset kubernetes.Interface) *ConfigMapWriter { return &ConfigMapWriter{ + clientset: clientset, manifestsDir: manifestsDir, configMapName: configMapName, namespace: namespace, - clientset: clientset, + gzip: gzip, } } @@ -71,6 +80,7 @@ func (c *ConfigMapWriter) Populate(maxDataSizeLimit uint64) error { return err } configMapPopulate.Data = map[string]string{} + configMapPopulate.BinaryData = map[string][]byte{} var totalSize uint64 for _, dir := range subDirs { @@ -84,24 +94,13 @@ func (c *ConfigMapWriter) Populate(maxDataSizeLimit uint64) error { for _, file := range files { log := logrus.WithField("file", completePath+file.Name()) log.Info("Reading file") + content, err := ioutil.ReadFile(completePath + file.Name()) if err != nil { log.Errorf("read failed: %v", err) return err } - totalSize += uint64(len(content)) - if totalSize > maxDataSizeLimit { - log.Errorf("File with size %v exceeded %v limit, aboring", len(content), maxDataSizeLimit) - return fmt.Errorf("file %v bigger than total allowed limit", file.Name()) - } - validConfigMapKey := TranslateInvalidChars(file.Name()) - if validConfigMapKey != file.Name() { - logrus.WithFields(logrus.Fields{ - "file.Name": file.Name(), - "validConfigMapKey": validConfigMapKey, - }).Info("translated filename for configmap comptability") - } if file.Name() == bundle.AnnotationsFile { var annotationsFile AnnotationsFile err := yaml.Unmarshal(content, &annotationsFile) @@ -116,6 +115,36 @@ func (c *ConfigMapWriter) Populate(maxDataSizeLimit uint64) error { bundle.ChannelsLabel: annotationsFile.Annotations.Channels, bundle.ChannelDefaultLabel: annotationsFile.Annotations.ChannelDefault, }) + + // annotations aren't accounted for the ConfigMap data size + // limit, and rather have their own limit of 262144 bytes. + continue + } + + if c.gzip { + content, err = encoding.GzipBase64Encode(content) + if err != nil { + log.Errorf("failed to gzip encode file %v: %v", file.Name(), err) + return err + } + } + + totalSize += uint64(len(content)) + if totalSize > maxDataSizeLimit { + log.Errorf("Bundle files exceeded %v bytes limit", maxDataSizeLimit) + return fmt.Errorf("bundle files exceeded %v bytes limit", maxDataSizeLimit) + } + + validConfigMapKey := TranslateInvalidChars(file.Name()) + if validConfigMapKey != file.Name() { + logrus.WithFields(logrus.Fields{ + "file.Name": file.Name(), + "validConfigMapKey": validConfigMapKey, + }).Info("translated filename for configmap comptability") + } + + if c.gzip { + configMapPopulate.BinaryData[validConfigMapKey] = content } else { configMapPopulate.Data[validConfigMapKey] = string(content) } @@ -123,10 +152,14 @@ func (c *ConfigMapWriter) Populate(maxDataSizeLimit uint64) error { } if sourceImage := os.Getenv(EnvContainerImage); sourceImage != "" { - annotations := configMapPopulate.GetAnnotations() + annotations := initAndGetAnnotations(configMapPopulate) annotations[ConfigMapImageAnnotationKey] = sourceImage } + if c.gzip { + setGzipEncodingAnnotation(configMapPopulate) + } + _, err = c.clientset.CoreV1().ConfigMaps(c.namespace).Update(context.TODO(), configMapPopulate, metav1.UpdateOptions{}) if err != nil { return err @@ -139,7 +172,7 @@ func (c *ConfigMapWriter) Populate(maxDataSizeLimit uint64) error { // the responsibility of the caller to delete the job, the pod, and the configmap // when done. This function is intended to be called from OLM, but is put here // for locality. -func LaunchBundleImage(kubeclient kubernetes.Interface, bundleImage, initImage, namespace string) (*corev1.ConfigMap, *batchv1.Job, error) { +func LaunchBundleImage(kubeclient kubernetes.Interface, bundleImage, initImage, namespace string, gzip bool) (*corev1.ConfigMap, *batchv1.Job, error) { // create configmap for bundle image data to write to (will be returned) newConfigMap, err := kubeclient.CoreV1().ConfigMaps(namespace).Create(context.TODO(), &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -150,6 +183,11 @@ func LaunchBundleImage(kubeclient kubernetes.Interface, bundleImage, initImage, return nil, nil, err } + opmCommand := []string{"/injected/opm", "alpha", "bundle", "extract", "-n", namespace, "-c", newConfigMap.GetName()} + if gzip { + opmCommand = append(opmCommand, "--gzip") + } + launchJob := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "deploy-bundle-image-", @@ -166,7 +204,7 @@ func LaunchBundleImage(kubeclient kubernetes.Interface, bundleImage, initImage, { Name: "bundle-image", Image: bundleImage, - Command: []string{"/injected/opm", "alpha", "bundle", "extract", "-n", namespace, "-c", newConfigMap.GetName()}, + Command: opmCommand, Env: []corev1.EnvVar{ { Name: EnvContainerImage, @@ -218,3 +256,23 @@ func LaunchBundleImage(kubeclient kubernetes.Interface, bundleImage, initImage, return newConfigMap, launchedJob, nil } + +func setGzipEncodingAnnotation(cm *corev1.ConfigMap) { + annotations := initAndGetAnnotations(cm) + annotations[ConfigMapEncodingAnnotationKey] = ConfigMapEncodingAnnotationGzip +} + +func hasGzipEncodingAnnotation(cm *corev1.ConfigMap) bool { + annotations := cm.GetAnnotations() + encoding, ok := annotations[ConfigMapEncodingAnnotationKey] + return ok && encoding == ConfigMapEncodingAnnotationGzip +} + +func initAndGetAnnotations(cm *corev1.ConfigMap) map[string]string { + annotations := cm.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + cm.SetAnnotations(annotations) + } + return annotations +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/containertools/runner.go b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/runner.go index 29a40b7520..f178d8c5d6 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/containertools/runner.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/containertools/runner.go @@ -22,14 +22,53 @@ type CommandRunner interface { type ContainerCommandRunner struct { logger *logrus.Entry containerTool ContainerTool + config *RunnerConfig +} + +type RunnerConfig struct { + SkipTLS bool +} + +type RunnerOption func(config *RunnerConfig) + +func SkipTLS(skip bool) RunnerOption { + return func(config *RunnerConfig) { + config.SkipTLS = skip + } +} + +func (r *RunnerConfig) apply(options []RunnerOption) { + for _, option := range options { + option(r) + } +} + +func (r *ContainerCommandRunner) argsForCmd(cmd string, args ...string) []string { + cmdArgs := []string{cmd} + switch r.containerTool { + case PodmanTool: + switch cmd { + case "pull", "push", "login", "search": + // --tls-verify is a valid flag for these podman subcommands + if r.config.SkipTLS { + cmdArgs = append(cmdArgs, "--tls-verify=false") + } + } + default: + } + cmdArgs = append(cmdArgs, args...) + return cmdArgs } // NewCommandRunner takes the containerTool as an input string and returns a // CommandRunner to run commands with that cli tool -func NewCommandRunner(containerTool ContainerTool, logger *logrus.Entry) *ContainerCommandRunner { +func NewCommandRunner(containerTool ContainerTool, logger *logrus.Entry, opts ...RunnerOption) *ContainerCommandRunner { + var config RunnerConfig + config.apply(opts) r := &ContainerCommandRunner{ logger: logger, containerTool: containerTool, + config: &config, } return r } @@ -42,7 +81,7 @@ func (r *ContainerCommandRunner) GetToolName() string { // Pull takes a container image path hosted on a container registry and runs the // pull command to download it onto the local environment func (r *ContainerCommandRunner) Pull(image string) error { - args := []string{"pull", image} + args := r.argsForCmd("pull", image) command := exec.Command(r.containerTool.String(), args...) @@ -84,7 +123,7 @@ func (r *ContainerCommandRunner) Build(dockerfile, tag string) error { // Unpack copies a directory from a local container image to a directory in the local filesystem. func (r *ContainerCommandRunner) Unpack(image, src, dst string) error { - args := []string{"create", image, ""} + args := r.argsForCmd("create", image, "") command := exec.Command(r.containerTool.String(), args...) @@ -98,7 +137,7 @@ func (r *ContainerCommandRunner) Unpack(image, src, dst string) error { } id := strings.TrimSuffix(string(out), "\n") - args = []string{"cp", id + ":" + src, dst} + args = r.argsForCmd("cp", id+":"+src, dst) command = exec.Command(r.containerTool.String(), args...) r.logger.Infof("running %s cp", r.containerTool) @@ -110,7 +149,7 @@ func (r *ContainerCommandRunner) Unpack(image, src, dst string) error { return fmt.Errorf("error copying container directory %s: %v", string(out), err) } - args = []string{"rm", id} + args = r.argsForCmd("rm", id) command = exec.Command(r.containerTool.String(), args...) r.logger.Infof("running %s rm", r.containerTool) @@ -128,7 +167,7 @@ func (r *ContainerCommandRunner) Unpack(image, src, dst string) error { // Inspect runs the 'inspect' command to get image metadata of a local container // image and returns a byte array of the command's output func (r *ContainerCommandRunner) Inspect(image string) ([]byte, error) { - args := []string{"inspect", image} + args := r.argsForCmd("inspect", image) command := exec.Command(r.containerTool.String(), args...) diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/registry.go b/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/registry.go index 78ae103aa3..6519539e5b 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/registry.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/registry.go @@ -8,6 +8,9 @@ import ( "fmt" "io" "os" + "regexp" + "strings" + "time" "github.com/containerd/containerd/archive" "github.com/containerd/containerd/archive/compression" @@ -18,6 +21,8 @@ import ( "github.com/containerd/containerd/remotes" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" "github.com/operator-framework/operator-registry/pkg/image" ) @@ -33,6 +38,8 @@ type Registry struct { var _ image.Registry = &Registry{} +var nonRetriablePullError = regexp.MustCompile("specified image is a docker schema v1 manifest, which is not supported") + // Pull fetches and stores an image by reference. func (r *Registry) Pull(ctx context.Context, ref image.Reference) error { // Set the default namespace if unset @@ -42,14 +49,30 @@ func (r *Registry) Pull(ctx context.Context, ref image.Reference) error { if err != nil { return fmt.Errorf("error resolving name %s: %v", name, err) } - r.log.Infof("resolved name: %s", name) + r.log.Debugf("resolved name: %s", name) fetcher, err := r.resolver.Fetcher(ctx, name) if err != nil { return err } - if err := r.fetch(ctx, fetcher, root); err != nil { + retryBackoff := wait.Backoff{ + Duration: 1 * time.Second, + Factor: 1.0, + Jitter: 0.1, + Steps: 5, + } + + if err := retry.OnError(retryBackoff, + func(pullErr error) bool { + if nonRetriablePullError.MatchString(pullErr.Error()) { + return false + } + r.log.Warnf("Error pulling image %q: %v. Retrying", ref.String(), pullErr) + return true + }, + func() error { return r.fetch(ctx, fetcher, root) }, + ); err != nil { return err } @@ -82,7 +105,7 @@ func (r *Registry) Unpack(ctx context.Context, ref image.Reference, dir string) } for _, layer := range manifest.Layers { - r.log.Infof("unpacking layer: %v", layer) + r.log.Debugf("unpacking layer: %v", layer) if err := r.unpackLayer(ctx, layer, dir); err != nil { return err } @@ -153,7 +176,7 @@ func (r *Registry) getImage(ctx context.Context, manifest ocispec.Manifest) (*oc func (r *Registry) fetch(ctx context.Context, fetcher remotes.Fetcher, root ocispec.Descriptor) error { visitor := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - r.log.WithField("digest", desc.Digest).Info("fetched") + r.log.WithField("digest", desc.Digest).Debug("fetched") r.log.Debug(desc) return nil, nil }) @@ -183,7 +206,9 @@ func (r *Registry) unpackLayer(ctx context.Context, layer ocispec.Descriptor, di if err != nil { return err } - _, err = archive.Apply(ctx, dir, decompressed, archive.WithFilter(adjustPerms)) + + filters := filterList{adjustPerms, dropXattrs} + _, err = archive.Apply(ctx, dir, decompressed, archive.WithFilter(filters.and)) return err } @@ -195,6 +220,19 @@ func ensureNamespace(ctx context.Context) context.Context { return ctx } +type filterList []archive.Filter + +func (f filterList) and(h *tar.Header) (bool, error) { + for _, filter := range f { + ok, err := filter(h) + if !ok || err != nil { + return ok, err + } + } + + return true, nil +} + func adjustPerms(h *tar.Header) (bool, error) { h.Uid = os.Getuid() h.Gid = os.Getgid() @@ -207,3 +245,19 @@ func adjustPerms(h *tar.Header) (bool, error) { return true, nil } + +// paxSchilyXattr contains the key prefix for xattrs stored in PAXRecords (see https://golang.org/src/archive/tar/common.go for more details). +const paxSchilyXattr = "SCHILY.xattr." + +// dropXattrs removes all xattrs from a Header. +// This is useful for unpacking on systems where writing certain xattrs is a restricted operation; e.g. "security.capability" on SELinux. +func dropXattrs(h *tar.Header) (bool, error) { + h.Xattrs = nil // Deprecated, but still in use, clear anyway. + for key := range h.PAXRecords { + if strings.HasPrefix(key, paxSchilyXattr) { // Xattrs are stored under keys with the "Schilly.xattr." prefix. + delete(h.PAXRecords, key) + } + } + + return true, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/resolver.go b/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/resolver.go index 3273641250..9ac771dde9 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/resolver.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/image/containerdregistry/resolver.go @@ -28,7 +28,7 @@ func NewResolver(configDir string, insecure bool, roots *x509.CertPool) (remotes ExpectContinueTimeout: 5 * time.Second, TLSClientConfig: &tls.Config{ InsecureSkipVerify: false, - RootCAs: roots, + RootCAs: roots, }, } @@ -40,8 +40,7 @@ func NewResolver(configDir string, insecure bool, roots *x509.CertPool) (remotes headers := http.Header{} headers.Set("User-Agent", "opm/alpha") - client := http.DefaultClient - client.Transport = transport + client := &http.Client{Transport: transport} cfg, err := loadConfig(configDir) if err != nil { @@ -61,7 +60,7 @@ func NewResolver(configDir string, insecure bool, roots *x509.CertPool) (remotes } opts := docker.ResolverOptions{ - Hosts: docker.ConfigureDefaultRegistries(regopts...), + Hosts: docker.ConfigureDefaultRegistries(regopts...), Headers: headers, } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/image/execregistry/registry.go b/vendor/github.com/operator-framework/operator-registry/pkg/image/execregistry/registry.go index f94a63b02e..40769d23e7 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/image/execregistry/registry.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/image/execregistry/registry.go @@ -26,10 +26,10 @@ type Registry struct { var _ image.Registry = &Registry{} // NewRegistry instantiates and returns a new registry which manipulates images via exec podman/docker commands. -func NewRegistry(tool containertools.ContainerTool, logger *logrus.Entry) (registry *Registry, err error) { +func NewRegistry(tool containertools.ContainerTool, logger *logrus.Entry, opts ...containertools.RunnerOption) (registry *Registry, err error) { return &Registry{ log: logger, - cmd: containertools.NewCommandRunner(tool, logger), + cmd: containertools.NewCommandRunner(tool, logger, opts...), }, nil } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/image/mock.go b/vendor/github.com/operator-framework/operator-registry/pkg/image/mock.go new file mode 100644 index 0000000000..6f856c30ec --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/image/mock.go @@ -0,0 +1,86 @@ +package image + +import ( + "context" + "errors" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "sync" +) + +var _ Registry = &MockRegistry{} + +type MockRegistry struct { + RemoteImages map[Reference]*MockImage + localImages map[Reference]*MockImage + m sync.RWMutex +} + +type MockImage struct { + Labels map[string]string + FS fs.FS +} + +func (i *MockImage) unpack(dir string) error { + return fs.WalkDir(i.FS, ".", func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + data, err := fs.ReadFile(i.FS, path) + if err != nil { + return err + } + path = filepath.Join(dir, path) + pathDir := filepath.Dir(path) + if err := os.MkdirAll(pathDir, 0777); err != nil { + return err + } + return ioutil.WriteFile(path, data, 0666) + }) +} + +func (m *MockRegistry) Pull(_ context.Context, ref Reference) error { + image, ok := m.RemoteImages[ref] + if !ok { + return errors.New("not found") + } + m.m.Lock() + defer m.m.Unlock() + if m.localImages == nil { + m.localImages = map[Reference]*MockImage{} + } + m.localImages[ref] = image + return nil +} + +func (m *MockRegistry) Unpack(_ context.Context, ref Reference, dir string) error { + m.m.RLock() + defer m.m.RUnlock() + image, ok := m.localImages[ref] + if !ok { + return errors.New("not found") + } + return image.unpack(dir) +} + +func (m *MockRegistry) Labels(_ context.Context, ref Reference) (map[string]string, error) { + m.m.RLock() + defer m.m.RUnlock() + image, ok := m.localImages[ref] + if !ok { + return nil, errors.New("not found") + } + return image.Labels, nil +} + +func (m *MockRegistry) Destroy() error { + m.m.Lock() + defer m.m.Unlock() + m.localImages = nil + return nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/image/registry.go b/vendor/github.com/operator-framework/operator-registry/pkg/image/registry.go index d47507ab2c..e7c3657202 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/image/registry.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/image/registry.go @@ -29,4 +29,3 @@ type Registry interface { // If it exists, it's used as the base image. // Pack(ctx context.Context, ref Reference, from io.Reader) (next string, err error) } - diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/exporter.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/exporter.go index 506e955ad6..53469f8682 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/exporter.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/exporter.go @@ -30,7 +30,7 @@ func NewExporterForBundle(image, directory string, containerTool containertools. } } -func (i *BundleExporter) Export() error { +func (i *BundleExporter) Export(skipTLS bool) error { log := logrus.WithField("img", i.image) @@ -44,11 +44,11 @@ func (i *BundleExporter) Export() error { var rerr error switch i.containerTool { case containertools.NoneTool: - reg, rerr = containerdregistry.NewRegistry(containerdregistry.WithLog(log)) + reg, rerr = containerdregistry.NewRegistry(containerdregistry.SkipTLS(skipTLS), containerdregistry.WithLog(log), containerdregistry.WithCacheDir(filepath.Join(tmpDir, "cacheDir"))) case containertools.PodmanTool: fallthrough case containertools.DockerTool: - reg, rerr = execregistry.NewRegistry(i.containerTool, log) + reg, rerr = execregistry.NewRegistry(i.containerTool, log, containertools.SkipTLS(skipTLS)) } if rerr != nil { return rerr diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/generate.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/generate.go index ed56d18a43..0e480c41a8 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/generate.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/generate.go @@ -112,9 +112,6 @@ func GenerateFunc(directory, outputDir, packageName, channels, channelDefault st if channelDefault == "" { channelDefault = i.GetDefaultChannel() - if !containsString(strings.Split(channels, ","), channelDefault) { - channelDefault = "" - } log.Infof("Inferred default channel: %s", channelDefault) } } @@ -299,16 +296,18 @@ func ValidateAnnotations(existing, expected []byte) error { func GenerateAnnotations(mediaType, manifests, metadata, packageName, channels, channelDefault string) ([]byte, error) { annotations := &AnnotationMetadata{ Annotations: map[string]string{ - MediatypeLabel: mediaType, - ManifestsLabel: manifests, - MetadataLabel: metadata, - PackageLabel: packageName, - ChannelsLabel: channels, - ChannelDefaultLabel: channelDefault, + MediatypeLabel: mediaType, + ManifestsLabel: manifests, + MetadataLabel: metadata, + PackageLabel: packageName, + ChannelsLabel: channels, }, } - annotations.Annotations[ChannelDefaultLabel] = channelDefault + // Only add defaultChannel annotation if present + if channelDefault != "" { + annotations.Annotations[ChannelDefaultLabel] = channelDefault + } afile, err := yaml.Marshal(annotations) if err != nil { @@ -328,11 +327,13 @@ func GenerateDockerfile(mediaType, manifests, metadata, copyManifestDir, copyMet if err != nil { return nil, err } + relativeManifestDirectory = filepath.ToSlash(relativeManifestDirectory) relativeMetadataDirectory, err := filepath.Rel(workingDir, copyMetadataDir) if err != nil { return nil, err } + relativeMetadataDirectory = filepath.ToSlash(relativeMetadataDirectory) // FROM fileContent += "FROM scratch\n\n" @@ -343,7 +344,11 @@ func GenerateDockerfile(mediaType, manifests, metadata, copyManifestDir, copyMet fileContent += fmt.Sprintf("LABEL %s=%s\n", MetadataLabel, metadata) fileContent += fmt.Sprintf("LABEL %s=%s\n", PackageLabel, packageName) fileContent += fmt.Sprintf("LABEL %s=%s\n", ChannelsLabel, channels) - fileContent += fmt.Sprintf("LABEL %s=%s\n\n", ChannelDefaultLabel, channelDefault) + + // Only add defaultChannel annotation if present + if channelDefault != "" { + fileContent += fmt.Sprintf("LABEL %s=%s\n\n", ChannelDefaultLabel, channelDefault) + } // CONTENT fileContent += fmt.Sprintf("COPY %s %s\n", relativeManifestDirectory, "/manifests/") @@ -352,7 +357,7 @@ func GenerateDockerfile(mediaType, manifests, metadata, copyManifestDir, copyMet return []byte(fileContent), nil } -// Write `fileName` file with `content` into a `directory` +// WriteFile writes `fileName` file with `content` into a `directory` // Note: Will overwrite the existing `fileName` file if it exists func WriteFile(fileName, directory string, content []byte) error { if _, err := os.Stat(directory); os.IsNotExist(err) { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/interfaces.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/interfaces.go index 426d61e08a..79a69159dd 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/interfaces.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/interfaces.go @@ -21,9 +21,10 @@ type BundleImageValidator interface { } // NewImageValidator is a constructor that returns an ImageValidator -func NewImageValidator(registry image.Registry, logger *logrus.Entry) BundleImageValidator { +func NewImageValidator(registry image.Registry, logger *logrus.Entry, options ...string) BundleImageValidator { return imageValidator{ registry: registry, logger: logger, + optional: options, } } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/supported_resources.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/supported_resources.go index 1ecf414ff4..534e6568c1 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/supported_resources.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/supported_resources.go @@ -16,6 +16,7 @@ const ( PodDisruptionBudgetKind = "PodDisruptionBudget" PriorityClassKind = "PriorityClass" VerticalPodAutoscalerKind = "VerticalPodAutoscaler" + ConsoleYamlSampleKind = "ConsoleYamlSample" ) // Namespaced indicates whether the resource is namespace scoped (true) or cluster-scoped (false). @@ -39,6 +40,7 @@ var supportedResources = map[string]Namespaced{ PodDisruptionBudgetKind: true, PriorityClassKind: false, VerticalPodAutoscalerKind: false, + ConsoleYamlSampleKind: false, } // IsSupported checks if the object kind is OLM-supported and if it is namespaced diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/validate.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/validate.go index 4309593e7b..0d0aa7a347 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/validate.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/bundle/validate.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "github.com/operator-framework/api/pkg/manifests" v1 "github.com/operator-framework/api/pkg/operators/v1alpha1" v "github.com/operator-framework/api/pkg/validation" "github.com/operator-framework/operator-registry/pkg/image" @@ -27,8 +28,10 @@ import ( ) const ( - v1CRDapiVersion = "apiextensions.k8s.io/v1" - v1beta1CRDapiVersion = "apiextensions.k8s.io/v1beta1" + v1CRDapiVersion = "apiextensions.k8s.io/v1" + v1beta1CRDapiVersion = "apiextensions.k8s.io/v1beta1" + validateOperatorHubKey = "operatorhub" + validateBundleObjectsKey = "bundle-objects" ) type Meta struct { @@ -40,6 +43,7 @@ type Meta struct { type imageValidator struct { registry image.Registry logger *log.Entry + optional []string } // PullBundleImage shells out to a container tool and pulls a given image tag @@ -142,7 +146,7 @@ func (i imageValidator) ValidateBundleFormat(directory string) error { if !annotationsFound { validationErrors = append(validationErrors, fmt.Errorf("Could not find annotations file")) } else { - i.logger.Info("Found annotations file") + i.logger.Debug("Found annotations file") errs := validateAnnotations(mediaType, fileAnnotations) if errs != nil { validationErrors = append(validationErrors, errs...) @@ -150,9 +154,9 @@ func (i imageValidator) ValidateBundleFormat(directory string) error { } if !dependenciesFound { - i.logger.Info("Could not find optional dependencies file") + i.logger.Debug("Could not find optional dependencies file") } else { - i.logger.Info("Found dependencies file") + i.logger.Debug("Found dependencies file") errs := validateDependencies(dependenciesFile) if errs != nil { validationErrors = append(validationErrors, errs...) @@ -180,7 +184,7 @@ func validateAnnotations(mediaType string, fileAnnotations *AnnotationMetadata) for label, item := range annotations { val, ok := fileAnnotations.Annotations[label] - if !ok { + if !ok && label != ChannelDefaultLabel { aErr := fmt.Errorf("Missing annotation %q", label) validationErrors = append(validationErrors, aErr) } @@ -205,11 +209,12 @@ func validateAnnotations(mediaType string, fileAnnotations *AnnotationMetadata) if val == "" { aErr := fmt.Errorf("Expecting annotation %q to have non-empty value", label) validationErrors = append(validationErrors, aErr) - } else { - annotations[label] = val } case ChannelDefaultLabel: - annotations[label] = val + if ok && val == "" { + aErr := fmt.Errorf("Expecting annotation %q to have non-empty value", label) + validationErrors = append(validationErrors, aErr) + } } } @@ -269,6 +274,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error { } var csvName string + csv := &v1.ClusterServiceVersion{} unstObjs := []*unstructured.Unstructured{} csvValidator := v.ClusterServiceVersionValidator crdValidator := v.CustomResourceDefinitionValidator @@ -306,7 +312,6 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error { } if gvk.Kind == CSVKind { - csv := &v1.ClusterServiceVersion{} err := runtime.DefaultUnstructuredConverter.FromUnstructured(k8sFile.Object, csv) if err != nil { validationErrors = append(validationErrors, err) @@ -364,7 +369,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error { // Validate the bundle object if len(unstObjs) > 0 { - bundle := registry.NewBundle(csvName, "", nil, unstObjs...) + bundle := registry.NewBundle(csvName, ®istry.Annotations{}, unstObjs...) bundleValidator := validation.BundleValidator results := bundleValidator.Validate(bundle) if len(results) > 0 { @@ -374,6 +379,32 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error { } } + // Determine if optional validations are enabled + optionalValidators := parseOptions(i.optional) + + // Run the operatorhub validation if specified + if _, ok := optionalValidators[validateOperatorHubKey]; ok { + i.logger.Debug("Performing operatorhub validation") + bundle := &manifests.Bundle{Name: csvName, CSV: csv} + results := v.OperatorHubValidator.Validate(bundle) + if len(results) > 0 { + for _, err := range results[0].Errors { + validationErrors = append(validationErrors, err) + } + } + } + + // Run the bundle object validation if specified + if _, ok := optionalValidators[validateBundleObjectsKey]; ok { + i.logger.Debug("Performing bundle objects validation") + results := v.ObjectValidator.Validate(unstObjs) + if len(results) > 0 { + for _, err := range results[0].Errors { + validationErrors = append(validationErrors, err) + } + } + } + if len(validationErrors) > 0 { return NewValidationError(validationErrors) } @@ -409,3 +440,18 @@ func validateKubectlable(fileBytes []byte) error { return nil } + +// parseOptions looks at the provided optional validators provided via a command line flag and returns an map +// example input: ["operatorhub,bundle-objects"] +// example output: {"operatorhub": {}, "bundle-objects": {}} +func parseOptions(args []string) map[string]struct{} { + validators := make(map[string]struct{}) + for _, arg := range args { + arr := strings.Split(arg, ",") + for _, key := range arr { + key = strings.TrimSpace(key) + validators[key] = struct{}{} + } + } + return validators +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/encoding/encoding.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/encoding/encoding.go new file mode 100644 index 0000000000..79cee4f0c7 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/encoding/encoding.go @@ -0,0 +1,50 @@ +package encoding + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "io" +) + +// GzipBase64Encode applies gzip compression to the given bytes, followed by base64 encoding. +func GzipBase64Encode(data []byte) ([]byte, error) { + buf := &bytes.Buffer{} + + bWriter := base64.NewEncoder(base64.StdEncoding, buf) + zWriter := gzip.NewWriter(bWriter) + _, err := zWriter.Write(data) + if err != nil { + zWriter.Close() + bWriter.Close() + return nil, err + } + + // Ensure all gzipped bytes are flushed to the underlying base64 encoder + err = zWriter.Close() + if err != nil { + return nil, err + } + + // Ensure all base64d bytes are flushed to the underlying buffer + err = bWriter.Close() + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// GzipBase64Decode applies base64 decoding to the given bytes, followed by gzip decompression. +func GzipBase64Decode(data []byte) ([]byte, error) { + bBuffer := bytes.NewReader(data) + + bReader := base64.NewDecoder(base64.StdEncoding, bBuffer) + zReader, err := gzip.NewReader(bReader) + if err != nil { + return nil, err + } + defer zReader.Close() + + return io.ReadAll(zReader) +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/semver/semver.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/semver/semver.go new file mode 100644 index 0000000000..033bb5419e --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/semver/semver.go @@ -0,0 +1,54 @@ +package semver + +import ( + "fmt" + + "github.com/blang/semver" +) + +// BuildIdCompare compares two versions and returns negative one if the first arg is less than the second arg, positive one if it is larger, and zero if they are equal. +// This comparison follows typical semver precedence rules, with one addition: whenever two versions are equal with the exception of their build-ids, the build-ids are compared using prerelease precedence rules. Further, versions with no build-id are always less than versions with build-ids; e.g. 1.0.0 < 1.0.0+1. +func BuildIdCompare(b semver.Version, v semver.Version) (int, error) { + if c := b.Compare(v); c != 0 { + return c, nil + } + + bPre, err := buildAsPrerelease(b) + if err != nil { + return 0, fmt.Errorf("failed to convert build-id of %s to prerelease version for comparison: %s", b, err) + } + + vPre, err := buildAsPrerelease(v) + if err != nil { + return 0, fmt.Errorf("failed to convert build-id of %s to prerelease version for comparison: %s", v, err) + } + + return bPre.Compare(*vPre), nil +} + +func buildAsPrerelease(v semver.Version) (*semver.Version, error) { + var pre []semver.PRVersion + for _, b := range v.Build { + p, err := semver.NewPRVersion(b) + if err != nil { + return nil, err + } + pre = append(pre, p) + } + + var major uint64 + if len(pre) > 0 { + // Adjust for the case where we compare a build-id prerelease analog to a version without a build-id. + // Without this `0.0.0+1` and `0.0.0` would become `0.0.0-1` and `0.0.0`, where the rules of prerelease comparison would + // end up giving us the wrong result; i.e. `0.0.0+1` < `0.0.0`. With this, `0.0.0+1` and `0.0.0` become `1.0.0-1` and `0.0.0` + // respectively, which does yield the intended result. + major = 1 + } + + return &semver.Version{ + Major: major, + Minor: 0, + Patch: 0, + Pre: pre, + }, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/lib/validation/bundle.go b/vendor/github.com/operator-framework/operator-registry/pkg/lib/validation/bundle.go index 309d3313aa..d8f6d5b8e5 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/lib/validation/bundle.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/lib/validation/bundle.go @@ -56,9 +56,9 @@ func validateOwnedCRDs(bundle *registry.Bundle, csv *registry.ClusterServiceVers for _, ownedKey := range ownedKeys { if _, ok := keySet[*ownedKey]; !ok { result.Add(errors.ErrInvalidBundle(fmt.Sprintf("owned CRD %s not found in bundle %q", keyToString(*ownedKey), bundle.Name), *ownedKey)) - } else { - delete(keySet, *ownedKey) + continue } + delete(keySet, *ownedKey) } // CRDs not defined in the CSV present in the bundle for key := range keySet { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go index eadb34fba8..b5fb28b941 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundle.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "fmt" "strings" @@ -41,25 +42,42 @@ type Bundle struct { Package string Channels []string BundleImage string + version string csv *ClusterServiceVersion v1beta1crds []*apiextensionsv1beta1.CustomResourceDefinition v1crds []*apiextensionsv1.CustomResourceDefinition Dependencies []*Dependency - Properties []*Property + Properties []Property + Annotations *Annotations cacheStale bool } -func NewBundle(name, pkgName string, channels []string, objs ...*unstructured.Unstructured) *Bundle { - bundle := &Bundle{Name: name, Package: pkgName, Channels: channels, cacheStale: false} +func NewBundle(name string, annotations *Annotations, objs ...*unstructured.Unstructured) *Bundle { + bundle := &Bundle{ + Name: name, + Package: annotations.PackageName, + Annotations: annotations, + } for _, o := range objs { bundle.Add(o) } + + if annotations == nil { + return bundle + } + bundle.Channels = strings.Split(annotations.Channels, ",") + return bundle } -func NewBundleFromStrings(name, pkgName string, channels []string, objs []string) (*Bundle, error) { +func NewBundleFromStrings(name, version, pkg, defaultChannel, channels, objs string) (*Bundle, error) { + objStrs, err := BundleStringToObjectStrings(objs) + if err != nil { + return nil, err + } + unstObjs := []*unstructured.Unstructured{} - for _, o := range objs { + for _, o := range objStrs { dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(o), 10) unst := &unstructured.Unstructured{} if err := dec.Decode(unst); err != nil { @@ -67,13 +85,21 @@ func NewBundleFromStrings(name, pkgName string, channels []string, objs []string } unstObjs = append(unstObjs, unst) } - return NewBundle(name, pkgName, channels, unstObjs...), nil + + annotations := &Annotations{ + PackageName: pkg, + Channels: channels, + DefaultChannelName: defaultChannel, + } + bundle := NewBundle(name, annotations, unstObjs...) + bundle.version = version + + return bundle, nil } func (b *Bundle) Size() int { return len(b.Objects) } - func (b *Bundle) Add(obj *unstructured.Unstructured) { b.Objects = append(b.Objects, obj) b.cacheStale = true @@ -87,10 +113,20 @@ func (b *Bundle) ClusterServiceVersion() (*ClusterServiceVersion, error) { } func (b *Bundle) Version() (string, error) { - if err := b.cache(); err != nil { + if b.version != "" { + return b.version, nil + } + + var err error + if err = b.cache(); err != nil { return "", err } - return b.csv.GetVersion() + + if b.csv != nil { + b.version, err = b.csv.GetVersion() + } + + return b.version, err } func (b *Bundle) SkipRange() (string, error) { @@ -114,6 +150,20 @@ func (b *Bundle) Skips() ([]string, error) { return b.csv.GetSkips() } +func (b *Bundle) Icons() ([]Icon, error) { + if err := b.cache(); err != nil { + return nil, err + } + return b.csv.GetIcons() +} + +func (b *Bundle) Description() (string, error) { + if err := b.cache(); err != nil { + return "", err + } + return b.csv.GetDescription() +} + func (b *Bundle) CustomResourceDefinitions() ([]runtime.Object, error) { if err := b.cache(); err != nil { return nil, err @@ -132,7 +182,7 @@ func (b *Bundle) ProvidedAPIs() (map[APIKey]struct{}, error) { provided := map[APIKey]struct{}{} crds, err := b.CustomResourceDefinitions() if err != nil { - return nil, err + return nil, fmt.Errorf("error getting crds: %s", err) } for _, c := range crds { @@ -160,7 +210,7 @@ func (b *Bundle) ProvidedAPIs() (map[APIKey]struct{}, error) { ownedAPIs, _, err := csv.GetApiServiceDefinitions() if err != nil { - return nil, err + return nil, fmt.Errorf("error getting apiservice definitions: %s", err) } for _, api := range ownedAPIs { provided[APIKey{Group: api.Group, Version: api.Version, Kind: api.Kind, Plural: api.Name}] = struct{}{} @@ -206,6 +256,7 @@ func (b *Bundle) AllProvidedAPIsInBundle() error { if err != nil { return err } + ownedCRDs, _, err := csv.GetCustomResourceDefintions() if err != nil { return err @@ -227,12 +278,12 @@ func (b *Bundle) AllProvidedAPIsInBundle() error { return nil } -func (b *Bundle) Serialize() (csvName, bundleImage string, csvBytes []byte, bundleBytes []byte, err error) { +func (b *Bundle) Serialize() (csvName, bundleImage string, csvBytes []byte, bundleBytes []byte, annotationBytes []byte, err error) { csvCount := 0 for _, obj := range b.Objects { objBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) if err != nil { - return "", "", nil, nil, err + return "", "", nil, nil, nil, err } bundleBytes = append(bundleBytes, objBytes...) @@ -240,38 +291,55 @@ func (b *Bundle) Serialize() (csvName, bundleImage string, csvBytes []byte, bund csvName = obj.GetName() csvBytes, err = runtime.Encode(unstructured.UnstructuredJSONScheme, obj) if err != nil { - return "", "", nil, nil, err + return "", "", nil, nil, nil, err } csvCount += 1 if csvCount > 1 { - return "", "", nil, nil, fmt.Errorf("two csvs found in one bundle") + return "", "", nil, nil, nil, fmt.Errorf("two csvs found in one bundle") } } } - return csvName, b.BundleImage, csvBytes, bundleBytes, nil + if b.Annotations != nil { + annotationBytes, err = json.Marshal(b.Annotations) + } + + return csvName, b.BundleImage, csvBytes, bundleBytes, annotationBytes, nil } func (b *Bundle) Images() (map[string]struct{}, error) { + result := make(map[string]struct{}) + + if b.BundleImage != "" { + result[b.BundleImage] = struct{}{} + } + csv, err := b.ClusterServiceVersion() if err != nil { return nil, err } + if csv == nil { + return result, nil + } + images, err := csv.GetOperatorImages() if err != nil { return nil, err } + for img := range images { + result[img] = struct{}{} + } relatedImages, err := csv.GetRelatedImages() if err != nil { return nil, err } for img := range relatedImages { - images[img] = struct{}{} + result[img] = struct{}{} } - return images, nil + return result, nil } func (b *Bundle) cache() error { @@ -318,3 +386,10 @@ func (b *Bundle) cache() error { b.cacheStale = false return nil } + +func (b *Bundle) SubstitutesFor() (string, error) { + if err := b.cache(); err != nil { + return "", err + } + return b.csv.GetSubstitutesFor(), nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundlegraphloader.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundlegraphloader.go index f03c128a39..05d8abf064 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundlegraphloader.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/bundlegraphloader.go @@ -13,7 +13,7 @@ type BundleGraphLoader struct { // AddBundleToGraph takes a bundle and an existing graph and updates the graph to insert the new bundle // into each channel it is included in -func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, newDefaultChannel string, skippatch bool) (*Package, error) { +func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, annotations *AnnotationsFile, skippatch bool) (*Package, error) { bundleVersion, err := bundle.Version() if err != nil { return nil, fmt.Errorf("Unable to extract bundle version from bundle %s, can't insert in semver mode", bundle.BundleImage) @@ -34,10 +34,20 @@ func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, new if graph.Name == "" { graph.Name = bundle.Package } + + newDefaultChannel := annotations.Annotations.DefaultChannelName if newDefaultChannel != "" { graph.DefaultChannel = newDefaultChannel } + if graph.DefaultChannel == "" { + // Infer default channel from channel list + if annotations.SelectDefaultChannel() == "" { + return nil, fmt.Errorf("Default channel is missing and can't be inferred") + } + graph.DefaultChannel = annotations.SelectDefaultChannel() + } + // generate the DAG for each channel the new bundle is being insert into for _, channel := range bundle.Channels { replaces := make(map[BundleKey]struct{}, 0) @@ -142,6 +152,9 @@ func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, new return graph, nil } +// isSkipPatchCandidate returns true if version is equal to toCompare +// in major and minor positions and strictly greater in all others, +// indicating that toCompare can be skipped over to version. func isSkipPatchCandidate(version, toCompare semver.Version) bool { - return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && (version.Patch > toCompare.Patch) + return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && version.GT(toCompare) } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/csv.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/csv.go index 28d59ce832..a3b5ebc191 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/csv.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/csv.go @@ -38,6 +38,17 @@ const ( // skips skips = "skips" + // The yaml attribute that points to the icon for the ClusterServiceVersion + icon = "icon" + + // The yaml attribute that points to the icon.base64data for the ClusterServiceVersion + base64data = "base64data" + + // The yaml attribute that points to the icon.mediatype for the ClusterServiceVersion + mediatype = "mediatype" + // The yaml attribute that points to the description for the ClusterServiceVersion + description = "description" + // The yaml attribute that specifies the version of the ClusterServiceVersion // expected to be semver and parseable by blang/semver version = "version" @@ -47,6 +58,9 @@ const ( // The yaml attribute that specifies the skipRange of the ClusterServiceVersion skipRangeAnnotationKey = "olm.skipRange" + + // The yaml attribute that specifies the optional substitutesfor of the ClusterServiceVersion + substitutesForAnnotationKey = "olm.substitutesFor" ) // ClusterServiceVersion is a structured representation of cluster service @@ -87,6 +101,7 @@ func ReadCSVFromBundleDirectory(bundleDir string) (*ClusterServiceVersion, error if err != nil { continue } + defer yamlReader.Close() unstructuredCSV := unstructured.Unstructured{} @@ -248,7 +263,7 @@ func (csv *ClusterServiceVersion) GetApiServiceDefinitions() (owned []*Definitio var objmap map[string]*json.RawMessage if err = json.Unmarshal(csv.Spec, &objmap); err != nil { - return + return nil, nil, fmt.Errorf("error unmarshaling into object map: %s", err) } rawValue, ok := objmap[apiServiceDefinitions] @@ -340,3 +355,60 @@ func (csv *ClusterServiceVersion) GetOperatorImages() (map[string]struct{}, erro return images, nil } + +type Icon struct { + MediaType string `json:"mediatype"` + Base64data []byte `json:"base64data"` +} + +// GetIcons returns the icons from the ClusterServiceVersion +func (csv *ClusterServiceVersion) GetIcons() ([]Icon, error) { + var objmap map[string]*json.RawMessage + if err := json.Unmarshal(csv.Spec, &objmap); err != nil { + return nil, err + } + + rawValue, ok := objmap[icon] + if !ok || rawValue == nil { + return nil, nil + } + var icons []Icon + if err := json.Unmarshal(*rawValue, &icons); err != nil { + return nil, err + } + return icons, nil +} + +// GetDescription returns the description from the ClusterServiceVersion +// If not defined, the function returns an empty string. +func (csv *ClusterServiceVersion) GetDescription() (string, error) { + var objmap map[string]*json.RawMessage + + if err := json.Unmarshal(csv.Spec, &objmap); err != nil { + return "", err + } + + rawValue, ok := objmap[description] + if !ok || rawValue == nil { + return "", nil + } + + var desc string + if err := json.Unmarshal(*rawValue, &desc); err != nil { + return "", err + } + + return desc, nil +} + +// GetSubstitutesFor returns the name of the ClusterServiceVersion object that +// is substituted by this ClusterServiceVersion object. +// +// If not defined, the function returns an empty string. +func (csv *ClusterServiceVersion) GetSubstitutesFor() string { + substitutesFor, ok := csv.Annotations[substitutesForAnnotationKey] + if !ok { + return "" + } + return substitutesFor +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/decode.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/decode.go index 4d418df765..392d5dd887 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/decode.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/decode.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "io" + "io/fs" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" ) @@ -41,3 +43,15 @@ func DecodePackageManifest(reader io.Reader) (manifest *PackageManifest, err err manifest = obj return } + +func decodeFileFS(root fs.FS, path string, into interface{}) error { + fileReader, err := root.Open(path) + if err != nil { + return fmt.Errorf("unable to read file %s: %s", path, err) + } + defer fileReader.Close() + + decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) + + return decoder.Decode(into) +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/directoryGraphLoader.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/directoryGraphLoader.go index bd336f9f86..03efa4d63f 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/directoryGraphLoader.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/directoryGraphLoader.go @@ -55,8 +55,6 @@ func (g *DirGraphLoader) Generate() (*Package, error) { return nil, fmt.Errorf("error geting CSVs from bundles in the package directory, %v", err) } - g.loadReplaces() - pkg, err := g.parsePackageYAMLFile() if err != nil { return nil, fmt.Errorf("error parsing package.yaml file in the package root directory, %v", err) @@ -120,22 +118,6 @@ func (g *DirGraphLoader) loadBundleCsvPathMap() error { return nil } -// loadReplaces converts skipRange to explicit csv name and write to replaces in CsvNameAndReplaceMap. -func (g *DirGraphLoader) loadReplaces() { - for csvName, replace := range g.CsvNameAndReplaceMap { - if replace.skipRange != nil { - for _, v := range g.SortedCSVs { - if replace.skipRange(v.version) { - g.CsvNameAndReplaceMap[csvName] = csvReplaces{ - replaces: append(g.CsvNameAndReplaceMap[csvName].replaces, v.name), - skipRange: g.CsvNameAndReplaceMap[csvName].skipRange, - } - } - } - } - } -} - // getChannelNodes follows the head of the channel csv through all replaces to fill the nodes until the tail of the // channel which will not replace anything. func (g *DirGraphLoader) getChannelNodes(channelHeadCsv string) *map[BundleKey]map[BundleKey]struct{} { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/empty.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/empty.go index a35bf3eb8b..fd4676eb6c 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/empty.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/empty.go @@ -104,6 +104,14 @@ func (EmptyQuery) GetDependenciesForBundle(ctx context.Context, name, version, p return nil, errors.New("empty querier: cannot get dependencies for bundle") } +func (EmptyQuery) GetBundlePathIfExists(ctx context.Context, csvName string) (bundlePath string, err error) { + return "", errors.New("empty querier: cannot get bundle path for bundle") +} + +func (EmptyQuery) ListRegistryBundles(ctx context.Context) ([]*Bundle, error) { + return nil, errors.New("empty querier: cannot list registry bundles") +} + var _ Query = &EmptyQuery{} func NewEmptyQuerier() *EmptyQuery { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/graph.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/graph.go index f62a30ab47..32185f1894 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/graph.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/graph.go @@ -2,6 +2,7 @@ package registry import ( "fmt" + "strings" ) type Package struct { @@ -10,11 +11,38 @@ type Package struct { Channels map[string]Channel } +func (p *Package) String() string { + var b strings.Builder + b.WriteString("name: ") + b.WriteString(p.Name) + b.WriteString("\ndefault channel: ") + b.WriteString(p.DefaultChannel) + b.WriteString("\nchannels:\n") + + for n, c := range p.Channels { + b.WriteString(n) + b.WriteString("\n") + b.WriteString(c.String()) + } + + return b.String() +} + type Channel struct { Head BundleKey Nodes map[BundleKey]map[BundleKey]struct{} } +func (c *Channel) String() string { + var b strings.Builder + for node, _ := range c.Nodes { + b.WriteString(node.String()) + b.WriteString("\n") + } + + return b.String() +} + type BundleKey struct { BundlePath string Version string //semver string diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/imageinput.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/imageinput.go index 6cfb905305..69fe210ef9 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/imageinput.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/imageinput.go @@ -1,128 +1,30 @@ package registry import ( - "fmt" - "io/ioutil" + "os" "path/filepath" - "strings" - - "github.com/sirupsen/logrus" "github.com/operator-framework/operator-registry/pkg/image" + "github.com/sirupsen/logrus" ) type ImageInput struct { - manifestsDir string - metadataDir string - to image.Reference - from string - annotationsFile *AnnotationsFile - dependenciesFile *DependenciesFile - bundle *Bundle + to image.Reference + from string + Bundle *Bundle } func NewImageInput(to image.Reference, from string) (*ImageInput, error) { - var annotationsFound, dependenciesFound bool - path := from - manifests := filepath.Join(path, "manifests") - metadata := filepath.Join(path, "metadata") - // Get annotations file - log := logrus.WithFields(logrus.Fields{"dir": from, "file": metadata, "load": "annotations"}) - files, err := ioutil.ReadDir(metadata) - if err != nil { - return nil, fmt.Errorf("unable to read directory %s: %s", metadata, err) - } - - // Look for the metadata and manifests sub-directories to find the annotations.yaml - // file that will inform how the manifests of the bundle should be loaded into the database. - // If dependencies.yaml which contains operator dependencies in metadata directory - // exists, parse and load it into the DB - annotationsFile := &AnnotationsFile{} - dependenciesFile := &DependenciesFile{} - for _, f := range files { - if !annotationsFound { - err = DecodeFile(filepath.Join(metadata, f.Name()), annotationsFile) - if err == nil && *annotationsFile != (AnnotationsFile{}) { - annotationsFound = true - continue - } - } - - if !dependenciesFound { - err = DecodeFile(filepath.Join(metadata, f.Name()), &dependenciesFile) - if err != nil { - return nil, err - } - if len(dependenciesFile.Dependencies) > 0 { - dependenciesFound = true - } - } - } - - if !annotationsFound { - return nil, fmt.Errorf("Could not find annotations file") - } - - if !dependenciesFound { - log.Info("Could not find optional dependencies file") - } - - imageInput := &ImageInput{ - manifestsDir: manifests, - metadataDir: metadata, - to: to, - from: from, - annotationsFile: annotationsFile, - dependenciesFile: dependenciesFile, - } - - err = imageInput.getBundleFromManifests() + parser := newBundleParser(logrus.WithFields(logrus.Fields{"with": from, "file": filepath.Join(from, "metadata"), "load": "annotations"})) + bundle, err := parser.Parse(os.DirFS(from)) if err != nil { return nil, err } + bundle.BundleImage = to.String() - return imageInput, nil -} - -func (i *ImageInput) getBundleFromManifests() error { - log := logrus.WithFields(logrus.Fields{"dir": i.from, "file": i.manifestsDir, "load": "bundle"}) - - csv, err := i.findCSV(i.manifestsDir) - if err != nil { - return err - } - - if csv.Object == nil { - return fmt.Errorf("csv is empty: %s", err) - } - - log.Info("found csv, loading bundle") - - csvName := csv.GetName() - - bundle, err := loadBundle(csvName, i.manifestsDir) - if err != nil { - return fmt.Errorf("error loading objs in directory: %s", err) - } - - if bundle == nil || bundle.Size() == 0 { - return fmt.Errorf("no bundle objects found") - } - - // set the bundleimage on the bundle - bundle.BundleImage = i.to.String() - // set the dependencies on the bundle - bundle.Dependencies = i.dependenciesFile.GetDependencies() - - bundle.Name = csvName - bundle.Package = i.annotationsFile.Annotations.PackageName - bundle.Channels = strings.Split(i.annotationsFile.Annotations.Channels, ",") - - if err := bundle.AllProvidedAPIsInBundle(); err != nil { - return fmt.Errorf("error checking provided apis in bundle %s: %s", bundle.Name, err) - } - - i.bundle = bundle - - return nil + return &ImageInput{ + to: to, + from: from, + Bundle: bundle, + }, nil } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go index 430905209a..fd77636117 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/interface.go @@ -12,28 +12,49 @@ type Load interface { AddPackageChannels(manifest PackageManifest) error AddBundlePackageChannels(manifest PackageManifest, bundle *Bundle) error RemovePackage(packageName string) error + RemoveStrandedBundles() error DeprecateBundle(path string) error ClearNonHeadBundles() error } -type Query interface { - ListTables(ctx context.Context) ([]string, error) +type GRPCQuery interface { + // List all available package names in the index ListPackages(ctx context.Context) ([]string, error) + + // List all available bundles in the index + ListBundles(ctx context.Context) (bundles []*api.Bundle, err error) + + // Get a package by name from the index GetPackage(ctx context.Context, name string) (*PackageManifest, error) - GetDefaultPackage(ctx context.Context, name string) (string, error) - GetChannelEntriesFromPackage(ctx context.Context, packageName string) ([]ChannelEntryAnnotated, error) + + // Get a bundle by its package name, channel name and csv name from the index GetBundle(ctx context.Context, pkgName, channelName, csvName string) (*api.Bundle, error) + + // Get the bundle in the specified package at the head of the specified channel GetBundleForChannel(ctx context.Context, pkgName string, channelName string) (*api.Bundle, error) + // Get all channel entries that say they replace this one GetChannelEntriesThatReplace(ctx context.Context, name string) (entries []*ChannelEntry, err error) + // Get the bundle in a package/channel that replace this one GetBundleThatReplaces(ctx context.Context, name, pkgName, channelName string) (*api.Bundle, error) + // Get all channel entries that provide an api GetChannelEntriesThatProvide(ctx context.Context, group, version, kind string) (entries []*ChannelEntry, err error) + // Get latest channel entries that provide an api GetLatestChannelEntriesThatProvide(ctx context.Context, group, version, kind string) (entries []*ChannelEntry, err error) + // Get the the latest bundle that provides the API in a default channel GetBundleThatProvides(ctx context.Context, group, version, kind string) (*api.Bundle, error) +} + +type Query interface { + GRPCQuery + + ListTables(ctx context.Context) ([]string, error) + GetDefaultPackage(ctx context.Context, name string) (string, error) + GetChannelEntriesFromPackage(ctx context.Context, packageName string) ([]ChannelEntryAnnotated, error) // List all images in the database ListImages(ctx context.Context) ([]string, error) // List all images for a particular bundle @@ -52,12 +73,16 @@ type Query interface { ListChannels(ctx context.Context, pkgName string) ([]string, error) // Get CurrentCSV name for channel and package GetCurrentCSVNameForChannel(ctx context.Context, pkgName, channel string) (string, error) - // List all available bundles in the database - ListBundles(ctx context.Context) (bundles []*api.Bundle, err error) // Get the list of dependencies for a bundle GetDependenciesForBundle(ctx context.Context, name, version, path string) (dependencies []*api.Dependency, err error) + // Get the bundle path if it exists + GetBundlePathIfExists(ctx context.Context, csvName string) (string, error) + // ListRegistryBundles returns a set of registry bundles. + ListRegistryBundles(ctx context.Context) ([]*Bundle, error) } +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . GraphLoader + // GraphLoader generates a graph // GraphLoader supports multiple different loading schemes // GraphLoader from SQL, GraphLoader from old format (filesystem), GraphLoader from SQL + input bundles diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/parse.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/parse.go new file mode 100644 index 0000000000..984517cc04 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/parse.go @@ -0,0 +1,252 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io/fs" + "strings" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" +) + +type bundleParser struct { + log *logrus.Entry +} + +func newBundleParser(log *logrus.Entry) *bundleParser { + return &bundleParser{ + log: log, + } +} + +// Parse parses the given FS into a Bundle. +func (b *bundleParser) Parse(root fs.FS) (*Bundle, error) { + if root == nil { + return nil, fmt.Errorf("filesystem is nil") + } + + bundle := &Bundle{} + manifests, err := fs.Sub(root, "manifests") + if err != nil { + return nil, fmt.Errorf("error opening manifests directory: %s", err) + } + if err := b.addManifests(manifests, bundle); err != nil { + return nil, err + } + + metadata, err := fs.Sub(root, "metadata") + if err != nil { + return nil, fmt.Errorf("error opening metadata directory: %s", err) + } + if err := b.addMetadata(metadata, bundle); err != nil { + return nil, err + } + + derived, err := b.derivedProperties(bundle) + if err != nil { + return nil, fmt.Errorf("failed to derive properties: %s", err) + } + + bundle.Properties = propertySet(append(bundle.Properties, derived...)) + + return bundle, nil +} + +// addManifests adds the result of parsing the manifests directory to a bundle. +func (b *bundleParser) addManifests(manifests fs.FS, bundle *Bundle) error { + files, err := fs.ReadDir(manifests, ".") + if err != nil { + return err + } + + var csvFound bool + for _, f := range files { + if f.IsDir() { + continue + } + + name := f.Name() + if strings.HasPrefix(name, ".") { + continue + } + + obj := &unstructured.Unstructured{} + if err = decodeFileFS(manifests, name, obj); err != nil { + b.log.Warnf("failed to decode: %s", err) + continue + } + + // Only include the first CSV we find in the + if obj.GetKind() == operatorsv1alpha1.ClusterServiceVersionKind { + if csvFound { + continue + } + csvFound = true + } + + if obj.Object != nil { + bundle.Add(obj) + } + } + + if bundle.Size() == 0 { + return fmt.Errorf("no bundle objects found") + } + + csv, err := bundle.ClusterServiceVersion() + if err != nil { + return err + } + if csv == nil { + return fmt.Errorf("no csv in bundle") + } + + bundle.Name = csv.GetName() + if err := bundle.AllProvidedAPIsInBundle(); err != nil { + return fmt.Errorf("error checking provided apis in bundle %s: %s", bundle.Name, err) + } + + return nil +} + +// addManifests adds the result of parsing the metadata directory to a bundle. +func (b *bundleParser) addMetadata(metadata fs.FS, bundle *Bundle) error { + files, err := fs.ReadDir(metadata, ".") + if err != nil { + return err + } + + var ( + af *AnnotationsFile + df *DependenciesFile + pf *PropertiesFile + ) + for _, f := range files { + name := f.Name() + if af == nil { + decoded := AnnotationsFile{} + if err = decodeFileFS(metadata, name, &decoded); err == nil { + if decoded != (AnnotationsFile{}) { + af = &decoded + } + } + } + if df == nil { + decoded := DependenciesFile{} + if err = decodeFileFS(metadata, name, &decoded); err == nil { + if len(decoded.Dependencies) > 0 { + df = &decoded + } + } + } + if pf == nil { + decoded := PropertiesFile{} + if err = decodeFileFS(metadata, name, &decoded); err == nil { + if len(decoded.Properties) > 0 { + pf = &decoded + } + } + } + } + + if af != nil { + bundle.Annotations = &af.Annotations + bundle.Package = af.Annotations.PackageName + bundle.Channels = af.GetChannels() + } else { + return fmt.Errorf("Could not find annotations file") + } + + if df != nil { + bundle.Dependencies = append(bundle.Dependencies, df.GetDependencies()...) + } else { + b.log.Info("Could not find optional dependencies file") + } + + if pf != nil { + bundle.Properties = append(bundle.Properties, pf.Properties...) + } else { + b.log.Info("Could not find optional properties file") + } + + return nil +} + +func (b *bundleParser) derivedProperties(bundle *Bundle) ([]Property, error) { + // Add properties from CSV annotations + csv, err := bundle.ClusterServiceVersion() + if err != nil { + return nil, fmt.Errorf("error getting csv: %s", err) + } + if csv == nil { + return nil, fmt.Errorf("bundle missing csv") + } + + var derived []Property + if len(csv.GetAnnotations()) > 0 { + properties, ok := csv.GetAnnotations()[PropertyKey] + if ok { + if err := json.Unmarshal([]byte(properties), &derived); err != nil { + b.log.Warnf("failed to unmarshal csv annotation properties: %s", err) + } + } + } + + if bundle.Annotations != nil && bundle.Annotations.PackageName != "" { + pkg := bundle.Annotations.PackageName + version, err := bundle.Version() + if err != nil { + return nil, err + } + + value, err := json.Marshal(PackageProperty{ + PackageName: pkg, + Version: version, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal package property: %s", err) + } + + // Annotations file takes precedent over CSV annotations + derived = append([]Property{{Type: PackageType, Value: value}}, derived...) + } + + providedAPIs, err := bundle.ProvidedAPIs() + if err != nil { + return nil, fmt.Errorf("error getting provided apis: %s", err) + } + + for api := range providedAPIs { + value, err := json.Marshal(GVKProperty{ + Group: api.Group, + Kind: api.Kind, + Version: api.Version, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal gvk property: %s", err) + } + derived = append(derived, Property{Type: GVKType, Value: value}) + } + + return propertySet(derived), nil +} + +// propertySet returns the deduplicated set of a property list. +func propertySet(properties []Property) []Property { + var ( + set []Property + visited = map[string]struct{}{} + ) + for _, p := range properties { + if _, ok := visited[p.String()]; ok { + continue + } + visited[p.String()] = struct{}{} + set = append(set, p) + } + + return set +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/populator.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/populator.go index 8e3e2eb5ab..ad65e78b2d 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/populator.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/populator.go @@ -4,17 +4,14 @@ import ( "context" "errors" "fmt" - "io/ioutil" "os" - "path/filepath" - "strings" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/blang/semver" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/yaml" "github.com/operator-framework/operator-registry/pkg/image" + libsemver "github.com/operator-framework/operator-registry/pkg/lib/semver" ) type Dependencies struct { @@ -23,18 +20,22 @@ type Dependencies struct { // DirectoryPopulator loads an unpacked operator bundle from a directory into the database. type DirectoryPopulator struct { - loader Load - graphLoader GraphLoader - querier Query - imageDirMap map[image.Reference]string + loader Load + graphLoader GraphLoader + querier Query + imageDirMap map[image.Reference]string + overwriteDirMap map[string]map[image.Reference]string + overwrite bool } -func NewDirectoryPopulator(loader Load, graphLoader GraphLoader, querier Query, imageDirMap map[image.Reference]string) *DirectoryPopulator { +func NewDirectoryPopulator(loader Load, graphLoader GraphLoader, querier Query, imageDirMap map[image.Reference]string, overwriteDirMap map[string]map[image.Reference]string, overwrite bool) *DirectoryPopulator { return &DirectoryPopulator{ - loader: loader, - graphLoader: graphLoader, - querier: querier, - imageDirMap: imageDirMap, + loader: loader, + graphLoader: graphLoader, + querier: querier, + imageDirMap: imageDirMap, + overwriteDirMap: overwriteDirMap, + overwrite: overwrite, } } @@ -51,11 +52,24 @@ func (i *DirectoryPopulator) Populate(mode Mode) error { imagesToAdd = append(imagesToAdd, imageInput) } + imagesToReAdd := make([]*ImageInput, 0) + for pkg := range i.overwriteDirMap { + for to, from := range i.overwriteDirMap[pkg] { + imageInput, err := NewImageInput(to, from) + if err != nil { + errs = append(errs, err) + continue + } + + imagesToReAdd = append(imagesToReAdd, imageInput) + } + } + if len(errs) > 0 { return utilerrors.NewAggregate(errs) } - err := i.loadManifests(imagesToAdd, mode) + err := i.loadManifests(imagesToAdd, imagesToReAdd, mode) if err != nil { return err } @@ -67,11 +81,13 @@ func (i *DirectoryPopulator) globalSanityCheck(imagesToAdd []*ImageInput) error var errs []error images := make(map[string]struct{}) for _, image := range imagesToAdd { - images[image.bundle.BundleImage] = struct{}{} + images[image.Bundle.BundleImage] = struct{}{} } + attemptedOverwritesPerPackage := map[string]struct{}{} for _, image := range imagesToAdd { - bundlePaths, err := i.querier.GetBundlePathsForPackage(context.TODO(), image.bundle.Package) + validOverwrite := false + bundlePaths, err := i.querier.GetBundlePathsForPackage(context.TODO(), image.Bundle.Package) if err != nil { // Assume that this means that the bundle is empty // Or that this is the first time the package is loaded. @@ -79,77 +95,89 @@ func (i *DirectoryPopulator) globalSanityCheck(imagesToAdd []*ImageInput) error } for _, bundlePath := range bundlePaths { if _, ok := images[bundlePath]; ok { - errs = append(errs, BundleImageAlreadyAddedErr{ErrorString: fmt.Sprintf("Bundle %s already exists", image.bundle.BundleImage)}) + errs = append(errs, BundleImageAlreadyAddedErr{ErrorString: fmt.Sprintf("Bundle %s already exists", image.Bundle.BundleImage)}) continue } } - for _, channel := range image.bundle.Channels { - bundle, err := i.querier.GetBundle(context.TODO(), image.bundle.Package, channel, image.bundle.csv.GetName()) + channels, err := i.querier.ListChannels(context.TODO(), image.Bundle.Package) + if err != nil { + return err + } + + for _, channel := range channels { + bundle, err := i.querier.GetBundle(context.TODO(), image.Bundle.Package, channel, image.Bundle.Name) if err != nil { // Assume that if we can not find a bundle for the package, channel and or CSV Name that this is safe to add continue } if bundle != nil { - // raise error that this package + channel + csv combo is already in the db - errs = append(errs, PackageVersionAlreadyAddedErr{ErrorString: "Bundle already added that provides package and csv"}) + if !i.overwrite { + // raise error that this package + channel + csv combo is already in the db + errs = append(errs, PackageVersionAlreadyAddedErr{ErrorString: "Bundle already added that provides package and csv"}) + break + } + // ensure overwrite is not in the middle of a channel (i.e. nothing replaces it) + _, err = i.querier.GetBundleThatReplaces(context.TODO(), image.Bundle.Name, image.Bundle.Package, channel) + if err != nil { + if err.Error() == fmt.Errorf("no entry found for %s %s", image.Bundle.Package, channel).Error() { + // overwrite is not replaced by any other bundle + validOverwrite = true + continue + } + errs = append(errs, err) + break + } + // This bundle is in this channel but is not the head of this channel + errs = append(errs, OverwriteErr{ErrorString: "Cannot overwrite a bundle that is not at the head of a channel using --overwrite-latest"}) + validOverwrite = false + break + } + } + if validOverwrite { + if _, ok := attemptedOverwritesPerPackage[image.Bundle.Package]; ok { + errs = append(errs, OverwriteErr{ErrorString: "Cannot overwrite more than one bundle at a time for a given package using --overwrite-latest"}) break } + attemptedOverwritesPerPackage[image.Bundle.Package] = struct{}{} } } return utilerrors.NewAggregate(errs) } -func (i *DirectoryPopulator) loadManifests(imagesToAdd []*ImageInput, mode Mode) error { +func (i *DirectoryPopulator) loadManifests(imagesToAdd []*ImageInput, imagesToReAdd []*ImageInput, mode Mode) error { // global sanity checks before insertion - err := i.globalSanityCheck(imagesToAdd) - if err != nil { + if err := i.globalSanityCheck(imagesToAdd); err != nil { return err } switch mode { case ReplacesMode: - // TODO: This is relatively inefficient. Ideally, we should be able to use a replaces - // graph loader to construct what the graph would look like with a set of new bundles - // and use that to return an error if it's not valid, rather than insert one at a time - // and reinspect the database. - // - // Additionally, it would be preferrable if there was a single database transaction - // that took the updated graph as a whole as input, rather than inserting bundles of the - // same package linearly. - var err error - var validImagesToAdd []*ImageInput - for len(imagesToAdd) > 0 { - validImagesToAdd, imagesToAdd, err = i.getNextReplacesImagesToAdd(imagesToAdd) - if err != nil { + for pkg := range i.overwriteDirMap { + // TODO: If this succeeds but the add fails there will be a disconnect between + // the registry and the index. Loading the bundles in a single transactions as + // described above would allow us to do the removable in that same transaction + // and ensure that rollback is possible. + if err := i.loader.RemovePackage(pkg); err != nil { return err } - for _, image := range validImagesToAdd { - err := i.loadManifestsReplaces(image.bundle, image.annotationsFile) - if err != nil { - return err - } - } } + + return i.loadManifestsReplaces(append(imagesToAdd, imagesToReAdd...)) case SemVerMode: for _, image := range imagesToAdd { - err := i.loadManifestsSemver(image.bundle, image.annotationsFile, false) - if err != nil { + if err := i.loadManifestsSemver(image.Bundle, false); err != nil { return err } } case SkipPatchMode: for _, image := range imagesToAdd { - err := i.loadManifestsSemver(image.bundle, image.annotationsFile, true) - if err != nil { + if err := i.loadManifestsSemver(image.Bundle, true); err != nil { return err } } default: - err := fmt.Errorf("Unsupported update mode") - if err != nil { - return err - } + return fmt.Errorf("Unsupported update mode") } // Finally let's delete all the old bundles @@ -160,98 +188,57 @@ func (i *DirectoryPopulator) loadManifests(imagesToAdd []*ImageInput, mode Mode) return nil } -func (i *DirectoryPopulator) loadManifestsReplaces(bundle *Bundle, annotationsFile *AnnotationsFile) error { - channels, err := i.querier.ListChannels(context.TODO(), annotationsFile.GetName()) - existingPackageChannels := map[string]string{} - for _, c := range channels { - current, err := i.querier.GetCurrentCSVNameForChannel(context.TODO(), annotationsFile.GetName(), c) - if err != nil { - return err - } - existingPackageChannels[c] = current - } - - bcsv, err := bundle.ClusterServiceVersion() - if err != nil { - return fmt.Errorf("error getting csv from bundle %s: %s", bundle.Name, err) - } - - packageManifest, err := translateAnnotationsIntoPackage(annotationsFile, bcsv, existingPackageChannels) - if err != nil { - return fmt.Errorf("Could not translate annotations file into packageManifest %s", err) - } +var packageContextKey = "package" - if err := i.loadOperatorBundle(packageManifest, bundle); err != nil { - return fmt.Errorf("Error adding package %s", err) - } - - return nil +// ContextWithPackage adds a package value to a context. +func ContextWithPackage(ctx context.Context, pkg string) context.Context { + return context.WithValue(ctx, packageContextKey, pkg) } -func (i *DirectoryPopulator) getNextReplacesImagesToAdd(imagesToAdd []*ImageInput) ([]*ImageInput, []*ImageInput, error) { - remainingImages := make([]*ImageInput, 0) - foundImages := make([]*ImageInput, 0) +// PackageFromContext returns the package value of the context if set, returns false if unset. +func PackageFromContext(ctx context.Context) (string, bool) { + pkg, ok := ctx.Value(packageContextKey).(string) + return pkg, ok +} +func (i *DirectoryPopulator) loadManifestsReplaces(images []*ImageInput) error { + packages := map[string][]*Bundle{} var errs []error - - // Separate these image sets per package, since multiple different packages have - // separate graph - imagesPerPackage := make(map[string][]*ImageInput, 0) - for _, image := range imagesToAdd { - pkg := image.bundle.Package - if _, ok := imagesPerPackage[pkg]; !ok { - newPkgImages := make([]*ImageInput, 0) - newPkgImages = append(newPkgImages, image) - imagesPerPackage[pkg] = newPkgImages - } else { - imagesPerPackage[pkg] = append(imagesPerPackage[pkg], image) + for _, img := range images { + // Add the bundle directly to the store + if err := i.loader.AddOperatorBundle(img.Bundle); err != nil { + errs = append(errs, err) + continue } - } - for pkg, pkgImages := range imagesPerPackage { - // keep a tally of valid and invalid images to ensure at least one - // image per package is valid. If not, throw an error - pkgRemainingImages := 0 - pkgFoundImages := 0 + packages[img.Bundle.Package] = append(packages[img.Bundle.Package], img.Bundle) + } - // first, try to pull the existing package graph from the database if it exists - graph, err := i.graphLoader.Generate(pkg) - if err != nil && !errors.Is(err, ErrPackageNotInDatabase) { - return nil, nil, err + // Regenerate the upgrade graphs for each package + for pkg, bundles := range packages { + // Add any existing bundles into the mix + ctx := ContextWithPackage(context.TODO(), pkg) + existing, err := i.querier.ListRegistryBundles(ctx) + if err != nil { + errs = append(errs, err) + continue } - var pkgErrs []error - // then check each image to see if it can be a replacement - replacesLoader := ReplacesGraphLoader{} - for _, pkgImage := range pkgImages { - canAdd, err := replacesLoader.CanAdd(pkgImage.bundle, graph) - if err != nil { - pkgErrs = append(pkgErrs, err) - } - if canAdd { - pkgFoundImages++ - foundImages = append(foundImages, pkgImage) - } else { - pkgRemainingImages++ - remainingImages = append(remainingImages, pkgImage) - } + packageManifest, err := SemverPackageManifest(append(existing, bundles...)) + if err != nil { + errs = append(errs, err) + continue } - // no new images can be added, the current iteration aggregates all the - // errors that describe invalid bundles - if pkgFoundImages == 0 && pkgRemainingImages > 0 { - errs = append(errs, utilerrors.NewAggregate(pkgErrs)) + if err = i.loader.AddPackageChannels(*packageManifest); err != nil { + errs = append(errs, err) } } - if len(errs) > 0 { - return nil, nil, utilerrors.NewAggregate(errs) - } - - return foundImages, remainingImages, nil + return utilerrors.NewAggregate(errs) } -func (i *DirectoryPopulator) loadManifestsSemver(bundle *Bundle, annotations *AnnotationsFile, skippatch bool) error { +func (i *DirectoryPopulator) loadManifestsSemver(bundle *Bundle, skippatch bool) error { graph, err := i.graphLoader.Generate(bundle.Package) if err != nil && !errors.Is(err, ErrPackageNotInDatabase) { return err @@ -259,7 +246,7 @@ func (i *DirectoryPopulator) loadManifestsSemver(bundle *Bundle, annotations *An // add to the graph bundleLoader := BundleGraphLoader{} - updatedGraph, err := bundleLoader.AddBundleToGraph(bundle, graph, annotations.Annotations.DefaultChannelName, skippatch) + updatedGraph, err := bundleLoader.AddBundleToGraph(bundle, graph, &AnnotationsFile{Annotations: *bundle.Annotations}, skippatch) if err != nil { return err } @@ -271,130 +258,138 @@ func (i *DirectoryPopulator) loadManifestsSemver(bundle *Bundle, annotations *An return nil } -// loadBundle takes the directory that a CSV is in and assumes the rest of the objects in that directory -// are part of the bundle. -func loadBundle(csvName string, dir string) (*Bundle, error) { - log := logrus.WithFields(logrus.Fields{"dir": dir, "load": "bundle"}) - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err +// loadOperatorBundle adds the package information to the loader's store +func (i *DirectoryPopulator) loadOperatorBundle(manifest PackageManifest, bundle *Bundle) error { + if manifest.PackageName == "" { + return nil } - bundle := &Bundle{ - Name: csvName, + if err := i.loader.AddBundlePackageChannels(manifest, bundle); err != nil { + return fmt.Errorf("error loading bundle into db: %s", err) } - for _, f := range files { - log = log.WithField("file", f.Name()) - if f.IsDir() { - log.Info("skipping directory") - continue - } - - if strings.HasPrefix(f.Name(), ".") { - log.Info("skipping hidden file") - continue - } - log.Info("loading bundle file") - var ( - obj = &unstructured.Unstructured{} - path = filepath.Join(dir, f.Name()) - ) - if err = DecodeFile(path, obj); err != nil { - log.WithError(err).Debugf("could not decode file contents for %s", path) - continue - } + return nil +} - // Don't include other CSVs in the bundle - if obj.GetKind() == "ClusterServiceVersion" && obj.GetName() != csvName { - continue - } +type bundleVersion struct { + name string + version semver.Version - if obj.Object != nil { - bundle.Add(obj) - } - } - - return bundle, nil + // Keep track of the number of times we visit each version so we can tell if a head is contested + count int } -// findCSV looks through the bundle directory to find a csv -func (i *ImageInput) findCSV(manifests string) (*unstructured.Unstructured, error) { - log := logrus.WithFields(logrus.Fields{"dir": i.from, "find": "csv"}) +// compare returns a value less than one if the receiver arg is less smaller the given version, greater than one if it is larger, and zero if they are equal. +// This comparison follows typical semver precedence rules, with one addition: whenever two versions are equal with the exception of their build-ids, the build-ids are compared using prerelease precedence rules. Further, versions with no build-id are always less than versions with build-ids; e.g. 1.0.0 < 1.0.0+1. +func (b bundleVersion) compare(v bundleVersion) (int, error) { + return libsemver.BuildIdCompare(b.version, v.version) +} - files, err := ioutil.ReadDir(manifests) - if err != nil { - return nil, fmt.Errorf("unable to read directory %s: %s", manifests, err) - } +// SemverPackageManifest generates a PackageManifest from a set of bundles, determining channel heads and the default channel using semver. +// Bundles with the highest version field (according to semver) are chosen as channel heads, and the default channel is taken from the last, +// highest versioned bundle in the entire set to define it. +// The given bundles must all belong to the same package or an error is thrown. +func SemverPackageManifest(bundles []*Bundle) (*PackageManifest, error) { + heads := map[string]bundleVersion{} + + var ( + pkgName string + defaultChannel string + maxVersion bundleVersion + ) + + for _, bundle := range bundles { + if pkgName != "" && pkgName != bundle.Package { + return nil, fmt.Errorf("more than one package in input") + } + pkgName = bundle.Package - for _, f := range files { - log = log.WithField("file", f.Name()) - if f.IsDir() { - log.Info("skipping directory") - continue + rawVersion, err := bundle.Version() + if err != nil { + return nil, fmt.Errorf("error getting bundle %s version: %s", bundle.Name, err) + } + if rawVersion == "" { + // If a version isn't provided by the bundle, give it a dummy zero version + // The thought is that properly versioned bundles will always be non-zero + rawVersion = "0.0.0-z" } - if strings.HasPrefix(f.Name(), ".") { - log.Info("skipping hidden file") - continue + version, err := semver.Parse(rawVersion) + if err != nil { + return nil, fmt.Errorf("error parsing bundle %s version %s: %s", bundle.Name, rawVersion, err) + } + current := bundleVersion{ + name: bundle.Name, + version: version, + count: 1, } - var ( - obj = &unstructured.Unstructured{} - path = filepath.Join(manifests, f.Name()) - ) - if err = DecodeFile(path, obj); err != nil { - log.WithError(err).Debugf("could not decode file contents for %s", path) - continue + for _, channel := range bundle.Channels { + head, ok := heads[channel] + if !ok { + heads[channel] = current + continue + } + + if c, err := current.compare(head); err != nil { + return nil, err + } else if c < 0 { + continue + } else if c == 0 { + // We have a duplicate version, add the count + current.count += head.count + } + + // Current >= head + heads[channel] = current } - if obj.GetKind() != clusterServiceVersionKind { + // Set max if bundle is greater + if c, err := current.compare(maxVersion); err != nil { + return nil, err + } else if c < 0 { continue + } else if c == 0 { + current.count += maxVersion.count } - return obj, nil - } - - return nil, fmt.Errorf("no csv found in bundle") -} - -// loadOperatorBundle adds the package information to the loader's store -func (i *DirectoryPopulator) loadOperatorBundle(manifest PackageManifest, bundle *Bundle) error { - if manifest.PackageName == "" { - return nil + // Current >= maxVersion + maxVersion = current + if annotations := bundle.Annotations; annotations != nil && annotations.DefaultChannelName != "" { + // Take it when you can get it + defaultChannel = annotations.DefaultChannelName + } } - if err := i.loader.AddBundlePackageChannels(manifest, bundle); err != nil { - return fmt.Errorf("error loading bundle into db: %s", err) + if maxVersion.count > 1 { + return nil, fmt.Errorf("more than one bundle with maximum version %s", maxVersion.version) } - return nil -} - -// translateAnnotationsIntoPackage attempts to translate the channels.yaml file at the given path into a package.yaml -func translateAnnotationsIntoPackage(annotations *AnnotationsFile, csv *ClusterServiceVersion, existingPackageChannels map[string]string) (PackageManifest, error) { - manifest := PackageManifest{} - - for _, ch := range annotations.GetChannels() { - existingPackageChannels[ch] = csv.GetName() + pkg := &PackageManifest{ + PackageName: pkgName, + DefaultChannelName: defaultChannel, } - - channels := []PackageChannel{} - for c, current := range existingPackageChannels { - channels = append(channels, - PackageChannel{ - Name: c, - CurrentCSVName: current, - }) + defaultFound := len(heads) == 1 && defaultChannel == "" + for channel, head := range heads { + if head.count > 1 { + return nil, fmt.Errorf("more than one potential channel head for %s", channel) + } + if len(heads) == 1 { + // Only one possible default channel + pkg.DefaultChannelName = channel + } + defaultFound = defaultFound || channel == defaultChannel + pkg.Channels = append(pkg.Channels, PackageChannel{ + Name: channel, + CurrentCSVName: head.name, + }) } - manifest = PackageManifest{ - PackageName: annotations.GetName(), - DefaultChannelName: annotations.GetDefaultChannelName(), - Channels: channels, + if !defaultFound { + return nil, fmt.Errorf("unable to determine default channel among channel heads: %+v", heads) } - return manifest, nil + return pkg, nil } // DecodeFile decodes the file at a path into the given interface. diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/query.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/query.go new file mode 100644 index 0000000000..9acd9d48fa --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/query.go @@ -0,0 +1,322 @@ +package registry + +import ( + "context" + "fmt" + "sort" + + "github.com/operator-framework/operator-registry/internal/model" + "github.com/operator-framework/operator-registry/pkg/api" +) + +type Querier struct { + pkgs model.Model +} + +var _ GRPCQuery = &Querier{} + +func NewQuerier(packages model.Model) *Querier { + return &Querier{ + pkgs: packages, + } +} + +func (q Querier) ListPackages(_ context.Context) ([]string, error) { + var packages []string + for pkgName := range q.pkgs { + packages = append(packages, pkgName) + } + return packages, nil +} + +func (q Querier) ListBundles(_ context.Context) ([]*api.Bundle, error) { + var bundles []*api.Bundle + + for _, pkg := range q.pkgs { + for _, ch := range pkg.Channels { + for _, b := range ch.Bundles { + apiBundle, err := api.ConvertModelBundleToAPIBundle(*b) + if err != nil { + return nil, fmt.Errorf("convert bundle %q: %v", b.Name, err) + } + bundles = append(bundles, apiBundle) + } + } + } + return bundles, nil +} + +func (q Querier) GetPackage(_ context.Context, name string) (*PackageManifest, error) { + pkg, ok := q.pkgs[name] + if !ok { + return nil, fmt.Errorf("package %q not found", name) + } + + var channels []PackageChannel + for _, ch := range pkg.Channels { + head, err := ch.Head() + if err != nil { + return nil, fmt.Errorf("package %q, channel %q has invalid head: %v", name, ch.Name, err) + } + channels = append(channels, PackageChannel{ + Name: ch.Name, + CurrentCSVName: head.Name, + }) + } + return &PackageManifest{ + PackageName: pkg.Name, + Channels: channels, + DefaultChannelName: pkg.DefaultChannel.Name, + }, nil +} + +func (q Querier) GetBundle(_ context.Context, pkgName, channelName, csvName string) (*api.Bundle, error) { + pkg, ok := q.pkgs[pkgName] + if !ok { + return nil, fmt.Errorf("package %q not found", pkgName) + } + ch, ok := pkg.Channels[channelName] + if !ok { + return nil, fmt.Errorf("package %q, channel %q not found", pkgName, channelName) + } + b, ok := ch.Bundles[csvName] + if !ok { + return nil, fmt.Errorf("package %q, channel %q, bundle %q not found", pkgName, channelName, csvName) + } + apiBundle, err := api.ConvertModelBundleToAPIBundle(*b) + if err != nil { + return nil, fmt.Errorf("convert bundle %q: %v", b.Name, err) + } + + // unset Replaces and Skips (sqlite query does not populate these fields) + apiBundle.Replaces = "" + apiBundle.Skips = nil + return apiBundle, nil +} + +func (q Querier) GetBundleForChannel(_ context.Context, pkgName string, channelName string) (*api.Bundle, error) { + pkg, ok := q.pkgs[pkgName] + if !ok { + return nil, fmt.Errorf("package %q not found", pkgName) + } + ch, ok := pkg.Channels[channelName] + if !ok { + return nil, fmt.Errorf("package %q, channel %q not found", pkgName, channelName) + } + head, err := ch.Head() + if err != nil { + return nil, fmt.Errorf("package %q, channel %q has invalid head: %v", pkgName, channelName, err) + } + apiBundle, err := api.ConvertModelBundleToAPIBundle(*head) + if err != nil { + return nil, fmt.Errorf("convert bundle %q: %v", head.Name, err) + } + + // unset Replaces and Skips (sqlite query does not populate these fields) + apiBundle.Replaces = "" + apiBundle.Skips = nil + return apiBundle, nil +} + +func (q Querier) GetChannelEntriesThatReplace(_ context.Context, name string) ([]*ChannelEntry, error) { + var entries []*ChannelEntry + + for _, pkg := range q.pkgs { + for _, ch := range pkg.Channels { + for _, b := range ch.Bundles { + entries = append(entries, channelEntriesThatReplace(*b, name)...) + } + } + } + if len(entries) == 0 { + return nil, fmt.Errorf("no channel entries found that replace %s", name) + } + return entries, nil +} + +func (q Querier) GetBundleThatReplaces(_ context.Context, name, pkgName, channelName string) (*api.Bundle, error) { + pkg, ok := q.pkgs[pkgName] + if !ok { + return nil, fmt.Errorf("package %s not found", pkgName) + } + ch, ok := pkg.Channels[channelName] + if !ok { + return nil, fmt.Errorf("package %q, channel %q not found", pkgName, channelName) + } + + // NOTE: iterating over a map is non-deterministic in Go, so if multiple bundles replace this one, + // the bundle returned by this function is also non-deterministic. The sqlite implementation + // is ALSO non-deterministic because it doesn't use ORDER BY, so its probably okay for this + // implementation to be non-deterministic as well. + for _, b := range ch.Bundles { + if bundleReplaces(*b, name) { + apiBundle, err := api.ConvertModelBundleToAPIBundle(*b) + if err != nil { + return nil, fmt.Errorf("convert bundle %q: %v", b.Name, err) + } + + // unset Replaces and Skips (sqlite query does not populate these fields) + apiBundle.Replaces = "" + apiBundle.Skips = nil + return apiBundle, nil + } + } + return nil, fmt.Errorf("no entry found for package %q, channel %q", pkgName, channelName) +} + +func (q Querier) GetChannelEntriesThatProvide(_ context.Context, group, version, kind string) ([]*ChannelEntry, error) { + var entries []*ChannelEntry + + for _, pkg := range q.pkgs { + for _, ch := range pkg.Channels { + for _, b := range ch.Bundles { + provides, err := doesModelBundleProvide(*b, group, version, kind) + if err != nil { + return nil, err + } + if provides { + // TODO(joelanford): It seems like the SQLite query returns + // invalid entries (i.e. where bundle `Replaces` isn't actually + // in channel `ChannelName`). Is that a bug? For now, this mimics + // the sqlite server and returns seemingly invalid channel entries. + // Don't worry about this. Not used anymore. + + entries = append(entries, channelEntriesForBundle(*b, true)...) + } + } + } + } + if len(entries) == 0 { + return nil, fmt.Errorf("no channel entries found that provide group:%q version:%q kind:%q", group, version, kind) + } + return entries, nil +} + +// TODO(joelanford): Need to review the expected functionality of this function. I ran +// some experiments with the sqlite version of this function and it seems to only return +// channel heads that provide the GVK (rather than searching down the graph if parent bundles +// don't provide the API). Based on that, this function currently looks at channel heads only. +// --- +// Separate, but possibly related, I noticed there are several channels in the channel entry +// table who's minimum depth is 1. What causes 1 to be minimum depth in some cases and 0 in others? +func (q Querier) GetLatestChannelEntriesThatProvide(_ context.Context, group, version, kind string) ([]*ChannelEntry, error) { + var entries []*ChannelEntry + + for _, pkg := range q.pkgs { + for _, ch := range pkg.Channels { + b, err := ch.Head() + if err != nil { + return nil, fmt.Errorf("package %q, channel %q has invalid head: %v", pkg.Name, ch.Name, err) + } + + provides, err := doesModelBundleProvide(*b, group, version, kind) + if err != nil { + return nil, err + } + if provides { + entries = append(entries, channelEntriesForBundle(*b, false)...) + } + } + } + if len(entries) == 0 { + return nil, fmt.Errorf("no channel entries found that provide group:%q version:%q kind:%q", group, version, kind) + } + return entries, nil +} + +func (q Querier) GetBundleThatProvides(ctx context.Context, group, version, kind string) (*api.Bundle, error) { + latestEntries, err := q.GetLatestChannelEntriesThatProvide(ctx, group, version, kind) + if err != nil { + return nil, err + } + + // It's possible for multiple packages to provide an API, but this function is forced to choose one. + // To do that deterministically, we'll pick the the bundle based on a lexicographical sort of its + // package name. + sort.Slice(latestEntries, func(i, j int) bool { + return latestEntries[i].PackageName < latestEntries[j].PackageName + }) + + for _, entry := range latestEntries { + pkg, ok := q.pkgs[entry.PackageName] + if !ok { + // This should never happen because the latest entries were + // collected based on iterating over the packages in q.pkgs. + continue + } + if entry.ChannelName == pkg.DefaultChannel.Name { + return q.GetBundle(ctx, entry.PackageName, entry.ChannelName, entry.BundleName) + } + } + return nil, fmt.Errorf("no entry found that provides group:%q version:%q kind:%q", group, version, kind) +} + +func doesModelBundleProvide(b model.Bundle, group, version, kind string) (bool, error) { + apiBundle, err := api.ConvertModelBundleToAPIBundle(b) + if err != nil { + return false, fmt.Errorf("convert bundle %q: %v", b.Name, err) + } + for _, gvk := range apiBundle.ProvidedApis { + if gvk.Group == group && gvk.Version == version && gvk.Kind == kind { + return true, nil + } + } + return false, nil +} + +func bundleReplaces(b model.Bundle, name string) bool { + if b.Replaces == name { + return true + } + for _, s := range b.Skips { + if s == name { + return true + } + } + return false +} + +func channelEntriesThatReplace(b model.Bundle, name string) []*ChannelEntry { + var entries []*ChannelEntry + if b.Replaces == name { + entries = append(entries, &ChannelEntry{ + PackageName: b.Package.Name, + ChannelName: b.Channel.Name, + BundleName: b.Name, + Replaces: b.Replaces, + }) + } + for _, s := range b.Skips { + if s == name && s != b.Replaces { + entries = append(entries, &ChannelEntry{ + PackageName: b.Package.Name, + ChannelName: b.Channel.Name, + BundleName: b.Name, + Replaces: b.Replaces, + }) + } + } + return entries +} + +func channelEntriesForBundle(b model.Bundle, ignoreChannel bool) []*ChannelEntry { + entries := []*ChannelEntry{{ + PackageName: b.Package.Name, + ChannelName: b.Channel.Name, + BundleName: b.Name, + Replaces: b.Replaces, + }} + for _, s := range b.Skips { + // Ignore skips that duplicate b.Replaces. Also, only add it if its + // in the same channel as b (or we're ignoring channel presence). + if _, inChannel := b.Channel.Bundles[s]; s != b.Replaces && (ignoreChannel || inChannel) { + entries = append(entries, &ChannelEntry{ + PackageName: b.Package.Name, + ChannelName: b.Channel.Name, + BundleName: b.Name, + Replaces: s, + }) + } + } + return entries +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/registry_to_model.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/registry_to_model.go new file mode 100644 index 0000000000..8d921a29e3 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/registry_to_model.go @@ -0,0 +1,259 @@ +package registry + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/operator-framework/operator-registry/internal/model" + "github.com/operator-framework/operator-registry/internal/property" +) + +func ConvertRegistryBundleToModelBundles(b *Bundle) ([]model.Bundle, error) { + var bundles []model.Bundle + desc, err := b.csv.GetDescription() + if err != nil { + return nil, fmt.Errorf("Could not get description from bundle CSV:%s", err) + } + + i, err := b.csv.GetIcons() + if err != nil { + return nil, fmt.Errorf("Could not get icon from bundle CSV:%s", err) + } + mIcon := &model.Icon{ + MediaType: "", + Data: []byte{}, + } + if len(i) > 0 { + mIcon.MediaType = i[0].MediaType + mIcon.Data = []byte(i[0].Base64data) + } + + pkg := &model.Package{ + Name: b.Annotations.PackageName, + Description: desc, + Icon: mIcon, + Channels: make(map[string]*model.Channel), + } + + mb, err := registryBundleToModelBundle(b) + mb.Package = pkg + if err != nil { + return nil, err + } + + for _, ch := range extractChannels(b.Annotations.Channels) { + newCh := &model.Channel{ + Name: ch, + } + chBundle := mb + chBundle.Channel = newCh + bundles = append(bundles, *chBundle) + } + return bundles, nil +} + +func registryBundleToModelBundle(b *Bundle) (*model.Bundle, error) { + bundleProps, err := PropertiesFromBundle(b) + if err != nil { + return nil, fmt.Errorf("error converting properties for internal model: %v", err) + } + + csv, err := b.ClusterServiceVersion() + if err != nil { + return nil, fmt.Errorf("Could not get CVS for bundle: %s", err) + } + replaces, err := csv.GetReplaces() + if err != nil { + return nil, fmt.Errorf("Could not get Replaces from CSV for bundle: %s", err) + } + skips, err := csv.GetSkips() + if err != nil { + return nil, fmt.Errorf("Could not get Skips from CSV for bundle: %s", err) + } + relatedImages, err := convertToModelRelatedImages(csv) + if err != nil { + return nil, fmt.Errorf("Could not get Related images from bundle: %v", err) + } + + return &model.Bundle{ + Name: csv.Name, + Image: b.BundleImage, + Replaces: replaces, + Skips: skips, + Properties: bundleProps, + RelatedImages: relatedImages, + }, nil +} + +func PropertiesFromBundle(b *Bundle) ([]property.Property, error) { + csv, err := b.ClusterServiceVersion() + if err != nil { + return nil, fmt.Errorf("get csv: %v", err) + } + + skips, err := csv.GetSkips() + if err != nil { + return nil, fmt.Errorf("get csv skips: %v", err) + } + + var graphProps []property.Property + replaces, err := csv.GetReplaces() + if err != nil { + return nil, fmt.Errorf("get csv replaces: %v", err) + } + for _, ch := range b.Channels { + graphProps = append(graphProps, property.MustBuildChannel(ch, replaces)) + } + + for _, skip := range skips { + graphProps = append(graphProps, property.MustBuildSkips(skip)) + } + + skipRange := csv.GetSkipRange() + if skipRange != "" { + graphProps = append(graphProps, property.MustBuildSkipRange(skipRange)) + } + + providedGVKs := map[property.GVK]struct{}{} + requiredGVKs := map[property.GVKRequired]struct{}{} + + var packageProvidedProperty *property.Property + var otherProps []property.Property + + for i, p := range b.Properties { + switch p.Type { + case property.TypeGVK: + var v property.GVK + if err := json.Unmarshal(p.Value, &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + k := property.GVK{Group: v.Group, Kind: v.Kind, Version: v.Version} + providedGVKs[k] = struct{}{} + case property.TypePackage: + var v property.Package + if err := json.Unmarshal(p.Value, &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + p := property.MustBuildPackage(v.PackageName, v.Version) + packageProvidedProperty = &p + default: + otherProps = append(otherProps, property.Property{ + Type: p.Type, + Value: p.Value, + }) + } + } + + var packageRequiredProps []property.Property + for i, p := range b.Dependencies { + switch p.Type { + case property.TypeGVK: + var v property.GVK + if err := json.Unmarshal(p.Value, &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + k := property.GVKRequired{Group: v.Group, Kind: v.Kind, Version: v.Version} + requiredGVKs[k] = struct{}{} + case property.TypePackage: + var v property.Package + if err := json.Unmarshal(p.Value, &v); err != nil { + return nil, property.ParseError{Idx: i, Typ: p.Type, Err: err} + } + packageRequiredProps = append(packageRequiredProps, property.MustBuildPackageRequired(v.PackageName, v.Version)) + } + } + + version, err := b.Version() + if err != nil { + return nil, fmt.Errorf("get version: %v", err) + } + + providedApis, err := b.ProvidedAPIs() + if err != nil { + return nil, fmt.Errorf("get provided apis: %v", err) + } + + for p := range providedApis { + k := property.GVK{Group: p.Group, Kind: p.Kind, Version: p.Version} + if _, ok := providedGVKs[k]; !ok { + providedGVKs[k] = struct{}{} + } + } + requiredApis, err := b.RequiredAPIs() + if err != nil { + return nil, fmt.Errorf("get required apis: %v", err) + } + for p := range requiredApis { + k := property.GVKRequired{Group: p.Group, Kind: p.Kind, Version: p.Version} + if _, ok := requiredGVKs[k]; !ok { + requiredGVKs[k] = struct{}{} + } + } + + var out []property.Property + if packageProvidedProperty == nil { + p := property.MustBuildPackage(b.Package, version) + packageProvidedProperty = &p + } + out = append(out, *packageProvidedProperty) + out = append(out, graphProps...) + + for p := range providedGVKs { + out = append(out, property.MustBuildGVK(p.Group, p.Version, p.Kind)) + } + + for p := range requiredGVKs { + out = append(out, property.MustBuildGVKRequired(p.Group, p.Version, p.Kind)) + } + + out = append(out, packageRequiredProps...) + out = append(out, otherProps...) + + sort.Slice(out, func(i, j int) bool { + if out[i].Type != out[j].Type { + return out[i].Type < out[j].Type + } + return string(out[i].Value) < string(out[j].Value) + }) + + return out, nil +} + +func convertToModelRelatedImages(csv *ClusterServiceVersion) ([]model.RelatedImage, error) { + var objmap map[string]*json.RawMessage + if err := json.Unmarshal(csv.Spec, &objmap); err != nil { + return nil, err + } + + rawValue, ok := objmap[relatedImages] + if !ok || rawValue == nil { + return nil, nil + } + + type relatedImage struct { + Name string `json:"name"` + Ref string `json:"image"` + } + var relatedImages []relatedImage + if err := json.Unmarshal(*rawValue, &relatedImages); err != nil { + return nil, err + } + mrelatedImages := []model.RelatedImage{} + for _, img := range relatedImages { + mrelatedImages = append(mrelatedImages, model.RelatedImage{Name: img.Name, Image: img.Ref}) + } + return mrelatedImages, nil +} + +func extractChannels(annotationChannels string) []string { + var channels []string + for _, ch := range strings.Split(annotationChannels, ",") { + c := strings.TrimSpace(ch) + if c != "" { + channels = append(channels, ch) + } + } + return channels +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/replacesgraphloader.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/replacesgraphloader.go deleted file mode 100644 index a57df6e13d..0000000000 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/replacesgraphloader.go +++ /dev/null @@ -1,40 +0,0 @@ -package registry - -import ( - "fmt" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" -) - -type ReplacesGraphLoader struct { -} - -// CanAdd checks that a new bundle can be added in replaces mode (i.e. the replaces -// defined for the bundle already exists) -func (r *ReplacesGraphLoader) CanAdd(bundle *Bundle, graph *Package) (bool, error) { - replaces, err := bundle.Replaces() - if err != nil { - return false, fmt.Errorf("Invalid content, unable to parse bundle") - } - - csvName := bundle.Name - - // adding the first bundle in the graph - if replaces == "" { - return true, nil - } - - var errs []error - - // check that the bundle can be added - if !graph.HasCsv(replaces) { - err := fmt.Errorf("Invalid bundle %s, bundle specifies a non-existent replacement %s", csvName, replaces) - errs = append(errs, err) - } - - if len(errs) > 0 { - return false, utilerrors.NewAggregate(errs) - } - - return true, nil -} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go b/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go index f5d12e02db..719127cfba 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/registry/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "github.com/blang/semver" @@ -39,6 +40,15 @@ func (e PackageVersionAlreadyAddedErr) Error() string { return e.ErrorString } +// OverwritesErr is an error that describes that an error with the add request with --force enabled. +type OverwriteErr struct { + ErrorString string +} + +func (e OverwriteErr) Error() string { + return e.ErrorString +} + const ( GVKType = "olm.gvk" PackageType = "olm.package" @@ -150,13 +160,13 @@ type Annotations struct { DefaultChannelName string `json:"operators.operatorframework.io.bundle.channel.default.v1" yaml:"operators.operatorframework.io.bundle.channel.default.v1"` } -// DependenciesFile holds dependency information about a bundle +// DependenciesFile holds dependency information about a bundle. type DependenciesFile struct { // Dependencies is a list of dependencies for a given bundle Dependencies []Dependency `json:"dependencies" yaml:"dependencies"` } -// Dependency specifies a single constraint that can be satisfied by a property on another bundle.. +// Dependency specifies a single constraint that can be satisfied by a property on another bundle. type Dependency struct { // The type of dependency. This field is required. Type string `json:"type" yaml:"type"` @@ -165,6 +175,12 @@ type Dependency struct { Value json.RawMessage `json:"value" yaml:"value"` } +// PropertiesFile holds the properties associated with a bundle. +type PropertiesFile struct { + // Properties is a list of properties. + Properties []Property `json:"properties" yaml:"properties"` +} + // Property defines a single piece of the public interface for a bundle. Dependencies are specified over properties. // The Type of the property determines how to interpret the Value, but the value is treated opaquely for // for non-first-party types. @@ -176,6 +192,10 @@ type Property struct { Value json.RawMessage `json:"value" yaml:"value"` } +func (p Property) String() string { + return fmt.Sprintf("type: %s, value: %s", p.Type, p.Value) +} + type GVKDependency struct { // The group of GVK based dependency Group string `json:"group" yaml:"group"` @@ -196,7 +216,7 @@ type PackageDependency struct { } type LabelDependency struct { - // The version range of dependency in semver range format + // The Label name of dependency Label string `json:"label" yaml:"label"` } @@ -224,7 +244,7 @@ type DeprecatedProperty struct { } type LabelProperty struct { - // The version range of dependency in semver range format + // The name of Label Label string `json:"label" yaml:"label"` } @@ -243,7 +263,7 @@ func (gd *GVKDependency) Validate() []error { return errs } -// Validate will validate GVK dependency type and return error(s) +// Validate will validate Label dependency type and return error(s) func (ld *LabelDependency) Validate() []error { errs := []error{} if *ld == (LabelDependency{}) { @@ -333,12 +353,22 @@ func (a *AnnotationsFile) GetChannels() []string { // GetDefaultChannelName returns the name of the default channel func (a *AnnotationsFile) GetDefaultChannelName() string { - if a.Annotations.DefaultChannelName != "" { - return a.Annotations.DefaultChannelName - } - channels := a.GetChannels() - if len(channels) == 1 { - return channels[0] + return a.Annotations.DefaultChannelName +} + +// SelectDefaultChannel returns the first item in channel list that is sorted +// in lexicographic order. +func (a *AnnotationsFile) SelectDefaultChannel() string { + return a.Annotations.SelectDefaultChannel() +} + +func (a Annotations) SelectDefaultChannel() string { + if len(a.Channels) < 1 { + return "" } - return "" + + channels := strings.Split(a.Channels, ",") + sort.Strings(channels) + + return channels[0] } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/server/server.go b/vendor/github.com/operator-framework/operator-registry/pkg/server/server.go index 190347862b..9fe1592c3e 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/server/server.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/server/server.go @@ -1,21 +1,21 @@ package server import ( + "golang.org/x/net/context" + "github.com/operator-framework/operator-registry/pkg/api" "github.com/operator-framework/operator-registry/pkg/registry" - - "golang.org/x/net/context" ) type RegistryServer struct { api.UnimplementedRegistryServer - store registry.Query + store registry.GRPCQuery } var _ api.RegistryServer = &RegistryServer{} -func NewRegistryServer(store registry.Query) *RegistryServer { - return &RegistryServer{UnimplementedRegistryServer: api.UnimplementedRegistryServer{}, store: store} +func NewRegistryServer(store registry.GRPCQuery) *RegistryServer { + return &RegistryServer{UnimplementedRegistryServer: api.UnimplementedRegistryServer{}, store: store} } func (s *RegistryServer) ListPackages(req *api.ListPackageRequest, stream api.Registry_ListPackagesServer) error { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/configmap.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/configmap.go index 687902b98b..0f727e1d45 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/configmap.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/configmap.go @@ -7,7 +7,7 @@ import ( "github.com/ghodss/yaml" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -127,7 +127,7 @@ func (c *ConfigMapLoader) Populate() error { continue } - bundle := registry.NewBundle(csv.GetName(), "", nil, &unstructured.Unstructured{Object: csvUnst}) + bundle := registry.NewBundle(csv.GetName(), ®istry.Annotations{}, &unstructured.Unstructured{Object: csvUnst}) ownedCRDs, _, err := csv.GetCustomResourceDefintions() if err != nil { errs = append(errs, err) diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/conversion.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/conversion.go new file mode 100644 index 0000000000..7c4fa49095 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/conversion.go @@ -0,0 +1,133 @@ +package sqlite + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/operator-framework/api/pkg/operators/v1alpha1" + "github.com/sirupsen/logrus" + + "github.com/operator-framework/operator-registry/internal/model" + "github.com/operator-framework/operator-registry/pkg/api" + "github.com/operator-framework/operator-registry/pkg/registry" +) + +func ToModel(ctx context.Context, q *SQLQuerier) (model.Model, error) { + pkgs, err := initializeModelPackages(ctx, q) + if err != nil { + return nil, err + } + if err := populateModelChannels(ctx, pkgs, q); err != nil { + return nil, fmt.Errorf("populate channels: %v", err) + } + if err := populatePackageIcons(ctx, pkgs, q); err != nil { + return nil, fmt.Errorf("populate package icons: %v", err) + } + if err := pkgs.Validate(); err != nil { + return nil, err + } + pkgs.Normalize() + return pkgs, nil +} + +func initializeModelPackages(ctx context.Context, q *SQLQuerier) (model.Model, error) { + pkgNames, err := q.ListPackages(ctx) + if err != nil { + return nil, err + } + + var rPkgs []registry.PackageManifest + for _, pkgName := range pkgNames { + rPkg, err := q.GetPackage(ctx, pkgName) + if err != nil { + return nil, err + } + rPkgs = append(rPkgs, *rPkg) + } + + pkgs := model.Model{} + for _, rPkg := range rPkgs { + pkg := model.Package{ + Name: rPkg.PackageName, + } + + pkg.Channels = map[string]*model.Channel{} + for _, ch := range rPkg.Channels { + channel := &model.Channel{ + Package: &pkg, + Name: ch.Name, + Bundles: map[string]*model.Bundle{}, + } + if ch.Name == rPkg.DefaultChannelName { + pkg.DefaultChannel = channel + } + pkg.Channels[ch.Name] = channel + } + pkgs[pkg.Name] = &pkg + } + return pkgs, nil +} + +func populateModelChannels(ctx context.Context, pkgs model.Model, q *SQLQuerier) error { + bundles, err := q.ListBundles(ctx) + if err != nil { + return err + } + for _, bundle := range bundles { + pkg, ok := pkgs[bundle.PackageName] + if !ok { + return fmt.Errorf("unknown package %q for bundle %q", bundle.PackageName, bundle.CsvName) + } + + pkgChannel, ok := pkg.Channels[bundle.ChannelName] + if !ok { + return fmt.Errorf("unknown channel %q for bundle %q", bundle.ChannelName, bundle.CsvName) + } + + mbundle, err := api.ConvertAPIBundleToModelBundle(bundle) + if err != nil { + return fmt.Errorf("convert bundle %q: %v", bundle.CsvName, err) + } + mbundle.Package = pkg + mbundle.Channel = pkgChannel + pkgChannel.Bundles[bundle.CsvName] = mbundle + } + return nil +} + +// populatePackageIcons populates the package icons from the icon of bundle of the head +// of the default channel of each of the pacakges in pkgs. +func populatePackageIcons(ctx context.Context, pkgs model.Model, q *SQLQuerier) error { + for _, pkg := range pkgs { + head, err := q.GetBundleForChannel(ctx, pkg.Name, pkg.DefaultChannel.Name) + if err != nil { + return fmt.Errorf("get default channel head for package %q: %v", pkg.Name, err) + } + var csv v1alpha1.ClusterServiceVersion + if err := json.Unmarshal([]byte(head.CsvJson), &csv); err != nil { + return fmt.Errorf("unmarshal CSV json for bundle %q: %v", head.CsvName, err) + } + if len(csv.Spec.Icon) == 0 { + continue + } + iconData, origErr := base64.StdEncoding.DecodeString(csv.Spec.Icon[0].Data) + if origErr != nil { + // Try decoding after removing spaces (this is a problem with the planetscale operator). + iconData, err = base64.StdEncoding.DecodeString(strings.ReplaceAll(csv.Spec.Icon[0].Data, " ", "")) + if err != nil { + logrus.WithError(err).Warnf("base64 decode CSV icon for bundle %q", head.CsvName) + continue + } + } + if len(iconData) > 0 { + pkg.Icon = &model.Icon{ + Data: iconData, + MediaType: csv.Spec.Icon[0].MediaType, + } + } + } + return nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db.go new file mode 100644 index 0000000000..6426e69445 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db.go @@ -0,0 +1,29 @@ +package sqlite + +import ( + "database/sql" + + _ "github.com/mattn/go-sqlite3" +) + +// Open opens a connection to a sqlite db. It should be used everywhere instead of sql.Open so that foreign keys are +// ensured. +func Open(fileName string) (*sql.DB, error) { + return sql.Open("sqlite3", EnableForeignKeys(fileName)) +} + +// Open opens a connection to a sqlite db. It is +func OpenReadOnly(fileName string) (*sql.DB, error) { + return sql.Open("sqlite3", EnableImmutable(fileName)) +} + +// EnableForeignKeys appends the option to enable foreign keys on connections +// note that without this option, PRAGMAs about foreign keys will lie. +func EnableForeignKeys(fileName string) string { + return "file:" + fileName + "?_foreign_keys=on" +} + +// Immutable appends the option to mark the db immutable on connections +func EnableImmutable(fileName string) string { + return "file:" + fileName + "?immutable=true" +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db_options.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db_options.go index 889f550e0d..e09bfbc036 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db_options.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/db_options.go @@ -7,6 +7,7 @@ import ( type DbOptions struct { // MigratorBuilder is a function that returns a migrator instance MigratorBuilder func(*sql.DB) (Migrator, error) + EnableAlpha bool } type DbOption func(*DbOptions) @@ -14,6 +15,7 @@ type DbOption func(*DbOptions) func defaultDBOptions() *DbOptions { return &DbOptions{ MigratorBuilder: NewSQLLiteMigrator, + EnableAlpha: false, } } @@ -22,3 +24,9 @@ func WithMigratorBuilder(m func(loader *sql.DB) (Migrator, error)) DbOption { o.MigratorBuilder = m } } + +func WithEnableAlpha(enableAlpha bool) DbOption { + return func(o *DbOptions) { + o.EnableAlpha = enableAlpha + } +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/deprecate.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/deprecate.go index 5d2a0897f5..a8cdb24fa9 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/deprecate.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/deprecate.go @@ -31,17 +31,15 @@ func NewSQLDeprecatorForBundles(store registry.Load, bundles []string) *BundleDe func (d *BundleDeprecator) Deprecate() error { log := logrus.WithField("bundles", d.bundles) - log.Info("deprecating bundles") var errs []error - for _, bundlePath := range d.bundles { - if err := d.store.DeprecateBundle(bundlePath); err != nil { - if !errors.Is(err, registry.ErrBundleImageNotInDatabase) && !errors.Is(err, registry.ErrRemovingDefaultChannelDuringDeprecation) { - return utilerrors.NewAggregate(append(errs, fmt.Errorf("error deprecating bundle %s: %s", bundlePath, err))) - } + if err := d.store.DeprecateBundle(bundlePath); err != nil && !errors.Is(err, registry.ErrBundleImageNotInDatabase) { errs = append(errs, fmt.Errorf("error deprecating bundle %s: %s", bundlePath, err)) + if !errors.Is(err, registry.ErrRemovingDefaultChannelDuringDeprecation) { + break + } } } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go index c79f66f939..f4b2e3e6ce 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/directory.go @@ -92,6 +92,7 @@ func (d *DirectoryLoader) LoadBundleWalkFunc(path string, f os.FileInfo, err err if err != nil { return fmt.Errorf("unable to load file %s: %s", path, err) } + defer fileReader.Close() decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) csv := unstructured.Unstructured{} @@ -155,6 +156,7 @@ func (d *DirectoryLoader) LoadPackagesWalkFunc(path string, f os.FileInfo, err e if err != nil { return fmt.Errorf("unable to load package from file %s: %s", path, err) } + defer fileReader.Close() decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) manifest := registry.PackageManifest{} @@ -207,6 +209,7 @@ func loadBundle(csvName string, dir string) (*registry.Bundle, error) { errs = append(errs, fmt.Errorf("unable to load file %s: %s", path, err)) continue } + defer fileReader.Close() decoder := yaml.NewYAMLOrJSONDecoder(fileReader, 30) obj := &unstructured.Unstructured{} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/graphloader.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/graphloader.go index e5795550f6..e89bd75cd2 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/graphloader.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/graphloader.go @@ -111,6 +111,12 @@ func graphFromEntries(channelEntries []registry.ChannelEntryAnnotated, existingB } if !replacesKey.IsEmpty() { + if _, ok := channelGraph[entry.ChannelName]; !ok { + channelGraph[entry.ChannelName] = replaces{key: {replacesKey: struct{}{}}} + } + if _, ok := channelGraph[entry.ChannelName][key]; !ok { + channelGraph[entry.ChannelName][key] = map[registry.BundleKey]struct{}{replacesKey: {}} + } channelGraph[entry.ChannelName][key][replacesKey] = struct{}{} } diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go index 708f7d205e..cfb3b7015a 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/load.go @@ -7,15 +7,18 @@ import ( "fmt" "strings" + "github.com/blang/semver" _ "github.com/mattn/go-sqlite3" utilerrors "k8s.io/apimachinery/pkg/util/errors" + libsemver "github.com/operator-framework/operator-registry/pkg/lib/semver" "github.com/operator-framework/operator-registry/pkg/registry" ) type sqlLoader struct { - db *sql.DB - migrator Migrator + db *sql.DB + migrator Migrator + enableAlpha bool } type MigratableLoader interface { @@ -31,7 +34,7 @@ func NewSQLLiteLoader(db *sql.DB, opts ...DbOption) (MigratableLoader, error) { o(options) } - if _, err := db.Exec("PRAGMA foreign_keys = ON", nil); err != nil { + if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil { return nil, err } @@ -40,7 +43,7 @@ func NewSQLLiteLoader(db *sql.DB, opts ...DbOption) (MigratableLoader, error) { return nil, err } - return &sqlLoader{db: db, migrator: migrator}, nil + return &sqlLoader{db: db, migrator: migrator, enableAlpha: options.EnableAlpha}, nil } func (s *sqlLoader) Migrate(ctx context.Context) error { @@ -67,7 +70,7 @@ func (s *sqlLoader) AddOperatorBundle(bundle *registry.Bundle) error { } func (s *sqlLoader) addOperatorBundle(tx *sql.Tx, bundle *registry.Bundle) error { - addBundle, err := tx.Prepare("insert into operatorbundle(name, csv, bundle, bundlepath, version, skiprange, replaces, skips) values(?, ?, ?, ?, ?, ?, ?, ?)") + addBundle, err := tx.Prepare("insert into operatorbundle(name, csv, bundle, bundlepath, version, skiprange, replaces, skips, substitutesfor) values(?, ?, ?, ?, ?, ?, ?, ?, ?)") if err != nil { return err } @@ -79,7 +82,7 @@ func (s *sqlLoader) addOperatorBundle(tx *sql.Tx, bundle *registry.Bundle) error } defer addImage.Close() - csvName, bundleImage, csvBytes, bundleBytes, err := bundle.Serialize() + csvName, bundleImage, csvBytes, bundleBytes, _, err := bundle.Serialize() if err != nil { return err } @@ -104,18 +107,26 @@ func (s *sqlLoader) addOperatorBundle(tx *sql.Tx, bundle *registry.Bundle) error if err != nil { return err } - - if _, err := addBundle.Exec(csvName, csvBytes, bundleBytes, bundleImage, version, skiprange, replaces, strings.Join(skips, ",")); err != nil { + substitutesFor, err := bundle.SubstitutesFor() + if err != nil { return err } + if substitutesFor != "" && !s.enableAlpha { + return fmt.Errorf("SubstitutesFor is an alpha-only feature. You must enable alpha features with the flag --enable-alpha in order to use this feature.") + } + + if _, err := addBundle.Exec(csvName, csvBytes, bundleBytes, bundleImage, version, skiprange, replaces, strings.Join(skips, ","), substitutesFor); err != nil { + return fmt.Errorf("failed to add bundle %q: %s", csvName, err.Error()) + } + imgs, err := bundle.Images() if err != nil { return err } for img := range imgs { if _, err := addImage.Exec(img, csvName); err != nil { - return err + return fmt.Errorf("failed to add related images %q for bundle %q: %s", img, csvName, err.Error()) } } @@ -130,9 +141,256 @@ func (s *sqlLoader) addOperatorBundle(tx *sql.Tx, bundle *registry.Bundle) error return err } + if s.enableAlpha { + err = s.addSubstitutesFor(tx, bundle) + if err != nil { + return err + } + } + return s.addAPIs(tx, bundle) } +func (s *sqlLoader) addSubstitutesFor(tx *sql.Tx, bundle *registry.Bundle) error { + + updateBundleReplaces, err := tx.Prepare("update operatorbundle set replaces = ? where replaces = ?") + if err != nil { + return err + } + defer updateBundleReplaces.Close() + + updateBundleSkips, err := tx.Prepare("update operatorbundle set skips = ? where name = ?") + if err != nil { + return err + } + defer updateBundleSkips.Close() + + updateBundleSubstitutesFor, err := tx.Prepare("update operatorbundle set substitutesfor = ? where name = ?") + if err != nil { + return err + } + defer updateBundleSubstitutesFor.Close() + + updateBundleReplacesSkips, err := tx.Prepare("update operatorbundle set replaces = ?, skips = ? where name = ?") + if err != nil { + return err + } + defer updateBundleReplacesSkips.Close() + + csvName := bundle.Name + + replaces, err := bundle.Replaces() + if err != nil { + return err + } + skips, err := bundle.Skips() + if err != nil { + return err + } + version, err := bundle.Version() + if err != nil { + return err + } + substitutesFor, err := bundle.SubstitutesFor() + if err != nil { + return err + } + if substitutesFor != "" { + // Update any replaces that reference the substituted-for bundle + _, err = updateBundleReplaces.Exec(csvName, substitutesFor) + if err != nil { + return err + } + // Check if any other bundle substitutes for the same bundle + otherSubstitutions, err := s.getBundlesThatSubstitutesFor(tx, substitutesFor) + if err != nil { + return err + } + for len(otherSubstitutions) > 0 { + // consume the slice of substitutions + otherSubstitution := otherSubstitutions[0] + otherSubstitutions = otherSubstitutions[1:] + if otherSubstitution != csvName { + // Another bundle is substituting for that same bundle + // Get other bundle's version + _, _, rawVersion, err := s.getBundleSkipsReplacesVersion(tx, otherSubstitution) + if err != nil { + return err + } + otherSubstitutionVersion, err := semver.Parse(rawVersion) + if err != nil { + return err + } + currentSubstitutionVersion, err := semver.Parse(version) + if err != nil { + return err + } + // Compare versions + c, err := libsemver.BuildIdCompare(otherSubstitutionVersion, currentSubstitutionVersion) + if err != nil { + return err + } + if c < 0 { + // Update the currentSubstitution substitutesFor to point to otherSubstitution + // since it is latest + _, err = updateBundleSubstitutesFor.Exec(otherSubstitution, csvName) + if err != nil { + return err + } + moreSubstitutions, err := s.getBundlesThatSubstitutesFor(tx, otherSubstitution) + if err != nil { + return err + } + otherSubstitutions = append(otherSubstitutions, moreSubstitutions...) + } else if c > 0 { + // Update the otherSubstitution's substitutesFor to point to csvName + // Since it is the latest + _, err = updateBundleSubstitutesFor.Exec(csvName, otherSubstitution) + if err != nil { + return err + } + // Update the otherSubstitution's skips to include csvName and its skips + err = s.appendSkips(tx, append(skips, csvName), otherSubstitution) + if err != nil { + return err + } + moreSubstitutions, err := s.getBundlesThatSubstitutesFor(tx, csvName) + if err != nil { + return err + } + if len(moreSubstitutions) > 1 { + return fmt.Errorf("programmer error: more than one substitution pointing to %s", csvName) + } + } else { + // the versions are equal + return fmt.Errorf("cannot determine latest substitution because of duplicate versions") + } + } + } + } + + // Get latest substitutesFor value of the current bundle + substitutesFor, err = s.getBundleSubstitution(tx, csvName) + if err != nil { + return err + } + + // If the substituted-for of the current bundle substitutes for another bundle + // it should also be added to the skips of the substitutesFor bundle + for substitutesFor != "" { + skips = append(skips, substitutesFor) + substitutesFor, err = s.getBundleSubstitution(tx, substitutesFor) + if err != nil { + return err + } + } + + // If the substitution (or substitution of substitution) is added before the + // substituted for bundle, (i.e. the bundle being added is substituted for by + // another bundle) then transfer the skips from the substitutedFor bundle (this + // bundle) over to the substitution's skips + var substitutesFors []string + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, csvName) + if err != nil || len(substitutesFors) > 1 { + return err + } + for len(substitutesFors) > 0 { + err = s.appendSkips(tx, append(skips, csvName), substitutesFors[0]) + if err != nil { + return err + } + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, substitutesFors[0]) + if err != nil || len(substitutesFors) > 1 { + return err + } + } + + // Bundles that skip a bundle that is substituted for + // should also skip the substituted-for bundle + if len(skips) != 0 { + // ensure slice of skips doesn't contain duplicates + substitutesSkips := make(map[string]struct{}) + skipsOverwrite := []string{} + for _, skip := range skips { + substitutesSkips[skip] = struct{}{} + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, skip) + if err != nil || len(substitutesFors) > 1 { + return err + } + for len(substitutesFors) > 0 { + // consume the slice of substitutions + substitutesFor = substitutesFors[0] + substitutesFors = substitutesFors[1:] + // shouldn't skip yourself + if substitutesFor != csvName { + substitutesSkips[substitutesFor] = struct{}{} + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, substitutesFor) + if err != nil || len(substitutesFors) > 1 { + return err + } + } + } + } + for s := range substitutesSkips { + skipsOverwrite = append(skipsOverwrite, s) + } + skips = skipsOverwrite + } + + // If the bundle being added replaces a bundle that is substituted for + // (for example it was the previous head of the channel), change + // the replaces to the substituted-for bundle + if replaces != "" { + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, replaces) + if err != nil { + return err + } + for len(substitutesFors) > 0 { + // update the replaces to a newer substitution + replaces = substitutesFors[0] + // try to get the substitution of the substitution + substitutesFors, err = s.getBundlesThatSubstitutesFor(tx, replaces) + if err != nil || len(substitutesFors) > 1 { + return err + } + } + } + + _, err = updateBundleReplacesSkips.Exec(replaces, strings.Join(skips, ","), csvName) + if err != nil { + return err + } + + return nil +} + +func (s *sqlLoader) appendSkips(tx *sql.Tx, skips []string, csvName string) error { + updateSkips, err := tx.Prepare("update operatorbundle set skips = ? where name = ?") + if err != nil { + return err + } + defer updateSkips.Close() + + _, currentSkips, _, err := s.getBundleSkipsReplacesVersion(tx, csvName) + if err != nil { + return err + } + + // ensure slice of skips doesn't contain duplicates + skipsMap := make(map[string]struct{}) + for _, skip := range currentSkips { + skipsMap[skip] = struct{}{} + } + for _, skip := range skips { + if _, ok := skipsMap[skip]; !ok { + currentSkips = append(currentSkips, skip) + } + } + + _, err = updateSkips.Exec(strings.Join(currentSkips, ","), csvName) + return err +} + func (s *sqlLoader) AddPackageChannelsFromGraph(graph *registry.Package) error { tx, err := s.db.Begin() if err != nil { @@ -263,6 +521,10 @@ func (s *sqlLoader) AddPackageChannels(manifest registry.PackageManifest) error tx.Rollback() }() + if err := s.rmPackage(tx, manifest.PackageName); err != nil { + return err + } + if err := s.addPackageChannels(tx, manifest); err != nil { return err } @@ -310,35 +572,41 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani } defer getReplaces.Close() - var errs []error - if _, err := addPackage.Exec(manifest.PackageName); err != nil { - errs = append(errs, err) - return utilerrors.NewAggregate(errs) + return fmt.Errorf("failed to add package %q: %s", manifest.PackageName, err.Error()) } - hasDefault := false + var ( + errs []error + channels []registry.PackageChannel + hasDefault bool + ) for _, c := range manifest.Channels { + if deprecated, err := s.deprecated(tx, c.CurrentCSVName); err != nil || deprecated { + // Elide channels that start with a deprecated bundle + continue + } if _, err := addChannel.Exec(c.Name, manifest.PackageName, c.CurrentCSVName); err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add channel %q in package %q: %s", c.Name, manifest.PackageName, err.Error())) continue } if c.IsDefaultChannel(manifest) { hasDefault = true if _, err := addDefaultChannel.Exec(c.Name, manifest.PackageName); err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add default channel %q in package %q: %s", c.Name, manifest.PackageName, err.Error())) continue } } + channels = append(channels, c) } if !hasDefault { errs = append(errs, fmt.Errorf("no default channel specified for %s", manifest.PackageName)) } - for _, c := range manifest.Channels { + for _, c := range channels { res, err := addChannelEntry.Exec(c.Name, manifest.PackageName, c.CurrentCSVName, 0) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add channel %q in package %q: %s", c.Name, manifest.PackageName, err.Error())) continue } currentID, err := res.LastInsertId() @@ -360,7 +628,14 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani break } - if err := s.addPackageProperty(tx, channelEntryCSVName, manifest.PackageName, version); err != nil { + bundlePath, err := s.getBundlePathIfExists(tx, channelEntryCSVName) + if err != nil { + // this should only happen on an SQL error, bundlepath just not being set is for backwards compatibility reasons + errs = append(errs, err) + break + } + + if err := s.addPackageProperty(tx, channelEntryCSVName, manifest.PackageName, version, bundlePath); err != nil { errs = append(errs, err) break } @@ -369,7 +644,7 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani // add dummy channel entry for the skipped version skippedChannelEntry, err := addChannelEntry.Exec(c.Name, manifest.PackageName, skip, depth) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add channel %q for skipped version %q in package %q: %s", c.Name, skip, manifest.PackageName, err.Error())) continue } @@ -382,7 +657,7 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani // add another channel entry for the parent, which replaces the skipped synthesizedChannelEntry, err := addChannelEntry.Exec(c.Name, manifest.PackageName, channelEntryCSVName, depth) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add channel %q for replaces %q in package %q: %s", c.Name, channelEntryCSVName, manifest.PackageName, err.Error())) continue } @@ -408,7 +683,7 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani replacedChannelEntry, err := addChannelEntry.Exec(c.Name, manifest.PackageName, replaces, depth) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("failed to add channel %q for replaces %q in package %q: %s", c.Name, replaces, manifest.PackageName, err.Error())) break } @@ -428,8 +703,17 @@ func (s *sqlLoader) addPackageChannels(tx *sql.Tx, manifest registry.PackageMani errs = append(errs, err) break } + deprecated, err := s.deprecated(tx, channelEntryCSVName) + if err != nil { + errs = append(errs, err) + break + } + if deprecated { + // The package is truncated below this point, we're done! + break + } if _, _, _, err := s.getBundleSkipsReplacesVersion(tx, replaces); err != nil { - errs = append(errs, fmt.Errorf("%s specifies replacement that couldn't be found", c.CurrentCSVName)) + errs = append(errs, fmt.Errorf("Invalid bundle %s, replaces nonexistent bundle %s", c.CurrentCSVName, replaces)) break } @@ -485,6 +769,7 @@ func (s *sqlLoader) getBundleSkipsReplacesVersion(tx *sql.Tx, bundleName string) err = rerr return } + defer rows.Close() if !rows.Next() { err = fmt.Errorf("no bundle found for bundlename %s", bundleName) return @@ -510,6 +795,39 @@ func (s *sqlLoader) getBundleSkipsReplacesVersion(tx *sql.Tx, bundleName string) return } +func (s *sqlLoader) getBundlePathIfExists(tx *sql.Tx, bundleName string) (bundlePath string, err error) { + getBundlePath, err := tx.Prepare(` + SELECT bundlepath + FROM operatorbundle + WHERE operatorbundle.name=? LIMIT 1`) + if err != nil { + return + } + defer getBundlePath.Close() + + rows, rerr := getBundlePath.Query(bundleName) + if err != nil { + err = rerr + return + } + defer rows.Close() + if !rows.Next() { + // no bundlepath set + return + } + + var bundlePathSQL sql.NullString + if err = rows.Scan(&bundlePathSQL); err != nil { + return + } + + if bundlePathSQL.Valid { + bundlePath = bundlePathSQL.String + } + + return +} + func (s *sqlLoader) addAPIs(tx *sql.Tx, bundle *registry.Bundle) error { if bundle.Name == "" { return fmt.Errorf("cannot add apis for bundle with no name: %#v", bundle) @@ -591,6 +909,9 @@ func (s *sqlLoader) getCSVNames(tx *sql.Tx, packageName string) ([]string, error for rows.Next() { err := rows.Scan(&csvName) if err != nil { + if nerr := rows.Close(); nerr != nil { + return nil, nerr + } return nil, err } csvNames = append(csvNames, csvName) @@ -604,36 +925,91 @@ func (s *sqlLoader) getCSVNames(tx *sql.Tx, packageName string) ([]string, error } func (s *sqlLoader) RemovePackage(packageName string) error { - tx, err := s.db.Begin() + if err := func() error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + csvNames, err := s.getCSVNames(tx, packageName) + if err != nil { + return err + } + for _, csvName := range csvNames { + if err := s.rmBundle(tx, csvName); err != nil { + return err + } + } + + deletePackage, err := tx.Prepare("DELETE FROM package WHERE package.name=?") + if err != nil { + return err + } + defer deletePackage.Close() + + if _, err := deletePackage.Exec(packageName); err != nil { + return err + } + + deleteChannel, err := tx.Prepare("DELETE FROM channel WHERE package_name = ?") + if err != nil { + return err + } + defer deleteChannel.Close() + + if _, err := deleteChannel.Exec(packageName); err != nil { + return err + } + return tx.Commit() + }(); err != nil { + return err + } + + // separate transaction so that we remove stranded bundles after the package has been cleared + return s.RemoveStrandedBundles() +} + +func (s *sqlLoader) rmBundle(tx *sql.Tx, csvName string) error { + deleteBundle, err := tx.Prepare("DELETE FROM operatorbundle WHERE operatorbundle.name=?") if err != nil { return err } - defer func() { - tx.Rollback() - }() + defer deleteBundle.Close() - csvNames, err := s.getCSVNames(tx, packageName) + if _, err := deleteBundle.Exec(csvName); err != nil { + return err + } + + deleteProvider, err := tx.Prepare("DELETE FROM api_provider WHERE api_provider.operatorbundle_name=?") if err != nil { return err } - for _, csvName := range csvNames { - err = s.rmBundle(tx, csvName) - if err != nil { - return err - } + defer deleteProvider.Close() + + if _, err := deleteProvider.Exec(csvName); err != nil { + return err } - return tx.Commit() -} + deleteRequirer, err := tx.Prepare("DELETE FROM api_requirer WHERE api_requirer.operatorbundle_name=?") + if err != nil { + return err + } + defer deleteRequirer.Close() -func (s *sqlLoader) rmBundle(tx *sql.Tx, csvName string) error { - stmt, err := tx.Prepare("DELETE FROM operatorbundle WHERE operatorbundle.name=?") + if _, err := deleteRequirer.Exec(csvName); err != nil { + return err + } + + deleteChannelEntries, err := tx.Prepare("DELETE FROM channel_entry WHERE channel_entry.operatorbundle_name=?") if err != nil { return err } - defer stmt.Close() + defer deleteChannelEntries.Close() - if _, err := stmt.Exec(csvName); err != nil { + if _, err := deleteChannelEntries.Exec(csvName); err != nil { return err } @@ -667,31 +1043,50 @@ func (s *sqlLoader) AddBundlePackageChannels(manifest registry.PackageManifest, return err } - // Delete package and channels (entries will cascade) - they will be recalculated - deletePkg, err := tx.Prepare("delete from package where name = ?") + if err := s.rmPackage(tx, manifest.PackageName); err != nil { + return err + } + + if err := s.addPackageChannels(tx, manifest); err != nil { + return err + } + + return tx.Commit() +} + +func (s *sqlLoader) rmPackage(tx *sql.Tx, pkg string) error { + // Delete package, channel, and entries - they will be recalculated + deletePkg, err := tx.Prepare("DELETE FROM package WHERE name = ?") if err != nil { return err } + defer deletePkg.Close() - _, err = deletePkg.Exec(manifest.PackageName) + _, err = deletePkg.Exec(pkg) if err != nil { return err } - deleteChan, err := tx.Prepare("delete from channel where package_name = ?") + + deleteChan, err := tx.Prepare("DELETE FROM channel WHERE package_name = ?") if err != nil { return err } + defer deleteChan.Close() - _, err = deleteChan.Exec(manifest.PackageName) + _, err = deleteChan.Exec(pkg) if err != nil { return err } - if err := s.addPackageChannels(tx, manifest); err != nil { + deleteChannelEntries, err := tx.Prepare("DELETE FROM channel_entry WHERE package_name = ?") + if err != nil { return err } - return tx.Commit() + defer deleteChannelEntries.Close() + _, err = deleteChannelEntries.Exec(pkg) + + return err } func (s *sqlLoader) addDependencies(tx *sql.Tx, bundle *registry.Bundle) error { @@ -756,7 +1151,7 @@ func (s *sqlLoader) addProperty(tx *sql.Tx, propType, value, bundleName, version return nil } -func (s *sqlLoader) addPackageProperty(tx *sql.Tx, bundleName, pkg, version string) error { +func (s *sqlLoader) addPackageProperty(tx *sql.Tx, bundleName, pkg, version, bundlePath string) error { // Add the package property prop := registry.PackageProperty{ PackageName: pkg, @@ -767,20 +1162,27 @@ func (s *sqlLoader) addPackageProperty(tx *sql.Tx, bundleName, pkg, version stri return err } - return s.addProperty(tx, registry.PackageType, string(value), bundleName, version, "") + return s.addProperty(tx, registry.PackageType, string(value), bundleName, version, bundlePath) } func (s *sqlLoader) addBundleProperties(tx *sql.Tx, bundle *registry.Bundle) error { + type propstring struct { + Type string + Value string + } + properties := make(map[propstring]struct{}) + bundleVersion, err := bundle.Version() if err != nil { return err } for _, prop := range bundle.Properties { - value, _ := json.Marshal(prop.Value) - if err := s.addProperty(tx, prop.Type, string(value), bundle.Name, bundleVersion, bundle.BundleImage); err != nil { + value, err := json.Marshal(prop.Value) + if err != nil { return err } + properties[propstring{Type: prop.Type, Value: string(value)}] = struct{}{} } // Look up providedAPIs in CSV and add them in properties table @@ -799,36 +1201,74 @@ func (s *sqlLoader) addBundleProperties(tx *sql.Tx, bundle *registry.Bundle) err if err != nil { return err } - if err := s.addProperty(tx, registry.GVKType, string(value), bundle.Name, bundleVersion, bundle.BundleImage); err != nil { - return err + properties[propstring{Type: registry.GVKType, Value: string(value)}] = struct{}{} + } + + // Add properties from annotations + csv, err := bundle.ClusterServiceVersion() + if err != nil { + // FIXME: Returning nil here is in line with the original implementation, but that was probably wrong. We should probably just bubble-up the error. + return nil + } + + if csv == nil { + // FIXME: Currently, a CSV is requirement of bundle addition. Should this return an error? + return nil + } + + var props []registry.Property + if csv.GetAnnotations() != nil { + v, ok := csv.GetAnnotations()[registry.PropertyKey] + if ok { + if err := json.Unmarshal([]byte(v), &props); err != nil { + return err + } } } - // Add label properties - if csv, err := bundle.ClusterServiceVersion(); err == nil { - annotations := csv.ObjectMeta.GetAnnotations() - if v, ok := annotations[registry.PropertyKey]; ok { - var props []registry.Property - if err := json.Unmarshal([]byte(v), &props); err == nil { - for _, prop := range props { - // Only add label type from the list - // TODO: Support more types such as GVK and package - if prop.Type == registry.LabelType { - var label registry.LabelProperty - err := json.Unmarshal(prop.Value, &label) - if err != nil { - continue - } - value, err := json.Marshal(label) - if err != nil { - continue - } - if err := s.addProperty(tx, registry.LabelType, string(value), bundle.Name, bundleVersion, bundle.BundleImage); err != nil { - continue - } - } - } + for _, prop := range props { + value, err := json.Marshal(&prop.Value) + if err != nil { + return err + } + + // validate if Type is known + switch prop.Type { + case registry.LabelType: + if err := json.Unmarshal(prop.Value, ®istry.LabelProperty{}); err != nil { + return err + } + case registry.PackageType: + if err := json.Unmarshal(prop.Value, ®istry.PackageProperty{}); err != nil { + return err } + case registry.GVKType: + if err := json.Unmarshal(prop.Value, ®istry.GVKProperty{}); err != nil { + return err + } + case registry.DeprecatedType: + // deprecated has no value + } + + properties[propstring{Type: prop.Type, Value: string(value)}] = struct{}{} + } + + // If the bundle has been deprecated before, readd the deprecated property + deprecated, err := s.deprecated(tx, bundle.Name) + if err != nil { + return err + } + if deprecated { + value, err := json.Marshal(registry.DeprecatedProperty{}) + if err != nil { + return err + } + properties[propstring{Type: registry.DeprecatedType, Value: string(value)}] = struct{}{} + } + + for prop := range properties { + if err := s.addProperty(tx, prop.Type, prop.Value, bundle.Name, bundleVersion, bundle.BundleImage); err != nil { + return err } } @@ -836,19 +1276,18 @@ func (s *sqlLoader) addBundleProperties(tx *sql.Tx, bundle *registry.Bundle) err } func (s *sqlLoader) rmChannelEntry(tx *sql.Tx, csvName string) error { - getEntryID := `SELECT entry_id FROM channel_entry WHERE operatorbundle_name=?` - rows, err := tx.QueryContext(context.TODO(), getEntryID, csvName) + rows, err := tx.Query(`SELECT entry_id FROM channel_entry WHERE operatorbundle_name=?`, csvName) if err != nil { return err } + var entryIDs []int64 for rows.Next() { var entryID sql.NullInt64 rows.Scan(&entryID) entryIDs = append(entryIDs, entryID.Int64) } - err = rows.Close() - if err != nil { + if err := rows.Close(); err != nil { return err } @@ -858,6 +1297,7 @@ func (s *sqlLoader) rmChannelEntry(tx *sql.Tx, csvName string) error { } for _, id := range entryIDs { if _, err := updateChannelEntry.Exec(id); err != nil { + updateChannelEntry.Close() return err } } @@ -866,77 +1306,89 @@ func (s *sqlLoader) rmChannelEntry(tx *sql.Tx, csvName string) error { return err } - stmt, err := tx.Prepare("DELETE FROM channel_entry WHERE operatorbundle_name=?") + _, err = tx.Exec(`DELETE FROM channel WHERE head_operatorbundle_name=?`, csvName) if err != nil { return err } - defer stmt.Close() - - if _, err := stmt.Exec(csvName); err != nil { - return err - } return nil } -func getTailFromBundle(tx *sql.Tx, name string) (bundles []string, err error) { +func getTailFromBundle(tx *sql.Tx, head string) (bundles []string, err error) { getReplacesSkips := `SELECT replaces, skips FROM operatorbundle WHERE name=?` - isDefaultChannelHead := `SELECT head_operatorbundle_name FROM channel - INNER JOIN package ON channel.name = package.default_channel + isDefaultChannelHead := `SELECT head_operatorbundle_name FROM channel + INNER JOIN package ON channel.name = package.default_channel WHERE channel.head_operatorbundle_name = ?` - tail := make(map[string]struct{}) - next := name + visited := map[string]struct{}{} + next := []string{head} + + for len(next) > 0 { + // Pop the next bundle off of the queue + bundle := next[0] + next = next[1:] // Potentially inefficient queue implementation, but this function is only used when deprecate is called + + // Check if next is the head of the defaultChannel + // If it is, the defaultChannel would be removed -- this is not allowed because we cannot know which channel to promote as the new default + var err error + if row := tx.QueryRow(isDefaultChannelHead, bundle); row != nil { + err = row.Scan(&sql.NullString{}) + } + if err == nil { + // A nil error indicates that next is the default channel head + return nil, registry.ErrRemovingDefaultChannelDuringDeprecation + } else if err != sql.ErrNoRows { + return nil, err + } - for next != "" { - rows, err := tx.QueryContext(context.TODO(), getReplacesSkips, next) + rows, err := tx.QueryContext(context.TODO(), getReplacesSkips, bundle) if err != nil { return nil, err } - var replaces sql.NullString - var skips sql.NullString + + var ( + replaces sql.NullString + skips sql.NullString + ) if rows.Next() { if err := rows.Scan(&replaces, &skips); err != nil { + if nerr := rows.Close(); nerr != nil { + return nil, nerr + } return nil, err } } - rows.Close() + if err := rows.Close(); err != nil { + return nil, err + } if skips.Valid && skips.String != "" { for _, skip := range strings.Split(skips.String, ",") { - tail[skip] = struct{}{} + if _, ok := visited[skip]; ok { + // We've already visited this bundle's subgraph + continue + } + visited[skip] = struct{}{} + next = append(next, skip) } } if replaces.Valid && replaces.String != "" { - // check if replaces is the head of the defaultChannel - // if it is, the defaultChannel will be removed - // this is not allowed because we cannot know which channel to promote as the new default - rows, err := tx.QueryContext(context.TODO(), isDefaultChannelHead, replaces.String) - if err != nil { - return nil, err - } - if rows.Next() { - var defaultChannelHead sql.NullString - err := rows.Scan(&defaultChannelHead) - if err != nil { - return nil, err - } - if defaultChannelHead.Valid { - return nil, registry.ErrRemovingDefaultChannelDuringDeprecation - } + r := replaces.String + if _, ok := visited[r]; ok { + // We've already visited this bundle's subgraph + continue } - next = replaces.String - tail[replaces.String] = struct{}{} - } else { - next = "" + visited[r] = struct{}{} + next = append(next, r) } } - var allTails []string - for k := range tail { - allTails = append(allTails, k) + // The tail is exactly the set of bundles we visited while traversing the graph from head + var tail []string + for v := range visited { + tail = append(tail, v) } - return allTails, nil + return tail, nil } @@ -980,16 +1432,20 @@ func (s *sqlLoader) DeprecateBundle(path string) error { } for _, bundle := range tailBundles { - err := s.rmBundle(tx, bundle) - if err != nil { + if err := s.rmChannelEntry(tx, bundle); err != nil { return err } - err = s.rmChannelEntry(tx, bundle) - if err != nil { + if err := s.rmBundle(tx, bundle); err != nil { return err } } + // Remove any channels that start with the deprecated bundle + _, err = tx.Exec(fmt.Sprintf(`DELETE FROM channel WHERE head_operatorbundle_name="%s"`, name)) + if err != nil { + return err + } + deprecatedValue, err := json.Marshal(registry.DeprecatedProperty{}) if err != nil { return err @@ -999,5 +1455,84 @@ func (s *sqlLoader) DeprecateBundle(path string) error { return err } + // Create a persistent record of the bundle's deprecation + // This lets us recover from losing the properties and augmented bundle rows + _, err = tx.Exec("INSERT OR REPLACE INTO deprecated(operatorbundle_name) VALUES(?)", name) + if err != nil { + return err + } + + return tx.Commit() +} + +func (s *sqlLoader) RemoveStrandedBundles() error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + if err := s.rmStrandedBundles(tx); err != nil { + return err + } + return tx.Commit() } + +func (s *sqlLoader) rmStrandedBundles(tx *sql.Tx) error { + _, err := tx.Exec("DELETE FROM operatorbundle WHERE name NOT IN(select operatorbundle_name from channel_entry)") + return err +} + +func (s *sqlLoader) getBundlesThatSubstitutesFor(tx *sql.Tx, replaces string) ([]string, error) { + query := `SELECT name FROM operatorbundle WHERE substitutesfor=?` + rows, err := tx.QueryContext(context.TODO(), query, replaces) + if err != nil { + return []string{}, err + } + defer rows.Close() + + var substitutesFor []string + var subsFor sql.NullString + for rows.Next() { + if err := rows.Scan(&subsFor); err != nil { + return []string{}, err + } + if subsFor.Valid && subsFor.String != "" { + substitutesFor = append(substitutesFor, subsFor.String) + } + } + return substitutesFor, nil +} + +func (s *sqlLoader) getBundleSubstitution(tx *sql.Tx, name string) (string, error) { + query := `SELECT substitutesfor FROM operatorbundle WHERE name=?` + rows, err := tx.QueryContext(context.TODO(), query, name) + if err != nil { + return "", err + } + defer rows.Close() + + var substitutesFor sql.NullString + if rows.Next() { + if err := rows.Scan(&substitutesFor); err != nil { + return "", err + } + } + return substitutesFor.String, nil +} + +func (s *sqlLoader) deprecated(tx *sql.Tx, name string) (bool, error) { + var err error + if row := tx.QueryRow(`SELECT * FROM deprecated WHERE operatorbundle_name = ?`, name); row != nil { + err = row.Scan(&sql.NullString{}) + } + if err == sql.ErrNoRows { + return false, nil + } + + // Ignore any deprecated bundles + return err == nil, err +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/006_associate_apis_with_bundle.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/006_associate_apis_with_bundle.go index 221d90e484..f70436f1d2 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/006_associate_apis_with_bundle.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/006_associate_apis_with_bundle.go @@ -3,6 +3,7 @@ package migrations import ( "context" "database/sql" + "github.com/operator-framework/operator-registry/pkg/registry" ) diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/010_set_bundlepath_pkg_property.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/010_set_bundlepath_pkg_property.go new file mode 100644 index 0000000000..bee961621c --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/010_set_bundlepath_pkg_property.go @@ -0,0 +1,42 @@ +package migrations + +import ( + "context" + "database/sql" +) + +const BundlePathPkgMigrationKey = 10 + +// Register this migration +func init() { + registerMigration(BundlePathPkgMigrationKey, bundlePathPkgPropertyMigration) +} + +var bundlePathPkgPropertyMigration = &Migration{ + Id: BundlePathPkgMigrationKey, + Up: func(ctx context.Context, tx *sql.Tx) error { + updatePropertiesSql := ` + UPDATE properties + SET operatorbundle_path = (SELECT bundlepath + FROM operatorbundle + WHERE operatorbundle_name = operatorbundle.name AND operatorbundle_version = operatorbundle.version)` + _, err := tx.ExecContext(ctx, updatePropertiesSql) + if err != nil { + return err + } + + return nil + }, + Down: func(ctx context.Context, tx *sql.Tx) error { + updatePropertiesSql := ` + UPDATE properties + SET operatorbundle_path = null + WHERE type = "olm.package"` + _, err := tx.ExecContext(ctx, updatePropertiesSql) + if err != nil { + return err + } + + return err + }, +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/011_susbtitutes_for.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/011_susbtitutes_for.go new file mode 100644 index 0000000000..9b199579cf --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/011_susbtitutes_for.go @@ -0,0 +1,55 @@ +package migrations + +import ( + "context" + "database/sql" +) + +const SubstitutesForMigrationKey = 11 + +// Register this migration +func init() { + registerMigration(SubstitutesForMigrationKey, substitutesForPropertyMigration) +} + +var substitutesForPropertyMigration = &Migration{ + Id: SubstitutesForMigrationKey, + Up: func(ctx context.Context, tx *sql.Tx) error { + sql := ` + ALTER TABLE operatorbundle + ADD COLUMN substitutesfor TEXT; + ` + _, err := tx.ExecContext(ctx, sql) + return err + }, + Down: func(ctx context.Context, tx *sql.Tx) error { + foreignKeyOff := `PRAGMA foreign_keys = 0` + createTempTable := `CREATE TABLE operatorbundle_backup (name TEXT, csv TEXT, bundle TEXT, bundlepath TEXT, version TEXT, skiprange TEXT, replaces TEXT, skips TEXT)` + backupTargetTable := `INSERT INTO operatorbundle_backup SELECT name, csv, bundle, bundlepath, version, skiprange, replaces, skips FROM operatorbundle` + dropTargetTable := `DROP TABLE operatorbundle` + renameBackUpTable := `ALTER TABLE operatorbundle_backup RENAME TO operatorbundle;` + foreignKeyOn := `PRAGMA foreign_keys = 1` + _, err := tx.ExecContext(ctx, foreignKeyOff) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, createTempTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, backupTargetTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, dropTargetTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, renameBackUpTable) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, foreignKeyOn) + return err + }, +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/012_deprecated.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/012_deprecated.go new file mode 100644 index 0000000000..760b381ff7 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/migrations/012_deprecated.go @@ -0,0 +1,42 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/operator-framework/operator-registry/pkg/registry" +) + +const DeprecatedMigrationKey = 12 + +// Register this migration +func init() { + registerMigration(DeprecatedMigrationKey, deprecatedMigration) +} + +var deprecatedMigration = &Migration{ + Id: DeprecatedMigrationKey, + Up: func(ctx context.Context, tx *sql.Tx) error { + // Purposefully forego a foreign key constraint so this table can survive operations that drop bundles and properties + // e.g. a lossy implementation of --overwrite-latest that relies on readding all bundles in a package + sql := ` + CREATE TABLE IF NOT EXISTS deprecated ( + operatorbundle_name TEXT PRIMARY KEY + ); + ` + if _, err := tx.ExecContext(ctx, sql); err != nil { + return err + } + + initDeprecated := fmt.Sprintf(`INSERT OR REPLACE INTO deprecated(operatorbundle_name) SELECT operatorbundle_name FROM properties WHERE properties.type='%s'`, registry.DeprecatedType) + _, err := tx.ExecContext(ctx, initDeprecated) + + return err + }, + Down: func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `DROP TABLE deprecated`) + + return err + }, +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go index 0ab6506b65..5b68340359 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/query.go @@ -1,3 +1,5 @@ +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o sqlitefakes/fake_rowscanner.go . RowScanner +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o sqlitefakes/fake_querier.go . Querier package sqlite import ( @@ -13,23 +15,45 @@ import ( "github.com/operator-framework/operator-registry/pkg/registry" ) -type SQLQuerier struct { +type RowScanner interface { + Next() bool + Close() error + Scan(dest ...interface{}) error +} + +type Querier interface { + QueryContext(ctx context.Context, query string, args ...interface{}) (RowScanner, error) +} + +type dbQuerierAdapter struct { db *sql.DB } +func (a dbQuerierAdapter) QueryContext(ctx context.Context, query string, args ...interface{}) (RowScanner, error) { + return a.db.QueryContext(ctx, query, args...) +} + +type SQLQuerier struct { + db Querier +} + var _ registry.Query = &SQLQuerier{} func NewSQLLiteQuerier(dbFilename string) (*SQLQuerier, error) { - db, err := sql.Open("sqlite3", "file:"+dbFilename+"?immutable=true") + db, err := OpenReadOnly(dbFilename) if err != nil { return nil, err } - return &SQLQuerier{db}, nil + return &SQLQuerier{dbQuerierAdapter{db}}, nil } func NewSQLLiteQuerierFromDb(db *sql.DB) *SQLQuerier { - return &SQLQuerier{db} + return &SQLQuerier{dbQuerierAdapter{db}} +} + +func NewSQLLiteQuerierFromDBQuerier(q Querier) *SQLQuerier { + return &SQLQuerier{q} } func (s *SQLQuerier) ListTables(ctx context.Context) ([]string, error) { @@ -621,7 +645,7 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryID int64) (provid kinds := map[string]struct{}{} versions := map[string]struct{}{} - providedQuery := `SELECT properties.value FROM properties + providedQuery := `SELECT properties.value FROM properties INNER JOIN channel_entry ON channel_entry.operatorbundle_name = properties.operatorbundle_name WHERE properties.type=? AND channel_entry.entry_id=?` @@ -797,11 +821,10 @@ func (s *SQLQuerier) GetBundlePathsForPackage(ctx context.Context, pkgName strin if err := rows.Scan(&imgName); err != nil { return nil, err } - if imgName.Valid && imgName.String != "" { - images = append(images, imgName.String) - } else { + if imgName.Valid && imgName.String == "" { return nil, fmt.Errorf("Index malformed: cannot find paths to bundle images") } + images = append(images, imgName.String) } return images, nil } @@ -900,71 +923,130 @@ func (s *SQLQuerier) GetCurrentCSVNameForChannel(ctx context.Context, pkgName, c return "", nil } -func (s *SQLQuerier) ListBundles(ctx context.Context) (bundles []*api.Bundle, err error) { - query := `SELECT DISTINCT channel_entry.entry_id, operatorbundle.bundle, operatorbundle.bundlepath, - channel_entry.operatorbundle_name, channel_entry.package_name, channel_entry.channel_name, operatorbundle.replaces, operatorbundle.skips, - operatorbundle.version, operatorbundle.skiprange, - dependencies.type, dependencies.value, - properties.type, properties.value - FROM channel_entry - INNER JOIN operatorbundle ON operatorbundle.name = channel_entry.operatorbundle_name - LEFT OUTER JOIN dependencies ON dependencies.operatorbundle_name = channel_entry.operatorbundle_name - LEFT OUTER JOIN properties ON properties.operatorbundle_name = channel_entry.operatorbundle_name - INNER JOIN package ON package.name = channel_entry.package_name` - - rows, err := s.db.QueryContext(ctx, query) +// Rows in the channel_entry table essentially represent inbound +// upgrade edges to a bundle. There may be no linear "replaces" chain +// (for example, when an index is populated using semver-skippatch +// mode), and there may be multiple inbound "skips" to a single +// bundle. The ListBundles query determines a single "replaces" value +// per bundle per channel by recursively following "replaces" +// references beginning from the entries with minimal depth, which +// represent channel heads. All other edges are merged into an +// aggregate "skips" column. The result contains one row per bundle +// for each channel in which the bundle appears. +const listBundlesQuery = ` +WITH RECURSIVE +tip (depth) AS ( + SELECT min(depth) + FROM channel_entry +), replaces_entry (entry_id, replaces) AS ( + SELECT entry_id, replaces + FROM channel_entry + INNER JOIN tip ON channel_entry.depth = tip.depth + UNION + SELECT channel_entry.entry_id, channel_entry.replaces + FROM channel_entry + INNER JOIN replaces_entry + ON channel_entry.entry_id = replaces_entry.replaces +), replaces_bundle (entry_id, operatorbundle_name, package_name, channel_name, replaces) AS ( + SELECT min(all_entry.entry_id), all_entry.operatorbundle_name, all_entry.package_name, all_entry.channel_name, max(replaced_entry.operatorbundle_name) + FROM channel_entry AS all_entry + LEFT OUTER JOIN replaces_entry + ON all_entry.entry_id = replaces_entry.entry_id + LEFT OUTER JOIN channel_entry AS replaced_entry + ON replaces_entry.replaces = replaced_entry.entry_id + GROUP BY all_entry.operatorbundle_name, all_entry.package_name, all_entry.channel_name +), skips_entry (entry_id, skips) AS ( + SELECT entry_id, replaces + FROM channel_entry + WHERE replaces IS NOT NULL + EXCEPT + SELECT entry_id, replaces + FROM replaces_entry +), skips_bundle (operatorbundle_name, package_name, channel_name, skips) AS ( + SELECT all_entry.operatorbundle_name, all_entry.package_name, all_entry.channel_name, group_concat(skipped_entry.operatorbundle_name, ",") + FROM skips_entry + INNER JOIN channel_entry AS all_entry + ON skips_entry.entry_id = all_entry.entry_id + INNER JOIN channel_entry AS skipped_entry + ON skips_entry.skips = skipped_entry.entry_id + GROUP BY all_entry.operatorbundle_name, all_entry.package_name, all_entry.channel_name +) +SELECT + replaces_bundle.entry_id, + operatorbundle.bundle, + operatorbundle.bundlepath, + operatorbundle.name, + replaces_bundle.package_name, + replaces_bundle.channel_name, + replaces_bundle.replaces, + skips_bundle.skips, + operatorbundle.version, + operatorbundle.skiprange, + dependencies.type, + dependencies.value, + properties.type, + properties.value + FROM replaces_bundle + INNER JOIN operatorbundle + ON replaces_bundle.operatorbundle_name = operatorbundle.name + LEFT OUTER JOIN skips_bundle + ON replaces_bundle.operatorbundle_name = skips_bundle.operatorbundle_name + AND replaces_bundle.package_name = skips_bundle.package_name + AND replaces_bundle.channel_name = skips_bundle.channel_name + LEFT OUTER JOIN dependencies + ON operatorbundle.name = dependencies.operatorbundle_name + LEFT OUTER JOIN properties + ON operatorbundle.name = properties.operatorbundle_name` + +func (s *SQLQuerier) ListBundles(ctx context.Context) ([]*api.Bundle, error) { + rows, err := s.db.QueryContext(ctx, listBundlesQuery) if err != nil { return nil, err } defer rows.Close() - bundles = []*api.Bundle{} + var bundles []*api.Bundle bundlesMap := map[string]*api.Bundle{} for rows.Next() { - var entryID sql.NullInt64 - var bundle sql.NullString - var bundlePath sql.NullString - var bundleName sql.NullString - var pkgName sql.NullString - var channelName sql.NullString - var replaces sql.NullString - var skips sql.NullString - var version sql.NullString - var skipRange sql.NullString - var depType sql.NullString - var depValue sql.NullString - var propType sql.NullString - var propValue sql.NullString + var ( + entryID sql.NullInt64 + bundle sql.NullString + bundlePath sql.NullString + bundleName sql.NullString + pkgName sql.NullString + channelName sql.NullString + replaces sql.NullString + skips sql.NullString + version sql.NullString + skipRange sql.NullString + depType sql.NullString + depValue sql.NullString + propType sql.NullString + propValue sql.NullString + ) if err := rows.Scan(&entryID, &bundle, &bundlePath, &bundleName, &pkgName, &channelName, &replaces, &skips, &version, &skipRange, &depType, &depValue, &propType, &propValue); err != nil { return nil, err } - bundleKey := fmt.Sprintf("%s/%s/%s", bundleName.String, version.String, bundlePath.String) + if !bundleName.Valid || !version.Valid || !bundlePath.Valid || !channelName.Valid { + continue + } + + bundleKey := fmt.Sprintf("%s/%s/%s/%s", bundleName.String, version.String, bundlePath.String, channelName.String) bundleItem, ok := bundlesMap[bundleKey] if ok { - // Create new dependency object if depType.Valid && depValue.Valid { - dep := &api.Dependency{} - dep.Type = depType.String - dep.Value = depValue.String - - // Add new dependency to the existing list - existingDeps := bundleItem.Dependencies - existingDeps = append(existingDeps, dep) - bundleItem.Dependencies = existingDeps + bundleItem.Dependencies = append(bundleItem.Dependencies, &api.Dependency{ + Type: depType.String, + Value: depValue.String, + }) } - - // Create new property object if propType.Valid && propValue.Valid { - prop := &api.Property{} - prop.Type = propType.String - prop.Value = propValue.String - - // Add new property to the existing list - existingProps := bundleItem.Properties - existingProps = append(existingProps, prop) - bundleItem.Properties = existingProps + bundleItem.Properties = append(bundleItem.Properties, &api.Property{ + Type: propType.String, + Value: propValue.String, + }) } } else { // Create new bundle @@ -983,30 +1065,34 @@ func (s *SQLQuerier) ListBundles(ctx context.Context) (bundles []*api.Bundle, er out.Version = version.String out.SkipRange = skipRange.String out.Replaces = replaces.String - out.Skips = strings.Split(skips.String, ",") + if skips.Valid { + out.Skips = strings.Split(skips.String, ",") + } provided, required, err := s.GetApisForEntry(ctx, entryID.Int64) if err != nil { return nil, err } - out.ProvidedApis = provided - out.RequiredApis = required - - // Create new dependency and dependency list - dep := &api.Dependency{} - dependencies := []*api.Dependency{} - dep.Type = depType.String - dep.Value = depValue.String - dependencies = append(dependencies, dep) - out.Dependencies = dependencies - - // Create new property and property list - prop := &api.Property{} - properties := []*api.Property{} - prop.Type = propType.String - prop.Value = propValue.String - properties = append(properties, prop) - out.Properties = properties + if len(provided) > 0 { + out.ProvidedApis = provided + } + if len(required) > 0 { + out.RequiredApis = required + } + + if depType.Valid && depValue.Valid { + out.Dependencies = []*api.Dependency{{ + Type: depType.String, + Value: depValue.String, + }} + } + + if propType.Valid && propValue.Valid { + out.Properties = []*api.Property{{ + Type: propType.String, + Value: propValue.String, + }} + } bundlesMap[bundleKey] = out } @@ -1024,16 +1110,16 @@ func (s *SQLQuerier) ListBundles(ctx context.Context) (bundles []*api.Bundle, er bundles = append(bundles, v) } - return + return bundles, nil } func unique(deps []*api.Dependency) []*api.Dependency { - keys := make(map[string]bool) - list := []*api.Dependency{} + keys := make(map[string]struct{}) + var list []*api.Dependency for _, entry := range deps { depKey := fmt.Sprintf("%s/%s", entry.Type, entry.Value) if _, value := keys[depKey]; !value { - keys[depKey] = true + keys[depKey] = struct{}{} list = append(list, entry) } } @@ -1041,12 +1127,12 @@ func unique(deps []*api.Dependency) []*api.Dependency { } func uniqueProps(props []*api.Property) []*api.Property { - keys := make(map[string]bool) - list := []*api.Property{} + keys := make(map[string]struct{}) + var list []*api.Property for _, entry := range props { propKey := fmt.Sprintf("%s/%s", entry.Type, entry.Value) if _, value := keys[propKey]; !value { - keys[propKey] = true + keys[propKey] = struct{}{} list = append(list, entry) } } @@ -1086,7 +1172,7 @@ func (s *SQLQuerier) GetDependenciesForBundle(ctx context.Context, name, version } func (s *SQLQuerier) GetPropertiesForBundle(ctx context.Context, name, version, path string) (properties []*api.Property, err error) { - propQuery := `SELECT DISTINCT type, value FROM properties + propQuery := `SELECT DISTINCT type, value FROM properties WHERE operatorbundle_name=? AND (operatorbundle_version=? OR operatorbundle_version is NULL) AND (operatorbundle_path=? OR operatorbundle_path is NULL)` @@ -1116,3 +1202,135 @@ func (s *SQLQuerier) GetPropertiesForBundle(ctx context.Context, name, version, return } + +func (s *SQLQuerier) GetBundlePathIfExists(ctx context.Context, bundleName string) (bundlePath string, err error) { + getBundlePathQuery := ` + SELECT bundlepath + FROM operatorbundle + WHERE operatorbundle.name=? LIMIT 1` + + rows, err := s.db.QueryContext(ctx, getBundlePathQuery, bundleName) + if err != nil { + return + } + defer rows.Close() + + if !rows.Next() { + // no bundlepath set + err = registry.ErrBundleImageNotInDatabase + return + } + + var bundlePathSQL sql.NullString + if err = rows.Scan(&bundlePathSQL); err != nil { + return + } + + if bundlePathSQL.Valid { + bundlePath = bundlePathSQL.String + } + + return +} + +// ListRegistryBundles returns a set of registry bundles. +// The set can be filtered by package by setting the given context's 'package' key to a desired package name. +// e.g. +// ctx := ContextWithPackage(context.TODO(), "etcd") +// bundles, err := querier.ListRegistryBundles(ctx) +// // ... +func (s *SQLQuerier) ListRegistryBundles(ctx context.Context) ([]*registry.Bundle, error) { + listBundlesQuery := ` + SELECT DISTINCT operatorbundle.name, operatorbundle.version, operatorbundle.bundle, channel_entry.package_name + FROM operatorbundle + LEFT OUTER JOIN channel_entry ON operatorbundle.name = channel_entry.operatorbundle_name` + + var ( + err error + rows RowScanner + ) + if pkg, ok := registry.PackageFromContext(ctx); ok { + listBundlesQuery += " WHERE channel_entry.package_name=?" + rows, err = s.db.QueryContext(ctx, listBundlesQuery, pkg) + } else { + rows, err = s.db.QueryContext(ctx, listBundlesQuery) + } + if err != nil { + return nil, err + } + defer rows.Close() + + var bundles []*registry.Bundle + for rows.Next() { + var ( + bundleName sql.NullString + bundleVersion sql.NullString + bundle sql.NullString + packageName sql.NullString + ) + if err := rows.Scan(&bundleName, &bundleVersion, &bundle, &packageName); err != nil { + return nil, err + } + + switch { + case !bundleName.Valid: + return nil, fmt.Errorf("bundle name column corrupted") + case !bundleVersion.Valid: + // Version field is currently nullable + case !bundle.Valid: + // Bundle field is currently nullable + case !packageName.Valid: + return nil, fmt.Errorf("package name column corrupted") + } + + // Allow the channel_entry table to be authoritative + channels, err := s.listBundleChannels(ctx, bundleName.String) + if err != nil { + return nil, fmt.Errorf("unable to list channels for bundle %s: %s", bundleName.String, err) + } + + defaultChannel, err := s.GetDefaultChannelForPackage(ctx, packageName.String) + if err != nil { + return nil, fmt.Errorf("unable to get default channel for package %s: %s", packageName.String, err) + } + + b, err := registry.NewBundleFromStrings(bundleName.String, bundleVersion.String, packageName.String, defaultChannel, strings.Join(channels, ","), bundle.String) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal bundle %s from database: %s", bundleName.String, err) + } + + bundles = append(bundles, b) + } + + return bundles, nil +} + +func (s *SQLQuerier) listBundleChannels(ctx context.Context, bundleName string) ([]string, error) { + listBundleChannelsQuery := ` + SELECT DISTINCT channel_entry.channel_name + FROM channel_entry + INNER JOIN operatorbundle ON channel_entry.operatorbundle_name = operatorbundle.name + WHERE operatorbundle.name = ?` + + rows, err := s.db.QueryContext(ctx, listBundleChannelsQuery, bundleName) + if err != nil { + return nil, err + } + defer rows.Close() + + var channels []string + for rows.Next() { + var channel sql.NullString + if err := rows.Scan(&channel); err != nil { + return nil, err + } + + if !channel.Valid { + return nil, fmt.Errorf("channel name column corrupt for bundle %s", bundleName) + } + + channels = append(channels, channel.String) + } + + return channels, nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go index 7d2cfdd49a..1d2a2cb4d4 100644 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/remove.go @@ -36,7 +36,6 @@ func (d *PackageRemover) Remove() error { var errs []error packages := sanitizePackageList(strings.Split(d.packages, ",")) - log.Info("input has been sanitized") log.Infof("packages: %s", packages) for _, pkg := range packages { diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/stranded.go b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/stranded.go new file mode 100644 index 0000000000..9bab472eb0 --- /dev/null +++ b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/stranded.go @@ -0,0 +1,36 @@ +package sqlite + +import ( + "github.com/sirupsen/logrus" + + "github.com/operator-framework/operator-registry/pkg/registry" +) + +type SQLStrandedBundleRemover interface { + Remove() error +} + +// StrandedBundleRemover removes stranded bundles from the database +type StrandedBundleRemover struct { + store registry.Load +} + +var _ SQLStrandedBundleRemover = &StrandedBundleRemover{} + +func NewSQLStrandedBundleRemover(store registry.Load) *StrandedBundleRemover { + return &StrandedBundleRemover{ + store: store, + } +} + +func (d *StrandedBundleRemover) Remove() error { + log := logrus.New() + + err := d.store.RemoveStrandedBundles() + if err != nil { + return err + } + log.Info("removing stranded bundles ") + + return nil +} diff --git a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/test.txt b/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/test.txt deleted file mode 100644 index f64226b0fd..0000000000 --- a/vendor/github.com/operator-framework/operator-registry/pkg/sqlite/test.txt +++ /dev/null @@ -1 +0,0 @@ -expected:[]registry.OperatorBundle{registry.OperatorBundle{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x9, Patch: 0x2, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.9.2", ReplacesBundles: []registry.OperatorBundle{}, Replaces: []registry.BundleRef{registry.BundleRef{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x9, Patch: 0x0, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.9.0"}}}, registry.OperatorBundle{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x0, Patch: 0x0, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.9.1", ReplacesBundles: []registry.OperatorBundle{}, Replaces: []registry.BundleRef{}}, registry.OperatorBundle{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x9, Patch: 0x0, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.9.0", ReplacesBundles: []registry.OperatorBundle{}, Replaces: []registry.BundleRef{registry.BundleRef{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x6, Patch: 0x1, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.6.1"}}}, registry.OperatorBundle{BundlePath: "", Version: semver.Version{Major: 0x0, Minor: 0x6, Patch: 0x1, Pre: []semver.PRVersion(nil), Build: []string(nil)}, CsvName: "etcdoperator.v0.6.1", ReplacesBundles: []registry.OperatorBundle{}, Replaces: []registry.BundleRef{}}} \ No newline at end of file diff --git a/vendor/golang.org/x/crypto/scrypt/scrypt.go b/vendor/golang.org/x/crypto/scrypt/scrypt.go index 2f81fe4148..bbe4494c6c 100644 --- a/vendor/golang.org/x/crypto/scrypt/scrypt.go +++ b/vendor/golang.org/x/crypto/scrypt/scrypt.go @@ -9,6 +9,7 @@ package scrypt // import "golang.org/x/crypto/scrypt" import ( "crypto/sha256" + "encoding/binary" "errors" "math/bits" @@ -143,36 +144,34 @@ func integer(b []uint32, r int) uint64 { func smix(b []byte, r, N int, v, xy []uint32) { var tmp [16]uint32 + R := 32 * r x := xy - y := xy[32*r:] + y := xy[R:] j := 0 - for i := 0; i < 32*r; i++ { - x[i] = uint32(b[j]) | uint32(b[j+1])<<8 | uint32(b[j+2])<<16 | uint32(b[j+3])<<24 + for i := 0; i < R; i++ { + x[i] = binary.LittleEndian.Uint32(b[j:]) j += 4 } for i := 0; i < N; i += 2 { - blockCopy(v[i*(32*r):], x, 32*r) + blockCopy(v[i*R:], x, R) blockMix(&tmp, x, y, r) - blockCopy(v[(i+1)*(32*r):], y, 32*r) + blockCopy(v[(i+1)*R:], y, R) blockMix(&tmp, y, x, r) } for i := 0; i < N; i += 2 { j := int(integer(x, r) & uint64(N-1)) - blockXOR(x, v[j*(32*r):], 32*r) + blockXOR(x, v[j*R:], R) blockMix(&tmp, x, y, r) j = int(integer(y, r) & uint64(N-1)) - blockXOR(y, v[j*(32*r):], 32*r) + blockXOR(y, v[j*R:], R) blockMix(&tmp, y, x, r) } j = 0 - for _, v := range x[:32*r] { - b[j+0] = byte(v >> 0) - b[j+1] = byte(v >> 8) - b[j+2] = byte(v >> 16) - b[j+3] = byte(v >> 24) + for _, v := range x[:R] { + binary.LittleEndian.PutUint32(b[j:], v) j += 4 } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 3ce3e4db3c..aa0d1d11c9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -307,6 +307,13 @@ github.com/gregjones/httpcache github.com/gregjones/httpcache/diskcache # github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/go-grpc-prometheus +# github.com/h2non/filetype v1.1.1 +github.com/h2non/filetype +github.com/h2non/filetype/matchers +github.com/h2non/filetype/matchers/isobmff +github.com/h2non/filetype/types +# github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c +github.com/h2non/go-is-svg # github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru @@ -498,8 +505,10 @@ github.com/operator-framework/api/pkg/validation github.com/operator-framework/api/pkg/validation/errors github.com/operator-framework/api/pkg/validation/interfaces github.com/operator-framework/api/pkg/validation/internal -# github.com/operator-framework/operator-registry v1.13.6 +# github.com/operator-framework/operator-registry v1.17.5 ## explicit +github.com/operator-framework/operator-registry/internal/model +github.com/operator-framework/operator-registry/internal/property github.com/operator-framework/operator-registry/pkg/api github.com/operator-framework/operator-registry/pkg/api/grpc_health_v1 github.com/operator-framework/operator-registry/pkg/client @@ -509,6 +518,8 @@ github.com/operator-framework/operator-registry/pkg/image github.com/operator-framework/operator-registry/pkg/image/containerdregistry github.com/operator-framework/operator-registry/pkg/image/execregistry github.com/operator-framework/operator-registry/pkg/lib/bundle +github.com/operator-framework/operator-registry/pkg/lib/encoding +github.com/operator-framework/operator-registry/pkg/lib/semver github.com/operator-framework/operator-registry/pkg/lib/validation github.com/operator-framework/operator-registry/pkg/registry github.com/operator-framework/operator-registry/pkg/server @@ -630,7 +641,7 @@ go.uber.org/zap/internal/color go.uber.org/zap/internal/exit go.uber.org/zap/zapcore go.uber.org/zap/zapgrpc -# golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 +# golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish golang.org/x/crypto/cast5