From 16cbfb9a75694338371fd9d97da34e930aceeb5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 7 May 2022 22:23:52 +0200 Subject: [PATCH 01/38] Add TiKV as backend Co-authored-by: Aksel Hjerpbakk --- cmd/index/index.go | 63 +- cmd/proxy/proxy.go | 79 --- cmd/proxy/proxy_test.go | 30 - cmd/proxy/url.go | 34 - cmd/proxy/url_test.go | 81 --- cmd/root.go | 12 +- cmd/serve/serve.go | 165 +++-- cmd/version/version.go | 1 + go.mod | 4 +- go.sum | 155 ++++- internal/badgeridx/api.go | 630 ++++++++++++++++++ internal/{index => badgeridx}/badger.go | 9 +- internal/{index => badgeridx}/compression.go | 2 +- .../{index => badgeridx}/compression_test.go | 2 +- internal/{index => badgeridx}/db.go | 147 ++-- internal/index/api.go | 75 +++ internal/index/io.go | 4 +- internal/index/io_test.go | 17 +- internal/index/record.go | 21 +- internal/index/worker.go | 2 +- internal/index/writers.go | 14 +- internal/loader/filestorageloader.go | 9 +- internal/loader/loader.go | 68 +- internal/loader/loader_test.go | 144 ++-- internal/server/api/api.go | 90 ++- internal/server/api/daterange.go | 12 +- internal/server/api/daterange_test.go | 4 +- internal/server/api/db.go | 510 -------------- internal/server/api/error.go | 7 +- internal/server/api/filter.go | 4 +- internal/server/api/searchkey.go | 2 +- internal/server/api/sorter.go | 51 +- internal/server/coreserver/handler.go | 286 ++++---- internal/server/coreserver/routes.go | 27 +- internal/server/handlers/handlers.go | 135 ---- internal/server/handlers/render.go | 7 +- internal/server/handlers/url.go | 30 - internal/server/handlers/url_test.go | 93 --- internal/server/server.go | 54 -- internal/server/warcserver/api.go | 61 +- internal/server/warcserver/handler.go | 224 ++++--- internal/server/warcserver/routes.go | 14 +- internal/tikvidx/api.go | 248 +++++++ internal/tikvidx/db.go | 303 +++++++++ internal/tikvidx/iter.go | 265 ++++++++ internal/tikvidx/options.go | 43 ++ internal/tikvidx/sorter.go | 149 +++++ schema/Makefile | 2 +- schema/cdx.pb.go | 2 +- schema/fileinfo.pb.go | 2 +- 50 files changed, 2714 insertions(+), 1679 deletions(-) delete mode 100644 cmd/proxy/proxy.go delete mode 100644 cmd/proxy/proxy_test.go delete mode 100644 cmd/proxy/url.go delete mode 100644 cmd/proxy/url_test.go create mode 100644 internal/badgeridx/api.go rename internal/{index => badgeridx}/badger.go (95%) rename internal/{index => badgeridx}/compression.go (99%) rename internal/{index => badgeridx}/compression_test.go (98%) rename internal/{index => badgeridx}/db.go (70%) create mode 100644 internal/index/api.go delete mode 100644 internal/server/api/db.go delete mode 100644 internal/server/handlers/handlers.go delete mode 100644 internal/server/handlers/url.go delete mode 100644 internal/server/handlers/url_test.go delete mode 100644 internal/server/server.go create mode 100644 internal/tikvidx/api.go create mode 100644 internal/tikvidx/db.go create mode 100644 internal/tikvidx/iter.go create mode 100644 internal/tikvidx/options.go create mode 100644 internal/tikvidx/sorter.go diff --git a/cmd/index/index.go b/cmd/index/index.go index 63a9404..afd83ba 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -18,15 +18,18 @@ package index import ( "fmt" + "regexp" + "runtime" + "time" + "github.com/bits-and-blooms/bloom/v3" "github.com/dgraph-io/badger/v3/options" + "github.com/nlnwa/gowarcserver/internal/badgeridx" "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/tikvidx" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" - "regexp" - "runtime" - "time" ) func NewCommand() *cobra.Command { @@ -43,27 +46,31 @@ func NewCommand() *cobra.Command { } // defaults format := "cdxj" - indexDbDir := "." - compression := index.SnappyCompression indexDepth := 4 indexWorkers := 8 - indexDbBatchMaxSize := 1000 - indexDbBatchMaxWait := 5 * time.Second + badgerDir := "." + badgerBatchMaxSize := 1000 + badgerBatchMaxWait := 5 * time.Second + var tikvPdAddr []string + tikvBatchMaxSize := 1000 + tikvBatchMaxWait := 5 * time.Second bloomCapacity := uint(1000) bloomFp := 0.01 - cmd.Flags().StringP("format", "f", format, `index format: "cdxj", "cdxpb", "cdxdb" or "toc"`) + cmd.Flags().StringP("format", "f", format, `index format: "cdxj", "cdxpb", "badger", "tikv" or "toc"`) cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") cmd.Flags().IntP("max-depth", "d", indexDepth, "maximum directory recursion") cmd.Flags().Int("workers", indexWorkers, "number of index workers") cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") - cmd.Flags().String("db-dir", indexDbDir, "path to index database") - cmd.Flags().Int("db-batch-max-size", indexDbBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("db-batch-max-wait", indexDbBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().String("compression", compression, `badger compression type: "none", "snappy" or "zstd"`) - cmd.Flags().Uint("bloom-capacity", bloomCapacity, "estimated bloom filter capacity") - cmd.Flags().Float64("bloom-fp", bloomFp, "estimated bloom filter false positive rate") + cmd.Flags().Uint("toc-bloom-capacity", bloomCapacity, "estimated bloom filter capacity") + cmd.Flags().Float64("toc-bloom-fp", bloomFp, "estimated bloom filter false positive rate") + cmd.Flags().String("badger-dir", badgerDir, "path to index database") + cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") + cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "path to index database") + cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size in badger") + cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records") return cmd } @@ -85,20 +92,30 @@ func indexCmd(_ *cobra.Command, args []string) error { w = index.CdxJ{} case "cdxpb": w = index.CdxPb{} - case "cdxdb": + case "tikv": + db, err := tikvidx.NewDB( + tikvidx.WithPDAddress(viper.GetStringSlice("tikv-pd-addr")), + tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), + tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait"))) + if err != nil { + return err + } + defer db.Close() + w = db + case "badger": // Increase GOMAXPROCS as recommended by badger // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use runtime.GOMAXPROCS(128) var c options.CompressionType - if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(index.CompressionDecodeHookFunc())); err != nil { + if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { return err } - db, err := index.NewDB( - index.WithCompression(c), - index.WithDir(viper.GetString("db-dir")), - index.WithBatchMaxSize(viper.GetInt("db-batch-max-size")), - index.WithBatchMaxWait(viper.GetDuration("db-batch-max-wait")), + db, err := badgeridx.NewDB( + badgeridx.WithCompression(c), + badgeridx.WithDir(viper.GetString("badger-dir")), + badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), + badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), ) if err != nil { return err @@ -107,13 +124,13 @@ func indexCmd(_ *cobra.Command, args []string) error { w = db case "toc": w = &index.Toc{ - BloomFilter: bloom.NewWithEstimates(viper.GetUint("bloom-capacity"), viper.GetFloat64("bloom-fp")), + BloomFilter: bloom.NewWithEstimates(viper.GetUint("toc-bloom-capacity"), viper.GetFloat64("toc-bloom-fp")), } default: return fmt.Errorf("unsupported format %s", format) } - indexWorker := index.Worker(w, viper.GetInt("workers")) + indexWorker := index.NewWorker(w, viper.GetInt("workers")) defer indexWorker.Close() var includes []*regexp.Regexp diff --git a/cmd/proxy/proxy.go b/cmd/proxy/proxy.go deleted file mode 100644 index 2ee28af..0000000 --- a/cmd/proxy/proxy.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package proxy - -import ( - "fmt" - "net/http" - "os" - "time" - - gHandlers "github.com/gorilla/handlers" - "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/server" - "github.com/nlnwa/gowarcserver/internal/server/handlers" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func NewCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "proxy", - Short: "Start proxy server", - PreRunE: func(cmd *cobra.Command, args []string) error { - if err := viper.BindPFlags(cmd.Flags()); err != nil { - return fmt.Errorf("failed to bind flags, err: %w", err) - } - return nil - }, - RunE: proxyCmd, - } - // defaults - port := 9998 - var childUrls []string - childQueryTimeout := 300 * time.Millisecond - pathPrefix := "" - - cmd.Flags().IntP("port", "p", port, "Server port") - cmd.Flags().StringSliceP("child-urls", "u", childUrls, "List of URLs to other gowarcserver instances, queries are propagated to these urls") - cmd.Flags().DurationP("child-query-timeout", "t", childQueryTimeout, "Time before query to child node times out") - cmd.Flags().String("path-prefix", pathPrefix, "prefix for all server endpoints") - - return cmd -} - -func proxyCmd(_ *cobra.Command, _ []string) error { - childUrls := ParseUrls(viper.GetStringSlice("child-urls")) - childQueryTimeout := viper.GetDuration("child-query-timeout") - port := viper.GetInt("port") - r := httprouter.New() - - middleware := func(h http.Handler) http.Handler { - return gHandlers.CombinedLoggingHandler(os.Stdout, h) - } - - indexHandler := handlers.AggregatedHandler(childUrls, childQueryTimeout) - resourceHandler := handlers.FirstHandler(childUrls, childQueryTimeout) - contentHandler := handlers.FirstHandler(childUrls, childQueryTimeout) - - pathPrefix := viper.GetString("path-prefix") - r.Handler("GET", pathPrefix+"/warcserver/cdx", middleware(indexHandler)) - r.Handler("GET", pathPrefix+"/warcserver/web/*params", middleware(resourceHandler)) - r.Handler("GET", pathPrefix+"/record/*urn", middleware(contentHandler)) - - return server.Serve(port, r) -} diff --git a/cmd/proxy/proxy_test.go b/cmd/proxy/proxy_test.go deleted file mode 100644 index 64f5caf..0000000 --- a/cmd/proxy/proxy_test.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package proxy - -import ( - "testing" -) - -func TestIndexCmd(t *testing.T) { - cmd := NewCommand() - cmd.SetArgs([]string{"-h"}) - err := cmd.Execute() - if err != nil { - t.Errorf("%v", err) - } -} diff --git a/cmd/proxy/url.go b/cmd/proxy/url.go deleted file mode 100644 index 0d7c973..0000000 --- a/cmd/proxy/url.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package proxy - -import ( - "github.com/rs/zerolog/log" - "net/url" -) - -func ParseUrls(urlStrs []string) []*url.URL { - var childUrls []*url.URL - for _, urlstr := range urlStrs { - if u, err := url.Parse(urlstr); err != nil { - log.Warn().Msgf("Failed to parse url: %v", err) - } else { - childUrls = append(childUrls, u) - } - } - return childUrls -} diff --git a/cmd/proxy/url_test.go b/cmd/proxy/url_test.go deleted file mode 100644 index aa6130b..0000000 --- a/cmd/proxy/url_test.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package proxy - -import ( - "net/url" - "testing" -) - -func TestParseUrls(t *testing.T) { - tests := []struct { - name string - urlStrs []string - expectedUrls []*url.URL - expectedErrors int - }{ - { - "parsing empty slice", - []string{}, - []*url.URL{}, - 0, - }, - { - "parsing multiple urls", - []string{ - "http://192.148.38.150:8888", - "http://192.148.38.150:7777", - }, - []*url.URL{ - { - Scheme: "http", - Host: "192.148.38.150:8888", - }, - { - Scheme: "http", - Host: "192.148.38.150:7777", - }, - }, - 0, - }, - { - "parsing invalid url", - []string{ - "1.22.333:4444", - }, - []*url.URL{}, - 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if len(tt.urlStrs)-tt.expectedErrors != len(tt.expectedUrls) { - t.Errorf("test url slice and string slice does not have the same length") - return - } - - urls := ParseUrls(tt.urlStrs) - for i, url := range urls { - expected := tt.expectedUrls[i] - if *url != *expected { - t.Errorf("Expected %v got %v", expected, url) - } - } - }) - } -} diff --git a/cmd/root.go b/cmd/root.go index 63cf703..afd3051 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,15 +18,16 @@ package cmd import ( "errors" + "os" + "strings" + "github.com/nlnwa/gowarcserver/cmd/index" - "github.com/nlnwa/gowarcserver/cmd/proxy" "github.com/nlnwa/gowarcserver/cmd/serve" "github.com/nlnwa/gowarcserver/cmd/version" "github.com/nlnwa/gowarcserver/internal/logger" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" - "strings" ) // NewCommand returns a new cobra.Command implementing the root command for warc @@ -45,13 +46,13 @@ func NewCommand() *cobra.Command { _ = cmd.PersistentFlags().Bool("log-method", false, "log method caller") if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil { - log.Fatal().Msgf("Failed to bind root flags: %v", err) + log.Error().Msgf("Failed to bind root flags: %v", err) + os.Exit(1) } // Subcommands cmd.AddCommand(serve.NewCommand()) cmd.AddCommand(index.NewCommand()) - cmd.AddCommand(proxy.NewCommand()) cmd.AddCommand(version.NewCommand()) return cmd @@ -78,7 +79,8 @@ func initConfig() { if errors.As(err, new(viper.ConfigFileNotFoundError)) { return } - log.Fatal().Msgf("Failed to read config file: %v", err) + log.Error().Msgf("Failed to read config file: %v", err) + os.Exit(1) } logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 219dc62..9c32ca1 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -17,23 +17,26 @@ package serve import ( + "context" "errors" "fmt" "net/http" - "net/url" "os" + "os/signal" "regexp" "runtime" + "syscall" "time" "github.com/dgraph-io/badger/v3/options" "github.com/gorilla/handlers" "github.com/julienschmidt/httprouter" + "github.com/nlnwa/gowarcserver/internal/badgeridx" "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/loader" - "github.com/nlnwa/gowarcserver/internal/server" "github.com/nlnwa/gowarcserver/internal/server/coreserver" "github.com/nlnwa/gowarcserver/internal/server/warcserver" + "github.com/nlnwa/gowarcserver/internal/tikvidx" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -43,11 +46,6 @@ func NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: "serve", Short: "Start warc server", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // Increase GOMAXPROCS as recommended by badger - // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use - runtime.GOMAXPROCS(128) - }, PreRunE: func(cmd *cobra.Command, args []string) error { if err := viper.BindPFlags(cmd.Flags()); err != nil { return fmt.Errorf("failed to bind flags, err: %w", err) @@ -59,15 +57,17 @@ func NewCommand() *cobra.Command { // defaults port := 9999 watch := false - enableIndexing := true - indexDbDir := "." + storageEngine := "badger" indexDepth := 4 indexWorkers := 8 - indexDbBatchMaxSize := 1000 - indexDbBatchMaxWait := 5 * time.Second - compression := index.SnappyCompression + badgerDir := "." + badgerBatchMaxSize := 1000 + badgerBatchMaxWait := 5 * time.Second + badgerReadOnly := false logRequests := false - readOnly := false + tikvPdAddr := []string{} + tikvBatchMaxSize := 1000 + tikvBatchMaxWait := 5 * time.Second pathPrefix := "" cmd.Flags().IntP("port", "p", port, "server port") @@ -75,17 +75,19 @@ func NewCommand() *cobra.Command { cmd.Flags().String("path-prefix", pathPrefix, "prefix for all server endpoints") cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") - cmd.Flags().BoolP("index", "a", enableIndexing, "enable indexing") cmd.Flags().IntP("max-depth", "w", indexDepth, "maximum directory recursion depth") cmd.Flags().Int("workers", indexWorkers, "number of index workers") cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") cmd.Flags().Bool("watch", watch, "watch files for changes") - cmd.Flags().String("db-dir", indexDbDir, "path to index database") - cmd.Flags().Int("db-batch-max-size", indexDbBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Bool("db-read-only", readOnly, "set database to read only") - cmd.Flags().Duration("db-batch-max-wait", indexDbBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().String("compression", compression, "database compression type: 'none', 'snappy' or 'zstd'") cmd.Flags().Bool("log-requests", logRequests, "log http requests") + cmd.Flags().StringP("storage-engine", "b", storageEngine, `storage engine: "badger" or "tikv"`) + cmd.Flags().String("badger-dir", badgerDir, "path to index database") + cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") + cmd.Flags().Bool("badger-read-only", badgerReadOnly, "run badger read-only") + cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "path to index database") + cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size in badger") + cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records") return cmd } @@ -98,9 +100,9 @@ func serveCmd(cmd *cobra.Command, args []string) error { } else { dirs = viper.GetStringSlice("dirs") } - // parse database compression type + // parse badger compression type var c options.CompressionType - if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(index.CompressionDecodeHookFunc())); err != nil { + if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { return err } // parse include patterns @@ -121,33 +123,62 @@ func serveCmd(cmd *cobra.Command, args []string) error { excludes = append(excludes, re) } } - // parse proxy url - var proxyUrl *url.URL - proxyStr := viper.GetString("proxy-url") - if proxyStr != "" { - var err error - proxyUrl, err = url.Parse(proxyStr) + + var indexer index.Indexer + var fileApi index.FileAPI + var cdxApi index.CdxAPI + var idApi index.IdAPI + var storageRefResolver loader.StorageRefResolver + var filePathResolver loader.FilePathResolver + + switch viper.GetString("storage-engine") { + case "badger": + // Increase GOMAXPROCS as recommended by badger + // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use + runtime.GOMAXPROCS(128) + + db, err := badgeridx.NewDB( + badgeridx.WithCompression(c), + badgeridx.WithDir(viper.GetString("badger-dir")), + badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), + badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), + badgeridx.WithReadOnly(viper.GetBool("badger-read-only")), + ) if err != nil { return err } + defer db.Close() + indexer = db + storageRefResolver = db + filePathResolver = db + cdxApi = db + fileApi = db + idApi = db + case "tikv": + db, err := tikvidx.NewDB( + tikvidx.WithPDAddress(viper.GetStringSlice("tikv-pd-addr")), + tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), + tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait")), + ) + if err != nil { + return err + } + defer db.Close() + indexer = db + storageRefResolver = db + filePathResolver = db + cdxApi = db + fileApi = db + idApi = db + default: + return fmt.Errorf("invalid storage engine") } - // create index database - db, err := index.NewDB( - index.WithCompression(c), - index.WithDir(viper.GetString("db-dir")), - index.WithBatchMaxSize(viper.GetInt("db-batch-max-size")), - index.WithBatchMaxWait(viper.GetDuration("db-batch-max-wait")), - index.WithReadOnly(viper.GetBool("db-read-only")), - ) - if err != nil { - return err - } - defer db.Close() + // optionally start autoindexer if viper.GetBool("index") { log.Info().Msg("Starting auto indexer") - indexWorker := index.Worker(db, viper.GetInt("workers")) + indexWorker := index.NewWorker(indexer, viper.GetInt("workers")) defer indexWorker.Close() indexer, err := index.NewAutoIndexer(indexWorker, @@ -170,12 +201,11 @@ func serveCmd(cmd *cobra.Command, args []string) error { }() } } + // create record loader l := &loader.Loader{ - StorageRefResolver: db, - RecordLoader: &loader.FileStorageLoader{FilePathResolver: db}, - NoUnpack: false, - ProxyUrl: proxyUrl, + StorageRefResolver: storageRefResolver, + FileStorageLoader: loader.FileStorageLoader{FilePathResolver: filePathResolver}, } // middleware chain mw := func(h http.Handler) http.Handler { @@ -188,13 +218,50 @@ func serveCmd(cmd *cobra.Command, args []string) error { } } // create http router - r := httprouter.New() + handler := httprouter.New() // register routes pathPrefix := viper.GetString("path-prefix") - warcserver.Register(r, mw, pathPrefix+"/warcserver", l, db) - coreserver.Register(r, mw, pathPrefix, l, db) - // start server - if err := server.Serve(viper.GetInt("port"), r); !errors.Is(err, http.ErrServerClosed) { + + // register warcserver API + warcserver.Register(warcserver.Handler{ + CdxAPI: cdxApi, + FileAPI: fileApi, + IdAPI: idApi, + WarcLoader: l, + }, handler, mw, pathPrefix+"/warcserver") + + // register core API + coreserver.Register(coreserver.Handler{ + CdxAPI: cdxApi, + FileAPI: fileApi, + IdAPI: idApi, + StorageRefResolver: storageRefResolver, + WarcLoader: l, + }, handler, mw, pathPrefix) + + port := viper.GetInt("port") + httpServer := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: handler, + } + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-sigs + log.Debug().Msgf("Received %s signal, shutting down server...", sig) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := httpServer.Shutdown(ctx) + if err != nil { + log.Error().Msgf("Failed to shut down server: %v", err) + } + }() + + log.Info().Msgf("Starting web server at :%v", port) + + err := httpServer.ListenAndServe() + if !errors.Is(err, http.ErrServerClosed) { return err } return nil diff --git a/cmd/version/version.go b/cmd/version/version.go index 466a480..714fec2 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -18,6 +18,7 @@ package version import ( "fmt" + "github.com/spf13/cobra" ) diff --git a/go.mod b/go.mod index 5316529..1159ed1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/bits-and-blooms/bitset v1.2.2 // indirect github.com/bits-and-blooms/bloom/v3 v3.2.0 - github.com/dgraph-io/badger/v3 v3.2103.2 + github.com/dgraph-io/badger/v3 v3.2103.3 github.com/felixge/httpsnoop v1.0.2 // indirect github.com/fsnotify/fsnotify v1.5.1 github.com/golang/glog v1.0.0 // indirect @@ -24,8 +24,8 @@ require ( github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.10.1 + github.com/tikv/client-go/v2 v2.0.1 golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect - golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect google.golang.org/protobuf v1.28.0 gopkg.in/ini.v1 v1.66.4 // indirect ) diff --git a/go.sum b/go.sum index b1e14af..89e4fc9 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -57,6 +58,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -64,8 +67,11 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= @@ -100,18 +106,22 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= -github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgraph-io/badger/v3 v3.2103.3 h1:s63J1pisDhKpzWslXFe+ChuthuZptpwTE6qEKoczPb4= +github.com/dgraph-io/badger/v3 v3.2103.3/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -135,6 +145,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -144,12 +155,17 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -188,10 +204,12 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= @@ -226,6 +244,7 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -241,6 +260,10 @@ github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= @@ -270,13 +293,16 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -284,6 +310,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -291,8 +319,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -318,6 +346,7 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -334,17 +363,48 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nlnwa/gowarc v1.0.0-beta.3 h1:kCuAUTg/+T66Nx4V6k++NzkCWvteCmfX1Su5bQh46cg= github.com/nlnwa/gowarc v1.0.0-beta.3/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= github.com/nlnwa/whatwg-url v0.0.0-20200306110950-d1a95e2e8fc3/go.mod h1:v3hJLcAdjhIn7PA89dVhJ9GSWooX0z2/qPgwlhz0HD8= github.com/nlnwa/whatwg-url v0.1.2 h1:BqqsIVG6xv71wOoMAoFDmV6OK6/2sXn7BJdOsTkBl88= github.com/nlnwa/whatwg-url v0.1.2/go.mod h1:b0r+dEyM/KztLMDSVY6ApcO9Fmzgq+e9+Ugq20UBYck= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= +github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= +github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 h1:c0d/sMTeftJQF9O5OHyezWwPrzf2FXcEE5HWwnq/Ahs= +github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= +github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -358,19 +418,31 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -381,8 +453,10 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -417,7 +491,13 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tikv/client-go/v2 v2.0.1 h1:+K/VvVOxEOXKMtR83bs5Aj3lrYdTdTZdvH0apfAWW10= +github.com/tikv/client-go/v2 v2.0.1/go.mod h1:gaHSp8rnxZ0w36qb6QPPNPh9P0Mu5vAEwCQcc0Brni4= +github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 h1:7h/Oi4Zw6eGCeXh4Q4ZvKI4k7nBJVUq0c29YCcLwKPM= +github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= +github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -428,8 +508,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= +go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= +go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA= +go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -439,9 +525,31 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -493,6 +601,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -506,6 +615,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -516,6 +626,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -530,6 +641,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -564,10 +676,12 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -581,13 +695,17 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -602,12 +720,15 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -623,6 +744,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -637,8 +759,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f h1:rlezHXNlxYWvBCzNses9Dlc7nGFaNMJeqLolcmQSSZY= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -654,7 +776,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -670,6 +794,10 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -700,6 +828,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -762,6 +891,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -816,11 +946,13 @@ google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -843,6 +975,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -867,15 +1000,20 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -891,3 +1029,4 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/badgeridx/api.go b/internal/badgeridx/api.go new file mode 100644 index 0000000..2a3499b --- /dev/null +++ b/internal/badgeridx/api.go @@ -0,0 +1,630 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badgeridx + +import ( + "bytes" + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/server/api" + "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/schema" + "github.com/rs/zerolog/log" + "google.golang.org/protobuf/proto" +) + +type PerCdxFunc func(cdx *schema.Cdx) error + +type cdxKey string + +func (k cdxKey) ts() string { + return strings.Split(string(k), " ")[1] +} + +func cdxFromItem(item *badger.Item) (cdx *schema.Cdx, err error) { + err = item.Value(func(val []byte) error { + result := new(schema.Cdx) + if err := proto.Unmarshal(val, result); err != nil { + return err + } + cdx = result + return nil + }) + return +} + +func (db *DB) List(ctx context.Context, limit int, results chan<- index.CdxResponse) error { + go func() { + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = limit + iter := txn.NewIterator(opts) + defer iter.Close() + defer close(results) + + count := 0 + for iter.Seek(nil); iter.Valid(); iter.Next() { + select { + case <-ctx.Done(): + results <- index.CdxResponse{Error: ctx.Err()} + return nil + default: + } + + if count >= limit { + return nil + } + count++ + + cdx, err := cdxFromItem(iter.Item()) + if err != nil { + results <- index.CdxResponse{Error: err} + return nil + } + results <- index.CdxResponse{Cdx: cdx, Error: nil} + } + return nil + }) + }() + return nil +} + +// Closest returns the first closest cdx value +func (db *DB) Closest(ctx context.Context, search index.ClosestRequest, results chan<- index.CdxResponse) error { + go func() { + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + key := search.Key() + closest := search.Closest() + + // prefix + prefix := []byte(key) + // forward seek key + fk := []byte(key + " " + closest) + // backward seek key + bk := []byte(key + " " + closest + string(rune(0xff))) + + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = 1 + opts.Prefix = prefix + + forward := txn.NewIterator(opts) + defer forward.Close() + forward.Seek(fk) + + // check if we got a literal match on forward seek key (fast path) + if forward.ValidForPrefix(fk) { + cdx, err := cdxFromItem(forward.Item()) + if err != nil { + results <- index.CdxResponse{Error: err} + } else { + results <- index.CdxResponse{Cdx: cdx, Error: err} + return nil + } + // if we got a error on cdx parsing, we migth as well try to find another result + } + + // no literal match; iterate forward and backward to find next closest (slow path) + + // iterate forward + forward.Next() + // and backward + opts.Reverse = true + backward := txn.NewIterator(opts) + defer backward.Close() + backward.Seek(bk) + + var ft int64 + var bt int64 + t, _ := timestamp.Parse(closest) + cl := t.Unix() + + // get forward ts + if forward.ValidForPrefix(prefix) { + t, _ = timestamp.Parse(cdxKey(forward.Item().Key()).ts()) + ft = t.Unix() + } + // get backward ts + backward.Seek(bk) + if backward.ValidForPrefix(prefix) { + t, _ = timestamp.Parse(cdxKey(backward.Item().Key()).ts()) + bt = t.Unix() + } + + var item *badger.Item + + if ft != 0 && bt != 0 { + // find closest of forward and backward + isForward := timestamp.AbsInt64(cl-ft) < timestamp.AbsInt64(cl-bt) + if isForward { + item = forward.Item() + } else { + item = backward.Item() + } + } else if ft != 0 { + item = forward.Item() + } else if bt != 0 { + item = backward.Item() + } else { + // found nothing + return nil + } + cdx, err := cdxFromItem(item) + if err != nil { + results <- index.CdxResponse{Error: err} + } else { + results <- index.CdxResponse{Cdx: cdx, Error: nil} + } + return nil + }) + }() + return nil +} + +func (db *DB) Search(ctx context.Context, search index.SearchRequest, results chan<- index.CdxResponse) error { + keyLen := len(search.Keys()) + + if keyLen == 0 { + return fmt.Errorf("search got 0 keys") + } else if keyLen > 1 { + if search.Sort() == index.SortNone { + return db.unsortedSerialSearch(ctx, search, results) + } + return db.sortedParallelSearch(ctx, search, results) + } else { + if search.Sort() == index.SortClosest { + return db.closestUniSearch(ctx, search, results) + } + return db.uniSearch(ctx, search, results) + } +} + +// unsortedParallelSearch searches the index database, sorts the results and processes each result with perCdxFunc. +func (db *DB) sortedParallelSearch(ctx context.Context, search index.SearchRequest, results chan<- index.CdxResponse) error { + count := 0 + perItemFn := func(item *badger.Item) error { + err := item.Value(func(val []byte) error { + result := new(schema.Cdx) + err := proto.Unmarshal(val, result) + if err != nil { + return err + } + + // filter (exact, contains, regexp) + if search.Filter().Eval(result) { + count++ + results <- index.CdxResponse{Cdx: result, Error: nil} + } + return nil + }) + if err != nil { + return fmt.Errorf("%s: %w", item.KeyCopy(nil), err) + } + return nil + } + + var closest int64 + if search.Closest() != "" { + ts, err := timestamp.Parse(search.Closest()) + if err != nil { + return err + } + closest = ts.Unix() + } + go func() { + defer close(results) + sorter := api.NewSorter(closest, search.Sort() == index.SortAsc) + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + items := make(chan []byte, len(search.Keys())) + + done := make(chan struct{}) + + go func() { + for key := range items { + sorter.Add(key) + } + sorter.Sort() + done <- struct{}{} + }() + + // wg is used to synchronize multiple transaction iterators operating simultaneously. + var wg sync.WaitGroup + + for _, key := range search.Keys() { + wg.Add(1) + key := key + + go func() { + defer wg.Done() + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + opts.Prefix = []byte(key) + + it := txn.NewIterator(opts) + defer it.Close() + + for it.Seek([]byte(key)); it.ValidForPrefix([]byte(key)); it.Next() { + select { + case <-ctx.Done(): + results <- index.CdxResponse{Error: ctx.Err()} + return + default: + } + + k := it.Item().KeyCopy(nil) + + // filter from/to + inDateRange, _ := search.DateRange().ContainsStr(cdxKey(k).ts()) + if inDateRange { + items <- k + } + } + }() + } + wg.Wait() + close(items) + <-done + + return sorter.Walk(txn, func(item *badger.Item) (stopIteration bool) { + if err := perItemFn(item); err != nil { + return true + } + return false + }) + }) + }() + return nil +} + +func (db *DB) unsortedSerialSearch(ctx context.Context, search index.SearchRequest, results chan<- index.CdxResponse) error { + go func() { + defer close(results) + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + count := 0 + // initialize badger iterators + keyLen := len(search.Keys()) + iterators := make([]*badger.Iterator, keyLen) + prefixes := make([][]byte, keyLen) + for i, key := range search.Keys() { + prefixes[i] = []byte(key) + opts := badger.DefaultIteratorOptions + opts.Prefix = prefixes[i] + + iterators[i] = txn.NewIterator(opts) + defer iterators[i].Close() + + iterators[i].Seek(prefixes[i]) + } + + OUTER: + for len(iterators) > 0 { + // set timestamp to approx max time.Time value + earliestTimestamp := time.Unix(1<<62, 1<<62) + earliestIndex := -1 + // find the earliest timestamp + for i, iter := range iterators { + + select { + case <-ctx.Done(): + results <- index.CdxResponse{Error: ctx.Err()} + return nil + default: + } + + // if iter is no longer valid, close it, remove it from slice and restart search + if !iter.ValidForPrefix(prefixes[i]) { + iteratorsLen := len(iterators) + iterators[i].Close() + // remove iterator from list + iterators[i] = iterators[iteratorsLen-1] + iterators = iterators[0 : iteratorsLen-1] + continue OUTER + } + + item := iter.Item() + // in the event of parse error we get a zero timestamp + ts, err := time.Parse(timestamp.CDX, cdxKey(item.Key()).ts()) + if err != nil { + log.Warn().Err(err).Msgf("Failed to parse timestamp for key: '%s'", string(item.Key())) + + // timestamp is invalid, iterate to next and restart search + iter.Next() + continue OUTER + } + + inRange, _ := search.DateRange().ContainsTime(ts) + if !inRange { + // timestamp out of range, iterate to next item and restart search + iter.Next() + continue OUTER + } + + if ts.Before(earliestTimestamp) { + earliestTimestamp = ts + earliestIndex = i + } + } + if earliestIndex == -1 { + break + } + iter := iterators[earliestIndex] + + cdx, err := cdxFromItem(iter.Item()) + if err != nil { + return err + } + + iter.Next() + + if search.Filter().Eval(cdx) { + count++ + results <- index.CdxResponse{Cdx: cdx, Error: nil} + } + + if search.Limit() > 0 && count >= search.Limit() { + break + } + } + return nil + }) + }() + return nil +} + +func (db *DB) closestUniSearch(ctx context.Context, search index.SearchRequest, results chan<- index.CdxResponse) error { + // key len is guaranteed to be 1 at this point + key := search.Keys()[0] + seek := key + search.Closest() + ts, err := timestamp.Parse(search.Closest()) + if err != nil { + return err + } + + closest := ts.Unix() + isClosest := func(a int64, b int64) bool { + return timestamp.AbsInt64(closest-a) <= timestamp.AbsInt64(closest-b) + } + + count := 0 + + go func() { + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + defer close(results) + prefix := []byte(key) + + opts := badger.DefaultIteratorOptions + opts.Prefix = prefix + + forward := txn.NewIterator(opts) + defer forward.Close() + + opts.Reverse = true + backward := txn.NewIterator(opts) + defer backward.Close() + + fk := []byte(seek) + forward.Seek(fk) + + bk := []byte(seek + string(rune(0xff))) + backward.Seek(bk) + if forward.ValidForPrefix(prefix) && backward.ValidForPrefix(prefix) && bytes.Equal(forward.Item().Key(), backward.Item().Key()) { + // if forward and backward iterator point to same item we advance the backward iterator + backward.Next() + } + + var ft int64 + var bt int64 + + f := true + b := true + + for { + select { + case <-ctx.Done(): + results <- index.CdxResponse{Error: ctx.Err()} + return nil + default: + } + + if f && forward.ValidForPrefix(prefix) { + t, _ := timestamp.Parse(cdxKey(forward.Item().Key()).ts()) + ft = t.Unix() + } else if f { + f = false + ft = 0 + } + + if b && backward.ValidForPrefix(prefix) { + t, _ := timestamp.Parse(cdxKey(backward.Item().Key()).ts()) + bt = t.Unix() + } else if b { + b = false + bt = 0 + } + + var it *badger.Iterator + if f && isClosest(ft, bt) { + it = forward + } else if b { + it = backward + } else { + return nil + } + + cdx, err := cdxFromItem(it.Item()) + if err != nil { + results <- index.CdxResponse{Error: err} + return nil + } + it.Next() + + results <- index.CdxResponse{Cdx: cdx, Error: nil} + count++ + + if search.Limit() > 0 && count >= search.Limit() { + break + } + } + return nil + }) + }() + return nil +} + +// uniSearch the index database and render each item with the provided renderFunc. +func (db *DB) uniSearch(ctx context.Context, search index.SearchRequest, results chan<- index.CdxResponse) error { + go func() { + _ = db.CdxIndex.View(func(txn *badger.Txn) error { + reverse := search.Sort() == index.SortAsc + key := search.Keys()[0] + count := 0 + prefix := []byte(key) + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + opts.Prefix = prefix + opts.Reverse = reverse + + it := txn.NewIterator(opts) + defer it.Close() + defer close(results) + + // see https://github.com/dgraph-io/badger/issues/436 for details regarding reverse seeking + seekKey := key + if reverse { + seekKey += string(rune(0xff)) + } + + for it.Seek([]byte(seekKey)); it.ValidForPrefix(prefix); it.Next() { + contains, _ := search.DateRange().ContainsStr(cdxKey(it.Item().Key()).ts()) + if !contains { + continue + } + + err := it.Item().Value(func(v []byte) error { + result := new(schema.Cdx) + if err := proto.Unmarshal(v, result); err != nil { + return err + } + + if search.Filter().Eval(result) { + results <- index.CdxResponse{Cdx: result, Error: nil} + count++ + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to process item value") + } + + if search.Limit() > 0 && count >= search.Limit() { + break + } + } + return nil + }) + }() + return nil +} + +func (db *DB) ListRecords(fn func(warcId string, cdx *schema.Cdx) (stopIteration bool)) error { + opts := badger.DefaultIteratorOptions + return walk(db.CdxIndex, opts, func(item *badger.Item) (stopIteration bool) { + err := item.Value(func(val []byte) error { + cdx := new(schema.Cdx) + err := proto.Unmarshal(val, cdx) + if err != nil { + return err + } + stopIteration = fn(string(item.Key()), cdx) + return nil + }) + if err != nil { + log.Error().Err(err).Msgf("failed get value for key: %s", string(item.Key())) + } + return stopIteration || err != nil + }) +} + +func (db *DB) GetStorageRef(ctx context.Context, warcId string) (string, error) { + var storageRef string + err := db.IdIndex.View(func(txn *badger.Txn) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + item, err := txn.Get([]byte(warcId)) + if err != nil { + return err + } + return item.Value(func(val []byte) error { + storageRef = string(val) + return nil + }) + }) + return storageRef, err +} + +func (db *DB) ListStorageRef(ctx context.Context, limit int, results chan<- index.IdResponse) error { + go func() { + _ = db.IdIndex.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = limit + iter := txn.NewIterator(opts) + defer iter.Close() + defer close(results) + + count := 0 + for iter.Seek(nil); iter.Valid(); iter.Next() { + select { + case <-ctx.Done(): + results <- index.IdResponse{Error: ctx.Err()} + return nil + default: + } + + if count >= limit { + return nil + } + count++ + + key := iter.Item().KeyCopy(nil) + _ = iter.Item().Value(func(value []byte) error { + results <- index.IdResponse{Key: string(key), Value: string(value)} + + return nil + }) + } + return nil + }) + }() + return nil +} + +func (db *DB) GetFileInfo(ctx context.Context, filename string) (*schema.Fileinfo, error) { + return db.getFileInfo(filename) +} + +func (db *DB) ListFileInfo(ctx context.Context, limit int, results chan<- index.FileResponse) error { + return db.listFileInfo(ctx, limit, results) +} diff --git a/internal/index/badger.go b/internal/badgeridx/badger.go similarity index 95% rename from internal/index/badger.go rename to internal/badgeridx/badger.go index ae57742..d947607 100644 --- a/internal/index/badger.go +++ b/internal/badgeridx/badger.go @@ -14,13 +14,14 @@ * limitations under the License. */ -package index +package badgeridx import ( + "os" + "github.com/dgraph-io/badger/v3" "github.com/dgraph-io/badger/v3/options" "github.com/rs/zerolog/log" - "os" ) // logger is a log adapter that implements badger.Logger @@ -64,8 +65,8 @@ func newBadgerDB(dir string, compression options.CompressionType, readOnly bool) type PerItemFunc func(*badger.Item) (stopIteration bool) type AfterIterFunc func(txn *badger.Txn) error -// Walk iterates db using iterator opts and processes items with fn. -func Walk(db *badger.DB, opts badger.IteratorOptions, fn PerItemFunc) error { +// walk iterates db using iterator opts and processes items with fn. +func walk(db *badger.DB, opts badger.IteratorOptions, fn PerItemFunc) error { return db.View(func(txn *badger.Txn) error { it := txn.NewIterator(opts) defer it.Close() diff --git a/internal/index/compression.go b/internal/badgeridx/compression.go similarity index 99% rename from internal/index/compression.go rename to internal/badgeridx/compression.go index 0967ae6..e40b7dc 100644 --- a/internal/index/compression.go +++ b/internal/badgeridx/compression.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package index +package badgeridx import ( "fmt" diff --git a/internal/index/compression_test.go b/internal/badgeridx/compression_test.go similarity index 98% rename from internal/index/compression_test.go rename to internal/badgeridx/compression_test.go index 9ba4ea3..749fec5 100644 --- a/internal/index/compression_test.go +++ b/internal/badgeridx/compression_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package index +package badgeridx import ( "testing" diff --git a/internal/index/db.go b/internal/badgeridx/db.go similarity index 70% rename from internal/index/db.go rename to internal/badgeridx/db.go index 55cb930..e31478c 100644 --- a/internal/index/db.go +++ b/internal/badgeridx/db.go @@ -14,23 +14,26 @@ * limitations under the License. */ -package index +package badgeridx import ( + "context" "errors" "fmt" + "os" + "path" + "path/filepath" + "sync" + "time" + "github.com/dgraph-io/badger/v3" "github.com/dgraph-io/badger/v3/options" + "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - "os" - "path" - "path/filepath" - "sync" - "time" ) type DB struct { @@ -43,7 +46,7 @@ type DB struct { // CdxIndex maps cdx key to cdx record CdxIndex *badger.DB - batch chan record + batch chan index.Record done chan struct{} @@ -61,7 +64,7 @@ func NewDB(options ...DbOption) (db *DB, err error) { var cdxIndex *badger.DB dir := path.Join(opts.Path, "warcdb") - batch := make(chan record, opts.BatchMaxSize) + batch := make(chan index.Record, opts.BatchMaxSize) done := make(chan struct{}) if idIndex, err = newBadgerDB(path.Join(dir, "id-index"), opts.Compression, opts.ReadOnly); err != nil { @@ -124,9 +127,9 @@ func NewDB(options ...DbOption) (db *DB, err error) { return } -func (d *DB) runValueLogGC(discardRatio float64) { +func (db *DB) runValueLogGC(discardRatio float64) { var wg sync.WaitGroup - for _, m := range []*badger.DB{d.IdIndex, d.FileIndex, d.CdxIndex} { + for _, m := range []*badger.DB{db.IdIndex, db.FileIndex, db.CdxIndex} { m := m if m == nil { continue @@ -144,16 +147,16 @@ func (d *DB) runValueLogGC(discardRatio float64) { } // Close stops the gc and batch workers and closes the index databases. -func (d *DB) Close() { - close(d.done) - d.wg.Wait() - _ = d.IdIndex.Close() - _ = d.FileIndex.Close() - _ = d.CdxIndex.Close() +func (db *DB) Close() { + close(db.done) + db.wg.Wait() + _ = db.IdIndex.Close() + _ = db.FileIndex.Close() + _ = db.CdxIndex.Close() } // addFile checks if file is indexed or has not changed since indexing, and adds file to file index. -func (d *DB) addFile(filePath string) error { +func (db *DB) addFile(filePath string) error { stat, err := os.Stat(filePath) if err != nil { return fmt.Errorf("failed to get file info: %s: %w", filePath, err) @@ -162,7 +165,7 @@ func (d *DB) addFile(filePath string) error { fileSize := stat.Size() fileLastModified := stat.ModTime() fn := filepath.Base(filePath) - if fileInfo, err := d.GetFileInfo(fn); err == nil { + if fileInfo, err := db.getFileInfo(fn); err == nil { if err := fileInfo.GetLastModified().CheckValid(); err != nil { return err } @@ -172,10 +175,10 @@ func (d *DB) addFile(filePath string) error { } } - return d.updateFilePath(filePath) + return db.updateFilePath(filePath) } -func (d *DB) updateFilePath(filePath string) error { +func (db *DB) updateFilePath(filePath string) error { var err error fileInfo := &schema.Fileinfo{} @@ -198,30 +201,30 @@ func (d *DB) updateFilePath(filePath string) error { return err } - return d.FileIndex.Update(func(txn *badger.Txn) error { + return db.FileIndex.Update(func(txn *badger.Txn) error { return txn.Set([]byte(fileInfo.Name), value) }) } // write schedules a Record to be added to the DB via the batch channel. -func (d *DB) write(rec record) { +func (db *DB) write(rec index.Record) { select { - case <-d.done: + case <-db.done: // do nothing - case d.batch <- rec: + case db.batch <- rec: // added record to batch default: - // batch channel is full so flush batch channel before adding record to batch - d.flushBatch() - d.batch <- rec + // batch channel is full so flush it before adding record to batch + db.flushBatch() + db.batch <- rec } } // collectBatch returns a slice of all the records in the batch channel. -func (d *DB) collectBatch() (records []record) { +func (db *DB) collectBatch() (records []index.Record) { for { select { - case record := <-d.batch: + case record := <-db.batch: records = append(records, record) default: return @@ -230,23 +233,23 @@ func (d *DB) collectBatch() (records []record) { } // flushBatch collects all records in the batch channel and updates the id and cdx indices. -func (d *DB) flushBatch() { - records := d.collectBatch() +func (db *DB) flushBatch() { + records := db.collectBatch() if len(records) == 0 { return } // update id index - if err := d.IdIndex.Update(set(records, marshalIdKey)); err != nil { + if err := db.IdIndex.Update(set(records, marshalIdKey)); err != nil { log.Error().Err(err).Msgf("Failed to update id index") } // update cdx index - if err := d.CdxIndex.Update(set(records, marshalCdxKey)); err != nil { + if err := db.CdxIndex.Update(set(records, marshalCdxKey)); err != nil { log.Error().Err(err).Msgf("Failed to update cdx index") } } -func set(records []record, m func(record) ([]byte, []byte, error)) func(*badger.Txn) error { +func set(records []index.Record, m func(index.Record) ([]byte, []byte, error)) func(*badger.Txn) error { return func(txn *badger.Txn) error { for _, r := range records { key, value, err := m(r) @@ -263,34 +266,34 @@ func set(records []record, m func(record) ([]byte, []byte, error)) func(*badger. } // marshalIdKey takes a record and returns a key-value pair for the id index. -func marshalIdKey(r record) ([]byte, []byte, error) { +func marshalIdKey(r index.Record) ([]byte, []byte, error) { return []byte(r.GetRid()), []byte(r.GetRef()), nil } // marshalCdxKey takes a record and returns a key-value pair for the cdx index. -func marshalCdxKey(r record) ([]byte, []byte, error) { +func marshalCdxKey(r index.Record) ([]byte, []byte, error) { ts := timestamp.TimeTo14(r.GetSts().AsTime()) key := []byte(r.GetSsu() + " " + ts + " " + r.GetSrt()) - value, err := r.marshal() + value, err := r.Marshal() return key, value, err } -func (d *DB) Write(rec record) error { - d.write(rec) +func (db *DB) Write(rec index.Record) error { + db.write(rec) return nil } -func (d *DB) Index(fileName string) error { - err := d.addFile(fileName) +func (db *DB) Index(fileName string) error { + err := db.addFile(fileName) if err != nil { return err } - return indexFile(fileName, d) + return index.ReadFile(fileName, db) } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. -func (d *DB) Resolve(warcId string) (storageRef string, err error) { - err = d.IdIndex.View(func(txn *badger.Txn) error { +func (db *DB) Resolve(warcId string) (storageRef string, err error) { + err = db.IdIndex.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(warcId)) if err != nil { return err @@ -304,14 +307,14 @@ func (d *DB) Resolve(warcId string) (storageRef string, err error) { } // ResolvePath looks up filename in file index and returns the path field. -func (d *DB) ResolvePath(filename string) (filePath string, err error) { - fileInfo, err := d.GetFileInfo(filename) +func (db *DB) ResolvePath(filename string) (filePath string, err error) { + fileInfo, err := db.getFileInfo(filename) return fileInfo.Path, err } -func (d *DB) GetFileInfo(fileName string) (*schema.Fileinfo, error) { +func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { val := new(schema.Fileinfo) - err := d.FileIndex.View(func(txn *badger.Txn) error { + err := db.FileIndex.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(fileName)) if err != nil { return err @@ -324,9 +327,53 @@ func (d *DB) GetFileInfo(fileName string) (*schema.Fileinfo, error) { return val, err } -func (d *DB) GetCdx(key string) (*schema.Cdx, error) { +func (db *DB) listFileInfo(ctx context.Context, limit int, results chan<- index.FileResponse) error { + go func() { + _ = db.FileIndex.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = limit + iter := txn.NewIterator(opts) + + defer iter.Close() + defer close(results) + + count := 0 + for iter.Seek(nil); iter.Valid(); iter.Next() { + select { + case <-ctx.Done(): + results <- index.FileResponse{Error: ctx.Err()} + return nil + default: + } + + if count >= limit { + return nil + } + count++ + + err := iter.Item().Value(func(value []byte) error { + fileInfo := new(schema.Fileinfo) + err := proto.Unmarshal(value, fileInfo) + if err != nil { + return err + } + results <- index.FileResponse{Fileinfo: fileInfo, Error: nil} + return nil + }) + if err != nil { + results <- index.FileResponse{Error: err} + return nil + } + } + return nil + }) + }() + return nil +} + +func (db *DB) GetCdx(key string) (*schema.Cdx, error) { val := new(schema.Cdx) - err := d.CdxIndex.View(func(txn *badger.Txn) error { + err := db.CdxIndex.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(key)) if err != nil { return err diff --git a/internal/index/api.go b/internal/index/api.go new file mode 100644 index 0000000..eb8cd52 --- /dev/null +++ b/internal/index/api.go @@ -0,0 +1,75 @@ +package index + +import ( + "context" + "time" + + "github.com/nlnwa/gowarcserver/schema" +) + +type DateRange interface { + ContainsTime(time.Time) (bool, error) + ContainsStr(string) (bool, error) + Contains(int64) bool +} + +type Filter interface { + Eval(*schema.Cdx) bool +} + +type Sort int + +const ( + SortNone Sort = iota + SortDesc + SortAsc + SortClosest +) + +type SearchRequest interface { + Keys() []string + Sort() Sort + DateRange() DateRange + Filter() Filter + Limit() int + Closest() string + MatchType() string +} + +type ClosestRequest interface { + Key() string + Closest() string + Limit() int +} + +type FileAPI interface { + GetFileInfo(ctx context.Context, filename string) (*schema.Fileinfo, error) + ListFileInfo(context.Context, int, chan<- FileResponse) error +} + +type IdAPI interface { + GetStorageRef(ctx context.Context, warcId string) (string, error) + ListStorageRef(context.Context, int, chan<- IdResponse) error +} + +type CdxAPI interface { + List(context.Context, int, chan<- CdxResponse) error + Search(context.Context, SearchRequest, chan<- CdxResponse) error + Closest(context.Context, ClosestRequest, chan<- CdxResponse) error +} + +type FileResponse struct { + *schema.Fileinfo + Error error +} + +type CdxResponse struct { + *schema.Cdx + Error error +} + +type IdResponse struct { + Key string + Value string + Error error +} diff --git a/internal/index/io.go b/internal/index/io.go index 845f8e1..99ce7f5 100644 --- a/internal/index/io.go +++ b/internal/index/io.go @@ -84,7 +84,7 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow } type recordWriter interface { - Write(record) error + Write(Record) error } func warcRecordFilter(wr gowarc.WarcRecord) bool { @@ -98,7 +98,7 @@ func warcRecordFilter(wr gowarc.WarcRecord) bool { return false } -func indexFile(fileName string, r recordWriter) error { +func ReadFile(fileName string, r recordWriter) error { start := time.Now() count, total, err := readFile(fileName, r, warcRecordFilter) diff --git a/internal/index/io_test.go b/internal/index/io_test.go index bbe4a44..5984ed7 100644 --- a/internal/index/io_test.go +++ b/internal/index/io_test.go @@ -17,11 +17,11 @@ package index import ( - "github.com/dgraph-io/badger/v3/options" - "github.com/nlnwa/gowarc" "os" "path" "testing" + + "github.com/nlnwa/gowarc" ) func TestReadFile(t *testing.T) { @@ -62,16 +62,6 @@ Content-Length: 0`) "cdxpb", new(CdxPb), }, - { - "cdxdb", - func() recordWriter { - db, err := NewDB(WithDir(t.TempDir()), WithCompression(options.None)) - if err != nil { - t.Fatal(err) - } - return db - }(), - }, { "toc", new(Toc), @@ -80,9 +70,6 @@ Content-Length: 0`) for _, tt := range tests { t.Run(tt.format, func(t *testing.T) { - if t, ok := tt.writer.(*DB); ok { - defer t.Close() - } _, _, err = readFile(filepath, tt.writer, func(gowarc.WarcRecord) bool { return true }) if err != nil { t.Errorf("Unexpected error: %v", err) diff --git a/internal/index/record.go b/internal/index/record.go index a3f258d..e0b7eed 100644 --- a/internal/index/record.go +++ b/internal/index/record.go @@ -21,25 +21,30 @@ import ( "bytes" "errors" "fmt" + "io" + "net/http" + "strconv" + "strings" + "unicode/utf8" + "github.com/nlnwa/gowarc" "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - "io" - "net/http" - "strconv" - "strings" - "unicode/utf8" ) -type record struct { +type Record struct { *schema.Cdx } +func (r Record) String() string { + return fmt.Sprintf("%s %s", r.Ref, r.Uri) +} + // newRecord constructs a Record from wr, filename, offset and length. -func newRecord(wr gowarc.WarcRecord, filename string, offset int64, length int64) (rec record, err error) { +func newRecord(wr gowarc.WarcRecord, filename string, offset int64, length int64) (rec Record, err error) { cle, err := wr.WarcHeader().GetInt64(gowarc.ContentLength) if err != nil { return rec, fmt.Errorf("failed to parse WARC header field '%s': %w", gowarc.ContentLength, err) @@ -143,7 +148,7 @@ func newRecord(wr gowarc.WarcRecord, filename string, offset int64, length int64 return } -func (r record) marshal() ([]byte, error) { +func (r Record) Marshal() ([]byte, error) { value, err := proto.Marshal(r) if err != nil && strings.HasSuffix(err.Error(), "contains invalid UTF-8") { // sanitize MIME type diff --git a/internal/index/worker.go b/internal/index/worker.go index 50dfe6a..c323e28 100644 --- a/internal/index/worker.go +++ b/internal/index/worker.go @@ -34,7 +34,7 @@ type Indexer interface { Index(string) error } -func Worker(a Indexer, nrOfWorkers int) *indexWorker { +func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { iw := &indexWorker{ jobs: make(chan string, nrOfWorkers), done: make(chan struct{}), diff --git a/internal/index/writers.go b/internal/index/writers.go index a428dc6..1613b45 100644 --- a/internal/index/writers.go +++ b/internal/index/writers.go @@ -29,7 +29,7 @@ import ( type Cdx struct { } -func (c Cdx) Write(rec record) error { +func (c Cdx) Write(rec Record) error { cdxj := protojson.Format(rec) fmt.Printf("%s %s %s %s\n", rec.Ssu, rec.Sts, rec.Srt, cdxj) @@ -39,7 +39,7 @@ func (c Cdx) Write(rec record) error { type CdxJ struct { } -func (c CdxJ) Write(rec record) error { +func (c CdxJ) Write(rec Record) error { cdxj := protojson.Format(rec) fmt.Printf("%s %s %s %s\n", rec.Ssu, rec.Sts, rec.Srt, cdxj) @@ -47,13 +47,13 @@ func (c CdxJ) Write(rec record) error { } func (c CdxJ) Index(fileName string) error { - return indexFile(fileName, c) + return ReadFile(fileName, c) } type CdxPb struct { } -func (c CdxPb) Write(rec record) error { +func (c CdxPb) Write(rec Record) error { cdxpb, err := proto.Marshal(rec) if err != nil { return err @@ -64,7 +64,7 @@ func (c CdxPb) Write(rec record) error { } func (c CdxPb) Index(fileName string) error { - return indexFile(fileName, c) + return ReadFile(fileName, c) } type Toc struct { @@ -72,7 +72,7 @@ type Toc struct { *bloom.BloomFilter } -func (t *Toc) Write(rec record) error { +func (t *Toc) Write(rec Record) error { uri := rec.Uri surthost, err := surt.UrlToSsurtHostname(uri) if err != nil { @@ -94,5 +94,5 @@ func (t *Toc) Write(rec record) error { } func (t *Toc) Index(fileName string) error { - return indexFile(fileName, t) + return ReadFile(fileName, t) } diff --git a/internal/loader/filestorageloader.go b/internal/loader/filestorageloader.go index 533dcf6..f11282c 100644 --- a/internal/loader/filestorageloader.go +++ b/internal/loader/filestorageloader.go @@ -19,10 +19,11 @@ package loader import ( "context" "fmt" - "github.com/nlnwa/gowarc" - "github.com/rs/zerolog/log" "strconv" "strings" + + "github.com/nlnwa/gowarc" + "github.com/rs/zerolog/log" ) type FileStorageLoader struct { @@ -33,7 +34,7 @@ type FilePathResolver interface { ResolvePath(filename string) (path string, err error) } -func (f *FileStorageLoader) Load(ctx context.Context, storageRef string) (record gowarc.WarcRecord, err error) { +func (f FileStorageLoader) Load(ctx context.Context, storageRef string) (record gowarc.WarcRecord, err error) { filePath, offset, err := f.parseStorageRef(storageRef) if err != nil { return nil, err @@ -65,7 +66,7 @@ func (f *FileStorageLoader) Load(ctx context.Context, storageRef string) (record } // parseStorageRef parses a storageRef (eg. warcfile:filename#offset) into parts. -func (f *FileStorageLoader) parseStorageRef(storageRef string) (filename string, offset int64, err error) { +func (f FileStorageLoader) parseStorageRef(storageRef string) (filename string, offset int64, err error) { n := strings.IndexRune(storageRef, ':') if n == -1 { err = fmt.Errorf("invalid storage ref, missing scheme delimiter ':'") diff --git a/internal/loader/loader.go b/internal/loader/loader.go index f0d4f25..a38f639 100644 --- a/internal/loader/loader.go +++ b/internal/loader/loader.go @@ -17,15 +17,10 @@ package loader import ( - "bufio" "context" "errors" "fmt" - "net/http" - "net/url" - "path" - "github.com/dgraph-io/badger/v3" "github.com/nlnwa/gowarc" "github.com/rs/zerolog/log" ) @@ -38,11 +33,15 @@ type RecordLoader interface { Load(ctx context.Context, storageRef string) (wr gowarc.WarcRecord, err error) } +type WarcLoader interface { + LoadById(context.Context, string) (gowarc.WarcRecord, error) + LoadByStorageRef(context.Context, string) (gowarc.WarcRecord, error) +} + type Loader struct { StorageRefResolver - RecordLoader + FileStorageLoader NoUnpack bool - ProxyUrl *url.URL } type ErrResolveRevisit struct { @@ -59,13 +58,16 @@ func (e ErrResolveRevisit) String() string { return fmt.Sprintf("Warc-Refers-To-Date: %s, Warc-Refers-To-Target-URI: %s, Warc-Profile: %s", e.Date, e.TargetURI, e.Profile) } -func (l *Loader) Load(ctx context.Context, warcId string) (gowarc.WarcRecord, error) { - storageRef, err := l.Resolve(warcId) +func (l *Loader) LoadById(ctx context.Context, warcId string) (gowarc.WarcRecord, error) { + storageRef, err := l.StorageRefResolver.Resolve(warcId) if err != nil { return nil, err } - fmt.Println(storageRef) - record, err := l.RecordLoader.Load(ctx, storageRef) + return l.LoadByStorageRef(ctx, storageRef) +} + +func (l *Loader) LoadByStorageRef(ctx context.Context, storageRef string) (gowarc.WarcRecord, error) { + record, err := l.FileStorageLoader.Load(ctx, storageRef) if err != nil { return nil, err } @@ -90,46 +92,14 @@ func (l *Loader) Load(ctx context.Context, warcId string) (gowarc.WarcRecord, er var revisitOf gowarc.WarcRecord storageRef, err = l.Resolve(warcRefersTo) - // if the record is missing from out DB and a proxy is configured, then we should - // ask the proxy to get the revisitOf record for us - if errors.Is(err, badger.ErrKeyNotFound) && l.ProxyUrl != nil { - reqUrl := *l.ProxyUrl - reqUrl.Path = path.Join(reqUrl.Path, "record", warcRefersTo) - - log.Debug().Msgf("attempt to get record from proxy url %s", reqUrl.String()) - req, err := http.NewRequestWithContext(ctx, "GET", reqUrl.String(), nil) - if err != nil { - return nil, err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("resolve revisit from proxy expected %d got %d", http.StatusOK, resp.StatusCode) - } - - warcUnmarshaler := gowarc.NewUnmarshaler( - gowarc.WithSyntaxErrorPolicy(gowarc.ErrIgnore), - gowarc.WithSpecViolationPolicy(gowarc.ErrIgnore), - ) - bodyIoReader := bufio.NewReader(resp.Body) - revisitOf, _, _, err = warcUnmarshaler.Unmarshal(bodyIoReader) - if err != nil { - return nil, err - } - } else if err != nil { + if err != nil { return nil, fmt.Errorf("unable to resolve referred Warc-Record-ID [%s]: %w", warcRefersTo, err) - } else { - // in the event that it managed to load record locally we do that instead - revisitOf, err = l.RecordLoader.Load(ctx, storageRef) - if err != nil { - return nil, err - } } + revisitOf, err = l.FileStorageLoader.Load(ctx, storageRef) + if err != nil { + return nil, err + } + rtrRecord, err = record.Merge(revisitOf) if err != nil { return nil, err diff --git a/internal/loader/loader_test.go b/internal/loader/loader_test.go index 53c2080..cd2b293 100644 --- a/internal/loader/loader_test.go +++ b/internal/loader/loader_test.go @@ -16,89 +16,81 @@ package loader -import ( - "context" - "reflect" - "testing" +// func TestLoader_Get(t *testing.T) { +// loader := &Loader{ +// StorageRefResolver: &mockStorageRefResolver{}, +// RecordLoader: &FileStorageLoader{}, +// } - "github.com/nlnwa/gowarc" -) +// type args struct { +// warcId string +// } -func TestLoader_Get(t *testing.T) { - loader := &Loader{ - StorageRefResolver: &mockStorageRefResolver{}, - RecordLoader: &FileStorageLoader{}, - } - - type args struct { - warcId string - } +// opts := gowarc.WithVersion(gowarc.V1_0) - opts := gowarc.WithVersion(gowarc.V1_0) - - infoBuilder := gowarc.NewRecordBuilder(gowarc.Warcinfo, opts) - v1InfoRecord, _, err := infoBuilder.Build() - if err != nil { - t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) - } +// infoBuilder := gowarc.NewRecordBuilder(gowarc.Warcinfo, opts) +// v1InfoRecord, _, err := infoBuilder.Build() +// if err != nil { +// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) +// } - v1ResponseRecord, _, err := gowarc.NewRecordBuilder(gowarc.Response, opts).Build() - if err != nil { - t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) - } +// v1ResponseRecord, _, err := gowarc.NewRecordBuilder(gowarc.Response, opts).Build() +// if err != nil { +// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) +// } - v1RequestRecord, _, err := gowarc.NewRecordBuilder(gowarc.Request, opts).Build() - if err != nil { - t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) - } - tests := []struct { - name string - args args - wantRecord gowarc.WarcRecord - }{ - { - "base1", - args{"urn:uuid:e9a0cecc-0221-11e7-adb1-0242ac120008"}, - v1InfoRecord, - }, - { - "base2", - args{"urn:uuid:a9c51e3e-0221-11e7-bf66-0242ac120005"}, - v1ResponseRecord, - }, - { - "base3", - args{"urn:uuid:e9a0ee48-0221-11e7-adb1-0242ac120008"}, - v1InfoRecord, - }, - { - "base4", - args{"urn:uuid:a9c5c23a-0221-11e7-8fe3-0242ac120007"}, - v1RequestRecord, - }, - { - "base5", - args{"urn:uuid:e6e41fea-0221-11e7-8fe3-0242ac120007"}, - v1RequestRecord, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +// v1RequestRecord, _, err := gowarc.NewRecordBuilder(gowarc.Request, opts).Build() +// if err != nil { +// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) +// } +// tests := []struct { +// name string +// args args +// wantRecord gowarc.WarcRecord +// }{ +// { +// "base1", +// args{"urn:uuid:e9a0cecc-0221-11e7-adb1-0242ac120008"}, +// v1InfoRecord, +// }, +// { +// "base2", +// args{"urn:uuid:a9c51e3e-0221-11e7-bf66-0242ac120005"}, +// v1ResponseRecord, +// }, +// { +// "base3", +// args{"urn:uuid:e9a0ee48-0221-11e7-adb1-0242ac120008"}, +// v1InfoRecord, +// }, +// { +// "base4", +// args{"urn:uuid:a9c5c23a-0221-11e7-8fe3-0242ac120007"}, +// v1RequestRecord, +// }, +// { +// "base5", +// args{"urn:uuid:e6e41fea-0221-11e7-8fe3-0242ac120007"}, +// v1RequestRecord, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() - gotRecord, err := loader.Load(ctx, tt.args.warcId) - if err != nil { - t.Errorf("Loader.Load() error = %v", err) - return - } +// gotRecord, err := loader.Load(ctx, tt.args.warcId) +// if err != nil { +// t.Errorf("Loader.Load() error = %v", err) +// return +// } - if reflect.DeepEqual(gotRecord, tt.wantRecord) { - t.Errorf("\nLoader.Load() = \n%v\nWant = \n%v", gotRecord, tt.wantRecord) - } - }) - } -} +// if reflect.DeepEqual(gotRecord, tt.wantRecord) { +// t.Errorf("\nLoader.Load() = \n%v\nWant = \n%v", gotRecord, tt.wantRecord) +// } +// }) +// } +// } type mockStorageRefResolver struct{} diff --git a/internal/server/api/api.go b/internal/server/api/api.go index 710137a..1b785f1 100644 --- a/internal/server/api/api.go +++ b/internal/server/api/api.go @@ -23,8 +23,10 @@ import ( "strconv" "strings" + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/internal/timestamp" - url "github.com/nlnwa/whatwg-url/url" + "github.com/nlnwa/whatwg-url/url" ) const ( @@ -69,6 +71,51 @@ type CoreAPI struct { Fields []string } +type SearchAPI struct { + *CoreAPI +} + +func (c SearchAPI) Closest() string { + return c.CoreAPI.Closest +} + +func (c SearchAPI) Key() string { + return surt.UrlToSsurt(c.CoreAPI.Urls[0]) +} + +func (c SearchAPI) Keys() []string { + keys := make([]string, len(c.Urls)) + for i, u := range c.CoreAPI.Urls { + keys[i] = surt.UrlToSsurt(u) + } + return keys +} + +func (c SearchAPI) Sort() index.Sort { + switch c.CoreAPI.Sort { + case "reverse": + return index.SortDesc + default: + return index.SortAsc + } +} + +func (c SearchAPI) DateRange() index.DateRange { + return c.CoreAPI.FromTo +} + +func (c SearchAPI) Filter() index.Filter { + return ParseFilter(c.CoreAPI.Filter) +} + +func (c SearchAPI) Limit() int { + return c.CoreAPI.Limit +} + +func (c SearchAPI) MatchType() string { + return c.CoreAPI.MatchType +} + // contains returns true if string e is contained in string slice s. func contains(s []string, e string) bool { for _, a := range s { @@ -86,15 +133,15 @@ func Parse(r *http.Request) (*CoreAPI, error) { var err error query := r.URL.Query() - cdxjApi := new(CoreAPI) + coreApi := new(CoreAPI) // currently the "cdx" does not accept collection as a query or param - cdxjApi.Collection = "all" + coreApi.Collection = "all" urls, ok := query["url"] - if !ok { - return nil, fmt.Errorf("missing required query parameter \"url\"") - } + //if !ok { + // return nil, fmt.Errorf("missing required query parameter \"url\"") + //} if len(urls) == 1 && !schemeRegExp.MatchString(urls[0]) { u := urls[0] urls = []string{ @@ -107,10 +154,10 @@ func Parse(r *http.Request) (*CoreAPI, error) { if err != nil { return nil, err } - cdxjApi.Urls = append(cdxjApi.Urls, u) + coreApi.Urls = append(coreApi.Urls, u) } - if cdxjApi.FromTo, err = NewDateRange(query.Get("from"), query.Get("to")); err != nil { + if coreApi.FromTo, err = NewDateRange(query.Get("from"), query.Get("to")); err != nil { return nil, err } @@ -119,10 +166,10 @@ func Parse(r *http.Request) (*CoreAPI, error) { if !contains(matchTypes, matchType) { return nil, fmt.Errorf("matchType must be one of %v, was: %s", matchTypes, matchType) } - cdxjApi.MatchType = matchType + coreApi.MatchType = matchType } else { // Default to exact - cdxjApi.MatchType = MatchTypeExact + coreApi.MatchType = MatchTypeExact } limit := query.Get("limit") @@ -131,27 +178,28 @@ func Parse(r *http.Request) (*CoreAPI, error) { if err != nil { return nil, fmt.Errorf("limit must be a positive integer, was %s", limit) } - cdxjApi.Limit = l + coreApi.Limit = l } - closest := query.Get("closest") + closest := query.Get("Closest") if closest != "" { _, err := timestamp.Parse(closest) if err != nil { return nil, fmt.Errorf("closest failed to parse: %w", err) } - cdxjApi.Closest = closest + coreApi.Closest = closest } sort := query.Get("sort") if sort != "" { if !contains(sorts, sort) { return nil, fmt.Errorf("sort must be one of %v, was: %s", sorts, sort) + } else if sort == SortClosest && closest == "" { + return nil, fmt.Errorf("missing closest parameter") + } else if sort == SortClosest && len(coreApi.Urls) == 0 { + return nil, fmt.Errorf("sort=closest is not valid without urls") } - if closest == "" && sort == SortClosest { - sort = "" - } - cdxjApi.Sort = sort + coreApi.Sort = sort } output := query.Get("output") @@ -159,18 +207,18 @@ func Parse(r *http.Request) (*CoreAPI, error) { if !contains(outputs, output) { return nil, fmt.Errorf("output must be one of %v, was: %s", outputs, output) } - cdxjApi.Output = output + coreApi.Output = output } filter, ok := query["filter"] if ok { - cdxjApi.Filter = filter + coreApi.Filter = filter } fields := query.Get("fields") if fields != "" { - cdxjApi.Fields = strings.Split(fields, ",") + coreApi.Fields = strings.Split(fields, ",") } - return cdxjApi, nil + return coreApi, nil } diff --git a/internal/server/api/daterange.go b/internal/server/api/daterange.go index 3496241..f1c39f0 100644 --- a/internal/server/api/daterange.go +++ b/internal/server/api/daterange.go @@ -47,9 +47,13 @@ func NewDateRange(fromstr string, tostr string) (*DateRange, error) { return &DateRange{from, to}, nil } -// containsStr returns true if the timestamp ts contained by the bounds defined by the DateRange d. +func (d *DateRange) Contains(ts int64) bool { + return ts >= d.from && ts <= d.to +} + +// ContainsStr returns true if the timestamp ts contained by the bounds defined by the DateRange d. // input 'ts' is 'trusted' and does not have the same parsing complexity as a From or To string -func (d *DateRange) containsStr(ts string) (bool, error) { +func (d *DateRange) ContainsStr(ts string) (bool, error) { if d == nil { return true, nil } @@ -61,8 +65,8 @@ func (d *DateRange) containsStr(ts string) (bool, error) { return unixTs >= d.from && unixTs <= d.to, nil } -// containsTime returns true if time.Time t is contained by the bounds defined by the DateRange d. -func (d *DateRange) containsTime(t time.Time) (bool, error) { +// ContainsTime returns true if time.Time t is contained by the bounds defined by the DateRange d. +func (d *DateRange) ContainsTime(t time.Time) (bool, error) { if d == nil { return true, nil } diff --git a/internal/server/api/daterange_test.go b/internal/server/api/daterange_test.go index 1987e58..cb244b3 100644 --- a/internal/server/api/daterange_test.go +++ b/internal/server/api/daterange_test.go @@ -86,7 +86,7 @@ func TestValidDateRangeContains(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - contains, err := tt.daterange.containsStr(tt.datestr) + contains, err := tt.daterange.ContainsStr(tt.datestr) if err != nil { t.Errorf("Testing %s; Unexpected error: %s", tt.name, err) } @@ -109,7 +109,7 @@ func TestInvalidContainData(t *testing.T) { } t.Run(test.name, func(t *testing.T) { - contains, err := test.daterange.containsStr(test.datestr) + contains, err := test.daterange.ContainsStr(test.datestr) if err == nil { t.Errorf("Expected error, got %v", err) } diff --git a/internal/server/api/db.go b/internal/server/api/db.go deleted file mode 100644 index d9dee41..0000000 --- a/internal/server/api/db.go +++ /dev/null @@ -1,510 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "bytes" - "fmt" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/rs/zerolog/log" - "strings" - "sync" - "time" - - "github.com/dgraph-io/badger/v3" - "github.com/nlnwa/gowarcserver/internal/surt" - "github.com/nlnwa/gowarcserver/internal/timestamp" - "github.com/nlnwa/gowarcserver/schema" - "google.golang.org/protobuf/proto" -) - -type PerCdxFunc func(cdx *schema.Cdx) error - -type cdxKey string - -func (k cdxKey) ts() string { - return strings.Split(string(k), " ")[1] -} - -// DbAdapter implements searching the index database via the CoreAPI -type DbAdapter struct { - *index.DB -} - -func cdxFromItem(item *badger.Item) (cdx *schema.Cdx, err error) { - err = item.Value(func(val []byte) error { - result := new(schema.Cdx) - if err := proto.Unmarshal(val, result); err != nil { - return err - } - cdx = result - return nil - }) - return -} - -// Closest returns the first closest cdx value -func (c DbAdapter) Closest(key string, closest string) (cdx *schema.Cdx, err error) { - err = c.CdxIndex.View(func(txn *badger.Txn) error { - // prefix - prefix := []byte(key) - // forward seek key - fk := []byte(key + " " + closest) - // backward seek key - bk := []byte(key + " " + closest + string(rune(0xff))) - - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = 1 - opts.Prefix = prefix - - forward := txn.NewIterator(opts) - defer forward.Close() - forward.Seek(fk) - - // check if we got a literal match on forward seek key (fast path) - if forward.ValidForPrefix(fk) { - var err error - cdx, err = cdxFromItem(forward.Item()) - if err != nil { - return err - } - return nil - } - - // no literal match; iterate forward and backward to find next closest (slow path) - - // iterate forward - forward.Next() - // and backward - opts.Reverse = true - backward := txn.NewIterator(opts) - defer backward.Close() - backward.Seek(bk) - - var ft int64 - var bt int64 - t, _ := timestamp.Parse(closest) - cl := t.Unix() - - // get forward ts - if forward.ValidForPrefix(prefix) { - t, _ = timestamp.Parse(cdxKey(forward.Item().Key()).ts()) - ft = t.Unix() - } - // get backward ts - backward.Seek(bk) - if backward.ValidForPrefix(prefix) { - t, _ = timestamp.Parse(cdxKey(backward.Item().Key()).ts()) - bt = t.Unix() - } - - var item *badger.Item - - if ft != 0 && bt != 0 { - // find closest of forward and backward - isForward := timestamp.AbsInt64(cl-ft) < timestamp.AbsInt64(cl-bt) - if isForward { - item = forward.Item() - } else { - item = backward.Item() - } - } else if ft != 0 { - item = forward.Item() - } else if bt != 0 { - item = backward.Item() - } else { - // found nothing - return nil - } - var err error - cdx, err = cdxFromItem(item) - return err - }) - return -} - -func (c DbAdapter) Search(api *CoreAPI, perCdxFunc PerCdxFunc) (int, error) { - if len(api.Urls) > 1 { - if api.Sort == "" { - return c.unsortedSerialSearch(api, perCdxFunc) - } - return c.sortedParallelSearch(api, perCdxFunc) - } else { - if api.Sort == SortClosest && api.MatchType == MatchTypeExact { - return c.closestUniSearch(api, perCdxFunc) - } - return c.uniSearch(api, perCdxFunc) - } -} - -// unsortedParallelSearch searches the index database, sorts the results and processes each result with perCdxFunc. -func (c DbAdapter) sortedParallelSearch(api *CoreAPI, perCdxFunc PerCdxFunc) (int, error) { - var searchKeys []string - for _, u := range api.Urls { - key := parseKey(surt.UrlToSsurt(u), api.MatchType) - searchKeys = append(searchKeys, key) - } - - filter := parseFilter(api.Filter) - - t, _ := timestamp.Parse(api.Closest) - closest := t.Unix() - s := sorter{ - reverse: api.Sort == SortReverse, - closest: closest, - } - - count := 0 - - perItemFn := func(item *badger.Item) error { - err := item.Value(func(val []byte) error { - result := new(schema.Cdx) - err := proto.Unmarshal(val, result) - if err != nil { - return err - } - - // filter (exact, contains, regexp) - if filter.eval(result) { - count++ - return perCdxFunc(result) - } - return nil - }) - if err != nil { - return fmt.Errorf("%s: %w", item.KeyCopy(nil), err) - } - return nil - } - - err := c.CdxIndex.View(func(txn *badger.Txn) error { - items := make(chan []byte, len(searchKeys)) - - done := make(chan struct{}) - - go func() { - for key := range items { - s.add(key) - } - s.sort() - done <- struct{}{} - }() - - // wg is used to synchronize multiple transaction iterators operating simultaneously. - var wg sync.WaitGroup - - for _, key := range searchKeys { - wg.Add(1) - key := key - - go func() { - defer wg.Done() - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - opts.Prefix = []byte(key) - - it := txn.NewIterator(opts) - defer it.Close() - - for it.Seek([]byte(key)); it.ValidForPrefix([]byte(key)); it.Next() { - k := it.Item().KeyCopy(nil) - - // filter from/to - inDateRange, _ := api.FromTo.containsStr(cdxKey(k).ts()) - if inDateRange { - items <- k - } - } - }() - } - wg.Wait() - close(items) - <-done - - return s.walk(txn, func(item *badger.Item) (stopIteration bool) { - if err := perItemFn(item); err != nil { - return true - } - return false - }) - }) - return count, err -} - -func (c DbAdapter) unsortedSerialSearch(api *CoreAPI, perCdxFunc PerCdxFunc) (int, error) { - keys := make([]string, len(api.Urls)) - for i, url := range api.Urls { - key := parseKey(surt.UrlToSsurt(url), api.MatchType) - keys[i] = key - } - filter := parseFilter(api.Filter) - - count := 0 - err := c.CdxIndex.View(func(txn *badger.Txn) error { - // initialize badger iterators - iterators := make([]*badger.Iterator, len(keys)) - prefixes := make([][]byte, len(keys)) - for i, key := range keys { - prefixes[i] = []byte(key) - opts := badger.DefaultIteratorOptions - opts.Prefix = prefixes[i] - - iterators[i] = txn.NewIterator(opts) - defer iterators[i].Close() - - iterators[i].Seek(prefixes[i]) - } - - OUTER: - for len(iterators) > 0 { - // set timestamp to approx max time.Time value - earliestTimestamp := time.Unix(1<<62, 1<<62) - earliestIndex := -1 - // find the earliest timestamp - for i, iter := range iterators { - // if iter is no longer valid, close it, remove it from slice and restart search - if !iter.ValidForPrefix(prefixes[i]) { - iteratorsLen := len(iterators) - iterators[i].Close() - // remove iterator from list - iterators[i] = iterators[iteratorsLen-1] - iterators = iterators[0 : iteratorsLen-1] - continue OUTER - } - - item := iter.Item() - // in the event of parse error we get a zero timestamp - ts, err := time.Parse(timeLayout, cdxKey(item.Key()).ts()) - if err != nil { - log.Warn().Err(err).Msgf("Failed to parse timestamp for key: '%s'", string(item.Key())) - - // timestamp is invalid, iterate to next and restart search - iter.Next() - continue OUTER - } - - inRange, _ := api.FromTo.containsTime(ts) - if !inRange { - // timestamp out of range, iterate to next item and restart search - iter.Next() - continue OUTER - } - - if ts.Before(earliestTimestamp) { - earliestTimestamp = ts - earliestIndex = i - } - } - if earliestIndex == -1 { - break - } - iter := iterators[earliestIndex] - - cdx, err := cdxFromItem(iter.Item()) - if err != nil { - return err - } - - iter.Next() - - if filter.eval(cdx) { - err = perCdxFunc(cdx) - if err != nil { - return err - } - count++ - } - - if api.Limit > 0 && count >= api.Limit { - break - } - } - return nil - }) - return count, err -} - -func (c DbAdapter) closestUniSearch(api *CoreAPI, perCdxFunc PerCdxFunc) (int, error) { - u := api.Urls[0] - s := surt.UrlToSsurt(u) - key := parseKey(s, api.MatchType) - closest := api.Closest - t, _ := timestamp.Parse(api.Closest) - cl := t.Unix() - - seek := key + closest - - isClosest := func(a int64, b int64) bool { - return timestamp.AbsInt64(cl-a) <= timestamp.AbsInt64(cl-b) - } - - count := 0 - - err := c.CdxIndex.View(func(txn *badger.Txn) error { - prefix := []byte(key) - - opts := badger.DefaultIteratorOptions - opts.Prefix = prefix - - forward := txn.NewIterator(opts) - defer forward.Close() - - opts.Reverse = true - backward := txn.NewIterator(opts) - defer backward.Close() - - fk := []byte(seek) - forward.Seek(fk) - - bk := []byte(seek + string(rune(0xff))) - backward.Seek(bk) - if forward.ValidForPrefix(prefix) && backward.ValidForPrefix(prefix) && bytes.Equal(forward.Item().Key(), backward.Item().Key()) { - // if forward and backward iterator point to same item we advance the backward iterator - backward.Next() - } - - var ft int64 - var bt int64 - - f := true - b := true - - for { - if f && forward.ValidForPrefix(prefix) { - t, _ := timestamp.Parse(cdxKey(forward.Item().Key()).ts()) - ft = t.Unix() - } else if f { - f = false - ft = 0 - } - - if b && backward.ValidForPrefix(prefix) { - t, _ := timestamp.Parse(cdxKey(backward.Item().Key()).ts()) - bt = t.Unix() - } else if b { - b = false - bt = 0 - } - - var it *badger.Iterator - if f && isClosest(ft, bt) { - it = forward - } else if b { - it = backward - } else { - return nil - } - - cdx, err := cdxFromItem(it.Item()) - if err != nil { - return err - } - - it.Next() - - err = perCdxFunc(cdx) - if err != nil { - return err - } - count++ - - if api.Limit > 0 && count >= api.Limit { - break - } - } - return nil - }) - return count, err -} - -// uniSearch the index database and render each item with the provided renderFunc. -func (c DbAdapter) uniSearch(api *CoreAPI, perCdxFunc PerCdxFunc) (int, error) { - u := api.Urls[0] - s := surt.UrlToSsurt(u) - - key := parseKey(s, api.MatchType) - filter := parseFilter(api.Filter) - reverse := api.Sort == SortReverse - count := 0 - - err := c.CdxIndex.View(func(txn *badger.Txn) error { - prefix := []byte(key) - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - opts.Prefix = prefix - opts.Reverse = reverse - - it := txn.NewIterator(opts) - defer it.Close() - - // see https://github.com/dgraph-io/badger/issues/436 for details regarding reverse seeking - seekKey := key - if reverse { - seekKey += string(rune(0xff)) - } - - for it.Seek([]byte(seekKey)); it.ValidForPrefix(prefix); it.Next() { - contains, _ := api.FromTo.containsStr(cdxKey(it.Item().Key()).ts()) - if !contains { - continue - } - - err := it.Item().Value(func(v []byte) error { - result := new(schema.Cdx) - if err := proto.Unmarshal(v, result); err != nil { - return err - } - - if filter.eval(result) { - if err := perCdxFunc(result); err != nil { - return err - } - count++ - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to process item value") - } - - if api.Limit > 0 && count > api.Limit { - break - } - } - return nil - }) - return count, err -} - -func (c DbAdapter) ListRecords(fn func(warcId string, cdx *schema.Cdx) (stopIteration bool)) error { - opts := badger.DefaultIteratorOptions - return index.Walk(c.CdxIndex, opts, func(item *badger.Item) (stopIteration bool) { - err := item.Value(func(val []byte) error { - cdx := new(schema.Cdx) - err := proto.Unmarshal(val, cdx) - if err != nil { - return err - } - stopIteration = fn(string(item.Key()), cdx) - return nil - }) - if err != nil { - log.Error().Err(err).Msgf("failed get value for key: %s", string(item.Key())) - } - return stopIteration || err != nil - }) -} diff --git a/internal/server/api/error.go b/internal/server/api/error.go index f7c925d..8e61899 100644 --- a/internal/server/api/error.go +++ b/internal/server/api/error.go @@ -1,16 +1,13 @@ package api import ( - "github.com/rs/zerolog/log" "net/http" "runtime" + + "github.com/rs/zerolog/log" ) func HandleError(w http.ResponseWriter, n int, err error) { - // assume nothing has been written to client yet - if n == 0 { - http.Error(w, err.Error(), http.StatusInternalServerError) - } // try to log error with caller information _, file, no, ok := runtime.Caller(1) if ok { diff --git a/internal/server/api/filter.go b/internal/server/api/filter.go index 74941d1..f5caf91 100644 --- a/internal/server/api/filter.go +++ b/internal/server/api/filter.go @@ -34,7 +34,7 @@ const ( type Filter []filter -func (f Filter) eval(c *schema.Cdx) bool { +func (f Filter) Eval(c *schema.Cdx) bool { for _, ff := range f { if !ff.eval(c) { return false @@ -43,7 +43,7 @@ func (f Filter) eval(c *schema.Cdx) bool { return true } -func parseFilter(filterStrings []string) Filter { +func ParseFilter(filterStrings []string) Filter { var filters Filter for _, f := range filterStrings { diff --git a/internal/server/api/searchkey.go b/internal/server/api/searchkey.go index cfb6dd7..0ce35bc 100644 --- a/internal/server/api/searchkey.go +++ b/internal/server/api/searchkey.go @@ -20,7 +20,7 @@ import ( "strings" ) -func parseKey(ssurt string, matchType string) string { +func ParseKey(ssurt string, matchType string) string { switch matchType { case MatchTypeExact: ssurt += " " diff --git a/internal/server/api/sorter.go b/internal/server/api/sorter.go index b8be45f..a7b715f 100644 --- a/internal/server/api/sorter.go +++ b/internal/server/api/sorter.go @@ -17,39 +17,54 @@ package api import ( + "sort" + "strings" + "github.com/dgraph-io/badger/v3" - "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/timestamp" - "sort" ) +type cdxKey string + +func (k cdxKey) ts() string { + return strings.Split(string(k), " ")[1] +} + type value struct { ts int64 k []byte } -type sorter struct { - closest int64 - reverse bool - values []value +type Sorter struct { + Closest int64 + Reverse bool + Values []value } -func (s *sorter) add(k []byte) { +func NewSorter(closest int64, reverse bool) Sorter { + return Sorter{ + Closest: closest, + Reverse: reverse, + Values: []value{}, + } +} + +func (s *Sorter) Add(k []byte) { t, _ := timestamp.Parse(cdxKey(k).ts()) ts := t.Unix() - s.values = append(s.values, value{ts, k}) + s.Values = append(s.Values, value{ts, k}) } -func (s *sorter) sort() { +func (s *Sorter) Sort() { var cmp func(int64, int64) bool - // sort closest, reverse or forward - if s.closest > 0 { + // sort Closest, Reverse or forward + if s.Closest > 0 { cmp = func(ts1 int64, ts2 int64) bool { - return timestamp.AbsInt64(s.closest-ts1) < timestamp.AbsInt64(s.closest-ts2) + return timestamp.AbsInt64(s.Closest-ts1) < timestamp.AbsInt64(s.Closest-ts2) } - } else if s.reverse { + } else if s.Reverse { cmp = func(ts1 int64, ts2 int64) bool { return ts2 < ts1 } @@ -59,13 +74,15 @@ func (s *sorter) sort() { } } - sort.Slice(s.values, func(i, j int) bool { - return cmp(s.values[i].ts, s.values[j].ts) + sort.Slice(s.Values, func(i, j int) bool { + return cmp(s.Values[i].ts, s.Values[j].ts) }) } -func (s *sorter) walk(txn *badger.Txn, perItemFn index.PerItemFunc) error { - for _, value := range s.values { +type PerItemFunc func(*badger.Item) (stopIteration bool) + +func (s *Sorter) Walk(txn *badger.Txn, perItemFn PerItemFunc) error { + for _, value := range s.Values { item, err := txn.Get(value.k) if err != nil { return err diff --git a/internal/server/coreserver/handler.go b/internal/server/coreserver/handler.go index e52c803..8ed761a 100644 --- a/internal/server/coreserver/handler.go +++ b/internal/server/coreserver/handler.go @@ -17,67 +17,31 @@ package coreserver import ( + "bytes" + "context" "encoding/json" "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + "github.com/julienschmidt/httprouter" "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/loader" "github.com/nlnwa/gowarcserver/internal/server/api" "github.com/nlnwa/gowarcserver/internal/server/handlers" - "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" - "net/http" - "strconv" - "strings" - "time" - - "github.com/dgraph-io/badger/v3" ) type Handler struct { - db api.DbAdapter - loader loader.RecordLoader -} - -// keyHandler returns a http.HandlerFunc that outputs the keys given in db. -func keyHandler(db *badger.DB) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - limit := parseLimit(r) - - start := time.Now() - count := 0 - - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - err := index.Walk(db, opts, func(item *badger.Item) (stopIteration bool) { - _, err := fmt.Fprintln(w, string(item.Key())) - count++ - if count >= limit { - return true - } - return err != nil - }) - if err != nil { - api.HandleError(w, count, err) - } - if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) - } - } -} - -func (h Handler) listId(w http.ResponseWriter, r *http.Request) { - keyHandler(h.db.IdIndex)(w, r) -} - -func (h Handler) listFile(w http.ResponseWriter, r *http.Request) { - keyHandler(h.db.FileIndex)(w, r) -} - -func (h Handler) listCdx(w http.ResponseWriter, r *http.Request) { - keyHandler(h.db.CdxIndex)(w, r) + index.CdxAPI + index.FileAPI + index.IdAPI + loader.StorageRefResolver + loader.WarcLoader } func (h Handler) search(w http.ResponseWriter, r *http.Request) { @@ -86,22 +50,43 @@ func (h Handler) search(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + response := make(chan index.CdxResponse) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + log.Debug().Msgf("%v", coreAPI) + if err = h.CdxAPI.Search(ctx, api.SearchAPI{CoreAPI: coreAPI}, response); err != nil { + log.Error().Err(err).Msg("failed to search") + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } start := time.Now() + count := 0 + defer func() { + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + }() - count, err := h.db.Search(coreAPI, func(cdx *schema.Cdx) error { - b, err := protojson.Marshal(cdx) + for res := range response { + if res.Error != nil { + log.Warn().Err(res.Error).Msg("failed result") + continue + } + v, err := protojson.Marshal(res) if err != nil { - return err + log.Warn().Err(err).Msg("failed to marshal result") + continue + } + if count > 0 { + _, _ = w.Write([]byte("\r\n")) + } + _, err = io.Copy(w, bytes.NewReader(v)) + if err != nil { + log.Warn().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write result") + return + } else { + count++ } - _, err = fmt.Fprintln(w, string(b)) - return err - }) - if err != nil { - api.HandleError(w, count, err) - } - if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) } } @@ -113,43 +98,48 @@ type storageRef struct { func (h Handler) listIds(w http.ResponseWriter, r *http.Request) { limit := parseLimit(r) + response := make(chan index.IdResponse) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + if err := h.IdAPI.ListStorageRef(ctx, limit, response); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } start := time.Now() count := 0 + defer func() { + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + }() - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = limit - err := index.Walk(h.db.IdIndex, opts, func(item *badger.Item) (stopIteration bool) { - err := item.Value(func(val []byte) error { - filename, offset, err := parseStorageRef(string(val)) - if err != nil { - return err - } - storageRef := &storageRef{ - Id: string(item.Key()), - Filename: filename, - Offset: offset, - } - b, err := json.Marshal(storageRef) - if err != nil { - return err - } - _, err = fmt.Fprintln(w, string(b)) - return err - }) + for res := range response { + if res.Error != nil { + log.Warn().Err(res.Error).Msg("failed result") + continue + } + filename, offset, err := parseStorageRef(res.Value) + storageRef := &storageRef{ + Id: res.Key, + Filename: filename, + Offset: offset, + } + v, err := json.Marshal(storageRef) if err != nil { - log.Warn().Err(err).Msgf("Failed to output id '%s'", string(item.Key())) + log.Warn().Err(err).Msg("failed to marshal storage ref") } - count++ - if count >= limit { - return true + if count > 0 { + _, _ = w.Write([]byte("\r\n")) } - return err != nil - }) - if err != nil { - api.HandleError(w, count, err) - } else if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + _, err = io.Copy(w, bytes.NewReader(v)) + if err != nil { + log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write storage ref") + if count == 0 { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return + } + count++ } } @@ -157,7 +147,7 @@ func (h Handler) getStorageRefByURN(w http.ResponseWriter, r *http.Request) { params := httprouter.ParamsFromContext(r.Context()) urn := params.ByName("urn") - storageRef, err := h.db.Resolve(urn) + storageRef, err := h.StorageRefResolver.Resolve(urn) if err != nil { msg := fmt.Sprintf("failed to resolve storage reference of urn: %v", err) http.Error(w, msg, http.StatusInternalServerError) @@ -170,40 +160,41 @@ func (h Handler) getStorageRefByURN(w http.ResponseWriter, r *http.Request) { } func (h Handler) listFiles(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + limit := parseLimit(r) + responses := make(chan index.FileResponse) + + if err := h.FileAPI.ListFileInfo(ctx, limit, responses); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } start := time.Now() count := 0 + defer func() { + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + }() - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = limit - - err := index.Walk(h.db.FileIndex, opts, func(item *badger.Item) (stopIteration bool) { - fileInfo := new(schema.Fileinfo) - err := item.Value(func(val []byte) error { - return proto.Unmarshal(val, fileInfo) - }) - if err != nil { - log.Warn().Err(err).Msgf("Failed to unmarshal FileInfo: %s", string(item.Key())) + for res := range responses { + if res.Error != nil { + log.Warn().Err(res.Error).Msg("failed result") + continue } - b, err := protojson.Marshal(fileInfo) + v, err := protojson.Marshal(res.Fileinfo) if err != nil { - log.Warn().Err(err).Msgf("Failed to marshal fileInfo to JSON: %s", string(item.Key())) + log.Warn().Err(err).Msg("failed to marshal storage ref") + } + if count > 0 { + _, _ = w.Write([]byte("\r\n")) } - _, err = fmt.Fprintln(w, string(b)) + _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Warn().Err(err).Msgf("Failed to write fileInfo: %s", string(item.Key())) + log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write storage ref") + return } count++ - if count >= limit { - return true - } - return err != nil - }) - if err != nil { - api.HandleError(w, count, err) - } else if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) } } @@ -211,7 +202,10 @@ func (h Handler) getFileInfoByFilename(w http.ResponseWriter, r *http.Request) { params := httprouter.ParamsFromContext(r.Context()) filename := params.ByName("filename") - fileInfo, err := h.db.GetFileInfo(filename) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + fileInfo, err := h.FileAPI.GetFileInfo(ctx, filename) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -224,39 +218,44 @@ func (h Handler) getFileInfoByFilename(w http.ResponseWriter, r *http.Request) { func (h Handler) listCdxs(w http.ResponseWriter, r *http.Request) { limit := parseLimit(r) + responses := make(chan index.CdxResponse) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + if err := h.CdxAPI.List(ctx, limit, responses); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } start := time.Now() count := 0 + defer func() { + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + }() - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = limit - - err := index.Walk(h.db.CdxIndex, opts, func(item *badger.Item) (stopIteration bool) { - cdx := new(schema.Cdx) - err := item.Value(func(val []byte) error { - return proto.Unmarshal(val, cdx) - }) - if err != nil { - log.Warn().Err(err).Msgf("Failed to unmarshal Cdx: %s", string(item.Key())) + for res := range responses { + if res.Error != nil { + log.Warn().Err(res.Error).Msg("failed result") + continue } - b, err := protojson.Marshal(cdx) + v, err := protojson.Marshal(res.Cdx) if err != nil { - log.Warn().Err(err).Msgf("Failed to marshal JSON: %s", string(item.Key())) + log.Warn().Err(err).Msg("failed to marshal cdx to json") + continue } - _, err = fmt.Fprintln(w, string(b)) + if count > 0 { + _, _ = w.Write([]byte("\r\n")) + } + _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Warn().Err(err).Msgf("Failed to write: %s", string(item.Key())) + log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write result") + if count == 0 { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return } count++ - if count >= limit { - return true - } - return err != nil - }) - if err != nil { - api.HandleError(w, count, err) - } else if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) } } @@ -264,13 +263,20 @@ func (h Handler) loadRecordByUrn(w http.ResponseWriter, r *http.Request) { params := httprouter.ParamsFromContext(r.Context()) warcId := params.ByName("urn") - record, err := h.loader.Load(r.Context(), warcId) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + record, err := h.WarcLoader.LoadById(ctx, warcId) if err != nil { msg := fmt.Sprintf("failed to load record '%s': %v", warcId, err) http.Error(w, msg, http.StatusInternalServerError) return } - defer record.Close() + defer func() { + if err := record.Close(); err != nil { + log.Warn().Err(err).Msgf("Closing record: %v", record) + } + }() n, err := handlers.RenderRecord(w, record) if n == 0 { diff --git a/internal/server/coreserver/routes.go b/internal/server/coreserver/routes.go index 2170a86..7a2f59a 100644 --- a/internal/server/coreserver/routes.go +++ b/internal/server/coreserver/routes.go @@ -17,24 +17,17 @@ package coreserver import ( - "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/loader" - "github.com/nlnwa/gowarcserver/internal/server/api" "net/http" -) -func Register(r *httprouter.Router, mw func(http.Handler) http.Handler, pathPrefix string, l loader.RecordLoader, db *index.DB) { - coreHandler := Handler{api.DbAdapter{DB: db}, l} + "github.com/julienschmidt/httprouter" +) - r.Handler("GET", pathPrefix+"/id", mw(http.HandlerFunc(coreHandler.listId))) - r.Handler("GET", pathPrefix+"/ids", mw(http.HandlerFunc(coreHandler.listIds))) - r.Handler("GET", pathPrefix+"/id/:urn", mw(http.HandlerFunc(coreHandler.getStorageRefByURN))) - r.Handler("GET", pathPrefix+"/file", mw(http.HandlerFunc(coreHandler.listFile))) - r.Handler("GET", pathPrefix+"/files", mw(http.HandlerFunc(coreHandler.listFiles))) - r.Handler("GET", pathPrefix+"/file/:filename", mw(http.HandlerFunc(coreHandler.getFileInfoByFilename))) - r.Handler("GET", pathPrefix+"/cdx", mw(http.HandlerFunc(coreHandler.listCdx))) - r.Handler("GET", pathPrefix+"/cdxs", mw(http.HandlerFunc(coreHandler.listCdxs))) - r.Handler("GET", pathPrefix+"/search", mw(http.HandlerFunc(coreHandler.search))) - r.Handler("GET", pathPrefix+"/record/:urn", mw(http.HandlerFunc(coreHandler.loadRecordByUrn))) +func Register(h Handler, r *httprouter.Router, mw func(http.Handler) http.Handler, pathPrefix string) { + r.Handler("GET", pathPrefix+"/id", mw(http.HandlerFunc(h.listIds))) + r.Handler("GET", pathPrefix+"/id/:urn", mw(http.HandlerFunc(h.getStorageRefByURN))) + r.Handler("GET", pathPrefix+"/file", mw(http.HandlerFunc(h.listFiles))) + r.Handler("GET", pathPrefix+"/file/:filename", mw(http.HandlerFunc(h.getFileInfoByFilename))) + r.Handler("GET", pathPrefix+"/cdx", mw(http.HandlerFunc(h.listCdxs))) + r.Handler("GET", pathPrefix+"/search", mw(http.HandlerFunc(h.search))) + r.Handler("GET", pathPrefix+"/record/:urn", mw(http.HandlerFunc(h.loadRecordByUrn))) } diff --git a/internal/server/handlers/handlers.go b/internal/server/handlers/handlers.go deleted file mode 100644 index 98cb7d2..0000000 --- a/internal/server/handlers/handlers.go +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package handlers - -import ( - "context" - "io" - "net/http" - "net/url" - "sync" - "time" - - "github.com/rs/zerolog/log" -) - -var client = &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, -} - -func AggregatedHandler(children []*url.URL, timeout time.Duration) http.Handler { - responseHandler := func(w http.ResponseWriter, r *http.Request, responses <-chan *http.Response) { - written := false - for resp := range responses { - if resp.StatusCode == http.StatusOK { - _, err := io.Copy(w, resp.Body) - if err != nil { - log.Error(). - Err(err). - Str("uri", resp.Request.RequestURI). - Str("status", resp.Status). - Msg("Failed to write response body") - } else { - written = true - } - } - _ = resp.Body.Close() - } - if !written { - http.Error(w, "", http.StatusOK) - } - } - - return ChildHandler(children, timeout, responseHandler) -} - -func FirstHandler(children []*url.URL, timeout time.Duration) http.Handler { - responseHandler := func(w http.ResponseWriter, r *http.Request, responses <-chan *http.Response) { - written := false - for response := range responses { - resp := response - if resp.StatusCode < 400 { - if !written { - // Write headers - for key, values := range resp.Header { - for i, value := range values { - if i == 0 { - w.Header().Set(key, value) - } else { - w.Header().Add(key, value) - } - } - } - w.WriteHeader(resp.StatusCode) - - if _, err := io.Copy(w, resp.Body); err != nil { - log.Error().Err(err). - Str("uri", resp.Request.RequestURI). - Str("status", resp.Status). - Msg("Failed to write response body") - } - written = true - } - } - _ = resp.Body.Close() - } - if !written { - http.Error(w, "Not found", http.StatusNotFound) - } - } - return ChildHandler(children, timeout, responseHandler) -} - -type ResponseHandler func(http.ResponseWriter, *http.Request, <-chan *http.Response) - -func ChildHandler(children []*url.URL, timeout time.Duration, responseHandler ResponseHandler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - responses := make(chan *http.Response, len(children)) - wg := new(sync.WaitGroup) - wg.Add(len(children)) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - go func() { - wg.Wait() - close(responses) - }() - - for _, childUrl := range children { - req := r.Clone(ctx) - req.RequestURI = "" - req.URL = buildChildURLString(childUrl, req.URL) - log.Debug().Msgf("request to child url %s", req.URL.String()) - go func() { - defer wg.Done() - - // The consumer of the http response is responsible for closing the response body - //nolint:bodyclose - resp, err := client.Do(req) - if err != nil { - log.Error().Msgf("request failed: %v: %v", req, err) - } else { - responses <- resp - } - }() - } - - responseHandler(w, r, responses) - } -} diff --git a/internal/server/handlers/render.go b/internal/server/handlers/render.go index 5a958a6..77af377 100644 --- a/internal/server/handlers/render.go +++ b/internal/server/handlers/render.go @@ -18,9 +18,10 @@ package handlers import ( "fmt" - "github.com/nlnwa/gowarc" "io" "net/http" + + "github.com/nlnwa/gowarc" ) // RenderRecord renders gowarc.WarcRecord rec as a binary stream. @@ -51,7 +52,7 @@ func RenderContent(w http.ResponseWriter, r gowarc.HttpResponseBlock) error { return fmt.Errorf("failed to retrieve payload bytes: %w", err) } - return render(w, *r.HttpHeader(), r.HttpStatusCode(), p) + return Render(w, *r.HttpHeader(), r.HttpStatusCode(), p) } func RenderRedirect(w http.ResponseWriter, location string) { @@ -63,7 +64,7 @@ func RenderRedirect(w http.ResponseWriter, location string) { w.WriteHeader(http.StatusFound) } -func render(w http.ResponseWriter, h http.Header, code int, r io.Reader) error { +func Render(w http.ResponseWriter, h http.Header, code int, r io.Reader) error { // Write headers for key, values := range h { for i, value := range values { diff --git a/internal/server/handlers/url.go b/internal/server/handlers/url.go deleted file mode 100644 index a0a2482..0000000 --- a/internal/server/handlers/url.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package handlers - -import "net/url" - -func buildChildURLString(base *url.URL, v *url.URL) *url.URL { - u := *base - - u.Path = v.Path - u.RawPath = v.RawPath - u.RawQuery = v.RawQuery - u.Fragment = v.Fragment - - return &u -} diff --git a/internal/server/handlers/url_test.go b/internal/server/handlers/url_test.go deleted file mode 100644 index ce7cf38..0000000 --- a/internal/server/handlers/url_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package handlers - -import ( - "log" - "net/url" - "testing" -) - -func TestConfigToDBMask(t *testing.T) { - tests := []struct { - name string - nodeUrlStr string - childUrlStr string - expectedUrl string - }{ - { - "simple nop", - "http://190.165.33.152:8080", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999", - }, - { - "dont include child non-host", - "http://190.165.33.152:8080", - "http://190.165.33.152:9999/yes?foo=bar#ez", - "http://190.165.33.152:9999", - }, - { - "path", - "http://190.165.33.152:8080/foo", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999/foo", - }, - { - "query", - "http://190.165.33.152:8080?foo=bar", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999?foo=bar", - }, - { - "fragment", - "http://190.165.33.152:8080#foo", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999#foo", - }, - { - "path&query", - "http://190.165.33.152:8080/path?foo=bar", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999/path?foo=bar", - }, - { - "path&query&fragment", - "http://190.165.33.152:8080/path?foo=bar#f", - "http://190.165.33.152:9999", - "http://190.165.33.152:9999/path?foo=bar#f", - }, - } - - for _, tt := range tests { - var err error - childUrl, err := url.Parse(tt.childUrlStr) - if err != nil { - log.Fatal("illegal child url test string, msg: ", err) - } - nodeUrl, err := url.Parse(tt.nodeUrlStr) - if err != nil { - log.Fatal("illegal node url test string, msg: ", err) - } - t.Run(tt.name, func(t *testing.T) { - result := buildChildURLString(childUrl, nodeUrl).String() - if result != tt.expectedUrl { - t.Errorf("Expected %s got %s", tt.expectedUrl, result) - } - }) - } -} diff --git a/internal/server/server.go b/internal/server/server.go deleted file mode 100644 index efc2414..0000000 --- a/internal/server/server.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package server - -import ( - "context" - "fmt" - "github.com/rs/zerolog/log" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - "time" -) - -func Serve(port int, h http.Handler) error { - portStr := strconv.Itoa(port) - httpServer := &http.Server{ - Addr: fmt.Sprintf(":%v", portStr), - Handler: h, - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) - go func() { - sig := <-sigs - log.Debug().Msgf("Received %s signal, shutting down server...", sig) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - err := httpServer.Shutdown(ctx) - if err != nil { - log.Warn().Msgf("Failed to shut down server: %v", err) - } - }() - - log.Info().Msgf("Starting web server at :%v", port) - - return httpServer.ListenAndServe() -} diff --git a/internal/server/warcserver/api.go b/internal/server/warcserver/api.go index 9428e72..5d3857f 100644 --- a/internal/server/warcserver/api.go +++ b/internal/server/warcserver/api.go @@ -17,11 +17,15 @@ package warcserver import ( + "fmt" + "net/http" + "strconv" + "github.com/julienschmidt/httprouter" + "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" - "net/http" - "strconv" + "github.com/nlnwa/whatwg-url/url" ) type pywbJson struct { @@ -36,28 +40,28 @@ type pywbJson struct { func cdxToPywbJson(cdx *schema.Cdx) *pywbJson { return &pywbJson{ - Urlkey: cdx.Ssu, - Timestamp: timestamp.TimeTo14(cdx.Sts.AsTime()), - Url: cdx.Uri, - Mime: cdx.Mct, - Status: strconv.Itoa(int(cdx.Hsc)), - Digest: cdx.Dig, - Length: strconv.Itoa(int(cdx.Rle)), + Urlkey: cdx.GetSsu(), + Timestamp: timestamp.TimeTo14(cdx.GetSts().AsTime()), + Url: cdx.GetUri(), + Mime: cdx.GetMct(), + Status: strconv.Itoa(int(cdx.GetHsc())), + Digest: cdx.GetDig(), + Length: strconv.Itoa(int(cdx.GetRle())), } } -func parseWeb(r *http.Request) (uri string, ts string) { +func parseResourceRequest(r *http.Request) (closestRequest, error) { params := httprouter.ParamsFromContext(r.Context()) // closest parameter p0 := params.ByName("timestamp") // remove trailing 'id_' - ts = p0[:len(p0)-3] + closest := p0[:len(p0)-3] // url parameter p1 := params.ByName("url") // remove leading '/' - uri = p1[1:] + uri := p1[1:] // we must add on any query parameters if q := r.URL.Query().Encode(); len(q) > 0 { @@ -68,5 +72,36 @@ func parseWeb(r *http.Request) (uri string, ts string) { // and fragment uri += "#" + r.URL.Fragment } - return + + u, err := url.Parse(uri) + if err != nil { + return closestRequest{}, fmt.Errorf("failed to parse uri: %w", err) + } + key := surt.UrlToSsurt(u) + + return closestRequest{ + rawUrl: uri, + key: key, + closest: closest, + limit: 1, + }, nil +} + +type closestRequest struct { + rawUrl string + key string + closest string + limit int +} + +func (c closestRequest) Key() string { + return c.key +} + +func (c closestRequest) Closest() string { + return c.closest +} + +func (c closestRequest) Limit() int { + return c.limit } diff --git a/internal/server/warcserver/handler.go b/internal/server/warcserver/handler.go index ce348db..2457af4 100644 --- a/internal/server/warcserver/handler.go +++ b/internal/server/warcserver/handler.go @@ -5,24 +5,27 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "strings" + "time" + "github.com/nlnwa/gowarc" + "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/loader" "github.com/nlnwa/gowarcserver/internal/server/api" "github.com/nlnwa/gowarcserver/internal/server/handlers" "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/internal/timestamp" - "github.com/nlnwa/gowarcserver/schema" urlErrors "github.com/nlnwa/whatwg-url/errors" "github.com/nlnwa/whatwg-url/url" "github.com/rs/zerolog/log" - "net/http" - "strings" - "time" ) type Handler struct { - db api.DbAdapter - loader loader.RecordLoader + index.CdxAPI + index.FileAPI + index.IdAPI + loader.WarcLoader } func (h Handler) index(w http.ResponseWriter, r *http.Request) { @@ -33,67 +36,103 @@ func (h Handler) index(w http.ResponseWriter, r *http.Request) { } start := time.Now() - n, err := h.db.Search(coreAPI, renderCdx(w, coreAPI)) - if err != nil { - api.HandleError(w, n, err) - } - if n > 0 { - log.Debug().Msgf("Found %d items in %s", n, time.Since(start)) + count := 0 + defer func() { + if count > 0 { + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) + } + }() + + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + response := make(chan index.CdxResponse) + searchApi := api.SearchAPI{CoreAPI: coreAPI} + + if err = h.CdxAPI.Search(ctx, searchApi, response); err != nil { + log.Error().Err(err).Msg("search failed") + http.Error(w, err.Error(), http.StatusInternalServerError) + return } -} -func renderCdx(w http.ResponseWriter, coreAPI *api.CoreAPI) api.PerCdxFunc { - return func(record *schema.Cdx) error { - cdxj, err := json.Marshal(cdxToPywbJson(record)) + for res := range response { + if res.Error != nil { + log.Warn().Err(res.Error).Msg("failed result") + continue + } + cdxj, err := json.Marshal(cdxToPywbJson(res.Cdx)) if err != nil { - return err + log.Warn().Err(err).Msg("failed to marshal result") + continue } switch coreAPI.Output { case api.OutputJson: _, err = fmt.Fprintln(w, cdxj) default: - sts := timestamp.TimeTo14(record.Sts.AsTime()) - _, err = fmt.Fprintf(w, "%s %s %s\n", record.Ssu, sts, cdxj) + ssu := res.GetSsu() + sts := timestamp.TimeTo14(res.GetSts().AsTime()) + _, err = fmt.Fprintf(w, "%s %s %s\n", ssu, sts, cdxj) + } + if err != nil { + log.Warn().Err(err).Msg("failed to write result") + } else { + count++ } - return err } } func (h Handler) resource(w http.ResponseWriter, r *http.Request) { - uri, ts := parseWeb(r) - - key, err := surt.StringToSsurt(uri) + // parse API + closestReq, err := parseResourceRequest(r) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadRequest) return } - // fetch cdx to access warc record id - cdx, err := h.db.Closest(key, ts) + ctx, cancelQuery := context.WithCancel(r.Context()) + defer cancelQuery() + + // query API + response := make(chan index.CdxResponse) + err = h.CdxAPI.Closest(ctx, closestReq, response) if err != nil { + log.Error().Err(err).Msg("failed to query closest cdx") http.Error(w, err.Error(), http.StatusInternalServerError) return } - if cdx == nil { + + var res index.CdxResponse + for res = range response { + if res.Error != nil { + log.Warn().Err(err).Msg("failed cdx response") + continue + } + cancelQuery() + break + } + if res.Cdx == nil { http.NotFound(w, r) return } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - // load warc record by id - warcRecord, err := h.loader.Load(ctx, cdx.Rid) + // load warc record by storage ref + warcRecord, err := h.LoadByStorageRef(ctx, res.GetRef()) + defer func() { + if warcRecord != nil { + _ = warcRecord.Close() + } + }() if err != nil { - var errWarcProfile loader.ErrResolveRevisit - if errors.As(err, &errWarcProfile) { - http.Error(w, errWarcProfile.Error(), http.StatusNotImplemented) + var errResolveRevisit loader.ErrResolveRevisit + if errors.As(err, &errResolveRevisit) { + http.Error(w, errResolveRevisit.Error(), http.StatusNotImplemented) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } - defer warcRecord.Close() block, ok := warcRecord.Block().(gowarc.HttpResponseBlock) if !ok { @@ -103,74 +142,81 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { s := block.HttpStatusCode() - // Handle redirects - if isRedirect(s) { - location := block.HttpHeader().Get("Location") - if location == "" { - http.NotFound(w, r) - return - } - locUrl, err := url.Parse(location) - if urlErrors.Code(err) == urlErrors.FailRelativeUrlWithNoBase { - locUrl, err = url.ParseRef(uri, location) - if err != nil { - http.Error(w, fmt.Sprintf("Failed to parse relative location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) - return - } - } + if !isRedirect(s) { + p, err := block.PayloadBytes() if err != nil { - http.Error(w, fmt.Sprintf("Failed to parse location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusInternalServerError) return } - - coreAPI := &api.CoreAPI{ - Urls: []*url.Url{locUrl}, - MatchType: api.MatchTypeExact, - Limit: 1, - Sort: api.SortClosest, - Closest: ts, + err = handlers.Render(w, *block.HttpHeader(), block.HttpStatusCode(), p) + if err != nil { + log.Warn().Err(err).Msg("Failed to render response") } + return + } - // the fields we need to rewrite the location header - var sts string - var uri string - - // Get timestamp and uri from cdx record - perCdxFunc := func(record *schema.Cdx) error { - sts = timestamp.TimeTo14(record.Sts.AsTime()) - uri = record.Uri - return nil - } + // handle redirect + location := block.HttpHeader().Get("Location") + if location == "" { + http.Error(w, "Redirected to empty location", http.StatusInternalServerError) + return + } - if n, err := h.db.Search(coreAPI, perCdxFunc); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if n == 0 || sts == "" || uri == "" { - http.NotFound(w, r) + locUrl, err := url.Parse(location) + if urlErrors.Code(err) == urlErrors.FailRelativeUrlWithNoBase { + locUrl, err = url.ParseRef(closestReq.rawUrl, location) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to parse relative location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) return } + } + if err != nil { + http.Error(w, fmt.Sprintf("Failed to parse location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) + return + } - prefix := r.URL.Path[:strings.Index(r.URL.Path, "id_")-14] - path := prefix + sts + "id_/" + uri + closestReq = closestRequest{ + key: surt.UrlToSsurt(locUrl), + closest: closestReq.closest, + limit: 1, + } + response = make(chan index.CdxResponse) + err = h.Closest(ctx, closestReq, response) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } - scheme := "http" - if r.TLS != nil { - scheme = "https" - } - host := r.Host - u, err := url.Parse(scheme + "://" + host + path) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + // fields needed to rewrite the location header + var sts string + var uri string - handlers.RenderRedirect(w, u.String()) - } else { - err := handlers.RenderContent(w, block) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + for res := range response { + if res.Error != nil { + log.Warn().Err(err).Msg("") + continue } + sts = timestamp.TimeTo14(res.GetSts().AsTime()) + uri = res.GetUri() + } + if uri == "" { + http.NotFound(w, r) + return + } + path := r.URL.Path[:strings.Index(r.URL.Path, "id_")-14] + sts + "id_/" + uri + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + host := r.Host + u, err := url.Parse(scheme + "://" + host) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return } + u.SetPathname(path) + + handlers.RenderRedirect(w, u.String()) } func isRedirect(code int) bool { diff --git a/internal/server/warcserver/routes.go b/internal/server/warcserver/routes.go index 2de9ad9..4026167 100644 --- a/internal/server/warcserver/routes.go +++ b/internal/server/warcserver/routes.go @@ -17,21 +17,13 @@ package warcserver import ( - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/server/api" "net/http" "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/loader" ) -func Register(r *httprouter.Router, mw func(http.Handler) http.Handler, pathPrefix string, loader *loader.Loader, db *index.DB) { - handler := Handler{ - db: api.DbAdapter{DB: db}, - loader: loader, - } - +func Register(h Handler, r *httprouter.Router, mw func(http.Handler) http.Handler, pathPrefix string) { // https://pywb.readthedocs.io/en/latest/manual/warcserver.html#warcserver-api - r.Handler("GET", pathPrefix+"/cdx", mw(http.HandlerFunc(handler.index))) - r.Handler("GET", pathPrefix+"/web/:timestamp/*url", mw(http.HandlerFunc(handler.resource))) + r.Handler("GET", pathPrefix+"/cdx", mw(http.HandlerFunc(h.index))) + r.Handler("GET", pathPrefix+"/web/:timestamp/*url", mw(http.HandlerFunc(h.resource))) } diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go new file mode 100644 index 0000000..ec1745f --- /dev/null +++ b/internal/tikvidx/api.go @@ -0,0 +1,248 @@ +package tikvidx + +import ( + "context" + + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/schema" + "google.golang.org/protobuf/proto" +) + +// Closest returns the first closest cdx value(s). +func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- index.CdxResponse) error { + // begin transaction + tx, err := db.client.Begin() + if err != nil { + return err + } + + it, err := NewIterClosest(ctx, tx, req.Key(), req.Closest()) + if err != nil { + return err + } + + go func() { + defer close(res) + defer it.Close() + + for it.Valid() { + select { + case <-ctx.Done(): + return + default: + } + + cdx, err := cdxFromValue(it.Value()) + if err != nil { + res <- index.CdxResponse{Error: err} + } else { + res <- index.CdxResponse{Cdx: cdx} + } + + if err := it.Next(); err != nil { + res <- index.CdxResponse{Error: err} + } + } + }() + + return nil +} + +func cdxFromValue(value []byte) (*schema.Cdx, error) { + result := new(schema.Cdx) + if err := proto.Unmarshal(value, result); err != nil { + return nil, err + } + return result, nil +} + +func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- index.CdxResponse) error { + // begin transaction + tx, err := db.client.Begin() + if err != nil { + return err + } + + // initialize iterator + it, err := NewIter(ctx, tx, req) + if err != nil { + return err + } + + limit := req.Limit() + if limit == 0 { + limit = 100 + } + + go func() { + defer close(res) + defer it.Close() + + for it.Valid() && limit > 0 { + select { + case <-ctx.Done(): + res <- index.CdxResponse{Error: ctx.Err()} + return + default: + limit-- + } + + func() { + inDateRange, err := req.DateRange().ContainsStr(cdxKey(it.Key()).ts()) + if err != nil { + res <- index.CdxResponse{Error: err} + return + } + if !inDateRange { + return + } + cdx := new(schema.Cdx) + if err := proto.Unmarshal(it.Value(), cdx); err != nil { + res <- index.CdxResponse{Error: err} + return + } + if req.Filter().Eval(cdx) { + res <- index.CdxResponse{Cdx: cdx} + } + }() + if err := it.Next(); err != nil { + res <- index.CdxResponse{Error: err} + break + } + } + }() + return nil +} + +func (db *DB) List(ctx context.Context, limit int, res chan<- index.CdxResponse) error { + tx, err := db.client.Begin() + + it, err := tx.Iter([]byte(cdxPrefix), []byte(cdxEOF)) + if err != nil { + return err + } + + go func() { + defer it.Close() + defer close(res) + + for it.Valid() && limit > 0 { + select { + case <-ctx.Done(): + res <- index.CdxResponse{Error: ctx.Err()} + return + default: + } + limit-- + cdx := new(schema.Cdx) + err := proto.Unmarshal(it.Value(), cdx) + if err != nil { + res <- index.CdxResponse{Error: err} + } else { + res <- index.CdxResponse{Cdx: cdx} + } + err = it.Next() + if err != nil { + res <- index.CdxResponse{Error: err} + break + } + } + }() + + return nil +} + +func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, error) { + return db.getFileInfo(filename) +} + +func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileResponse) error { + tx, err := db.client.Begin() + + it, err := tx.Iter([]byte(filePrefix), []byte(fileEOF)) + if err != nil { + return err + } + + go func() { + defer it.Close() // close iterator + defer close(res) // close response channel + + for it.Valid() && limit > 0 { + limit-- + fileInfo := new(schema.Fileinfo) + err := proto.Unmarshal(it.Value(), fileInfo) + if err != nil { + res <- index.FileResponse{Error: err} + } else { + res <- index.FileResponse{Fileinfo: fileInfo} + } + err = it.Next() + if err != nil { + res <- index.FileResponse{Error: err} + break + } + } + }() + + return nil +} + +func (db *DB) GetStorageRef(ctx context.Context, id string) (string, error) { + tx, err := db.client.Begin() + if err != nil { + return "", err + } + + b, err := tx.Get(ctx, []byte(id)) + return string(b), err +} + +func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdResponse) error { + tx, err := db.client.Begin() + + it, err := tx.Iter([]byte(idPrefix), []byte(idEOF)) + if err != nil { + return err + } + + go func() { + defer it.Close() + defer close(res) + + for it.Valid() && limit > 0 { + limit-- + if err != nil { + res <- index.IdResponse{Error: err} + } else { + res <- index.IdResponse{Key: string(it.Key()[1:]), Value: string(it.Value())} + } + err = it.Next() + if err != nil { + res <- index.IdResponse{Error: err} + break + } + } + }() + + return nil +} + +// Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. +func (db *DB) Resolve(warcId string) (storageRef string, err error) { + key := []byte(idPrefix + warcId) + + var kv KV + kv, err = db.get(key) + if err != nil { + return + } + storageRef = string(kv.V) + return +} + +// ResolvePath looks up filename in file index and returns the path field. +func (db *DB) ResolvePath(filename string) (filePath string, err error) { + fileInfo, err := db.getFileInfo(filename) + return fileInfo.Path, err +} diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go new file mode 100644 index 0000000..65e6c40 --- /dev/null +++ b/internal/tikvidx/db.go @@ -0,0 +1,303 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tikvidx + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/schema" + "github.com/rs/zerolog/log" + "github.com/tikv/client-go/v2/txnkv" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// KV represents a Key-Value pair. +type KV struct { + K, V []byte +} + +func (kv KV) String() string { + return fmt.Sprintf("%s => %s (%v)", kv.K, kv.V, kv.V) +} + +func (kv KV) ts() int64 { + b := bytes.Split(kv.K, []byte{32})[1] + ts, _ := time.Parse(timestamp.CDX, string(b)) + return ts.Unix() +} + +type cdxKey string + +func (k cdxKey) ts() string { + return strings.Split(string(k), " ")[1] +} + +// tikv does not (yet) have a notion of keyspace, so we use key prefixes +const ( + idPrefix = "i" + idEOF = "j" + filePrefix = "f" + fileEOF = "g" + cdxPrefix = "c" + cdxEOF = "d" +) + +type DB struct { + client *txnkv.Client + + batch chan index.Record + + done chan struct{} + + wg sync.WaitGroup +} + +func NewDB(options ...Option) (db *DB, err error) { + opts := defaultOptions() + for _, opt := range options { + opt(opts) + } + + client, err := txnkv.NewClient(opts.PdAddr) + if err != nil { + return nil, err + } + + done := make(chan struct{}) + + db = &DB{ + client: client, + done: done, + } + + if opts.ReadOnly { + return + } + + db.batch = make(chan index.Record, opts.BatchMaxSize) + + // start batch worker + db.wg.Add(1) + go func() { + defer db.wg.Done() + ticker := time.NewTimer(opts.BatchMaxWait) + defer ticker.Stop() + for { + select { + case <-done: + db.flushBatch() + return + case <-ticker.C: + db.flushBatch() + } + } + }() + + return +} + +// Close stops the batch workers and closes the index databases. +func (db *DB) Close() { + close(db.done) + db.wg.Wait() +} + +// addFile checks if file referenced by filePath is indexed or has changed and adds/updates the index accordingly. +func (db *DB) addFile(filePath string) error { + stat, err := os.Stat(filePath) + if err != nil { + return fmt.Errorf("failed to get file info: %s: %w", filePath, err) + } + + fileSize := stat.Size() + fileLastModified := stat.ModTime() + fn := filepath.Base(filePath) + if fileInfo, err := db.getFileInfo(fn); err == nil { + if err := fileInfo.GetLastModified().CheckValid(); err != nil { + return err + } + fileInfoLastModified := fileInfo.LastModified.AsTime() + if fileInfo.Size == fileSize && fileInfoLastModified.Equal(fileLastModified) { + return errors.New("already indexed") + } + } + + return db.updateFilePath(filePath) +} + +func (db *DB) updateFilePath(filePath string) error { + var err error + fileInfo := new(schema.Fileinfo) + + fileInfo.Path, err = filepath.Abs(filePath) + if err != nil { + return err + } + + fileInfo.Name = filepath.Base(fileInfo.Path) + stat, err := os.Stat(fileInfo.Path) + if err != nil { + return err + } + + fileInfo.Size = stat.Size() + fileInfo.LastModified = timestamppb.New(stat.ModTime()) + + return db.putFileInfo(fileInfo) +} + +func (db *DB) get(k []byte) (KV, error) { + tx, err := db.client.Begin() + if err != nil { + return KV{}, err + } + v, err := tx.Get(context.TODO(), k) + if err != nil { + return KV{}, err + } + return KV{K: k, V: v}, nil +} + +func (db *DB) puts(kvs ...KV) error { + tx, err := db.client.Begin() + if err != nil { + return err + } + + for _, kv := range kvs { + err := tx.Set(kv.K, kv.V) + if err != nil { + return err + } + } + return tx.Commit(context.Background()) +} + +func (db *DB) putFileInfo(fi *schema.Fileinfo) error { + k := []byte(filePrefix + fi.Name) + v, err := proto.Marshal(fi) + if err != nil { + return err + } + return db.puts(KV{K: k, V: v}) +} + +func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { + key := []byte(filePrefix + fileName) + val, err := db.get(key) + if err != nil { + return nil, err + } + fi := new(schema.Fileinfo) + err = proto.Unmarshal(val.V, fi) + if err != nil { + return nil, err + } + return fi, nil +} + +// write schedules a Record to be added to the DB via the batch channel. +func (db *DB) write(rec index.Record) { + select { + case <-db.done: + // do nothing + case db.batch <- rec: + // added record to batch + default: + // batch channel is full so flush batch channel before adding record to batch + db.flushBatch() + db.batch <- rec + } +} + +// collectBatch returns a slice of all the records in the batch channel. +func (db *DB) collectBatch() (records []index.Record) { + for { + select { + case record := <-db.batch: + records = append(records, record) + default: + return + } + } +} + +// flushBatch collects all records in the batch channel and updates the id and cdx indices. +func (db *DB) flushBatch() { + records := db.collectBatch() + if len(records) == 0 { + return + } + + var kvs []KV + for _, r := range records { + id := idKV(r) + cdx, err := cdxKV(r) + if err != nil { + log.Error().Err(err).Msgf("failed to marshal record: %v", r) + continue + } + kvs = append(kvs, id, cdx) + } + err := db.puts(kvs...) + if err != nil { + log.Error().Err(err).Msgf("failed to commit batch (first batch ref is %s)", records[0].Ref) + } +} + +// idKV takes a record and returns a key-value pair for the id index. +func idKV(r index.Record) KV { + return KV{ + K: []byte(idPrefix + r.GetRid()), + V: []byte(r.GetRef()), + } +} + +// cdxKV takes a record and returns a key-value pair for the cdx index. +func cdxKV(r index.Record) (KV, error) { + ts := timestamp.TimeTo14(r.GetSts().AsTime()) + k := []byte(cdxPrefix + r.GetSsu() + " " + ts + " " + r.GetSrt()) + v, err := r.Marshal() + if err != nil { + return KV{}, err + } + return KV{K: k, V: v}, nil +} + +func (db *DB) Write(rec index.Record) error { + db.write(rec) + return nil +} + +func (db *DB) Index(fileName string) error { + err := db.addFile(fileName) + if err != nil { + return err + } + return index.ReadFile(fileName, db) +} diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go new file mode 100644 index 0000000..db768c1 --- /dev/null +++ b/internal/tikvidx/iter.go @@ -0,0 +1,265 @@ +package tikvidx + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/tikv/client-go/v2/txnkv/transaction" +) + +// iterator mimics tikv's internal iterator interface +type iterator interface { + Next() error + Key() []byte + Value() []byte + Valid() bool + Close() +} + +type iterClosest struct { + forward iterator + backward iterator + prefix []byte + key []byte + value []byte + valid bool + cmp func(int64, int64) bool +} + +func NewIterClosest(_ context.Context, tx *transaction.KVTxn, key string, closest string) (*iterClosest, error) { + ic := new(iterClosest) + if t, err := time.Parse(timestamp.CDX, closest); err != nil { + return nil, err + } else { + ic.cmp = CompareClosest(t.Unix()) + } + + ic.prefix = []byte(cdxPrefix + key) + k := []byte(cdxPrefix + key + " " + closest) + + // initialize forward iterator + forward, err := tx.Iter(k, []byte(cdxEOF)) + if err != nil { + return nil, err + } + ic.forward = forward + + // initialize backward iterator + backward, err := tx.IterReverse(k) + if err != nil { + return nil, err + } + ic.backward = backward + + return ic, ic.Next() +} + +func (ic *iterClosest) Next() error { + var ft int64 + var bt int64 + + // get forward ts + if ic.forward.Valid() && bytes.HasPrefix(ic.forward.Key(), ic.prefix) { + ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.forward.Key()).ts()) + ft = ts.Unix() + } + // get backward ts + if ic.backward.Valid() && bytes.HasPrefix(ic.backward.Key(), ic.prefix) { + ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.backward.Key()).ts()) + bt = ts.Unix() + } + + var it iterator + if ft != 0 && bt != 0 { + // find closest of forward and backward + isForward := ic.cmp(ft, bt) + if isForward { + it = ic.forward + } else { + it = ic.backward + } + } else if ft != 0 { + it = ic.forward + } else if bt != 0 { + it = ic.backward + } else { + ic.valid = false + return nil + } + ic.key = it.Key() + ic.value = it.Value() + + return it.Next() +} + +func (ic *iterClosest) Key() []byte { + return ic.key +} + +func (ic *iterClosest) Value() []byte { + return ic.value +} + +func (ic *iterClosest) Valid() bool { + return ic.valid +} + +func (ic *iterClosest) Close() { + ic.forward.Close() + ic.backward.Close() +} + +type maybeKV struct { + kv KV + error error +} + +type iterSort struct { + iterators []iterator + keys [][]byte + key []byte + value []byte + valid bool + cmp func([]byte, []byte) bool + next <-chan maybeKV + // done <-chan struct{} +} + +func NewIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest) (iterator, error) { + is := new(iterSort) + // iter.done = ctx.Done() + + // initialize iterators + var results []chan *maybeKV + var err error + var it iterator + + if len(req.Keys()) == 0 { + k := []byte(cdxPrefix) + switch req.Sort() { + case index.SortDesc: + it, err = tx.IterReverse(k) + case index.SortAsc: + it, err = tx.Iter(k, []byte(cdxEOF)) + case index.SortClosest: + return nil, fmt.Errorf("invalid request: sort=closest with no keys") + default: + it, err = tx.Iter(k, []byte(cdxEOF)) + } + return it, nil + } + + for _, key := range req.Keys() { + k := []byte(cdxPrefix + key) + var it iterator + switch req.Sort() { + case index.SortDesc: + it, err = tx.IterReverse(k) + case index.SortClosest: + it, err = NewIterClosest(ctx, tx, key, req.Closest()) + case index.SortAsc: + fallthrough + default: + it, err = tx.Iter(k, []byte(cdxEOF)) + } + if err != nil { + break + } + is.iterators = append(is.iterators, it) + results = append(results, make(chan *maybeKV)) + } + if err != nil { + defer is.Close() + return nil, err + } + + // initialize comparator + var cmp func(a KV, b KV) bool + + switch req.Sort() { + case index.SortDesc: + cmp = func(a KV, b KV) bool { + return CompareDesc(a.ts(), b.ts()) + } + case index.SortClosest: + var t time.Time + t, err = time.Parse(timestamp.CDX, req.Closest()) + cmp = func(a KV, b KV) bool { + return CompareClosest(t.Unix())(a.ts(), b.ts()) + } + case index.SortAsc: + fallthrough + default: + cmp = func(a KV, b KV) bool { + return CompareAsc(a.ts(), b.ts()) + } + } + + is.next = mergeIter(ctx.Done(), cmp, results...) + + for i, iter := range is.iterators { + i := i + go func(iter iterator, ch chan<- *maybeKV) { + defer close(ch) + for iter.Valid() && bytes.HasPrefix(iter.Key(), []byte(cdxPrefix)) { + select { + case <-ctx.Done(): + return + case ch <- &maybeKV{kv: KV{K: iter.Key(), V: iter.Value()}}: + } + err := iter.Next() + if err != nil { + select { + case <-ctx.Done(): + case ch <- &maybeKV{error: err}: + } + return + } + } + }(iter, results[i]) + } + + return is, is.Next() +} + +// Next updates the next key, value and validity. +func (is *iterSort) Next() error { + mkv, ok := <-is.next + if !ok { + is.valid = false + return nil + } + is.valid = true + + if mkv.error != nil { + return mkv.error + } + is.key = mkv.kv.K + is.value = mkv.kv.V + + return nil +} + +func (is *iterSort) Key() []byte { + return is.key +} + +func (is *iterSort) Value() []byte { + return is.value +} + +func (is *iterSort) Valid() bool { + return is.valid +} + +func (is *iterSort) Close() { + for _, it := range is.iterators { + if it != nil { + it.Close() + } + } +} diff --git a/internal/tikvidx/options.go b/internal/tikvidx/options.go new file mode 100644 index 0000000..6a110ae --- /dev/null +++ b/internal/tikvidx/options.go @@ -0,0 +1,43 @@ +package tikvidx + +import "time" + +func defaultOptions() *Options { + return &Options{ + BatchMaxSize: 10000, + BatchMaxWait: 5 * time.Second, + } +} + +type Options struct { + BatchMaxSize int + BatchMaxWait time.Duration + ReadOnly bool + PdAddr []string +} + +type Option func(opts *Options) + +func WithPDAddress(pdAddr []string) Option { + return func(opts *Options) { + opts.PdAddr = pdAddr + } +} + +func WithReadOnly(readOnly bool) Option { + return func(opts *Options) { + opts.ReadOnly = readOnly + } +} + +func WithBatchMaxSize(size int) Option { + return func(opts *Options) { + opts.BatchMaxSize = size + } +} + +func WithBatchMaxWait(t time.Duration) Option { + return func(opts *Options) { + opts.BatchMaxWait = t + } +} diff --git a/internal/tikvidx/sorter.go b/internal/tikvidx/sorter.go new file mode 100644 index 0000000..6f35eb6 --- /dev/null +++ b/internal/tikvidx/sorter.go @@ -0,0 +1,149 @@ +/* + * Copyright 2020 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tikvidx + +import ( + "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/rs/zerolog/log" +) + +func CompareClosest(ts int64) func(int64, int64) bool { + return func(ts1 int64, ts2 int64) bool { + return timestamp.AbsInt64(ts-ts1) < timestamp.AbsInt64(ts-ts2) + } +} + +func CompareAsc(a int64, b int64) bool { + return a <= b +} + +func CompareDesc(a int64, b int64) bool { + return a > b +} + +// mergeSort sorts elements of input channels to output channel according to sort direction. +func mergeSort(done <-chan struct{}, cmp func(*index.CdxResponse, *index.CdxResponse) bool, in ...chan index.CdxResponse) <-chan index.CdxResponse { + out := make(chan index.CdxResponse) + cords := make([]*index.CdxResponse, len(in)) + + go func() { + var zombie []int + for { + var curr *index.CdxResponse + for i, cord := range cords { + if cord == nil { + v, ok := <-in[i] + if !ok { + // closed channel becomes a zombie + zombie = append(zombie, i) + continue + } + cord = &v + } + if cmp(curr, cord) { + curr = cord + } + } + if curr == nil { + return + } + select { + case out <- *curr: + case <-done: + return + } + if len(zombie) > 0 { + // kill zombies to avoid checking on them every round + for _, i := range zombie { + cords[i] = cords[len(cords)-1] + cords = cords[:len(cords)-1] + in[i] = in[len(in)-1] + in = in[:len(in)-1] + } + zombie = nil + } + } + }() + + return out +} + +// mergeSort merges sorted input channels to a sorted output channel +// +// Sorting is done by comparing keys from key-value pairs. +// +// The input channels are closed externally +func mergeIter(done <-chan struct{}, cmp func(KV, KV) bool, in ...chan *maybeKV) <-chan maybeKV { + out := make(chan maybeKV) + cords := make([]*maybeKV, len(in)) + go func() { + defer close(out) + var zombie []int + for { + curr := -1 + for i, cord := range cords { + if cord == nil { + select { + case cord = <-in[i]: + cords[i] = cord + case <-done: + return + } + // closed channel becomes zombie + if cord == nil { + zombie = append(zombie, i) + continue + } + } + if cord.error != nil { + log.Error().Err(cord.error).Msg("cord error") + // prioritize errors + curr = i + break + } + if curr == -1 { + curr = i + } else if cmp(cords[i].kv, cord.kv) { + curr = i + } + } + if curr == -1 { + return + } + select { + case <-done: + return + case out <- *cords[curr]: + cords[curr] = nil + curr = -1 + } + // if zombie, then kill + if len(zombie) > 0 { + for _, i := range zombie { + cords[i] = cords[len(cords)-1] + cords = cords[:len(cords)-1] + in[i] = in[len(in)-1] + in = in[:len(in)-1] + } + zombie = nil + } + } + }() + + return out +} diff --git a/schema/Makefile b/schema/Makefile index 4096053..a649afe 100644 --- a/schema/Makefile +++ b/schema/Makefile @@ -1,4 +1,4 @@ -PROTOC_VERSION=3.19.1 +PROTOC_VERSION=21.8 .PHONY: all .PHONY: clean diff --git a/schema/cdx.pb.go b/schema/cdx.pb.go index d92d48d..37fb541 100644 --- a/schema/cdx.pb.go +++ b/schema/cdx.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.19.1 +// protoc v3.21.8 // source: cdx.proto package schema diff --git a/schema/fileinfo.pb.go b/schema/fileinfo.pb.go index b175656..1215437 100644 --- a/schema/fileinfo.pb.go +++ b/schema/fileinfo.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.27.1 -// protoc v3.19.1 +// protoc v3.21.8 // source: fileinfo.proto package schema From 6f1c7e78fca21a543e38f58a2fcc927655cd30d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 20:53:07 +0100 Subject: [PATCH 02/38] Promote index, server, loader and surt packages from internal --- {internal/index => index}/api.go | 0 {internal/index => index}/autoindex.go | 0 {internal/index => index}/io.go | 0 {internal/index => index}/io_test.go | 0 {internal/index => index}/options.go | 0 {internal/index => index}/record.go | 0 {internal/index => index}/watcher.go | 3 +- {internal/index => index}/worker.go | 6 +- {internal/index => index}/writers.go | 5 +- internal/server/api/error.go | 18 ---- internal/server/api/sorter.go | 95 ------------------- .../loader => loader}/filestorageloader.go | 0 {internal/loader => loader}/loader.go | 0 {internal/loader => loader}/loader_test.go | 0 {internal/server => server}/api/api.go | 4 +- {internal/server => server}/api/daterange.go | 0 .../server => server}/api/daterange_test.go | 0 {internal/server => server}/api/filter.go | 0 {internal/server => server}/api/searchkey.go | 10 +- {internal/server => server}/coreserver/api.go | 0 .../server => server}/coreserver/handler.go | 0 .../server => server}/coreserver/routes.go | 0 .../server => server}/handlers/render.go | 0 {internal/server => server}/warcserver/api.go | 0 .../server => server}/warcserver/handler.go | 0 .../server => server}/warcserver/routes.go | 0 {internal/surt => surt}/ssurt.go | 0 {internal/surt => surt}/ssurt_test.go | 0 {internal/surt => surt}/surt.go | 0 {internal/surt => surt}/surt_test.go | 0 30 files changed, 17 insertions(+), 124 deletions(-) rename {internal/index => index}/api.go (100%) rename {internal/index => index}/autoindex.go (100%) rename {internal/index => index}/io.go (100%) rename {internal/index => index}/io_test.go (100%) rename {internal/index => index}/options.go (100%) rename {internal/index => index}/record.go (100%) rename {internal/index => index}/watcher.go (99%) rename {internal/index => index}/worker.go (93%) rename {internal/index => index}/writers.go (99%) delete mode 100644 internal/server/api/error.go delete mode 100644 internal/server/api/sorter.go rename {internal/loader => loader}/filestorageloader.go (100%) rename {internal/loader => loader}/loader.go (100%) rename {internal/loader => loader}/loader_test.go (100%) rename {internal/server => server}/api/api.go (97%) rename {internal/server => server}/api/daterange.go (100%) rename {internal/server => server}/api/daterange_test.go (100%) rename {internal/server => server}/api/filter.go (100%) rename {internal/server => server}/api/searchkey.go (87%) rename {internal/server => server}/coreserver/api.go (100%) rename {internal/server => server}/coreserver/handler.go (100%) rename {internal/server => server}/coreserver/routes.go (100%) rename {internal/server => server}/handlers/render.go (100%) rename {internal/server => server}/warcserver/api.go (100%) rename {internal/server => server}/warcserver/handler.go (100%) rename {internal/server => server}/warcserver/routes.go (100%) rename {internal/surt => surt}/ssurt.go (100%) rename {internal/surt => surt}/ssurt_test.go (100%) rename {internal/surt => surt}/surt.go (100%) rename {internal/surt => surt}/surt_test.go (100%) diff --git a/internal/index/api.go b/index/api.go similarity index 100% rename from internal/index/api.go rename to index/api.go diff --git a/internal/index/autoindex.go b/index/autoindex.go similarity index 100% rename from internal/index/autoindex.go rename to index/autoindex.go diff --git a/internal/index/io.go b/index/io.go similarity index 100% rename from internal/index/io.go rename to index/io.go diff --git a/internal/index/io_test.go b/index/io_test.go similarity index 100% rename from internal/index/io_test.go rename to index/io_test.go diff --git a/internal/index/options.go b/index/options.go similarity index 100% rename from internal/index/options.go rename to index/options.go diff --git a/internal/index/record.go b/index/record.go similarity index 100% rename from internal/index/record.go rename to index/record.go diff --git a/internal/index/watcher.go b/index/watcher.go similarity index 99% rename from internal/index/watcher.go rename to index/watcher.go index 414f04a..a3355f7 100644 --- a/internal/index/watcher.go +++ b/index/watcher.go @@ -17,9 +17,10 @@ package index import ( + "os" + "github.com/fsnotify/fsnotify" "github.com/rs/zerolog/log" - "os" ) func newWatcher() (*watcher, error) { diff --git a/internal/index/worker.go b/index/worker.go similarity index 93% rename from internal/index/worker.go rename to index/worker.go index c323e28..0950894 100644 --- a/internal/index/worker.go +++ b/index/worker.go @@ -19,6 +19,8 @@ package index import ( "sync" "time" + + "github.com/rs/zerolog/log" ) type indexWorker struct { @@ -46,7 +48,9 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { for { select { case job := <-iw.jobs: - _ = a.Index(job) + if err := a.Index(job); err != nil { + log.Error().Err(err).Msgf("Index job: %s", job) + } iw.wg.Done() iw.mx.Lock() delete(iw.jobMap, job) diff --git a/internal/index/writers.go b/index/writers.go similarity index 99% rename from internal/index/writers.go rename to index/writers.go index 1613b45..c3fe3b4 100644 --- a/internal/index/writers.go +++ b/index/writers.go @@ -18,12 +18,13 @@ package index import ( "fmt" + "strconv" + "sync" + "github.com/bits-and-blooms/bloom/v3" "github.com/nlnwa/gowarcserver/internal/surt" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "strconv" - "sync" ) type Cdx struct { diff --git a/internal/server/api/error.go b/internal/server/api/error.go deleted file mode 100644 index 8e61899..0000000 --- a/internal/server/api/error.go +++ /dev/null @@ -1,18 +0,0 @@ -package api - -import ( - "net/http" - "runtime" - - "github.com/rs/zerolog/log" -) - -func HandleError(w http.ResponseWriter, n int, err error) { - // try to log error with caller information - _, file, no, ok := runtime.Caller(1) - if ok { - log.Error().Err(err).Msgf("%s#%d\n", file, no) - } else { - log.Error().Err(err).Msg("") - } -} diff --git a/internal/server/api/sorter.go b/internal/server/api/sorter.go deleted file mode 100644 index a7b715f..0000000 --- a/internal/server/api/sorter.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2020 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package api - -import ( - "sort" - "strings" - - "github.com/dgraph-io/badger/v3" - "github.com/nlnwa/gowarcserver/internal/timestamp" -) - -type cdxKey string - -func (k cdxKey) ts() string { - return strings.Split(string(k), " ")[1] -} - -type value struct { - ts int64 - k []byte -} - -type Sorter struct { - Closest int64 - Reverse bool - Values []value -} - -func NewSorter(closest int64, reverse bool) Sorter { - return Sorter{ - Closest: closest, - Reverse: reverse, - Values: []value{}, - } -} - -func (s *Sorter) Add(k []byte) { - t, _ := timestamp.Parse(cdxKey(k).ts()) - ts := t.Unix() - - s.Values = append(s.Values, value{ts, k}) -} - -func (s *Sorter) Sort() { - var cmp func(int64, int64) bool - - // sort Closest, Reverse or forward - if s.Closest > 0 { - cmp = func(ts1 int64, ts2 int64) bool { - return timestamp.AbsInt64(s.Closest-ts1) < timestamp.AbsInt64(s.Closest-ts2) - } - } else if s.Reverse { - cmp = func(ts1 int64, ts2 int64) bool { - return ts2 < ts1 - } - } else { - cmp = func(ts1 int64, ts2 int64) bool { - return ts1 < ts2 - } - } - - sort.Slice(s.Values, func(i, j int) bool { - return cmp(s.Values[i].ts, s.Values[j].ts) - }) -} - -type PerItemFunc func(*badger.Item) (stopIteration bool) - -func (s *Sorter) Walk(txn *badger.Txn, perItemFn PerItemFunc) error { - for _, value := range s.Values { - item, err := txn.Get(value.k) - if err != nil { - return err - } - if perItemFn(item) { - break - } - } - return nil -} diff --git a/internal/loader/filestorageloader.go b/loader/filestorageloader.go similarity index 100% rename from internal/loader/filestorageloader.go rename to loader/filestorageloader.go diff --git a/internal/loader/loader.go b/loader/loader.go similarity index 100% rename from internal/loader/loader.go rename to loader/loader.go diff --git a/internal/loader/loader_test.go b/loader/loader_test.go similarity index 100% rename from internal/loader/loader_test.go rename to loader/loader_test.go diff --git a/internal/server/api/api.go b/server/api/api.go similarity index 97% rename from internal/server/api/api.go rename to server/api/api.go index 1b785f1..7d04b41 100644 --- a/internal/server/api/api.go +++ b/server/api/api.go @@ -80,13 +80,13 @@ func (c SearchAPI) Closest() string { } func (c SearchAPI) Key() string { - return surt.UrlToSsurt(c.CoreAPI.Urls[0]) + return MatchType(surt.UrlToSsurt(c.CoreAPI.Urls[0]), c.CoreAPI.MatchType) } func (c SearchAPI) Keys() []string { keys := make([]string, len(c.Urls)) for i, u := range c.CoreAPI.Urls { - keys[i] = surt.UrlToSsurt(u) + keys[i] = MatchType(surt.UrlToSsurt(u), c.CoreAPI.MatchType) } return keys } diff --git a/internal/server/api/daterange.go b/server/api/daterange.go similarity index 100% rename from internal/server/api/daterange.go rename to server/api/daterange.go diff --git a/internal/server/api/daterange_test.go b/server/api/daterange_test.go similarity index 100% rename from internal/server/api/daterange_test.go rename to server/api/daterange_test.go diff --git a/internal/server/api/filter.go b/server/api/filter.go similarity index 100% rename from internal/server/api/filter.go rename to server/api/filter.go diff --git a/internal/server/api/searchkey.go b/server/api/searchkey.go similarity index 87% rename from internal/server/api/searchkey.go rename to server/api/searchkey.go index 0ce35bc..c28f459 100644 --- a/internal/server/api/searchkey.go +++ b/server/api/searchkey.go @@ -20,24 +20,24 @@ import ( "strings" ) -func ParseKey(ssurt string, matchType string) string { +func MatchType(ssurt string, matchType string) string { switch matchType { case MatchTypeExact: - ssurt += " " + return ssurt + " " case MatchTypePrefix: i := strings.IndexAny(ssurt, "?#") if i > 0 { - ssurt = ssurt[:i] + return ssurt[:i] } case MatchTypeHost: i := strings.Index(ssurt, "//") if i > 0 { - ssurt = ssurt[:i+2] + return ssurt[:i+2] } case MatchTypeDomain: i := strings.Index(ssurt, "//") if i > 0 { - ssurt = ssurt[:i] + return ssurt[:i] } } return ssurt diff --git a/internal/server/coreserver/api.go b/server/coreserver/api.go similarity index 100% rename from internal/server/coreserver/api.go rename to server/coreserver/api.go diff --git a/internal/server/coreserver/handler.go b/server/coreserver/handler.go similarity index 100% rename from internal/server/coreserver/handler.go rename to server/coreserver/handler.go diff --git a/internal/server/coreserver/routes.go b/server/coreserver/routes.go similarity index 100% rename from internal/server/coreserver/routes.go rename to server/coreserver/routes.go diff --git a/internal/server/handlers/render.go b/server/handlers/render.go similarity index 100% rename from internal/server/handlers/render.go rename to server/handlers/render.go diff --git a/internal/server/warcserver/api.go b/server/warcserver/api.go similarity index 100% rename from internal/server/warcserver/api.go rename to server/warcserver/api.go diff --git a/internal/server/warcserver/handler.go b/server/warcserver/handler.go similarity index 100% rename from internal/server/warcserver/handler.go rename to server/warcserver/handler.go diff --git a/internal/server/warcserver/routes.go b/server/warcserver/routes.go similarity index 100% rename from internal/server/warcserver/routes.go rename to server/warcserver/routes.go diff --git a/internal/surt/ssurt.go b/surt/ssurt.go similarity index 100% rename from internal/surt/ssurt.go rename to surt/ssurt.go diff --git a/internal/surt/ssurt_test.go b/surt/ssurt_test.go similarity index 100% rename from internal/surt/ssurt_test.go rename to surt/ssurt_test.go diff --git a/internal/surt/surt.go b/surt/surt.go similarity index 100% rename from internal/surt/surt.go rename to surt/surt.go diff --git a/internal/surt/surt_test.go b/surt/surt_test.go similarity index 100% rename from internal/surt/surt_test.go rename to surt/surt_test.go From 25979724667f5ee724472bfdfdf1e63d021eee91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 20:56:08 +0100 Subject: [PATCH 03/38] Fix flag descriptions --- cmd/index/index.go | 9 ++++----- cmd/serve/serve.go | 14 +++++++------- cmd/version/version.go | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index afd83ba..441aecb 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -24,8 +24,8 @@ import ( "github.com/bits-and-blooms/bloom/v3" "github.com/dgraph-io/badger/v3/options" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/badgeridx" - "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/tikvidx" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -68,10 +68,9 @@ func NewCommand() *cobra.Command { cmd.Flags().String("badger-dir", badgerDir, "path to index database") cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "path to index database") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records") - + cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") return cmd } diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 9c32ca1..d09f467 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -31,12 +31,12 @@ import ( "github.com/dgraph-io/badger/v3/options" "github.com/gorilla/handlers" "github.com/julienschmidt/httprouter" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/badgeridx" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/loader" - "github.com/nlnwa/gowarcserver/internal/server/coreserver" - "github.com/nlnwa/gowarcserver/internal/server/warcserver" "github.com/nlnwa/gowarcserver/internal/tikvidx" + "github.com/nlnwa/gowarcserver/loader" + "github.com/nlnwa/gowarcserver/server/coreserver" + "github.com/nlnwa/gowarcserver/server/warcserver" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -85,9 +85,9 @@ func NewCommand() *cobra.Command { cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") cmd.Flags().Bool("badger-read-only", badgerReadOnly, "run badger read-only") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "path to index database") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records") + cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") return cmd } diff --git a/cmd/version/version.go b/cmd/version/version.go index 714fec2..6573d04 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -27,7 +27,7 @@ var Version string func NewCommand() *cobra.Command { return &cobra.Command{ Use: "version", - Short: "Print version number", + Short: "Print version", Run: func(cmd *cobra.Command, args []string) { fmt.Println(Version) }, From 8a59e80bfaa10c1e21a6c58955cf35c175b3212f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 21:33:50 +0100 Subject: [PATCH 04/38] Fix config example --- exampleconfig.yaml | 54 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/exampleconfig.yaml b/exampleconfig.yaml index 4a2c64e..7d02e72 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -1,16 +1,30 @@ -port: 8880 +# GLOBAL + log-level: "debug" log-formatter: "log-fmt" log-method: false # SERVER +# server port +port: 8880 +# prefix for server endpoint paths +path-prefix: "" +# start autoindexer index: true +# watch for file changes (NFS does not support this) watch: false # log server requests log-requests: true +# index storage engine: "badger" or "tikv" +storage-engine: badger # INDEX + +# index format: "cdxj", "cdxpb", "badger", "tikv" or "toc" +format: cdxj + +# directories to search for warc files in dirs: - "./testdata" # include only files matching regular expressions @@ -18,21 +32,29 @@ include: - ".warc.gz$" # exclude files matching regular expressions exclude: [] +# max number of directory recursions max-depth: 4 +# number of index workers workers: 8 -# format: toc -bloom-capacity: 10000 -bloom-fp: 0.1 - -# format: cdxdb -db-dir: . -compression: "snappy" - - -# PROXY - -child-urls: - - "http://localhost:8888" - - "http://localhost:8889" -child-query-timeout: "300ms" +# path to badger database +badger-dir: ./badger +# run badger read-only +badger-read-only: false +# max transaction batch size +badger-batch-max-size: 1000 +# max wait time before flushing batched records regardless of max batch size +badger-batch-max-wait: 5s +# badger compression algorithm +badger-compression: "snappy" + +# address of tikv placement driver +tikv-pd-addr: "127.0.0.1: 2379" +# max transaction batch size +tikv-batch-max-size: 1000 +# max wait time before flushing batched records regardless of max batch size +tikv-batch-max-wait: 5s + +# estimated bloom filter capacity +toc-bloom-capacity: 10000 +toc-bloom-fp: 0.1 From 58effc17a388c893497641f1381c1caeb8331a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 21:36:25 +0100 Subject: [PATCH 05/38] Don't log before logger is initialized --- cmd/root.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index afd3051..1764f4b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,6 +18,7 @@ package cmd import ( "errors" + "fmt" "os" "strings" @@ -46,7 +47,7 @@ func NewCommand() *cobra.Command { _ = cmd.PersistentFlags().Bool("log-method", false, "log method caller") if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil { - log.Error().Msgf("Failed to bind root flags: %v", err) + _, _ = fmt.Fprintf(os.Stderr, "Failed to bind global flags: %v", err) os.Exit(1) } @@ -79,7 +80,7 @@ func initConfig() { if errors.As(err, new(viper.ConfigFileNotFoundError)) { return } - log.Error().Msgf("Failed to read config file: %v", err) + _, _ = fmt.Fprintf(os.Stderr, "Failed to read config file: %v", err) os.Exit(1) } logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) From 1842610b93ce30aebd1a0a307af242d494cd8e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 21:37:38 +0100 Subject: [PATCH 06/38] Fix flags --- cmd/index/index.go | 4 +++- cmd/serve/serve.go | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index 441aecb..3620ea8 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -51,6 +51,7 @@ func NewCommand() *cobra.Command { badgerDir := "." badgerBatchMaxSize := 1000 badgerBatchMaxWait := 5 * time.Second + badgerCompression := "snappy" var tikvPdAddr []string tikvBatchMaxSize := 1000 tikvBatchMaxWait := 5 * time.Second @@ -68,6 +69,7 @@ func NewCommand() *cobra.Command { cmd.Flags().String("badger-dir", badgerDir, "path to index database") cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") + cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") @@ -107,7 +109,7 @@ func indexCmd(_ *cobra.Command, args []string) error { runtime.GOMAXPROCS(128) var c options.CompressionType - if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { + if err := viper.UnmarshalKey("badger-compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { return err } db, err := badgeridx.NewDB( diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index d09f467..4afa6b8 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -58,9 +58,11 @@ func NewCommand() *cobra.Command { port := 9999 watch := false storageEngine := "badger" + autoIndex := true indexDepth := 4 indexWorkers := 8 badgerDir := "." + badgerCompression := "snappy" badgerBatchMaxSize := 1000 badgerBatchMaxWait := 5 * time.Second badgerReadOnly := false @@ -71,17 +73,18 @@ func NewCommand() *cobra.Command { pathPrefix := "" cmd.Flags().IntP("port", "p", port, "server port") - cmd.Flags().String("proxy-url", "", "url to a gowarc server proxy that will be used to resolve records") cmd.Flags().String("path-prefix", pathPrefix, "prefix for all server endpoints") cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") cmd.Flags().IntP("max-depth", "w", indexDepth, "maximum directory recursion depth") cmd.Flags().Int("workers", indexWorkers, "number of index workers") cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") + cmd.Flags().Bool("index", autoIndex, "run indexing") cmd.Flags().Bool("watch", watch, "watch files for changes") cmd.Flags().Bool("log-requests", logRequests, "log http requests") cmd.Flags().StringP("storage-engine", "b", storageEngine, `storage engine: "badger" or "tikv"`) cmd.Flags().String("badger-dir", badgerDir, "path to index database") + cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") cmd.Flags().Bool("badger-read-only", badgerReadOnly, "run badger read-only") @@ -102,7 +105,7 @@ func serveCmd(cmd *cobra.Command, args []string) error { } // parse badger compression type var c options.CompressionType - if err := viper.UnmarshalKey("compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { + if err := viper.UnmarshalKey("badger-compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { return err } // parse include patterns From 551cb8543ee1b0b0f6b265620062b318e2ca4dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 21:38:00 +0100 Subject: [PATCH 07/38] Upgrade tikv go client --- go.mod | 2 +- go.sum | 39 ++++++++++++++------------------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 1159ed1..7b06854 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.10.1 - github.com/tikv/client-go/v2 v2.0.1 + github.com/tikv/client-go/v2 v2.0.2 golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect google.golang.org/protobuf v1.28.0 gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 89e4fc9..b6f31d3 100644 --- a/go.sum +++ b/go.sum @@ -165,7 +165,6 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -209,8 +208,9 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -311,7 +311,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -389,9 +388,6 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= -github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -400,11 +396,10 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 h1:c0d/sMTeftJQF9O5OHyezWwPrzf2FXcEE5HWwnq/Ahs= -github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= -github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= +github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -480,6 +475,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= +github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= +github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -491,10 +488,10 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tikv/client-go/v2 v2.0.1 h1:+K/VvVOxEOXKMtR83bs5Aj3lrYdTdTZdvH0apfAWW10= -github.com/tikv/client-go/v2 v2.0.1/go.mod h1:gaHSp8rnxZ0w36qb6QPPNPh9P0Mu5vAEwCQcc0Brni4= -github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 h1:7h/Oi4Zw6eGCeXh4Q4ZvKI4k7nBJVUq0c29YCcLwKPM= -github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8= +github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= +github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= +github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc= +github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= @@ -525,9 +522,7 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -537,15 +532,10 @@ go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= @@ -778,7 +768,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -795,8 +784,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1030,3 +1017,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= From b5387ff177fcc8ec0e44f00ee0f03474b5834e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Fri, 4 Nov 2022 21:40:57 +0100 Subject: [PATCH 08/38] Be explicit about it context --- internal/badgeridx/api.go | 14 +++--- internal/badgeridx/db.go | 2 +- internal/badgeridx/sorter.go | 86 ++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 internal/badgeridx/sorter.go diff --git a/internal/badgeridx/api.go b/internal/badgeridx/api.go index 2a3499b..c943e9a 100644 --- a/internal/badgeridx/api.go +++ b/internal/badgeridx/api.go @@ -25,8 +25,7 @@ import ( "time" "github.com/dgraph-io/badger/v3" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/server/api" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" @@ -90,7 +89,7 @@ func (db *DB) List(ctx context.Context, limit int, results chan<- index.CdxRespo } // Closest returns the first closest cdx value -func (db *DB) Closest(ctx context.Context, search index.ClosestRequest, results chan<- index.CdxResponse) error { +func (db *DB) Closest(_ context.Context, search index.ClosestRequest, results chan<- index.CdxResponse) error { go func() { _ = db.CdxIndex.View(func(txn *badger.Txn) error { key := search.Key() @@ -232,7 +231,7 @@ func (db *DB) sortedParallelSearch(ctx context.Context, search index.SearchReque } go func() { defer close(results) - sorter := api.NewSorter(closest, search.Sort() == index.SortAsc) + sorter := NewSorter(closest, search.Sort() == index.SortAsc) _ = db.CdxIndex.View(func(txn *badger.Txn) error { items := make(chan []byte, len(search.Keys())) @@ -514,6 +513,11 @@ func (db *DB) uniSearch(ctx context.Context, search index.SearchRequest, results } for it.Seek([]byte(seekKey)); it.ValidForPrefix(prefix); it.Next() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } contains, _ := search.DateRange().ContainsStr(cdxKey(it.Item().Key()).ts()) if !contains { continue @@ -621,7 +625,7 @@ func (db *DB) ListStorageRef(ctx context.Context, limit int, results chan<- inde return nil } -func (db *DB) GetFileInfo(ctx context.Context, filename string) (*schema.Fileinfo, error) { +func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, error) { return db.getFileInfo(filename) } diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index e31478c..8dd5071 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -28,7 +28,7 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/dgraph-io/badger/v3/options" - "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" diff --git a/internal/badgeridx/sorter.go b/internal/badgeridx/sorter.go new file mode 100644 index 0000000..b5cf2d8 --- /dev/null +++ b/internal/badgeridx/sorter.go @@ -0,0 +1,86 @@ +/* + * Copyright 2020 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badgeridx + +import ( + "sort" + + "github.com/dgraph-io/badger/v3" + "github.com/nlnwa/gowarcserver/internal/timestamp" +) + +type value struct { + ts int64 + k []byte +} + +type Sorter struct { + Closest int64 + Reverse bool + Values []value +} + +func NewSorter(closest int64, reverse bool) Sorter { + return Sorter{ + Closest: closest, + Reverse: reverse, + Values: []value{}, + } +} + +func (s *Sorter) Add(k []byte) { + t, _ := timestamp.Parse(cdxKey(k).ts()) + ts := t.Unix() + + s.Values = append(s.Values, value{ts, k}) +} + +func (s *Sorter) Sort() { + var cmp func(int64, int64) bool + + // sort Closest, Reverse or forward + if s.Closest > 0 { + cmp = func(ts1 int64, ts2 int64) bool { + return timestamp.AbsInt64(s.Closest-ts1) < timestamp.AbsInt64(s.Closest-ts2) + } + } else if s.Reverse { + cmp = func(ts1 int64, ts2 int64) bool { + return ts2 < ts1 + } + } else { + cmp = func(ts1 int64, ts2 int64) bool { + return ts1 < ts2 + } + } + + sort.Slice(s.Values, func(i, j int) bool { + return cmp(s.Values[i].ts, s.Values[j].ts) + }) +} + +func (s *Sorter) Walk(txn *badger.Txn, perItemFn PerItemFunc) error { + for _, value := range s.Values { + item, err := txn.Get(value.k) + if err != nil { + return err + } + if perItemFn(item) { + break + } + } + return nil +} From 39649283de771f3c2428a5166866c1edbd3fa626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 14:18:31 +0100 Subject: [PATCH 09/38] Remove unused function mergeSort --- index/io.go | 5 ++-- index/record.go | 2 +- index/writers.go | 2 +- internal/tikvidx/api.go | 2 +- internal/tikvidx/db.go | 2 +- internal/tikvidx/iter.go | 2 +- internal/tikvidx/sorter.go | 50 +----------------------------------- server/api/api.go | 4 +-- server/coreserver/handler.go | 8 +++--- server/warcserver/api.go | 2 +- server/warcserver/handler.go | 10 ++++---- 11 files changed, 21 insertions(+), 68 deletions(-) diff --git a/index/io.go b/index/io.go index 99ce7f5..27614ae 100644 --- a/index/io.go +++ b/index/io.go @@ -19,12 +19,13 @@ package index import ( "errors" "fmt" - "github.com/nlnwa/gowarc" - "github.com/rs/zerolog/log" "io" "path/filepath" "strings" "time" + + "github.com/nlnwa/gowarc" + "github.com/rs/zerolog/log" ) type recordFilter func(gowarc.WarcRecord) bool diff --git a/index/record.go b/index/record.go index e0b7eed..8102e4d 100644 --- a/index/record.go +++ b/index/record.go @@ -28,8 +28,8 @@ import ( "unicode/utf8" "github.com/nlnwa/gowarc" - "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/schema" + "github.com/nlnwa/gowarcserver/surt" "github.com/rs/zerolog/log" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/index/writers.go b/index/writers.go index c3fe3b4..9bfecad 100644 --- a/index/writers.go +++ b/index/writers.go @@ -22,7 +22,7 @@ import ( "sync" "github.com/bits-and-blooms/bloom/v3" - "github.com/nlnwa/gowarcserver/internal/surt" + "github.com/nlnwa/gowarcserver/surt" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index ec1745f..e0640d6 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -3,7 +3,7 @@ package tikvidx import ( "context" - "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" "google.golang.org/protobuf/proto" ) diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 65e6c40..d4622e5 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -27,7 +27,7 @@ import ( "sync" "time" - "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" "github.com/rs/zerolog/log" diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index db768c1..89d04ac 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "github.com/nlnwa/gowarcserver/internal/index" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/tikv/client-go/v2/txnkv/transaction" ) diff --git a/internal/tikvidx/sorter.go b/internal/tikvidx/sorter.go index 6f35eb6..d1fefd5 100644 --- a/internal/tikvidx/sorter.go +++ b/internal/tikvidx/sorter.go @@ -17,7 +17,6 @@ package tikvidx import ( - "github.com/nlnwa/gowarcserver/internal/index" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/rs/zerolog/log" ) @@ -36,54 +35,7 @@ func CompareDesc(a int64, b int64) bool { return a > b } -// mergeSort sorts elements of input channels to output channel according to sort direction. -func mergeSort(done <-chan struct{}, cmp func(*index.CdxResponse, *index.CdxResponse) bool, in ...chan index.CdxResponse) <-chan index.CdxResponse { - out := make(chan index.CdxResponse) - cords := make([]*index.CdxResponse, len(in)) - - go func() { - var zombie []int - for { - var curr *index.CdxResponse - for i, cord := range cords { - if cord == nil { - v, ok := <-in[i] - if !ok { - // closed channel becomes a zombie - zombie = append(zombie, i) - continue - } - cord = &v - } - if cmp(curr, cord) { - curr = cord - } - } - if curr == nil { - return - } - select { - case out <- *curr: - case <-done: - return - } - if len(zombie) > 0 { - // kill zombies to avoid checking on them every round - for _, i := range zombie { - cords[i] = cords[len(cords)-1] - cords = cords[:len(cords)-1] - in[i] = in[len(in)-1] - in = in[:len(in)-1] - } - zombie = nil - } - } - }() - - return out -} - -// mergeSort merges sorted input channels to a sorted output channel +// mergeIter merges sorted input channels into a sorted output channel // // Sorting is done by comparing keys from key-value pairs. // diff --git a/server/api/api.go b/server/api/api.go index 7d04b41..7eecdbe 100644 --- a/server/api/api.go +++ b/server/api/api.go @@ -23,9 +23,9 @@ import ( "strconv" "strings" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/surt" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/surt" "github.com/nlnwa/whatwg-url/url" ) diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index 8ed761a..e763097 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -28,10 +28,10 @@ import ( "time" "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/loader" - "github.com/nlnwa/gowarcserver/internal/server/api" - "github.com/nlnwa/gowarcserver/internal/server/handlers" + "github.com/nlnwa/gowarcserver/index" + "github.com/nlnwa/gowarcserver/loader" + "github.com/nlnwa/gowarcserver/server/api" + "github.com/nlnwa/gowarcserver/server/handlers" "github.com/rs/zerolog/log" "google.golang.org/protobuf/encoding/protojson" ) diff --git a/server/warcserver/api.go b/server/warcserver/api.go index 5d3857f..aac7603 100644 --- a/server/warcserver/api.go +++ b/server/warcserver/api.go @@ -22,9 +22,9 @@ import ( "strconv" "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/surt" "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" + "github.com/nlnwa/gowarcserver/surt" "github.com/nlnwa/whatwg-url/url" ) diff --git a/server/warcserver/handler.go b/server/warcserver/handler.go index 2457af4..d95f2c1 100644 --- a/server/warcserver/handler.go +++ b/server/warcserver/handler.go @@ -10,12 +10,12 @@ import ( "time" "github.com/nlnwa/gowarc" - "github.com/nlnwa/gowarcserver/internal/index" - "github.com/nlnwa/gowarcserver/internal/loader" - "github.com/nlnwa/gowarcserver/internal/server/api" - "github.com/nlnwa/gowarcserver/internal/server/handlers" - "github.com/nlnwa/gowarcserver/internal/surt" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/loader" + "github.com/nlnwa/gowarcserver/server/api" + "github.com/nlnwa/gowarcserver/server/handlers" + "github.com/nlnwa/gowarcserver/surt" urlErrors "github.com/nlnwa/whatwg-url/errors" "github.com/nlnwa/whatwg-url/url" "github.com/rs/zerolog/log" From 97a2e79c221cc170a358664e5d9fbfa4a00fc1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 14:21:41 +0100 Subject: [PATCH 10/38] Promote timestamp package from internal --- internal/badgeridx/api.go | 2 +- internal/badgeridx/db.go | 2 +- internal/badgeridx/sorter.go | 2 +- internal/tikvidx/db.go | 2 +- internal/tikvidx/iter.go | 2 +- internal/tikvidx/sorter.go | 2 +- server/api/api.go | 2 +- server/warcserver/api.go | 2 +- server/warcserver/handler.go | 2 +- {internal/timestamp => timestamp}/timestamp.go | 0 {internal/timestamp => timestamp}/timestamp_test.go | 3 ++- 11 files changed, 11 insertions(+), 10 deletions(-) rename {internal/timestamp => timestamp}/timestamp.go (100%) rename {internal/timestamp => timestamp}/timestamp_test.go (97%) diff --git a/internal/badgeridx/api.go b/internal/badgeridx/api.go index c943e9a..bf9c028 100644 --- a/internal/badgeridx/api.go +++ b/internal/badgeridx/api.go @@ -26,8 +26,8 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" "google.golang.org/protobuf/proto" ) diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index 8dd5071..7298733 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -29,8 +29,8 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/dgraph-io/badger/v3/options" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/internal/badgeridx/sorter.go b/internal/badgeridx/sorter.go index b5cf2d8..b12870f 100644 --- a/internal/badgeridx/sorter.go +++ b/internal/badgeridx/sorter.go @@ -20,7 +20,7 @@ import ( "sort" "github.com/dgraph-io/badger/v3" - "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/timestamp" ) type value struct { diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index d4622e5..07c1dd7 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -28,8 +28,8 @@ import ( "time" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" "github.com/tikv/client-go/v2/txnkv" "google.golang.org/protobuf/proto" diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index 89d04ac..d62db18 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -7,7 +7,7 @@ import ( "time" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/tikv/client-go/v2/txnkv/transaction" ) diff --git a/internal/tikvidx/sorter.go b/internal/tikvidx/sorter.go index d1fefd5..7e918d1 100644 --- a/internal/tikvidx/sorter.go +++ b/internal/tikvidx/sorter.go @@ -17,7 +17,7 @@ package tikvidx import ( - "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" ) diff --git a/server/api/api.go b/server/api/api.go index 7eecdbe..4d00cbe 100644 --- a/server/api/api.go +++ b/server/api/api.go @@ -24,8 +24,8 @@ import ( "strings" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/surt" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/nlnwa/whatwg-url/url" ) diff --git a/server/warcserver/api.go b/server/warcserver/api.go index aac7603..51f8a2c 100644 --- a/server/warcserver/api.go +++ b/server/warcserver/api.go @@ -22,9 +22,9 @@ import ( "strconv" "github.com/julienschmidt/httprouter" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/schema" "github.com/nlnwa/gowarcserver/surt" + "github.com/nlnwa/gowarcserver/timestamp" "github.com/nlnwa/whatwg-url/url" ) diff --git a/server/warcserver/handler.go b/server/warcserver/handler.go index d95f2c1..cadf63b 100644 --- a/server/warcserver/handler.go +++ b/server/warcserver/handler.go @@ -11,11 +11,11 @@ import ( "github.com/nlnwa/gowarc" "github.com/nlnwa/gowarcserver/index" - "github.com/nlnwa/gowarcserver/internal/timestamp" "github.com/nlnwa/gowarcserver/loader" "github.com/nlnwa/gowarcserver/server/api" "github.com/nlnwa/gowarcserver/server/handlers" "github.com/nlnwa/gowarcserver/surt" + "github.com/nlnwa/gowarcserver/timestamp" urlErrors "github.com/nlnwa/whatwg-url/errors" "github.com/nlnwa/whatwg-url/url" "github.com/rs/zerolog/log" diff --git a/internal/timestamp/timestamp.go b/timestamp/timestamp.go similarity index 100% rename from internal/timestamp/timestamp.go rename to timestamp/timestamp.go diff --git a/internal/timestamp/timestamp_test.go b/timestamp/timestamp_test.go similarity index 97% rename from internal/timestamp/timestamp_test.go rename to timestamp/timestamp_test.go index 849c8ba..74376ad 100644 --- a/internal/timestamp/timestamp_test.go +++ b/timestamp/timestamp_test.go @@ -17,7 +17,8 @@ package timestamp_test import ( - "github.com/nlnwa/gowarcserver/internal/timestamp" + "github.com/nlnwa/gowarcserver/timestamp" + "testing" ) From d51b84adb4c1ccf6e1df24e2d8f331f731c8ad9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 14:54:54 +0100 Subject: [PATCH 11/38] Build docker image with golang 1.19 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 94d9a39..2280dc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16 as build +FROM golang:1.19 as build WORKDIR /build From dc01fc983fc48a097506239cbafe141e4ae0a3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 14:56:29 +0100 Subject: [PATCH 12/38] Build and push image on PR to main branch --- .github/workflows/release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 537b3f8..2ab78c7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,9 +1,12 @@ name: release on: + pull_request: + branches: + - main push: branches: - - master + - main tags: - v* From dc3d4ace848e9531ff7f1120c8da0cd185cdc9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 15:51:04 +0100 Subject: [PATCH 13/38] Update dependencies --- go.mod | 54 +++++-- go.sum | 494 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 532 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 7b06854..b95fb17 100644 --- a/go.mod +++ b/go.mod @@ -3,29 +3,51 @@ module github.com/nlnwa/gowarcserver go 1.16 require ( - github.com/bits-and-blooms/bitset v1.2.2 // indirect - github.com/bits-and-blooms/bloom/v3 v3.2.0 - github.com/dgraph-io/badger/v3 v3.2103.3 - github.com/felixge/httpsnoop v1.0.2 // indirect - github.com/fsnotify/fsnotify v1.5.1 + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/bits-and-blooms/bitset v1.3.3 // indirect + github.com/bits-and-blooms/bloom/v3 v3.3.1 + github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect + github.com/coreos/go-systemd/v22 v22.4.0 // indirect + github.com/dgraph-io/badger/v3 v3.2103.4 + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 github.com/golang/glog v1.0.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v2.0.6+incompatible // indirect + github.com/google/flatbuffers v22.10.26+incompatible // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/handlers v1.5.1 + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/julienschmidt/httprouter v1.3.0 - github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/compress v1.15.12 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mitchellh/mapstructure v1.4.3 - github.com/nlnwa/gowarc v1.0.0-beta.3 + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/mapstructure v1.5.0 + github.com/nlnwa/gowarc v1.0.0-beta.4 github.com/nlnwa/whatwg-url v0.1.2 + github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect + github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect + github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 // indirect + github.com/prometheus/client_golang v1.13.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect - github.com/rs/zerolog v1.26.1 - github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cobra v1.4.0 - github.com/spf13/viper v1.10.1 + github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect + github.com/rs/zerolog v1.28.0 + github.com/spf13/afero v1.9.2 // indirect + github.com/spf13/cobra v1.6.1 + github.com/spf13/viper v1.13.0 github.com/tikv/client-go/v2 v2.0.2 - golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect - google.golang.org/protobuf v1.28.0 - gopkg.in/ini.v1 v1.66.4 // indirect + github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a // indirect + go.etcd.io/etcd/client/v3 v3.5.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.23.0 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sync v0.1.0 // indirect + google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect + google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index b6f31d3..751edec 100644 --- a/go.sum +++ b/go.sum @@ -28,25 +28,246 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -69,6 +290,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -78,8 +301,13 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg= +github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.2.0 h1:N+g3GTQ0TVbghahYyzwkQbMZR+IwIwFFC8dpIChtN0U= github.com/bits-and-blooms/bloom/v3 v3.2.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= +github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= +github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -110,8 +338,12 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= @@ -120,10 +352,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v3 v3.2103.3 h1:s63J1pisDhKpzWslXFe+ChuthuZptpwTE6qEKoczPb4= github.com/dgraph-io/badger/v3 v3.2103.3/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= +github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -136,6 +372,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -144,10 +381,16 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -156,10 +399,13 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -214,6 +460,8 @@ github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl76 github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= +github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -227,6 +475,10 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -253,15 +505,26 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -273,6 +536,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -293,12 +557,15 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -317,6 +584,8 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= +github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -325,6 +594,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -339,14 +609,20 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= @@ -356,6 +632,8 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -365,6 +643,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nlnwa/gowarc v1.0.0-beta.3 h1:kCuAUTg/+T66Nx4V6k++NzkCWvteCmfX1Su5bQh46cg= github.com/nlnwa/gowarc v1.0.0-beta.3/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= +github.com/nlnwa/gowarc v1.0.0-beta.4 h1:XxGNxZnW6Stmebsw458hM9NPf0YYp9siOiNYU6ONww4= +github.com/nlnwa/gowarc v1.0.0-beta.4/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= github.com/nlnwa/whatwg-url v0.0.0-20200306110950-d1a95e2e8fc3/go.mod h1:v3hJLcAdjhIn7PA89dVhJ9GSWooX0z2/qPgwlhz0HD8= github.com/nlnwa/whatwg-url v0.1.2 h1:BqqsIVG6xv71wOoMAoFDmV6OK6/2sXn7BJdOsTkBl88= github.com/nlnwa/whatwg-url v0.1.2/go.mod h1:b0r+dEyM/KztLMDSVY6ApcO9Fmzgq+e9+Ugq20UBYck= @@ -388,16 +668,26 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8= +github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= +github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= +github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -416,37 +706,56 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= +github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= +github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= +github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -460,12 +769,18 @@ github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY52 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -475,10 +790,14 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= +github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= +github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -486,15 +805,26 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= +github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a h1:YMq0aKR30gBQBlUCqwnB4oBGNDcl0ddYPXQ1KzU3TTw= +github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -504,15 +834,26 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -521,12 +862,16 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -535,11 +880,15 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -551,8 +900,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -589,6 +940,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -634,11 +986,27 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -656,6 +1024,15 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -668,6 +1045,12 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -749,10 +1132,29 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -763,6 +1165,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -826,11 +1231,17 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -863,6 +1274,25 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -892,6 +1322,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -912,6 +1343,7 @@ google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -935,6 +1367,51 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -962,8 +1439,19 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -980,6 +1468,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -991,6 +1481,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1006,6 +1498,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 81324ac8f870324b17cea1504059a76b87fff4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 15:52:21 +0100 Subject: [PATCH 14/38] Simplify worker scheduling --- cmd/index/index.go | 2 +- index/worker.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index 3620ea8..91dae9e 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -62,7 +62,7 @@ func NewCommand() *cobra.Command { cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") cmd.Flags().IntP("max-depth", "d", indexDepth, "maximum directory recursion") - cmd.Flags().Int("workers", indexWorkers, "number of index workers") + cmd.Flags().Int("workers", indexWorkers, "number of files") cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") cmd.Flags().Uint("toc-bloom-capacity", bloomCapacity, "estimated bloom filter capacity") cmd.Flags().Float64("toc-bloom-fp", bloomFp, "estimated bloom filter false positive rate") diff --git a/index/worker.go b/index/worker.go index 0950894..e3fc7df 100644 --- a/index/worker.go +++ b/index/worker.go @@ -66,28 +66,31 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { } func (iw *indexWorker) Close() { - // Wait for all timers to complete + // Wait for all queued jobs to complete iw.wg.Wait() // before closing workers. close(iw.done) } +// Schedule schedules a job to be processed by a worker func (iw *indexWorker) Schedule(job string, batchWindow time.Duration) { + if batchWindow == 0 { + iw.wg.Add(1) + iw.jobs <- job + return + } + iw.mx.Lock() + defer iw.mx.Unlock() timer, ok := iw.jobMap[job] - iw.mx.Unlock() if ok { timer.Stop() timer.Reset(batchWindow) } else { iw.wg.Add(1) - if batchWindow == 0 { + iw.jobMap[job] = time.AfterFunc(batchWindow, func() { iw.jobs <- job - } else { - iw.jobMap[job] = time.AfterFunc(batchWindow, func() { - iw.jobs <- job - }) - } + }) } } From 001402bd352b6394a123a82887e1700d2e7fb764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 16:22:03 +0100 Subject: [PATCH 15/38] Do not allow search with no key --- internal/badgeridx/api.go | 3 ++- internal/tikvidx/api.go | 25 ++++++++++++++++--------- internal/tikvidx/iter.go | 26 ++++---------------------- 3 files changed, 22 insertions(+), 32 deletions(-) diff --git a/internal/badgeridx/api.go b/internal/badgeridx/api.go index bf9c028..5a2a585 100644 --- a/internal/badgeridx/api.go +++ b/internal/badgeridx/api.go @@ -19,6 +19,7 @@ package badgeridx import ( "bytes" "context" + "errors" "fmt" "strings" "sync" @@ -183,7 +184,7 @@ func (db *DB) Search(ctx context.Context, search index.SearchRequest, results ch keyLen := len(search.Keys()) if keyLen == 0 { - return fmt.Errorf("search got 0 keys") + return errors.New("search request is missing keys") } else if keyLen > 1 { if search.Sort() == index.SortNone { return db.unsortedSerialSearch(ctx, search, results) diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index e0640d6..f9a8723 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -2,9 +2,11 @@ package tikvidx import ( "context" + "errors" "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" + "github.com/tikv/client-go/v2/txnkv/transaction" "google.golang.org/protobuf/proto" ) @@ -57,27 +59,32 @@ func cdxFromValue(value []byte) (*schema.Cdx, error) { } func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- index.CdxResponse) error { - // begin transaction - tx, err := db.client.Begin() + var err error + var tx *transaction.KVTxn + var it iterator + + if len(req.Keys()) == 0 { + return errors.New("search request has no keys") + } + tx, err = db.client.Begin() if err != nil { return err } - // initialize iterator - it, err := NewIter(ctx, tx, req) + it, err = newIter(ctx, tx, req) if err != nil { return err } - limit := req.Limit() - if limit == 0 { - limit = 100 - } - go func() { defer close(res) defer it.Close() + limit := req.Limit() + if limit == 0 { + limit = 100 + } + for it.Valid() && limit > 0 { select { case <-ctx.Done(): diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index d62db18..3f42578 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -3,7 +3,6 @@ package tikvidx import ( "bytes" "context" - "fmt" "time" "github.com/nlnwa/gowarcserver/index" @@ -126,33 +125,14 @@ type iterSort struct { valid bool cmp func([]byte, []byte) bool next <-chan maybeKV - // done <-chan struct{} } -func NewIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest) (iterator, error) { +func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest) (*iterSort, error) { is := new(iterSort) - // iter.done = ctx.Done() // initialize iterators var results []chan *maybeKV var err error - var it iterator - - if len(req.Keys()) == 0 { - k := []byte(cdxPrefix) - switch req.Sort() { - case index.SortDesc: - it, err = tx.IterReverse(k) - case index.SortAsc: - it, err = tx.Iter(k, []byte(cdxEOF)) - case index.SortClosest: - return nil, fmt.Errorf("invalid request: sort=closest with no keys") - default: - it, err = tx.Iter(k, []byte(cdxEOF)) - } - return it, nil - } - for _, key := range req.Keys() { k := []byte(cdxPrefix + key) var it iterator @@ -176,7 +156,9 @@ func NewIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest defer is.Close() return nil, err } - + if len(results) == 0 { + return nil, nil + } // initialize comparator var cmp func(a KV, b KV) bool From c1318946a4fbd31b692e7ed5b567c4fe00e8a5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 17:00:25 +0100 Subject: [PATCH 16/38] Updated go.* --- go.mod | 5 -- go.sum | 101 ++----------------------------------- loader/loader_test.go | 113 ------------------------------------------ 3 files changed, 3 insertions(+), 216 deletions(-) delete mode 100644 loader/loader_test.go diff --git a/go.mod b/go.mod index b95fb17..fc4bcec 100644 --- a/go.mod +++ b/go.mod @@ -6,23 +6,18 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/bits-and-blooms/bitset v1.3.3 // indirect github.com/bits-and-blooms/bloom/v3 v3.3.1 - github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/dgraph-io/badger/v3 v3.2103.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 github.com/golang/glog v1.0.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v22.10.26+incompatible // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/handlers v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/julienschmidt/httprouter v1.3.0 github.com/klauspost/compress v1.15.12 // indirect - github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 diff --git a/go.sum b/go.sum index 751edec..226551f 100644 --- a/go.sum +++ b/go.sum @@ -288,7 +288,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -297,19 +296,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= -github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg= github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bloom/v3 v3.2.0 h1:N+g3GTQ0TVbghahYyzwkQbMZR+IwIwFFC8dpIChtN0U= -github.com/bits-and-blooms/bloom/v3 v3.2.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -330,19 +323,16 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -350,13 +340,10 @@ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v3 v3.2103.3 h1:s63J1pisDhKpzWslXFe+ChuthuZptpwTE6qEKoczPb4= -github.com/dgraph-io/badger/v3 v3.2103.3/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -371,23 +358,18 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= -github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -402,7 +384,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= @@ -458,8 +439,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= -github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -473,7 +452,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -521,7 +499,6 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= @@ -535,7 +512,6 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -559,10 +535,8 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -582,8 +556,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -592,16 +564,14 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -619,7 +589,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -630,7 +599,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -641,8 +609,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nlnwa/gowarc v1.0.0-beta.3 h1:kCuAUTg/+T66Nx4V6k++NzkCWvteCmfX1Su5bQh46cg= -github.com/nlnwa/gowarc v1.0.0-beta.3/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= github.com/nlnwa/gowarc v1.0.0-beta.4 h1:XxGNxZnW6Stmebsw458hM9NPf0YYp9siOiNYU6ONww4= github.com/nlnwa/gowarc v1.0.0-beta.4/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= github.com/nlnwa/whatwg-url v0.0.0-20200306110950-d1a95e2e8fc3/go.mod h1:v3hJLcAdjhIn7PA89dVhJ9GSWooX0z2/qPgwlhz0HD8= @@ -666,25 +632,20 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8= github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= @@ -694,7 +655,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -704,7 +664,6 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -713,7 +672,6 @@ github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= @@ -721,7 +679,6 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= @@ -730,7 +687,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= @@ -738,23 +694,19 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -765,20 +717,13 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -788,8 +733,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= @@ -803,25 +746,20 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= -github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a h1:YMq0aKR30gBQBlUCqwnB4oBGNDcl0ddYPXQ1KzU3TTw= github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= @@ -833,23 +771,16 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= -go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= @@ -860,7 +791,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -868,7 +798,6 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -878,14 +807,12 @@ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= @@ -894,15 +821,12 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -939,7 +863,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -987,13 +910,10 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1043,7 +963,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1123,8 +1042,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1148,7 +1065,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1163,7 +1079,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= @@ -1230,17 +1145,14 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1365,7 +1277,6 @@ google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1440,7 +1351,6 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= @@ -1466,7 +1376,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -1478,9 +1387,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= -gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1496,7 +1402,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/loader/loader_test.go b/loader/loader_test.go deleted file mode 100644 index cd2b293..0000000 --- a/loader/loader_test.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package loader - -// func TestLoader_Get(t *testing.T) { -// loader := &Loader{ -// StorageRefResolver: &mockStorageRefResolver{}, -// RecordLoader: &FileStorageLoader{}, -// } - -// type args struct { -// warcId string -// } - -// opts := gowarc.WithVersion(gowarc.V1_0) - -// infoBuilder := gowarc.NewRecordBuilder(gowarc.Warcinfo, opts) -// v1InfoRecord, _, err := infoBuilder.Build() -// if err != nil { -// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) -// } - -// v1ResponseRecord, _, err := gowarc.NewRecordBuilder(gowarc.Response, opts).Build() -// if err != nil { -// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) -// } - -// v1RequestRecord, _, err := gowarc.NewRecordBuilder(gowarc.Request, opts).Build() -// if err != nil { -// t.Fatalf("Warcinfo WarcRecord creation failed: %s", err) -// } -// tests := []struct { -// name string -// args args -// wantRecord gowarc.WarcRecord -// }{ -// { -// "base1", -// args{"urn:uuid:e9a0cecc-0221-11e7-adb1-0242ac120008"}, -// v1InfoRecord, -// }, -// { -// "base2", -// args{"urn:uuid:a9c51e3e-0221-11e7-bf66-0242ac120005"}, -// v1ResponseRecord, -// }, -// { -// "base3", -// args{"urn:uuid:e9a0ee48-0221-11e7-adb1-0242ac120008"}, -// v1InfoRecord, -// }, -// { -// "base4", -// args{"urn:uuid:a9c5c23a-0221-11e7-8fe3-0242ac120007"}, -// v1RequestRecord, -// }, -// { -// "base5", -// args{"urn:uuid:e6e41fea-0221-11e7-8fe3-0242ac120007"}, -// v1RequestRecord, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// gotRecord, err := loader.Load(ctx, tt.args.warcId) -// if err != nil { -// t.Errorf("Loader.Load() error = %v", err) -// return -// } - -// if reflect.DeepEqual(gotRecord, tt.wantRecord) { -// t.Errorf("\nLoader.Load() = \n%v\nWant = \n%v", gotRecord, tt.wantRecord) -// } -// }) -// } -// } - -type mockStorageRefResolver struct{} - -func (m mockStorageRefResolver) Resolve(warcId string) (storageRef string, err error) { - switch warcId { - case "urn:uuid:e9a0cecc-0221-11e7-adb1-0242ac120008": - storageRef = "warcfile:../../testdata/example.warc#0" - case "urn:uuid:e9a0ee48-0221-11e7-adb1-0242ac120008": - storageRef = "warcfile:../../testdata/example.warc#488" - case "urn:uuid:a9c51e3e-0221-11e7-bf66-0242ac120005": - storageRef = "warcfile:../../testdata/example.warc#1197" - case "urn:uuid:a9c5c23a-0221-11e7-8fe3-0242ac120007": - storageRef = "warcfile:../../testdata/example.warc#3078" - case "urn:uuid:e6e395ca-0221-11e7-a18d-0242ac120005": - storageRef = "warcfile:../../testdata/example.warc#3370" - case "urn:uuid:e6e41fea-0221-11e7-8fe3-0242ac120007": - storageRef = "warcfile:../../testdata/example.warc#4828" - } - return -} From a1d6c7977ddf163ddac6c23fbf14deb4528b8f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 17:00:37 +0100 Subject: [PATCH 17/38] Linter --- internal/tikvidx/api.go | 12 +++++++++--- internal/tikvidx/iter.go | 10 ++++++++-- internal/tikvidx/sorter.go | 1 - server/api/api.go | 5 +---- server/coreserver/handler.go | 8 ++++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index f9a8723..f490c57 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -123,7 +123,9 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in func (db *DB) List(ctx context.Context, limit int, res chan<- index.CdxResponse) error { tx, err := db.client.Begin() - + if err != nil { + return err + } it, err := tx.Iter([]byte(cdxPrefix), []byte(cdxEOF)) if err != nil { return err @@ -165,7 +167,9 @@ func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileResponse) error { tx, err := db.client.Begin() - + if err != nil { + return err + } it, err := tx.Iter([]byte(filePrefix), []byte(fileEOF)) if err != nil { return err @@ -207,7 +211,9 @@ func (db *DB) GetStorageRef(ctx context.Context, id string) (string, error) { func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdResponse) error { tx, err := db.client.Begin() - + if err != nil { + return err + } it, err := tx.Iter([]byte(idPrefix), []byte(idEOF)) if err != nil { return err diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index 3f42578..42c5427 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -3,6 +3,7 @@ package tikvidx import ( "bytes" "context" + "fmt" "time" "github.com/nlnwa/gowarcserver/index" @@ -119,11 +120,9 @@ type maybeKV struct { type iterSort struct { iterators []iterator - keys [][]byte key []byte value []byte valid bool - cmp func([]byte, []byte) bool next <-chan maybeKV } @@ -143,6 +142,8 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest it, err = NewIterClosest(ctx, tx, key, req.Closest()) case index.SortAsc: fallthrough + case index.SortNone: + fallthrough default: it, err = tx.Iter(k, []byte(cdxEOF)) } @@ -170,11 +171,16 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest case index.SortClosest: var t time.Time t, err = time.Parse(timestamp.CDX, req.Closest()) + if err != nil { + return nil, fmt.Errorf("failed to parse closest timestamp: %w", err) + } cmp = func(a KV, b KV) bool { return CompareClosest(t.Unix())(a.ts(), b.ts()) } case index.SortAsc: fallthrough + case index.SortNone: + fallthrough default: cmp = func(a KV, b KV) bool { return CompareAsc(a.ts(), b.ts()) diff --git a/internal/tikvidx/sorter.go b/internal/tikvidx/sorter.go index 7e918d1..3590964 100644 --- a/internal/tikvidx/sorter.go +++ b/internal/tikvidx/sorter.go @@ -82,7 +82,6 @@ func mergeIter(done <-chan struct{}, cmp func(KV, KV) bool, in ...chan *maybeKV) return case out <- *cords[curr]: cords[curr] = nil - curr = -1 } // if zombie, then kill if len(zombie) > 0 { diff --git a/server/api/api.go b/server/api/api.go index 4d00cbe..0e31356 100644 --- a/server/api/api.go +++ b/server/api/api.go @@ -138,10 +138,7 @@ func Parse(r *http.Request) (*CoreAPI, error) { // currently the "cdx" does not accept collection as a query or param coreApi.Collection = "all" - urls, ok := query["url"] - //if !ok { - // return nil, fmt.Errorf("missing required query parameter \"url\"") - //} + urls := query["url"] if len(urls) == 1 && !schemeRegExp.MatchString(urls[0]) { u := urls[0] urls = []string{ diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index e763097..77d1c61 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -118,12 +118,16 @@ func (h Handler) listIds(w http.ResponseWriter, r *http.Request) { continue } filename, offset, err := parseStorageRef(res.Value) - storageRef := &storageRef{ + if err != nil { + log.Warn().Err(err).Msgf("failed to parse storage ref: %s", res.Value) + continue + } + ref := &storageRef{ Id: res.Key, Filename: filename, Offset: offset, } - v, err := json.Marshal(storageRef) + v, err := json.Marshal(ref) if err != nil { log.Warn().Err(err).Msg("failed to marshal storage ref") } From 9b85ceee6a5c771c9851fdfce3f2889e7277dd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 17:03:05 +0100 Subject: [PATCH 18/38] Update distroless base image tag --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2280dc3..c2b4ce1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -ldflags "-s -w -X github.com/nlnwa/gowarcserver/cmd/version.Version=${VERSION}" -FROM gcr.io/distroless/base +FROM gcr.io/distroless/base-debian11 COPY --from=build /build/gowarcserver / EXPOSE 9999 From e56c735a696ac283475c788078b781251baa9280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sat, 5 Nov 2022 17:14:13 +0100 Subject: [PATCH 19/38] Indexer log warning on errors --- index/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/worker.go b/index/worker.go index e3fc7df..caed305 100644 --- a/index/worker.go +++ b/index/worker.go @@ -49,7 +49,7 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { select { case job := <-iw.jobs: if err := a.Index(job); err != nil { - log.Error().Err(err).Msgf("Index job: %s", job) + log.Warn().Err(err).Msgf("Index: %s", job) } iw.wg.Done() iw.mx.Lock() From b01a353966eb5527add92a711e82a17cf45a772a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sun, 6 Nov 2022 19:15:05 +0100 Subject: [PATCH 20/38] Add concept of database to badger and tikv --- cmd/index/index.go | 6 +++++- cmd/serve/serve.go | 14 +++++++++----- internal/badgeridx/api.go | 2 ++ internal/badgeridx/db.go | 2 +- internal/tikvidx/api.go | 4 ++-- internal/tikvidx/db.go | 31 +++++++++++++++++++++++-------- internal/tikvidx/options.go | 7 +++++++ loader/loader.go | 7 ++++--- server/coreserver/handler.go | 2 +- server/warcserver/handler.go | 11 +++++------ 10 files changed, 59 insertions(+), 27 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index 91dae9e..d652170 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -55,6 +55,7 @@ func NewCommand() *cobra.Command { var tikvPdAddr []string tikvBatchMaxSize := 1000 tikvBatchMaxWait := 5 * time.Second + tikvDatabase := "" bloomCapacity := uint(1000) bloomFp := 0.01 @@ -73,6 +74,7 @@ func NewCommand() *cobra.Command { cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", tikvDatabase, "name of database") return cmd } @@ -97,7 +99,9 @@ func indexCmd(_ *cobra.Command, args []string) error { db, err := tikvidx.NewDB( tikvidx.WithPDAddress(viper.GetStringSlice("tikv-pd-addr")), tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), - tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait"))) + tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait")), + tikvidx.WithDatabase(viper.GetString("tikv-database")), + ) if err != nil { return err } diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 4afa6b8..d0de967 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -70,6 +70,7 @@ func NewCommand() *cobra.Command { tikvPdAddr := []string{} tikvBatchMaxSize := 1000 tikvBatchMaxWait := 5 * time.Second + tikvDatabase := "" pathPrefix := "" cmd.Flags().IntP("port", "p", port, "server port") @@ -91,6 +92,7 @@ func NewCommand() *cobra.Command { cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", tikvDatabase, "name of database") return cmd } @@ -103,11 +105,6 @@ func serveCmd(cmd *cobra.Command, args []string) error { } else { dirs = viper.GetStringSlice("dirs") } - // parse badger compression type - var c options.CompressionType - if err := viper.UnmarshalKey("badger-compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { - return err - } // parse include patterns var includes []*regexp.Regexp for _, r := range viper.GetStringSlice("include") { @@ -140,6 +137,12 @@ func serveCmd(cmd *cobra.Command, args []string) error { // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use runtime.GOMAXPROCS(128) + // parse badger compression type + var c options.CompressionType + if err := viper.UnmarshalKey("badger-compression", &c, viper.DecodeHook(badgeridx.CompressionDecodeHookFunc())); err != nil { + return err + } + db, err := badgeridx.NewDB( badgeridx.WithCompression(c), badgeridx.WithDir(viper.GetString("badger-dir")), @@ -162,6 +165,7 @@ func serveCmd(cmd *cobra.Command, args []string) error { tikvidx.WithPDAddress(viper.GetStringSlice("tikv-pd-addr")), tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait")), + tikvidx.WithDatabase(viper.GetString("tikv-database")), ) if err != nil { return err diff --git a/internal/badgeridx/api.go b/internal/badgeridx/api.go index 5a2a585..8c939e0 100644 --- a/internal/badgeridx/api.go +++ b/internal/badgeridx/api.go @@ -93,6 +93,8 @@ func (db *DB) List(ctx context.Context, limit int, results chan<- index.CdxRespo func (db *DB) Closest(_ context.Context, search index.ClosestRequest, results chan<- index.CdxResponse) error { go func() { _ = db.CdxIndex.View(func(txn *badger.Txn) error { + defer close(results) + key := search.Key() closest := search.Closest() diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index 7298733..d42312c 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -292,7 +292,7 @@ func (db *DB) Index(fileName string) error { } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. -func (db *DB) Resolve(warcId string) (storageRef string, err error) { +func (db *DB) Resolve(_ context.Context, warcId string) (storageRef string, err error) { err = db.IdIndex.View(func(txn *badger.Txn) error { item, err := txn.Get([]byte(warcId)) if err != nil { diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index f490c57..1f5250d 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -242,11 +242,11 @@ func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdRe } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. -func (db *DB) Resolve(warcId string) (storageRef string, err error) { +func (db *DB) Resolve(ctx context.Context, warcId string) (storageRef string, err error) { key := []byte(idPrefix + warcId) var kv KV - kv, err = db.get(key) + kv, err = db.get(ctx, key) if err != nil { return } diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 07c1dd7..2207fd0 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -58,7 +58,7 @@ func (k cdxKey) ts() string { } // tikv does not (yet) have a notion of keyspace, so we use key prefixes -const ( +var ( idPrefix = "i" idEOF = "j" filePrefix = "f" @@ -83,6 +83,15 @@ func NewDB(options ...Option) (db *DB, err error) { opt(opts) } + // prefix all keys with name of database + // TODO use keyspace when TiKV support it + idPrefix = opts.Database + idPrefix + idEOF = opts.Database + idEOF + filePrefix = opts.Database + filePrefix + fileEOF = opts.Database + fileEOF + cdxPrefix = opts.Database + cdxPrefix + cdxEOF = opts.Database + cdxEOF + client, err := txnkv.NewClient(opts.PdAddr) if err != nil { return nil, err @@ -171,19 +180,19 @@ func (db *DB) updateFilePath(filePath string) error { return db.putFileInfo(fileInfo) } -func (db *DB) get(k []byte) (KV, error) { +func (db *DB) get(ctx context.Context, k []byte) (KV, error) { tx, err := db.client.Begin() if err != nil { return KV{}, err } - v, err := tx.Get(context.TODO(), k) + v, err := tx.Get(ctx, k) if err != nil { return KV{}, err } return KV{K: k, V: v}, nil } -func (db *DB) puts(kvs ...KV) error { +func (db *DB) puts(ctx context.Context, kvs ...KV) error { tx, err := db.client.Begin() if err != nil { return err @@ -195,21 +204,25 @@ func (db *DB) puts(kvs ...KV) error { return err } } - return tx.Commit(context.Background()) + return tx.Commit(ctx) } func (db *DB) putFileInfo(fi *schema.Fileinfo) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() k := []byte(filePrefix + fi.Name) v, err := proto.Marshal(fi) if err != nil { return err } - return db.puts(KV{K: k, V: v}) + return db.puts(ctx, KV{K: k, V: v}) } func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() key := []byte(filePrefix + fileName) - val, err := db.get(key) + val, err := db.get(ctx, key) if err != nil { return nil, err } @@ -264,7 +277,9 @@ func (db *DB) flushBatch() { } kvs = append(kvs, id, cdx) } - err := db.puts(kvs...) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err := db.puts(ctx, kvs...) if err != nil { log.Error().Err(err).Msgf("failed to commit batch (first batch ref is %s)", records[0].Ref) } diff --git a/internal/tikvidx/options.go b/internal/tikvidx/options.go index 6a110ae..4392383 100644 --- a/internal/tikvidx/options.go +++ b/internal/tikvidx/options.go @@ -14,6 +14,7 @@ type Options struct { BatchMaxWait time.Duration ReadOnly bool PdAddr []string + Database string } type Option func(opts *Options) @@ -41,3 +42,9 @@ func WithBatchMaxWait(t time.Duration) Option { opts.BatchMaxWait = t } } + +func WithDatabase(db string) Option { + return func(opts *Options) { + opts.Database = db + } +} diff --git a/loader/loader.go b/loader/loader.go index a38f639..fefc09f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -26,7 +26,7 @@ import ( ) type StorageRefResolver interface { - Resolve(warcId string) (storageRef string, err error) + Resolve(ctx context.Context, warcId string) (storageRef string, err error) } type RecordLoader interface { @@ -59,7 +59,7 @@ func (e ErrResolveRevisit) String() string { } func (l *Loader) LoadById(ctx context.Context, warcId string) (gowarc.WarcRecord, error) { - storageRef, err := l.StorageRefResolver.Resolve(warcId) + storageRef, err := l.StorageRefResolver.Resolve(ctx, warcId) if err != nil { return nil, err } @@ -91,7 +91,7 @@ func (l *Loader) LoadByStorageRef(ctx context.Context, storageRef string) (gowar } var revisitOf gowarc.WarcRecord - storageRef, err = l.Resolve(warcRefersTo) + storageRef, err = l.Resolve(ctx, warcRefersTo) if err != nil { return nil, fmt.Errorf("unable to resolve referred Warc-Record-ID [%s]: %w", warcRefersTo, err) } @@ -105,6 +105,7 @@ func (l *Loader) LoadByStorageRef(ctx context.Context, storageRef string) (gowar return nil, err } case gowarc.Continuation: + log.Warn().Msg("Not implemented: storage ref resolved to a continuation record") // TODO continuation not implemented fallthrough default: diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index 77d1c61..4442f34 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -151,7 +151,7 @@ func (h Handler) getStorageRefByURN(w http.ResponseWriter, r *http.Request) { params := httprouter.ParamsFromContext(r.Context()) urn := params.ByName("urn") - storageRef, err := h.StorageRefResolver.Resolve(urn) + storageRef, err := h.StorageRefResolver.Resolve(r.Context(), urn) if err != nil { msg := fmt.Sprintf("failed to resolve storage reference of urn: %v", err) http.Error(w, msg, http.StatusInternalServerError) diff --git a/server/warcserver/handler.go b/server/warcserver/handler.go index cadf63b..ac3b070 100644 --- a/server/warcserver/handler.go +++ b/server/warcserver/handler.go @@ -88,7 +88,7 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { return } - ctx, cancelQuery := context.WithCancel(r.Context()) + ctx, cancelQuery := context.WithTimeout(r.Context(), 5*time.Second) defer cancelQuery() // query API @@ -106,13 +106,12 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { log.Warn().Err(err).Msg("failed cdx response") continue } - cancelQuery() + if res.Cdx == nil { + http.NotFound(w, r) + return + } break } - if res.Cdx == nil { - http.NotFound(w, r) - return - } ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() From 0d4d8453f7383837e4ea9f0ac24a7937d15adf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sun, 6 Nov 2022 20:41:54 +0100 Subject: [PATCH 21/38] Add database concept to badger index --- cmd/index/index.go | 5 +++- cmd/serve/serve.go | 5 +++- exampleconfig.yaml | 6 ++++- internal/badgeridx/db.go | 14 ++++++++--- internal/tikvidx/api.go | 4 ++- internal/tikvidx/db.go | 54 ++++++++++++++++++++-------------------- 6 files changed, 53 insertions(+), 35 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index d652170..a301457 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -52,6 +52,7 @@ func NewCommand() *cobra.Command { badgerBatchMaxSize := 1000 badgerBatchMaxWait := 5 * time.Second badgerCompression := "snappy" + badgerDatabase := "" var tikvPdAddr []string tikvBatchMaxSize := 1000 tikvBatchMaxWait := 5 * time.Second @@ -68,13 +69,14 @@ func NewCommand() *cobra.Command { cmd.Flags().Uint("toc-bloom-capacity", bloomCapacity, "estimated bloom filter capacity") cmd.Flags().Float64("toc-bloom-fp", bloomFp, "estimated bloom filter false positive rate") cmd.Flags().String("badger-dir", badgerDir, "path to index database") + cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of database") + cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") return cmd } @@ -121,6 +123,7 @@ func indexCmd(_ *cobra.Command, args []string) error { badgeridx.WithDir(viper.GetString("badger-dir")), badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), + badgeridx.WithDatabase(viper.GetString("badger-database")), ) if err != nil { return err diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index d0de967..dab9d4e 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -63,6 +63,7 @@ func NewCommand() *cobra.Command { indexWorkers := 8 badgerDir := "." badgerCompression := "snappy" + badgerDatabase := "" badgerBatchMaxSize := 1000 badgerBatchMaxWait := 5 * time.Second badgerReadOnly := false @@ -85,6 +86,7 @@ func NewCommand() *cobra.Command { cmd.Flags().Bool("log-requests", logRequests, "log http requests") cmd.Flags().StringP("storage-engine", "b", storageEngine, `storage engine: "badger" or "tikv"`) cmd.Flags().String("badger-dir", badgerDir, "path to index database") + cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") @@ -92,7 +94,7 @@ func NewCommand() *cobra.Command { cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of database") + cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") return cmd } @@ -149,6 +151,7 @@ func serveCmd(cmd *cobra.Command, args []string) error { badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), badgeridx.WithReadOnly(viper.GetBool("badger-read-only")), + badgeridx.WithDatabase(viper.GetString("badger-database")), ) if err != nil { return err diff --git a/exampleconfig.yaml b/exampleconfig.yaml index 7d02e72..07dca9b 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -47,13 +47,17 @@ badger-batch-max-size: 1000 badger-batch-max-wait: 5s # badger compression algorithm badger-compression: "snappy" +# badger database to use +badger-database: "" # address of tikv placement driver -tikv-pd-addr: "127.0.0.1: 2379" +tikv-pd-addr: "127.0.0.1:2379" # max transaction batch size tikv-batch-max-size: 1000 # max wait time before flushing batched records regardless of max batch size tikv-batch-max-wait: 5s +# tikv database to use +tikv-database: "" # estimated bloom filter capacity toc-bloom-capacity: 10000 diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index d42312c..3a8a268 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -67,13 +67,13 @@ func NewDB(options ...DbOption) (db *DB, err error) { batch := make(chan index.Record, opts.BatchMaxSize) done := make(chan struct{}) - if idIndex, err = newBadgerDB(path.Join(dir, "id-index"), opts.Compression, opts.ReadOnly); err != nil { + if idIndex, err = newBadgerDB(path.Join(dir, opts.Database, "id-index"), opts.Compression, opts.ReadOnly); err != nil { return } - if fileIndex, err = newBadgerDB(path.Join(dir, "file-index"), opts.Compression, opts.ReadOnly); err != nil { + if fileIndex, err = newBadgerDB(path.Join(dir, opts.Database, "file-index"), opts.Compression, opts.ReadOnly); err != nil { return } - if cdxIndex, err = newBadgerDB(path.Join(dir, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil { + if cdxIndex, err = newBadgerDB(path.Join(dir, opts.Database, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil { return } @@ -393,7 +393,6 @@ func defaultDbOptions() *dbOptions { BatchMaxWait: 5 * time.Second, GcInterval: 15 * time.Second, Path: ".", - ReadOnly: false, } } @@ -404,6 +403,7 @@ type dbOptions struct { GcInterval time.Duration Path string ReadOnly bool + Database string } type DbOption func(opts *dbOptions) @@ -438,6 +438,12 @@ func WithGcInterval(t time.Duration) DbOption { } } +func WithDatabase(db string) DbOption { + return func(opts *dbOptions) { + opts.Database = db + } +} + func WithReadOnly(readOnly bool) DbOption { return func(opts *dbOptions) { opts.ReadOnly = readOnly diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 1f5250d..44a23cf 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -3,6 +3,7 @@ package tikvidx import ( "context" "errors" + "strings" "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" @@ -228,7 +229,8 @@ func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdRe if err != nil { res <- index.IdResponse{Error: err} } else { - res <- index.IdResponse{Key: string(it.Key()[1:]), Value: string(it.Value())} + k := strings.TrimPrefix(string(it.Key()), idPrefix) + res <- index.IdResponse{Key: k, Value: string(it.Value())} } err = it.Next() if err != nil { diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 2207fd0..6306bbf 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -180,33 +180,6 @@ func (db *DB) updateFilePath(filePath string) error { return db.putFileInfo(fileInfo) } -func (db *DB) get(ctx context.Context, k []byte) (KV, error) { - tx, err := db.client.Begin() - if err != nil { - return KV{}, err - } - v, err := tx.Get(ctx, k) - if err != nil { - return KV{}, err - } - return KV{K: k, V: v}, nil -} - -func (db *DB) puts(ctx context.Context, kvs ...KV) error { - tx, err := db.client.Begin() - if err != nil { - return err - } - - for _, kv := range kvs { - err := tx.Set(kv.K, kv.V) - if err != nil { - return err - } - } - return tx.Commit(ctx) -} - func (db *DB) putFileInfo(fi *schema.Fileinfo) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -234,6 +207,33 @@ func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { return fi, nil } +func (db *DB) get(ctx context.Context, k []byte) (KV, error) { + tx, err := db.client.Begin() + if err != nil { + return KV{}, err + } + v, err := tx.Get(ctx, k) + if err != nil { + return KV{}, err + } + return KV{K: k, V: v}, nil +} + +func (db *DB) puts(ctx context.Context, kvs ...KV) error { + tx, err := db.client.Begin() + if err != nil { + return err + } + + for _, kv := range kvs { + err := tx.Set(kv.K, kv.V) + if err != nil { + return err + } + } + return tx.Commit(ctx) +} + // write schedules a Record to be added to the DB via the batch channel. func (db *DB) write(rec index.Record) { select { From 99885a2610f931ed72d7bb63183fb4830dcefc63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sun, 6 Nov 2022 22:17:09 +0100 Subject: [PATCH 22/38] Differentiate between already indexed errors and other errors --- cmd/index/index.go | 3 ++- cmd/root.go | 2 +- cmd/serve/serve.go | 2 +- index/api.go | 8 ++++++++ index/worker.go | 7 ++++++- internal/badgeridx/badger.go | 4 ++-- internal/badgeridx/db.go | 10 ++++------ internal/tikvidx/db.go | 3 +-- 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cmd/index/index.go b/cmd/index/index.go index a301457..7816bf0 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -48,7 +48,7 @@ func NewCommand() *cobra.Command { format := "cdxj" indexDepth := 4 indexWorkers := 8 - badgerDir := "." + badgerDir := "./warcdb" badgerBatchMaxSize := 1000 badgerBatchMaxWait := 5 * time.Second badgerCompression := "snappy" @@ -159,6 +159,7 @@ func indexCmd(_ *cobra.Command, args []string) error { } } + log.Info().Msg("Starting auto indexer") indexer, err := index.NewAutoIndexer(indexWorker, index.WithMaxDepth(viper.GetInt("max-depth")), index.WithIncludes(includes...), diff --git a/cmd/root.go b/cmd/root.go index 1764f4b..535d78a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -85,5 +85,5 @@ func initConfig() { } logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) - log.Debug().Msgf("Using config file: %s", viper.ConfigFileUsed()) + log.Info().Msgf("Using config file: %s", viper.ConfigFileUsed()) } diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index dab9d4e..1cc6dd5 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -61,7 +61,7 @@ func NewCommand() *cobra.Command { autoIndex := true indexDepth := 4 indexWorkers := 8 - badgerDir := "." + badgerDir := "./warcdb" badgerCompression := "snappy" badgerDatabase := "" badgerBatchMaxSize := 1000 diff --git a/index/api.go b/index/api.go index eb8cd52..debb468 100644 --- a/index/api.go +++ b/index/api.go @@ -7,6 +7,14 @@ import ( "github.com/nlnwa/gowarcserver/schema" ) +type indexError string + +const AlreadyIndexedError indexError = "already indexed" + +func (a indexError) Error() string { + return string(a) +} + type DateRange interface { ContainsTime(time.Time) (bool, error) ContainsStr(string) (bool, error) diff --git a/index/worker.go b/index/worker.go index caed305..12b9260 100644 --- a/index/worker.go +++ b/index/worker.go @@ -17,6 +17,7 @@ package index import ( + "errors" "sync" "time" @@ -49,7 +50,11 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { select { case job := <-iw.jobs: if err := a.Index(job); err != nil { - log.Warn().Err(err).Msgf("Index: %s", job) + if errors.Is(err, AlreadyIndexedError) { + log.Debug().Err(err).Msg(job) + } else { + log.Warn().Err(err).Msgf(job) + } } iw.wg.Done() iw.mx.Lock() diff --git a/internal/badgeridx/badger.go b/internal/badgeridx/badger.go index d947607..b44d750 100644 --- a/internal/badgeridx/badger.go +++ b/internal/badgeridx/badger.go @@ -38,11 +38,11 @@ func (l logger) Warningf(fmt string, args ...interface{}) { } func (l logger) Infof(fmt string, args ...interface{}) { - log.Debug().Msgf(l.prefix+fmt, args...) + log.Trace().Msgf(l.prefix+fmt, args...) } func (l logger) Debugf(fmt string, args ...interface{}) { - log.Debug().Msgf(l.prefix+fmt, args...) + log.Trace().Msgf(l.prefix+fmt, args...) } func newBadgerDB(dir string, compression options.CompressionType, readOnly bool) (*badger.DB, error) { diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index 3a8a268..011381f 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -18,7 +18,6 @@ package badgeridx import ( "context" - "errors" "fmt" "os" "path" @@ -63,17 +62,16 @@ func NewDB(options ...DbOption) (db *DB, err error) { var fileIndex *badger.DB var cdxIndex *badger.DB - dir := path.Join(opts.Path, "warcdb") batch := make(chan index.Record, opts.BatchMaxSize) done := make(chan struct{}) - if idIndex, err = newBadgerDB(path.Join(dir, opts.Database, "id-index"), opts.Compression, opts.ReadOnly); err != nil { + if idIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "id-index"), opts.Compression, opts.ReadOnly); err != nil { return } - if fileIndex, err = newBadgerDB(path.Join(dir, opts.Database, "file-index"), opts.Compression, opts.ReadOnly); err != nil { + if fileIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "file-index"), opts.Compression, opts.ReadOnly); err != nil { return } - if cdxIndex, err = newBadgerDB(path.Join(dir, opts.Database, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil { + if cdxIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil { return } @@ -171,7 +169,7 @@ func (db *DB) addFile(filePath string) error { } fileInfoLastModified := fileInfo.LastModified.AsTime() if fileInfo.Size == fileSize && fileInfoLastModified.Equal(fileLastModified) { - return errors.New("already indexed") + return index.AlreadyIndexedError } } diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 6306bbf..fdd3447 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -19,7 +19,6 @@ package tikvidx import ( "bytes" "context" - "errors" "fmt" "os" "path/filepath" @@ -152,7 +151,7 @@ func (db *DB) addFile(filePath string) error { } fileInfoLastModified := fileInfo.LastModified.AsTime() if fileInfo.Size == fileSize && fileInfoLastModified.Equal(fileLastModified) { - return errors.New("already indexed") + return index.AlreadyIndexedError } } From 5efd9d6c0a70aa26319e6d2d1d2c91cf96ceeb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Sun, 6 Nov 2022 22:46:35 +0100 Subject: [PATCH 23/38] Update GitHub actions --- .github/workflows/release.yaml | 4 ++-- .github/workflows/test.yaml | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2ab78c7..b2d8cb3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,14 +30,14 @@ jobs: type=ref,event=pr - name: Log in to the container registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: true build-args: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fdad08e..3df2546 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,14 +13,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: - go-version: '^1.16' + go-version: '^1.19' - name: Cache go modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cache/go-build @@ -35,8 +35,11 @@ jobs: name: Linting runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: golangci/golangci-lint-action@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '^1.19' + - uses: golangci/golangci-lint-action@v3 with: version: latest # Enable additional linters (see: https://golangci-lint.run/usage/linters/) From 4f735fa79b950a4a8e77667a29d00e73c2389333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Mon, 7 Nov 2022 00:48:35 +0100 Subject: [PATCH 24/38] Be lenient when sort=closest and closest="" --- server/api/api.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/api/api.go b/server/api/api.go index 0e31356..ca2561a 100644 --- a/server/api/api.go +++ b/server/api/api.go @@ -93,8 +93,10 @@ func (c SearchAPI) Keys() []string { func (c SearchAPI) Sort() index.Sort { switch c.CoreAPI.Sort { - case "reverse": + case SortReverse: return index.SortDesc + case SortClosest: + return index.SortClosest default: return index.SortAsc } @@ -178,7 +180,7 @@ func Parse(r *http.Request) (*CoreAPI, error) { coreApi.Limit = l } - closest := query.Get("Closest") + closest := query.Get("closest") if closest != "" { _, err := timestamp.Parse(closest) if err != nil { @@ -192,7 +194,7 @@ func Parse(r *http.Request) (*CoreAPI, error) { if !contains(sorts, sort) { return nil, fmt.Errorf("sort must be one of %v, was: %s", sorts, sort) } else if sort == SortClosest && closest == "" { - return nil, fmt.Errorf("missing closest parameter") + sort = "" } else if sort == SortClosest && len(coreApi.Urls) == 0 { return nil, fmt.Errorf("sort=closest is not valid without urls") } From 02fc725ca8ec8156d287e0c58749f43cca8941e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Mon, 7 Nov 2022 01:25:54 +0100 Subject: [PATCH 25/38] tikv: Don't use invalid iterators --- internal/tikvidx/iter.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index 42c5427..40f1672 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -145,11 +145,14 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest case index.SortNone: fallthrough default: - it, err = tx.Iter(k, []byte(cdxEOF)) + it, err = tx.Iter(k, nil) } if err != nil { break } + if !it.Valid() { + continue + } is.iterators = append(is.iterators, it) results = append(results, make(chan *maybeKV)) } From 9341acf9777e8e2472b24d71f2f8489b7ebdd84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Mon, 7 Nov 2022 10:10:56 +0100 Subject: [PATCH 26/38] tikv: Fix iter prefix check --- internal/tikvidx/iter.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index 40f1672..b8d9d1e 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -130,6 +130,7 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest is := new(iterSort) // initialize iterators + var prefixes [][]byte var results []chan *maybeKV var err error for _, key := range req.Keys() { @@ -145,7 +146,7 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest case index.SortNone: fallthrough default: - it, err = tx.Iter(k, nil) + it, err = tx.Iter(k, []byte(cdxEOF)) } if err != nil { break @@ -154,6 +155,7 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest continue } is.iterators = append(is.iterators, it) + prefixes = append(prefixes, k[:]) results = append(results, make(chan *maybeKV)) } if err != nil { @@ -194,9 +196,9 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest for i, iter := range is.iterators { i := i - go func(iter iterator, ch chan<- *maybeKV) { + go func(iter iterator, prefix []byte, ch chan<- *maybeKV) { defer close(ch) - for iter.Valid() && bytes.HasPrefix(iter.Key(), []byte(cdxPrefix)) { + for iter.Valid() && bytes.HasPrefix(iter.Key(), prefix) { select { case <-ctx.Done(): return @@ -211,7 +213,7 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest return } } - }(iter, results[i]) + }(iter, prefixes[i], results[i]) } return is, is.Next() From 9014d7bb4733a89e82f02f417d0712f550ce0df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Mon, 7 Nov 2022 14:43:39 +0100 Subject: [PATCH 27/38] tikv: Fix closest iterator Iterator could never be valid --- internal/tikvidx/api.go | 5 ++++- internal/tikvidx/iter.go | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 44a23cf..2b70061 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -76,7 +76,10 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in if err != nil { return err } - + if it == nil { + close(res) + return nil + } go func() { defer close(res) defer it.Close() diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index b8d9d1e..f841e5f 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -92,6 +92,7 @@ func (ic *iterClosest) Next() error { } ic.key = it.Key() ic.value = it.Value() + ic.valid = true return it.Next() } From f124ee3b60ed045aa86d516eb4e78b0689cbae69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 8 Nov 2022 00:20:46 +0100 Subject: [PATCH 28/38] tikv: snapshot for reads, retry failed commits and fix reverse sort --- cmd/serve/serve.go | 2 +- index/io.go | 7 +- index/record.go | 4 +- internal/tikvidx/api.go | 54 +++++++--------- internal/tikvidx/db.go | 53 ++++++++++----- internal/tikvidx/iter.go | 126 +++++++++++++++++++++++++++++++----- internal/tikvidx/options.go | 16 +++-- internal/tikvidx/sorter.go | 100 ---------------------------- 8 files changed, 186 insertions(+), 176 deletions(-) delete mode 100644 internal/tikvidx/sorter.go diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 1cc6dd5..7ca0df6 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -69,7 +69,7 @@ func NewCommand() *cobra.Command { badgerReadOnly := false logRequests := false tikvPdAddr := []string{} - tikvBatchMaxSize := 1000 + tikvBatchMaxSize := 255 tikvBatchMaxWait := 5 * time.Second tikvDatabase := "" pathPrefix := "" diff --git a/index/io.go b/index/io.go index 27614ae..61b628e 100644 --- a/index/io.go +++ b/index/io.go @@ -41,7 +41,6 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow }() filename := filepath.Base(path) - var prevOffset int64 var prevWr gowarc.WarcRecord // write record from the previous iteration because we need to calculate record length @@ -60,7 +59,7 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow wr, offset, validation, err := wf.Next() if prevWr != nil { if err := write(offset); err != nil { - log.Warn().Err(err).Str("path", path).Int64("offset", offset).Msgf("Error writing index record") + log.Warn().Err(err).Msgf("Failed to index record: %s#%d", filename, prevOffset) } else { count++ } @@ -70,10 +69,10 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow break } if err != nil { - return count, total, fmt.Errorf("failed to get record number %d in %s at offset %d: %w", count, path, offset, err) + return count, total, fmt.Errorf("failed to get record number %d in %s at offset %d: %w", count, filename, offset, err) } if !validation.Valid() { - log.Warn().Err(validation).Str("path", path).Int64("offset", offset).Msgf("Invalid %s record: %s", wr.Type(), wr.RecordId()) + log.Warn().Err(validation).Str("filename", filename).Int64("offset", offset).Msgf("Invalid %s record: %s", wr.Type(), wr.RecordId()) } if filter(wr) { prevWr = wr diff --git a/index/record.go b/index/record.go index 8102e4d..7d279e1 100644 --- a/index/record.go +++ b/index/record.go @@ -87,7 +87,7 @@ func newRecord(wr gowarc.WarcRecord, filename string, offset int64, length int64 var err error rec.Ple, err = strconv.ParseInt(cl, 10, 64) if err != nil { - log.Warn().Msgf("failed to parse HTTP header field 'Content-Length' as int64: %v", err) + log.Warn().Msgf("Failed to parse HTTP header field 'Content-Length' as int64: %v", err) } } } @@ -135,7 +135,7 @@ func newRecord(wr gowarc.WarcRecord, filename string, offset int64, length int64 var err error rec.Ple, err = strconv.ParseInt(cl, 10, 64) if err != nil { - log.Warn().Msgf("failed to parse HTTP header field 'Content-Length' as int64: %v", err) + log.Warn().Msgf("Failed to parse HTTP header field 'Content-Length' as int64: %v", err) } } break diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 2b70061..627f6b5 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -2,24 +2,21 @@ package tikvidx import ( "context" - "errors" "strings" "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" - "github.com/tikv/client-go/v2/txnkv/transaction" "google.golang.org/protobuf/proto" ) // Closest returns the first closest cdx value(s). func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- index.CdxResponse) error { - // begin transaction - tx, err := db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return err } - - it, err := NewIterClosest(ctx, tx, req.Key(), req.Closest()) + snapshot := db.client.GetSnapshot(ts) + it, err := NewIterClosest(ctx, snapshot, req.Key(), req.Closest()) if err != nil { return err } @@ -60,19 +57,13 @@ func cdxFromValue(value []byte) (*schema.Cdx, error) { } func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- index.CdxResponse) error { - var err error - var tx *transaction.KVTxn - var it iterator - - if len(req.Keys()) == 0 { - return errors.New("search request has no keys") - } - tx, err = db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return err } + snapshot := db.client.GetSnapshot(ts) - it, err = newIter(ctx, tx, req) + it, err := newIter(ctx, snapshot, req) if err != nil { return err } @@ -126,11 +117,13 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in } func (db *DB) List(ctx context.Context, limit int, res chan<- index.CdxResponse) error { - tx, err := db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return err } - it, err := tx.Iter([]byte(cdxPrefix), []byte(cdxEOF)) + snapshot := db.client.GetSnapshot(ts) + + it, err := snapshot.Iter([]byte(cdxPrefix), []byte(cdxEOF)) if err != nil { return err } @@ -170,11 +163,13 @@ func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, } func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileResponse) error { - tx, err := db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return err } - it, err := tx.Iter([]byte(filePrefix), []byte(fileEOF)) + snapshot := db.client.GetSnapshot(ts) + + it, err := snapshot.Iter([]byte(filePrefix), []byte(fileEOF)) if err != nil { return err } @@ -204,21 +199,24 @@ func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileRe } func (db *DB) GetStorageRef(ctx context.Context, id string) (string, error) { - tx, err := db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return "", err } + snapshot := db.client.GetSnapshot(ts) - b, err := tx.Get(ctx, []byte(id)) + b, err := snapshot.Get(ctx, []byte(id)) return string(b), err } func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdResponse) error { - tx, err := db.client.Begin() + ts, err := db.client.CurrentTimestamp("global") if err != nil { return err } - it, err := tx.Iter([]byte(idPrefix), []byte(idEOF)) + snapshot := db.client.GetSnapshot(ts) + + it, err := snapshot.Iter([]byte(idPrefix), []byte(idEOF)) if err != nil { return err } @@ -247,16 +245,14 @@ func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdRe } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. -func (db *DB) Resolve(ctx context.Context, warcId string) (storageRef string, err error) { +func (db *DB) Resolve(ctx context.Context, warcId string) (string, error) { key := []byte(idPrefix + warcId) - var kv KV - kv, err = db.get(ctx, key) + kv, err := db.get(ctx, key) if err != nil { - return + return "", err } - storageRef = string(kv.V) - return + return string(kv.V), nil } // ResolvePath looks up filename in file index and returns the path field. diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index fdd3447..320512c 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -73,6 +73,8 @@ type DB struct { done chan struct{} + opts *Options + wg sync.WaitGroup } @@ -81,7 +83,6 @@ func NewDB(options ...Option) (db *DB, err error) { for _, opt := range options { opt(opts) } - // prefix all keys with name of database // TODO use keyspace when TiKV support it idPrefix = opts.Database + idPrefix @@ -101,6 +102,7 @@ func NewDB(options ...Option) (db *DB, err error) { db = &DB{ client: client, done: done, + opts: opts, } if opts.ReadOnly { @@ -206,31 +208,46 @@ func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { return fi, nil } -func (db *DB) get(ctx context.Context, k []byte) (KV, error) { - tx, err := db.client.Begin() +func (db *DB) get(ctx context.Context, k []byte) (*KV, error) { + ts, err := db.client.CurrentTimestamp("global") if err != nil { - return KV{}, err + return nil, err } - v, err := tx.Get(ctx, k) + snapshot := db.client.GetSnapshot(ts) + if err != nil { - return KV{}, err + return nil, err } - return KV{K: k, V: v}, nil -} - -func (db *DB) puts(ctx context.Context, kvs ...KV) error { - tx, err := db.client.Begin() + v, err := snapshot.Get(ctx, k) if err != nil { - return err + return nil, err } + return &KV{K: k, V: v}, nil +} - for _, kv := range kvs { - err := tx.Set(kv.K, kv.V) +func (db *DB) puts(ctx context.Context, kvs ...KV) error { + retries := db.opts.BatchMaxRetries + for { + tx, err := db.client.Begin() if err != nil { return err } + for _, kv := range kvs { + err := tx.Set(kv.K, kv.V) + if err != nil { + return err + } + } + err = tx.Commit(ctx) + if err == nil { + return nil + } else if retries > 0 { + retries-- + log.Warn().Err(err).Msgf("Commit failed, will try again %d times", retries) + } else { + return err + } } - return tx.Commit(ctx) } // write schedules a Record to be added to the DB via the batch channel. @@ -276,11 +293,15 @@ func (db *DB) flushBatch() { } kvs = append(kvs, id, cdx) } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + err := db.puts(ctx, kvs...) if err != nil { - log.Error().Err(err).Msgf("failed to commit batch (first batch ref is %s)", records[0].Ref) + for _, r := range records { + log.Warn().Err(err).Msgf("failed to commit record: %s", r.Ref) + } } } diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index f841e5f..f5f1d8d 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -8,9 +8,23 @@ import ( "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/timestamp" - "github.com/tikv/client-go/v2/txnkv/transaction" + "github.com/tikv/client-go/v2/txnkv" ) +func CompareClosest(ts int64) func(int64, int64) bool { + return func(ts1 int64, ts2 int64) bool { + return timestamp.AbsInt64(ts-ts1) < timestamp.AbsInt64(ts-ts2) + } +} + +func CompareAsc(a int64, b int64) bool { + return a <= b +} + +func CompareDesc(a int64, b int64) bool { + return a > b +} + // iterator mimics tikv's internal iterator interface type iterator interface { Next() error @@ -23,6 +37,7 @@ type iterator interface { type iterClosest struct { forward iterator backward iterator + curr iterator prefix []byte key []byte value []byte @@ -30,7 +45,7 @@ type iterClosest struct { cmp func(int64, int64) bool } -func NewIterClosest(_ context.Context, tx *transaction.KVTxn, key string, closest string) (*iterClosest, error) { +func NewIterClosest(_ context.Context, snapshot *txnkv.KVSnapshot, key string, closest string) (*iterClosest, error) { ic := new(iterClosest) if t, err := time.Parse(timestamp.CDX, closest); err != nil { return nil, err @@ -42,23 +57,31 @@ func NewIterClosest(_ context.Context, tx *transaction.KVTxn, key string, closes k := []byte(cdxPrefix + key + " " + closest) // initialize forward iterator - forward, err := tx.Iter(k, []byte(cdxEOF)) + forward, err := snapshot.Iter(k, []byte(cdxEOF)) if err != nil { return nil, err } ic.forward = forward // initialize backward iterator - backward, err := tx.IterReverse(k) + backward, err := snapshot.IterReverse(k) if err != nil { return nil, err } ic.backward = backward - return ic, ic.Next() + return ic, ic.next() } func (ic *iterClosest) Next() error { + err := ic.curr.Next() + if err != nil { + return err + } + return ic.next() +} + +func (ic *iterClosest) next() error { var ft int64 var bt int64 @@ -90,11 +113,13 @@ func (ic *iterClosest) Next() error { ic.valid = false return nil } + + ic.curr = it ic.key = it.Key() ic.value = it.Value() ic.valid = true - return it.Next() + return nil } func (ic *iterClosest) Key() []byte { @@ -127,23 +152,24 @@ type iterSort struct { next <-chan maybeKV } -func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest) (*iterSort, error) { +func newIter(ctx context.Context, tx *txnkv.KVSnapshot, req index.SearchRequest) (iterator, error) { is := new(iterSort) - - // initialize iterators var prefixes [][]byte var results []chan *maybeKV + var it iterator var err error for _, key := range req.Keys() { k := []byte(cdxPrefix + key) - var it iterator switch req.Sort() { - case index.SortDesc: - it, err = tx.IterReverse(k) + case index.SortAsc: + it, err = tx.Iter(k, []byte(cdxEOF)) case index.SortClosest: it, err = NewIterClosest(ctx, tx, key, req.Closest()) - case index.SortAsc: - fallthrough + case index.SortDesc: + if len(req.Keys()) == 1 { + return tx.IterReverse(k) + } + it, err = tx.Iter(k, []byte(cdxEOF)) case index.SortNone: fallthrough default: @@ -156,16 +182,19 @@ func newIter(ctx context.Context, tx *transaction.KVTxn, req index.SearchRequest continue } is.iterators = append(is.iterators, it) - prefixes = append(prefixes, k[:]) + prefixes = append(prefixes, k) results = append(results, make(chan *maybeKV)) } if err != nil { - defer is.Close() + is.Close() return nil, err } - if len(results) == 0 { + if len(is.iterators) == 0 { return nil, nil } + if len(is.iterators) == 1 { + return is.iterators[0], nil + } // initialize comparator var cmp func(a KV, b KV) bool @@ -257,3 +286,66 @@ func (is *iterSort) Close() { } } } + +// mergeIter merges sorted input channels into a sorted output channel +// +// Sorting is done by comparing keys from key-value pairs. +// +// The input channels are closed externally +func mergeIter(done <-chan struct{}, cmp func(KV, KV) bool, in ...chan *maybeKV) <-chan maybeKV { + out := make(chan maybeKV) + cords := make([]*maybeKV, len(in)) + go func() { + defer close(out) + var zombie []int + for { + curr := -1 + for i, cord := range cords { + if cord == nil { + select { + case cord = <-in[i]: + cords[i] = cord + case <-done: + return + } + // closed channel becomes zombie + if cord == nil { + zombie = append(zombie, i) + continue + } + } + if cord.error != nil { + // prioritize errors + curr = i + break + } + if curr == -1 { + curr = i + } else if cmp(cords[i].kv, cord.kv) { + curr = i + } + } + if curr == -1 { + return + } + select { + case <-done: + return + case out <- *cords[curr]: + cords[curr] = nil + } + // if zombie, then kill + if len(zombie) > 0 { + for _, i := range zombie { + cords[i] = cords[len(cords)-1] + cords = cords[:len(cords)-1] + in[i] = in[len(in)-1] + in = in[:len(in)-1] + } + zombie = nil + } + } + }() + + return out +} diff --git a/internal/tikvidx/options.go b/internal/tikvidx/options.go index 4392383..164f04e 100644 --- a/internal/tikvidx/options.go +++ b/internal/tikvidx/options.go @@ -4,17 +4,19 @@ import "time" func defaultOptions() *Options { return &Options{ - BatchMaxSize: 10000, - BatchMaxWait: 5 * time.Second, + BatchMaxSize: 255, + BatchMaxWait: 5 * time.Second, + BatchMaxRetries: 3, } } type Options struct { - BatchMaxSize int - BatchMaxWait time.Duration - ReadOnly bool - PdAddr []string - Database string + BatchMaxSize int + BatchMaxWait time.Duration + BatchMaxRetries int + ReadOnly bool + PdAddr []string + Database string } type Option func(opts *Options) diff --git a/internal/tikvidx/sorter.go b/internal/tikvidx/sorter.go deleted file mode 100644 index 3590964..0000000 --- a/internal/tikvidx/sorter.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2020 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package tikvidx - -import ( - "github.com/nlnwa/gowarcserver/timestamp" - "github.com/rs/zerolog/log" -) - -func CompareClosest(ts int64) func(int64, int64) bool { - return func(ts1 int64, ts2 int64) bool { - return timestamp.AbsInt64(ts-ts1) < timestamp.AbsInt64(ts-ts2) - } -} - -func CompareAsc(a int64, b int64) bool { - return a <= b -} - -func CompareDesc(a int64, b int64) bool { - return a > b -} - -// mergeIter merges sorted input channels into a sorted output channel -// -// Sorting is done by comparing keys from key-value pairs. -// -// The input channels are closed externally -func mergeIter(done <-chan struct{}, cmp func(KV, KV) bool, in ...chan *maybeKV) <-chan maybeKV { - out := make(chan maybeKV) - cords := make([]*maybeKV, len(in)) - go func() { - defer close(out) - var zombie []int - for { - curr := -1 - for i, cord := range cords { - if cord == nil { - select { - case cord = <-in[i]: - cords[i] = cord - case <-done: - return - } - // closed channel becomes zombie - if cord == nil { - zombie = append(zombie, i) - continue - } - } - if cord.error != nil { - log.Error().Err(cord.error).Msg("cord error") - // prioritize errors - curr = i - break - } - if curr == -1 { - curr = i - } else if cmp(cords[i].kv, cord.kv) { - curr = i - } - } - if curr == -1 { - return - } - select { - case <-done: - return - case out <- *cords[curr]: - cords[curr] = nil - } - // if zombie, then kill - if len(zombie) > 0 { - for _, i := range zombie { - cords[i] = cords[len(cords)-1] - cords = cords[:len(cords)-1] - in[i] = in[len(in)-1] - in = in[:len(in)-1] - } - zombie = nil - } - } - }() - - return out -} From a5470ce7843a0809218f09d3c1387eac5a7e9c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Wed, 9 Nov 2022 11:02:05 +0100 Subject: [PATCH 29/38] tikv: Remove default limit Should be done at a higher level --- internal/tikvidx/api.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 627f6b5..448d89b 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -76,9 +76,6 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in defer it.Close() limit := req.Limit() - if limit == 0 { - limit = 100 - } for it.Valid() && limit > 0 { select { From 97bbbca275993fa54244b04a37c01124eac13f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 29 Nov 2022 19:45:26 +0100 Subject: [PATCH 30/38] Add kafka index source --- index/kafka.go | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 index/kafka.go diff --git a/index/kafka.go b/index/kafka.go new file mode 100644 index 0000000..b9ad227 --- /dev/null +++ b/index/kafka.go @@ -0,0 +1,104 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package index + +import ( + "context" + "errors" + "io" + "time" + + "github.com/segmentio/kafka-go" +) + +type KafkaIndexOption func(options *kafka.ReaderConfig) + +func WithBrokers(brokers []string) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.Brokers = brokers + } +} + +func WithGroupID(groupID string) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.GroupID = groupID + } +} + +func WithTopic(topic string) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.Topic = topic + } +} + +func WithMinBytes(minBytes int) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.MinBytes = minBytes + } +} + +func WithMaxBytes(maxBytes int) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.MaxBytes = maxBytes + } +} + +func WithMaxWait(maxWait time.Duration) KafkaIndexOption { + return func(options *kafka.ReaderConfig) { + options.MaxWait = maxWait + } +} + +type KafkaIndexer struct { + kafka.ReaderConfig + Queue +} + +func (k KafkaIndexer) Run(ctx context.Context) (err error) { + defer func() { + r := recover() + switch v := r.(type) { + case error: + err = v + } + }() + + r := kafka.NewReader(k.ReaderConfig) + defer r.Close() + + for { + msg, err := r.ReadMessage(ctx) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + k.Add(string(msg.Value)) + } +} + +func NewKafkaIndexer(q Queue, options ...KafkaIndexOption) KafkaIndexer { + readerConfig := new(kafka.ReaderConfig) + for _, apply := range options { + apply(readerConfig) + } + return KafkaIndexer{ + ReaderConfig: kafka.ReaderConfig{}, + Queue: q, + } +} From f77ad379e649a77f8fa8b5b2fb53d6d88dacdade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 29 Nov 2022 19:45:56 +0100 Subject: [PATCH 31/38] Refactor and consolidate Remove file watcher --- .github/workflows/release.yaml | 8 +- .github/workflows/test.yaml | 24 +++ cmd/index/index.go | 154 ++++++++------- cmd/index/index_test.go | 29 --- cmd/root.go | 18 +- cmd/root_test.go | 16 ++ cmd/serve/serve.go | 176 +++++++++-------- exampleconfig.yaml | 47 +++-- go.mod | 30 ++- go.sum | 214 ++++++++++++++------ index/api.go | 4 + index/autoindex.go | 148 +++++++------- index/index.go | 88 +++++++++ index/io.go | 60 ++---- index/io_test.go | 4 +- index/options.go | 32 +-- index/watcher.go | 82 -------- index/worker.go | 76 ++------ index/writers.go | 29 +-- internal/badgeridx/compression.go | 3 +- internal/badgeridx/db.go | 91 ++------- internal/badgeridx/options.go | 94 +++++++++ internal/logger/initlog.go | 5 +- internal/tikvidx/api.go | 171 +++++++--------- internal/tikvidx/db.go | 123 ++++-------- internal/tikvidx/iter.go | 312 +++++++++++++++--------------- internal/tikvidx/options.go | 20 +- main.go | 27 +-- server/coreserver/handler.go | 5 +- 29 files changed, 1060 insertions(+), 1030 deletions(-) delete mode 100644 cmd/index/index_test.go create mode 100644 index/index.go delete mode 100644 index/watcher.go create mode 100644 internal/badgeridx/options.go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b2d8cb3..443849b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,9 +1,6 @@ name: release on: - pull_request: - branches: - - main push: branches: - main @@ -17,11 +14,14 @@ env: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3df2546..e6eabb3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,6 +6,10 @@ on: permissions: contents: read +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: unit_test: name: Golang unit tests @@ -44,3 +48,23 @@ jobs: version: latest # Enable additional linters (see: https://golangci-lint.run/usage/linters/) args: -E "bodyclose" -E "dogsled" -E "durationcheck" -E "errorlint" -E "forcetypeassert" -E "noctx" -E "exhaustive" -E "exportloopref" --timeout 3m0s + build: + runs-on: ubuntu-latest + steps: + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + build-args: | + VERSION=${{ steps.meta.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/cmd/index/index.go b/cmd/index/index.go index 7816bf0..495b917 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -17,9 +17,13 @@ package index import ( + "context" "fmt" + "os" + "os/signal" "regexp" "runtime" + "syscall" "time" "github.com/bits-and-blooms/bloom/v3" @@ -34,7 +38,7 @@ import ( func NewCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "index [dir] ...", + Use: "index", Short: "Index warc file(s)", PreRunE: func(cmd *cobra.Command, args []string) error { if err := viper.BindPFlags(cmd.Flags()); err != nil { @@ -44,55 +48,51 @@ func NewCommand() *cobra.Command { }, RunE: indexCmd, } - // defaults - format := "cdxj" - indexDepth := 4 - indexWorkers := 8 - badgerDir := "./warcdb" - badgerBatchMaxSize := 1000 - badgerBatchMaxWait := 5 * time.Second - badgerCompression := "snappy" - badgerDatabase := "" - var tikvPdAddr []string - tikvBatchMaxSize := 1000 - tikvBatchMaxWait := 5 * time.Second - tikvDatabase := "" - bloomCapacity := uint(1000) - bloomFp := 0.01 - - cmd.Flags().StringP("format", "f", format, `index format: "cdxj", "cdxpb", "badger", "tikv" or "toc"`) - cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") - cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") - cmd.Flags().IntP("max-depth", "d", indexDepth, "maximum directory recursion") - cmd.Flags().Int("workers", indexWorkers, "number of files") - cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") - cmd.Flags().Uint("toc-bloom-capacity", bloomCapacity, "estimated bloom filter capacity") - cmd.Flags().Float64("toc-bloom-fp", bloomFp, "estimated bloom filter false positive rate") - cmd.Flags().String("badger-dir", badgerDir, "path to index database") - cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") - cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") + // index options + cmd.Flags().StringP("index-source", "s", "file", `index source: "file" or "kafka"`) + cmd.Flags().StringP("index-format", "o", "cdxj", `index format: "cdxj", "cdxpb", "toc", badger", "tikv"`) + cmd.Flags().StringSlice("index-include", nil, "only include files matching these regular expressions") + cmd.Flags().StringSlice("index-exclude", nil, "exclude files matching these regular expressions") + cmd.Flags().Int("index-workers", 8, "number of index workers") + + // auto indexer options + cmd.Flags().StringSliceP("file-paths", "f", []string{"./testdata"}, "directories to search for warc files in") + cmd.Flags().Int("file-max-depth", 4, "maximum directory recursion") + + // kafka indexer options + cmd.Flags().StringSlice("kafka-brokers", nil, "the list of broker addresses used to connect to the kafka cluster") + cmd.Flags().String("kafka-group-id", "", "optional consumer group id") + cmd.Flags().String("kafka-topic", "", "the topic to read messages from") + cmd.Flags().Int("kafka-min-bytes", 0, "indicates to the broker the minimum batch size that the consumer will accept") + cmd.Flags().Int("kafka-max-bytes", 0, "indicates to the broker the maximum batch size that the consumer will accept") + cmd.Flags().Duration("kafka-max-wait", 0, "maximum amount of time to wait for new data to come when fetching batches of messages from kafka") + + // toc indexer options + cmd.Flags().Uint("toc-bloom-capacity", uint(1000), "estimated bloom filter capacity") + cmd.Flags().Float64("toc-bloom-fp", 0.01, "estimated bloom filter false positive rate") + + // badger options + cmd.Flags().String("badger-dir", "./warcdb", "path to index database") + cmd.Flags().String("badger-database", "", "name of badger database") + cmd.Flags().Int("badger-batch-max-size", 1000, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", 5*time.Second, "max wait time before flushing batched records") + cmd.Flags().String("badger-compression", "snappy", "compression algorithm") + cmd.Flags().Bool("badger-read-only", false, "run badger in read-only mode") + + // tikv options + cmd.Flags().StringSlice("tikv-pd-addr", nil, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", 1000, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", 5*time.Second, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", "", "name of tikv database") + return cmd } -func indexCmd(_ *cobra.Command, args []string) error { - // collect paths from args or flag - var dirs []string - if len(args) > 0 { - dirs = append(dirs, args...) - } else { - dirs = viper.GetStringSlice("dirs") - } - - var w index.Indexer +func indexCmd(_ *cobra.Command, _ []string) error { + var w index.RecordWriter - format := viper.GetString("format") - switch format { + indexFormat := viper.GetString("index-format") + switch indexFormat { case "cdxj": w = index.CdxJ{} case "cdxpb": @@ -135,14 +135,11 @@ func indexCmd(_ *cobra.Command, args []string) error { BloomFilter: bloom.NewWithEstimates(viper.GetUint("toc-bloom-capacity"), viper.GetFloat64("toc-bloom-fp")), } default: - return fmt.Errorf("unsupported format %s", format) + return fmt.Errorf("unknown index format: %s", indexFormat) } - indexWorker := index.NewWorker(w, viper.GetInt("workers")) - defer indexWorker.Close() - var includes []*regexp.Regexp - for _, r := range viper.GetStringSlice("include") { + for _, r := range viper.GetStringSlice("index-include") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -151,7 +148,7 @@ func indexCmd(_ *cobra.Command, args []string) error { } var excludes []*regexp.Regexp - for _, r := range viper.GetStringSlice("exclude") { + for _, r := range viper.GetStringSlice("index-exclude") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -159,23 +156,50 @@ func indexCmd(_ *cobra.Command, args []string) error { } } - log.Info().Msg("Starting auto indexer") - indexer, err := index.NewAutoIndexer(indexWorker, - index.WithMaxDepth(viper.GetInt("max-depth")), + indexer := index.NewIndexer(w, index.WithIncludes(includes...), index.WithExcludes(excludes...), ) - if err != nil { - return err - } - defer indexer.Close() + queue := index.NewWorkQueue(indexer, + viper.GetInt("index-workers"), + ) + defer queue.Close() - for _, dir := range dirs { - err := indexer.Index(dir) - if err != nil { - log.Warn().Msgf(`Error indexing "%s": %v`, dir, err) - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + sig := <-sigs + log.Info().Msgf("Received %s signal, shutting down...", sig) + cancel() + }() + + var runner index.Runner + + indexSource := viper.GetString("index-source") + switch indexSource { + case "file": + runner = index.NewAutoIndexer(queue, + index.WithMaxDepth(viper.GetInt("file-max-depth")), + index.WithPaths(viper.GetStringSlice("file-paths")), + index.WithExcludeDirs(excludes...), + ) + case "kafka": + runner = index.NewKafkaIndexer(queue, + index.WithBrokers(viper.GetStringSlice("kafka-brokers")), + index.WithGroupID(viper.GetString("kafka-group-id")), + index.WithTopic(viper.GetString("kafka-topic")), + index.WithMinBytes(viper.GetInt("kafka-min-bytes")), + index.WithMaxBytes(viper.GetInt("kafka-max-bytes")), + index.WithMaxWait(viper.GetDuration("kafka-max-wait")), + ) + default: + return fmt.Errorf("unknown index source: %s", indexSource) } - return nil + log.Info().Msg("Starting indexer") + + return runner.Run(ctx) } diff --git a/cmd/index/index_test.go b/cmd/index/index_test.go deleted file mode 100644 index d02c51f..0000000 --- a/cmd/index/index_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package index - -import ( - "testing" -) - -func TestIndexCmd(t *testing.T) { - cmd := NewCommand() - err := cmd.Execute() - if err != nil { - t.Errorf("%v", err) - } -} diff --git a/cmd/root.go b/cmd/root.go index 535d78a..18c73a4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,7 +33,7 @@ import ( // NewCommand returns a new cobra.Command implementing the root command for warc func NewCommand() *cobra.Command { - cobra.OnInitialize(func() { initConfig() }) + cobra.OnInitialize(initConfig) cmd := &cobra.Command{ Use: "gowarcserver", @@ -41,7 +41,7 @@ func NewCommand() *cobra.Command { } // Global flags - _ = cmd.PersistentFlags().StringP("config", "c", "", `path to config file, default paths are "./config.yaml", "$HOME/.gowarcserver/config.yaml" or "/etc/gowarcserver/config.yaml"`) + _ = cmd.PersistentFlags().String("config", "", `path to config file, default paths are "./config.yaml", "$HOME/.gowarcserver/config.yaml" or "/etc/gowarcserver/config.yaml"`) _ = cmd.PersistentFlags().StringP("log-level", "l", "info", `set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace"`) _ = cmd.PersistentFlags().String("log-formatter", "logfmt", "log formatter, available values are logfmt and json") _ = cmd.PersistentFlags().Bool("log-method", false, "log method caller") @@ -76,14 +76,14 @@ func initConfig() { viper.AddConfigPath("/etc/gowarcserver/") // global configuration directory } - if err := viper.ReadInConfig(); err != nil { - if errors.As(err, new(viper.ConfigFileNotFoundError)) { - return - } + defer func() { + logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) + log.Debug().Msgf("Using config file: %s", viper.ConfigFileUsed()) + }() + + err := viper.ReadInConfig() + if err != nil && !errors.As(err, new(viper.ConfigFileNotFoundError)) { _, _ = fmt.Fprintf(os.Stderr, "Failed to read config file: %v", err) os.Exit(1) } - logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) - - log.Info().Msgf("Using config file: %s", viper.ConfigFileUsed()) } diff --git a/cmd/root_test.go b/cmd/root_test.go index a0cff70..88115d5 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -1,3 +1,19 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package cmd import ( diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 7ca0df6..aff41e4 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -54,62 +54,51 @@ func NewCommand() *cobra.Command { }, RunE: serveCmd, } - // defaults - port := 9999 - watch := false - storageEngine := "badger" - autoIndex := true - indexDepth := 4 - indexWorkers := 8 - badgerDir := "./warcdb" - badgerCompression := "snappy" - badgerDatabase := "" - badgerBatchMaxSize := 1000 - badgerBatchMaxWait := 5 * time.Second - badgerReadOnly := false - logRequests := false - tikvPdAddr := []string{} - tikvBatchMaxSize := 255 - tikvBatchMaxWait := 5 * time.Second - tikvDatabase := "" - pathPrefix := "" - cmd.Flags().IntP("port", "p", port, "server port") - cmd.Flags().String("path-prefix", pathPrefix, "prefix for all server endpoints") - cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") - cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") - cmd.Flags().IntP("max-depth", "w", indexDepth, "maximum directory recursion depth") - cmd.Flags().Int("workers", indexWorkers, "number of index workers") - cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") - cmd.Flags().Bool("index", autoIndex, "run indexing") - cmd.Flags().Bool("watch", watch, "watch files for changes") - cmd.Flags().Bool("log-requests", logRequests, "log http requests") - cmd.Flags().StringP("storage-engine", "b", storageEngine, `storage engine: "badger" or "tikv"`) - cmd.Flags().String("badger-dir", badgerDir, "path to index database") - cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") - cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") - cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().Bool("badger-read-only", badgerReadOnly, "run badger read-only") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") + // server options + cmd.Flags().IntP("port", "p", 9999, "server port") + cmd.Flags().String("path-prefix", "", "path prefix for all server endpoints") + cmd.Flags().Bool("log-requests", false, "log incoming http requests") + + // index options + cmd.Flags().StringP("index-source", "s", "file", `index source: "file" or "kafka"`) + cmd.Flags().StringP("index-format", "o", "badger", `index format: "badger", "tikv"`) + cmd.Flags().StringSlice("index-include", nil, "only include files matching these regular expressions") + cmd.Flags().StringSlice("index-exclude", nil, "exclude files matching these regular expressions") + cmd.Flags().Int("index-workers", 8, "number of index workers") + + // auto indexer options + cmd.Flags().StringSlice("file-paths", []string{"./testdata"}, "list of paths to warc files or directories containing warc files") + cmd.Flags().Int("file-max-depth", 4, "maximum directory recursion depth") + + // kafka indexer options + cmd.Flags().StringSlice("kafka-brokers", nil, "the list of broker addresses used to connect to the kafka cluster") + cmd.Flags().String("kafka-group-id", "", "optional consumer group id") + cmd.Flags().String("kafka-topic", "", "the topic to read messages from") + cmd.Flags().Int("kafka-min-bytes", 0, "indicates to the broker the minimum batch size that the consumer will accept") + cmd.Flags().Int("kafka-max-bytes", 0, "indicates to the broker the maximum batch size that the consumer will accept") + cmd.Flags().Duration("kafka-max-wait", 0, "maximum amount of time to wait for new data to come when fetching batches of messages from kafka") + + // badger options + cmd.Flags().String("badger-dir", "./warcdb", "path to index database") + cmd.Flags().String("badger-database", "", "name of badger database") + cmd.Flags().Int("badger-batch-max-size", 1000, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", 5*time.Second, "max wait time before flushing batched records") + cmd.Flags().String("badger-compression", "snappy", "compression algorithm") + + // tikv options + cmd.Flags().StringSlice("tikv-pd-addr", nil, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", 1000, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", 5*time.Second, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", "", "name of tikv database") return cmd } -func serveCmd(cmd *cobra.Command, args []string) error { - // collect paths from args or flag - var dirs []string - if len(args) > 0 { - dirs = append(dirs, args...) - } else { - dirs = viper.GetStringSlice("dirs") - } +func serveCmd(_ *cobra.Command, _ []string) error { // parse include patterns var includes []*regexp.Regexp - for _, r := range viper.GetStringSlice("include") { + for _, r := range viper.GetStringSlice("index-include") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -118,7 +107,7 @@ func serveCmd(cmd *cobra.Command, args []string) error { } // parse exclude patterns var excludes []*regexp.Regexp - for _, r := range viper.GetStringSlice("exclude") { + for _, r := range viper.GetStringSlice("index-exclude") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -126,14 +115,15 @@ func serveCmd(cmd *cobra.Command, args []string) error { } } - var indexer index.Indexer + var writer index.RecordWriter var fileApi index.FileAPI var cdxApi index.CdxAPI var idApi index.IdAPI var storageRefResolver loader.StorageRefResolver var filePathResolver loader.FilePathResolver - switch viper.GetString("storage-engine") { + indexFormat := viper.GetString("index-format") + switch indexFormat { case "badger": // Increase GOMAXPROCS as recommended by badger // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use @@ -150,14 +140,14 @@ func serveCmd(cmd *cobra.Command, args []string) error { badgeridx.WithDir(viper.GetString("badger-dir")), badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), - badgeridx.WithReadOnly(viper.GetBool("badger-read-only")), + badgeridx.WithReadOnly(viper.GetString("index-source") == ""), badgeridx.WithDatabase(viper.GetString("badger-database")), ) if err != nil { return err } defer db.Close() - indexer = db + writer = db storageRefResolver = db filePathResolver = db cdxApi = db @@ -169,47 +159,63 @@ func serveCmd(cmd *cobra.Command, args []string) error { tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait")), tikvidx.WithDatabase(viper.GetString("tikv-database")), + tikvidx.WithReadOnly(viper.GetString("index-source") == ""), ) if err != nil { return err } defer db.Close() - indexer = db + writer = db storageRefResolver = db filePathResolver = db cdxApi = db fileApi = db idApi = db default: - return fmt.Errorf("invalid storage engine") + return fmt.Errorf("unknown index format: %s", indexFormat) } - // optionally start autoindexer - if viper.GetBool("index") { - log.Info().Msg("Starting auto indexer") + ctx, cancelIndexer := context.WithCancel(context.Background()) + defer cancelIndexer() - indexWorker := index.NewWorker(indexer, viper.GetInt("workers")) - defer indexWorker.Close() - - indexer, err := index.NewAutoIndexer(indexWorker, - index.WithWatch(viper.GetBool("watch")), - index.WithMaxDepth(viper.GetInt("max-depth")), + indexSource := viper.GetString("index-source") + if indexSource != "" { + indexer := index.NewIndexer(writer, index.WithIncludes(includes...), index.WithExcludes(excludes...), ) - if err != nil { - return err - } - defer indexer.Close() + queue := index.NewWorkQueue(indexer, + viper.GetInt("index-workers"), + ) + defer queue.Close() - for _, dir := range dirs { - dir := dir - go func() { - if err := indexer.Index(dir); err != nil { - log.Warn().Msgf(`Error indexing "%s": %v`, dir, err) - } - }() + var runner index.Runner + switch indexSource { + case "file": + runner = index.NewAutoIndexer(queue, + index.WithMaxDepth(viper.GetInt("file-max-depth")), + index.WithPaths(viper.GetStringSlice("file-paths")), + ) + case "kafka": + runner = index.NewKafkaIndexer(queue, + index.WithBrokers(viper.GetStringSlice("kafka-brokers")), + index.WithGroupID(viper.GetString("kafka-group-id")), + index.WithTopic(viper.GetString("kafka-topic")), + index.WithMinBytes(viper.GetInt("kafka-min-bytes")), + index.WithMaxBytes(viper.GetInt("kafka-max-bytes")), + index.WithMaxWait(viper.GetDuration("kafka-max-wait")), + ) + default: + return fmt.Errorf("unknown index source: %s", indexSource) } + go func() { + log.Info().Msg("Starting indexer") + err := runner.Run(ctx) + if err != nil { + log.Error().Err(err).Msg("Indexer has stopped") + } + }() + } // create record loader @@ -255,24 +261,28 @@ func serveCmd(cmd *cobra.Command, args []string) error { Handler: handler, } - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) sig := <-sigs - log.Debug().Msgf("Received %s signal, shutting down server...", sig) + log.Info().Msgf("Received %s signal, shutting down...", sig) + + cancelIndexer() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + err := httpServer.Shutdown(ctx) if err != nil { log.Error().Msgf("Failed to shut down server: %v", err) } }() - log.Info().Msgf("Starting web server at :%v", port) + log.Info().Msgf("Starting server at :%v", port) err := httpServer.ListenAndServe() - if !errors.Is(err, http.ErrServerClosed) { - return err + if errors.Is(err, http.ErrServerClosed) { + return nil } - return nil + return err } diff --git a/exampleconfig.yaml b/exampleconfig.yaml index 07dca9b..2518cc8 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -10,10 +10,6 @@ log-method: false port: 8880 # prefix for server endpoint paths path-prefix: "" -# start autoindexer -index: true -# watch for file changes (NFS does not support this) -watch: false # log server requests log-requests: true # index storage engine: "badger" or "tikv" @@ -21,21 +17,42 @@ storage-engine: badger # INDEX -# index format: "cdxj", "cdxpb", "badger", "tikv" or "toc" -format: cdxj - -# directories to search for warc files in -dirs: - - "./testdata" +# index format (index): "cdxj", "cdxpb", "badger", "tikv" or "toc" +# index format (serve): "badger" or "tikv" +index-format: cdxj +# index source: "file" or "kafka" +index-source: file # include only files matching regular expressions -include: +index-include: - ".warc.gz$" # exclude files matching regular expressions -exclude: [] -# max number of directory recursions -max-depth: 4 +index-exclude: [] # number of index workers -workers: 8 +index-workers: 8 + +# FILE TRAVERSAL INDEX SOURCE + +# warc files or directories to search for warc files in +file-paths: + - "./testdata" +# max number of directory recursions +file-max-depth: 4 + +# KAFKA INDEX SOURCE + +# the list of broker addresses used to connect to the kafka cluster +kafka-brokers: + - 127.0.0.1:3939 +# optional consumer group id +kafka-group-id: "my-group-id" +# the topic to read messages from +kafka-topic: "my-topic" +# indicates to the broker the minimum batch size that the consumer will accept +kafka-min-bytes: 0 +# indicates to the broker the maximum batch size that the consumer will accept +kafka-max-bytes: 0 +# maximum amount of time to wait for new data to come when fetching batches of messages from kafka +kafka-max-wait: 0 # path to badger database badger-dir: ./badger diff --git a/go.mod b/go.mod index fc4bcec..91a573a 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,14 @@ go 1.16 require ( github.com/benbjohnson/clock v1.3.0 // indirect - github.com/bits-and-blooms/bitset v1.3.3 // indirect + github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.3.1 - github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/dgraph-io/badger/v3 v3.2103.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.6.0 github.com/golang/glog v1.0.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v22.10.26+incompatible // indirect + github.com/google/flatbuffers v22.11.23+incompatible // indirect github.com/gorilla/handlers v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/julienschmidt/httprouter v1.3.0 @@ -23,26 +21,26 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nlnwa/gowarc v1.0.0-beta.4 github.com/nlnwa/whatwg-url v0.1.2 + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect - github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 // indirect - github.com/prometheus/client_golang v1.13.1 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 // indirect + github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c // indirect + github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect github.com/rs/zerolog v1.28.0 - github.com/spf13/afero v1.9.2 // indirect + github.com/segmentio/kafka-go v0.4.38 + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cobra v1.6.1 - github.com/spf13/viper v1.13.0 + github.com/spf13/viper v1.14.0 github.com/tikv/client-go/v2 v2.0.2 - github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect + github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/sync v0.1.0 // indirect - google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect + golang.org/x/net v0.2.0 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index 226551f..8dd4858 100644 --- a/go.sum +++ b/go.sum @@ -34,33 +34,45 @@ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34h cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -68,19 +80,27 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -88,23 +108,34 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -112,81 +143,114 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.8.0/go.mod h1:r3KB8cAdRIe8znzoPWLw8S6gpDVd9treohhn8b09424= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -198,42 +262,59 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOj cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -244,30 +325,44 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -286,6 +381,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -298,8 +394,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg= -github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= +github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -326,16 +422,12 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -370,7 +462,6 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -439,8 +530,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= -github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v22.11.23+incompatible h1:334TygA7iuxt0hoamawsM36xoui01YiouEZnr0qeFMI= +github.com/google/flatbuffers v22.11.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -505,25 +596,31 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/consul/api v1.15.3/go.mod h1:/g/qgcoBcEXALCNZgRRisyTW0nY86++L0KbeAMXYCeY= +github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -532,8 +629,9 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/serf v0.9.8/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -556,6 +654,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -594,12 +693,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -634,8 +734,12 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -647,10 +751,12 @@ github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 h1:IjRv6eg+44iApCtFKPA+POfq+6pU4xCG8pdrw7xUuD8= +github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYhW5Piw+kYNKZqFQqfC2puUf6gMI= +github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -667,8 +773,8 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -695,8 +801,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= @@ -707,8 +811,11 @@ github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6us github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.8.0/go.mod h1:TmKwZAo97S4Fy4sfMH/HX/cQP5D+ijra2NyLpNNmttY= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/kafka-go v0.4.38 h1:iQdOBbUSdfuYlFpvjuALgj7N6DrdPA0HfB4AhREOdtg= +github.com/segmentio/kafka-go v0.4.38/go.mod h1:ikyuGon/60MN/vXFgykf7Zm8P5Be49gJU6vezwjnnhU= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -717,9 +824,9 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -733,8 +840,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= -github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -757,14 +864,18 @@ github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= -github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a h1:YMq0aKR30gBQBlUCqwnB4oBGNDcl0ddYPXQ1KzU3TTw= -github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= +github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b h1:7BbO/1w3JUnU8ul270MhLXB/V8enVR2ugP45qvxCUSQ= +github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b/go.mod h1:V94ltDCVCbxtzkM3wK9ksXfppIlJ4TL043xU9mYZOUs= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -773,17 +884,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= -go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -814,6 +919,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -827,7 +933,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -911,22 +1019,23 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -942,7 +1051,6 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= @@ -964,7 +1072,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1001,7 +1108,6 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1045,7 +1151,6 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1066,11 +1171,11 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1086,6 +1191,7 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1183,7 +1289,6 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6 google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= @@ -1194,7 +1299,6 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69 google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= @@ -1205,6 +1309,7 @@ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1273,8 +1378,6 @@ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1294,7 +1397,6 @@ google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= @@ -1321,8 +1423,9 @@ google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1360,8 +1463,9 @@ google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/index/api.go b/index/api.go index debb468..8fb20d2 100644 --- a/index/api.go +++ b/index/api.go @@ -7,6 +7,10 @@ import ( "github.com/nlnwa/gowarcserver/schema" ) +type Runner interface { + Run(context.Context) error +} + type indexError string const AlreadyIndexedError indexError = "already indexed" diff --git a/index/autoindex.go b/index/autoindex.go index 6a7d23a..b1c9093 100644 --- a/index/autoindex.go +++ b/index/autoindex.go @@ -17,130 +17,112 @@ package index import ( + "context" "fmt" "os" "path/filepath" - "time" + "regexp" + + "github.com/rs/zerolog/log" ) -type autoIndexer struct { - done chan struct{} - perFileFn func(string) - perDirFn func(string) bool - watcher *watcher - settings *Options +type AutoIndexOptions struct { + MaxDepth int + Paths []string + Options } +type AutoIndexOption func(*AutoIndexOptions) -type Scheduler interface { - Schedule(job string, batchWindow time.Duration) +func WithMaxDepth(depth int) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.MaxDepth = depth + } } -func NewAutoIndexer(s Scheduler, opts ...Option) (*autoIndexer, error) { - a := new(autoIndexer) +func WithPaths(paths []string) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.Paths = paths + } +} - settings := defaultOptions() - for _, opt := range opts { - opt(settings) +func WithExcludeDirs(res ...*regexp.Regexp) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.Excludes = res } - a.settings = settings +} - done := make(chan struct{}) - a.done = done +type Queue interface { + Add(path string) +} - isDone := func() bool { - select { - case <-done: - return true - default: - return false - } - } +type AutoIndexer struct { + Queue + opts *AutoIndexOptions +} - perDirFn := func(name string) bool { - if isDone() { - return false - } - return !settings.isExcluded(name) +func NewAutoIndexer(s Queue, options ...AutoIndexOption) AutoIndexer { + opts := new(AutoIndexOptions) + for _, apply := range options { + apply(opts) } - perFileFn := func(name string) { - if isDone() { - return - } - if settings.filter(name) { - s.Schedule(name, 0) - } + return AutoIndexer{ + Queue: s, + opts: opts, } +} - if settings.Watch { - w, err := newWatcher() +func (a AutoIndexer) Run(ctx context.Context) error { + for _, path := range a.opts.Paths { + err := a.index(ctx.Done(), path) if err != nil { - return nil, err - } - a.watcher = w - - perDirFn = func(name string) bool { - if isDone() { - return false - } - if settings.isExcluded(name) { - return false - } - _ = w.Add(name) - return true + log.Warn().Msgf(`Error indexing "%s": %v`, path, err) } - - onFileChanged := func(name string) { - if isDone() { - return - } - if settings.filter(name) { - s.Schedule(name, 10*time.Second) - } - } - go w.Watch(onFileChanged) } - a.perFileFn = perFileFn - a.perDirFn = perDirFn - - return a, nil + return nil } -func (a *autoIndexer) Index(path string) error { +func (a AutoIndexer) index(done <-chan struct{}, path string) error { + select { + case <-done: + return nil + default: + } info, err := os.Stat(path) if err != nil { return fmt.Errorf("failed to get file info: %w", err) } if info.IsDir() { - if err := a.walk(path, 0); err != nil { + if err := a.walk(done, path, 0); err != nil { return err } } else { - a.perFileFn(path) + a.Add(path) } return nil } -func (a *autoIndexer) Close() { - close(a.done) - a.watcher.Close() -} - -func (a *autoIndexer) walk(dir string, currentDepth int) error { +func (a AutoIndexer) walk(done <-chan struct{}, dir string, currentDepth int) error { + if a.opts.isExcluded(dir) { + return nil + } entries, err := os.ReadDir(dir) if err != nil { return fmt.Errorf(`failed to read directory "%s": %w`, dir, err) } for _, entry := range entries { - name := filepath.Join(dir, entry.Name()) + select { + case <-done: + return nil + default: + } + path := filepath.Join(dir, entry.Name()) if !entry.IsDir() { - a.perFileFn(name) - } else if currentDepth < a.settings.MaxDepth { - if a.perDirFn(name) { - err = a.walk(name, currentDepth+1) - if err != nil { - return err - } + a.Queue.Add(path) + } else if currentDepth < a.opts.MaxDepth { + err = a.walk(done, path, currentDepth+1) + if err != nil { + return err } } } diff --git a/index/index.go b/index/index.go new file mode 100644 index 0000000..1dff00b --- /dev/null +++ b/index/index.go @@ -0,0 +1,88 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package index + +import ( + "errors" + "path/filepath" + "strings" + "time" + + "github.com/nlnwa/gowarc" + "github.com/rs/zerolog/log" +) + +type Indexer interface { + Index(string) error +} + +func NewIndexer(w RecordWriter, options ...Option) func(string) { + opts := new(Options) + for _, apply := range options { + apply(opts) + } + + indexer, ok := w.(Indexer) + + return func(path string) { + // file + dir := filepath.Dir(path) + base := filepath.Base(path) + if opts.isExcluded(dir) || !opts.filter(base) { + return + } + + // if writer implements Index interface we call its index interface + if ok { + if err := indexer.Index(path); err != nil { + if errors.Is(err, AlreadyIndexedError) { + log.Debug().Err(err).Msgf("%s", path) + } else if err != nil { + log.Error().Err(err).Msgf("%s", path) + } + return + } + } + + index(path, w, opts) + } +} + +func index(filename string, r RecordWriter, opts *Options) { + start := time.Now() + + filter := func(wr gowarc.WarcRecord, validation *gowarc.Validation) bool { + if !validation.Valid() { + log.Debug().Msg(validation.Error()) + } + // only index response and revisit records of type application/http + if wr.Type() == gowarc.Response || wr.Type() == gowarc.Revisit { + // of type application/http + if strings.HasPrefix(wr.WarcHeader().Get(gowarc.ContentType), gowarc.ApplicationHttp) { + return true + } + } + return false + } + + count, total, err := readFile(filename, r, filter, opts.warcRecordOption...) + if err != nil { + log.Error().Err(err).Msgf("Indexing failed: %s", filename) + } + + log.Info().Msgf("Indexed %5d of %5d records in %10v: %s\n", count, total, time.Since(start), filename) +} diff --git a/index/io.go b/index/io.go index 61b628e..d3992b6 100644 --- a/index/io.go +++ b/index/io.go @@ -21,17 +21,21 @@ import ( "fmt" "io" "path/filepath" - "strings" - "time" "github.com/nlnwa/gowarc" "github.com/rs/zerolog/log" ) -type recordFilter func(gowarc.WarcRecord) bool +type recordFilter func(gowarc.WarcRecord, *gowarc.Validation) bool + +type RecordWriter interface { + Write(Record) error +} + +// readFile reads, filters and writes records of a warc file to a record writer +func readFile(path string, writer RecordWriter, filter recordFilter, opts ...gowarc.WarcRecordOption) (int, int, error) { + filename := filepath.Base(path) -// readFile reads a file using the supplied config and writes with a IndexWriter. -func readFile(path string, writer recordWriter, filter recordFilter, opts ...gowarc.WarcRecordOption) (int, int, error) { wf, err := gowarc.NewWarcFileReader(path, 0, opts...) if err != nil { return 0, 0, err @@ -40,17 +44,8 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow _ = wf.Close() }() - filename := filepath.Base(path) var prevOffset int64 var prevWr gowarc.WarcRecord - // write record from the previous iteration because we need to calculate record length - write := func(offset int64) error { - r, err := newRecord(prevWr, filename, prevOffset, offset-prevOffset) - if err != nil { - return err - } - return writer.Write(r) - } count := 0 total := 0 @@ -58,8 +53,10 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow for { wr, offset, validation, err := wf.Next() if prevWr != nil { - if err := write(offset); err != nil { - log.Warn().Err(err).Msgf("Failed to index record: %s#%d", filename, prevOffset) + if r, err := newRecord(prevWr, filename, prevOffset, offset-prevOffset); err != nil { + log.Error().Err(err).Msgf("Failed to create index record %s#%d", filename, prevOffset) + } else if err = writer.Write(r); err != nil { + log.Error().Err(err).Msgf("Failed to index record: %s#%d", filename, prevOffset) } else { count++ } @@ -69,12 +66,9 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow break } if err != nil { - return count, total, fmt.Errorf("failed to get record number %d in %s at offset %d: %w", count, filename, offset, err) + return count, total, fmt.Errorf("failed to read record #%d at %s#%d: %w", total, filename, offset, err) } - if !validation.Valid() { - log.Warn().Err(validation).Str("filename", filename).Int64("offset", offset).Msgf("Invalid %s record: %s", wr.Type(), wr.RecordId()) - } - if filter(wr) { + if filter(wr, validation) { prevWr = wr } total++ @@ -82,27 +76,3 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow } return count, total, err } - -type recordWriter interface { - Write(Record) error -} - -func warcRecordFilter(wr gowarc.WarcRecord) bool { - // only write response and revisit records - if wr.Type() == gowarc.Response || wr.Type() == gowarc.Revisit { - // of type application/http - if strings.HasPrefix(wr.WarcHeader().Get(gowarc.ContentType), gowarc.ApplicationHttp) { - return true - } - } - return false -} - -func ReadFile(fileName string, r recordWriter) error { - start := time.Now() - - count, total, err := readFile(fileName, r, warcRecordFilter) - - log.Info().Msgf("Indexed %5d of %5d records in %10v: %s\n", count, total, time.Since(start), fileName) - return err -} diff --git a/index/io_test.go b/index/io_test.go index 5984ed7..8da27e3 100644 --- a/index/io_test.go +++ b/index/io_test.go @@ -51,7 +51,7 @@ Content-Length: 0`) tests := []struct { format string - writer recordWriter + writer RecordWriter }{ { "cdxj", @@ -70,7 +70,7 @@ Content-Length: 0`) for _, tt := range tests { t.Run(tt.format, func(t *testing.T) { - _, _, err = readFile(filepath, tt.writer, func(gowarc.WarcRecord) bool { return true }) + _, _, err = readFile(filepath, tt.writer, func(gowarc.WarcRecord, *gowarc.Validation) bool { return true }) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/index/options.go b/index/options.go index 545723e..df30313 100644 --- a/index/options.go +++ b/index/options.go @@ -18,15 +18,18 @@ package index import ( "regexp" + + "github.com/nlnwa/gowarc" ) type Options struct { - Watch bool - Includes []*regexp.Regexp - Excludes []*regexp.Regexp - MaxDepth int + Includes []*regexp.Regexp + Excludes []*regexp.Regexp + warcRecordOption []gowarc.WarcRecordOption } +type Option func(*Options) + func (o *Options) filter(name string) bool { return o.isIncluded(name) && !o.isExcluded(name) } @@ -52,21 +55,6 @@ func (o *Options) isIncluded(name string) bool { return false } -func defaultOptions() *Options { - return &Options{ - Watch: false, - MaxDepth: 4, - } -} - -type Option func(*Options) - -func WithMaxDepth(depth int) Option { - return func(opts *Options) { - opts.MaxDepth = depth - } -} - func WithIncludes(res ...*regexp.Regexp) Option { return func(opts *Options) { opts.Includes = res @@ -78,9 +66,3 @@ func WithExcludes(res ...*regexp.Regexp) Option { opts.Excludes = res } } - -func WithWatch(watch bool) Option { - return func(opts *Options) { - opts.Watch = watch - } -} diff --git a/index/watcher.go b/index/watcher.go deleted file mode 100644 index a3355f7..0000000 --- a/index/watcher.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package index - -import ( - "os" - - "github.com/fsnotify/fsnotify" - "github.com/rs/zerolog/log" -) - -func newWatcher() (*watcher, error) { - w, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &watcher{ - Watcher: w, - }, nil -} - -type watcher struct { - *fsnotify.Watcher -} - -func (w *watcher) Close() { - if w == nil { - return - } - _ = w.Watcher.Close() -} - -func (w *watcher) Add(dir string) error { - if w == nil { - return nil - } - return w.Watcher.Add(dir) -} - -func (w *watcher) Watch(perFileFn func(string)) { - for { - select { - case event, ok := <-w.Events: - if !ok { - return - } - if event.Op&fsnotify.Write == fsnotify.Write { - perFileFn(event.Name) - } else if event.Op&fsnotify.Create == fsnotify.Create { - if info, err := os.Stat(event.Name); err != nil { - log.Warn().Msgf("Watcher failed to stat new file event: %v: %v", event.Name, err) - continue - } else if !info.Mode().IsDir() { - continue - } - - if err := w.Add(event.Name); err != nil { - log.Warn().Msgf("Watcher failed to add new directory: %s, %v", event.Name, err) - } - } - case err, ok := <-w.Errors: - if !ok { - return - } - log.Warn().Msgf("Watcher error: %v", err) - } - } -} diff --git a/index/worker.go b/index/worker.go index 12b9260..9d502de 100644 --- a/index/worker.go +++ b/index/worker.go @@ -17,49 +17,31 @@ package index import ( - "errors" "sync" - "time" - - "github.com/rs/zerolog/log" ) -type indexWorker struct { - jobs chan string - done chan struct{} - jobMap map[string]*time.Timer - mx sync.Mutex - wg sync.WaitGroup +type WorkQueue struct { + queue chan string + done chan struct{} + wg sync.WaitGroup } -// Indexer is the interface that wraps the Index method -type Indexer interface { - Index(string) error -} +type Worker func(string) -func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { - iw := &indexWorker{ - jobs: make(chan string, nrOfWorkers), - done: make(chan struct{}), - jobMap: map[string]*time.Timer{}, +func NewWorkQueue(execute Worker, nrOfWorkers int) *WorkQueue { + iw := &WorkQueue{ + queue: make(chan string, nrOfWorkers), + done: make(chan struct{}), } for i := 0; i < nrOfWorkers; i++ { + iw.wg.Add(1) go func() { + defer iw.wg.Done() for { select { - case job := <-iw.jobs: - if err := a.Index(job); err != nil { - if errors.Is(err, AlreadyIndexedError) { - log.Debug().Err(err).Msg(job) - } else { - log.Warn().Err(err).Msgf(job) - } - } - iw.wg.Done() - iw.mx.Lock() - delete(iw.jobMap, job) - iw.mx.Unlock() + case job := <-iw.queue: + execute(job) case <-iw.done: return } @@ -70,32 +52,14 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { return iw } -func (iw *indexWorker) Close() { - // Wait for all queued jobs to complete - iw.wg.Wait() - // before closing workers. +func (iw *WorkQueue) Close() { + // stop workers close(iw.done) + // and wait for them to complete + iw.wg.Wait() } -// Schedule schedules a job to be processed by a worker -func (iw *indexWorker) Schedule(job string, batchWindow time.Duration) { - if batchWindow == 0 { - iw.wg.Add(1) - iw.jobs <- job - return - } - - iw.mx.Lock() - defer iw.mx.Unlock() - timer, ok := iw.jobMap[job] - - if ok { - timer.Stop() - timer.Reset(batchWindow) - } else { - iw.wg.Add(1) - iw.jobMap[job] = time.AfterFunc(batchWindow, func() { - iw.jobs <- job - }) - } +// Add job to work queue +func (iw *WorkQueue) Add(job string) { + iw.queue <- job } diff --git a/index/writers.go b/index/writers.go index 9bfecad..c2733db 100644 --- a/index/writers.go +++ b/index/writers.go @@ -18,7 +18,6 @@ package index import ( "fmt" - "strconv" "sync" "github.com/bits-and-blooms/bloom/v3" @@ -27,8 +26,7 @@ import ( "google.golang.org/protobuf/proto" ) -type Cdx struct { -} +type Cdx struct{} func (c Cdx) Write(rec Record) error { cdxj := protojson.Format(rec) @@ -37,8 +35,7 @@ func (c Cdx) Write(rec Record) error { return nil } -type CdxJ struct { -} +type CdxJ struct{} func (c CdxJ) Write(rec Record) error { cdxj := protojson.Format(rec) @@ -47,12 +44,7 @@ func (c CdxJ) Write(rec Record) error { return nil } -func (c CdxJ) Index(fileName string) error { - return ReadFile(fileName, c) -} - -type CdxPb struct { -} +type CdxPb struct{} func (c CdxPb) Write(rec Record) error { cdxpb, err := proto.Marshal(rec) @@ -64,10 +56,6 @@ func (c CdxPb) Write(rec Record) error { return nil } -func (c CdxPb) Index(fileName string) error { - return ReadFile(fileName, c) -} - type Toc struct { m sync.Mutex *bloom.BloomFilter @@ -79,21 +67,14 @@ func (t *Toc) Write(rec Record) error { if err != nil { return nil } - ts := rec.GetSts().AsTime() - year := strconv.Itoa(ts.Year()) - key := surthost + " " + year t.m.Lock() - hasSurt := t.BloomFilter.TestOrAddString(key) + hasSurt := t.BloomFilter.TestOrAddString(surthost) t.m.Unlock() if !hasSurt { - fmt.Println(key) + fmt.Println(surthost) } return nil } - -func (t *Toc) Index(fileName string) error { - return ReadFile(fileName, t) -} diff --git a/internal/badgeridx/compression.go b/internal/badgeridx/compression.go index e40b7dc..99d9290 100644 --- a/internal/badgeridx/compression.go +++ b/internal/badgeridx/compression.go @@ -18,10 +18,11 @@ package badgeridx import ( "fmt" - "github.com/mitchellh/mapstructure" "reflect" "strings" + "github.com/mitchellh/mapstructure" + "github.com/dgraph-io/badger/v3/options" ) diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index 011381f..9c33856 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -26,7 +26,6 @@ import ( "time" "github.com/dgraph-io/badger/v3" - "github.com/dgraph-io/badger/v3/options" "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" "github.com/nlnwa/gowarcserver/timestamp" @@ -52,7 +51,7 @@ type DB struct { wg sync.WaitGroup } -func NewDB(options ...DbOption) (db *DB, err error) { +func NewDB(options ...Option) (db *DB, err error) { opts := defaultDbOptions() for _, opt := range options { opt(opts) @@ -83,7 +82,7 @@ func NewDB(options ...DbOption) (db *DB, err error) { done: done, } - // If read-only return. We don't need to run batch and gc workers when read-only. + // We don't need to run batch and gc workers when operating in read-only mode. if opts.ReadOnly { return } @@ -154,15 +153,15 @@ func (db *DB) Close() { } // addFile checks if file is indexed or has not changed since indexing, and adds file to file index. -func (db *DB) addFile(filePath string) error { - stat, err := os.Stat(filePath) +func (db *DB) addFile(path string) error { + stat, err := os.Stat(path) if err != nil { - return fmt.Errorf("failed to get file info: %s: %w", filePath, err) + return fmt.Errorf("failed to get file info: %s: %w", path, err) } fileSize := stat.Size() fileLastModified := stat.ModTime() - fn := filepath.Base(filePath) + fn := filepath.Base(path) if fileInfo, err := db.getFileInfo(fn); err == nil { if err := fileInfo.GetLastModified().CheckValid(); err != nil { return err @@ -173,14 +172,14 @@ func (db *DB) addFile(filePath string) error { } } - return db.updateFilePath(filePath) + return db.updateFilePath(path) } -func (db *DB) updateFilePath(filePath string) error { +func (db *DB) updateFilePath(path string) error { var err error fileInfo := &schema.Fileinfo{} - fileInfo.Path, err = filepath.Abs(filePath) + fileInfo.Path, err = filepath.Abs(path) if err != nil { return err } @@ -281,12 +280,8 @@ func (db *DB) Write(rec index.Record) error { return nil } -func (db *DB) Index(fileName string) error { - err := db.addFile(fileName) - if err != nil { - return err - } - return index.ReadFile(fileName, db) +func (db *DB) Index(path string) error { + return db.addFile(path) } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. @@ -383,67 +378,3 @@ func (db *DB) GetCdx(key string) (*schema.Cdx, error) { }) return val, err } - -func defaultDbOptions() *dbOptions { - return &dbOptions{ - Compression: options.Snappy, - BatchMaxSize: 10000, - BatchMaxWait: 5 * time.Second, - GcInterval: 15 * time.Second, - Path: ".", - } -} - -type dbOptions struct { - Compression options.CompressionType - BatchMaxSize int - BatchMaxWait time.Duration - GcInterval time.Duration - Path string - ReadOnly bool - Database string -} - -type DbOption func(opts *dbOptions) - -func WithCompression(c options.CompressionType) DbOption { - return func(opts *dbOptions) { - opts.Compression = c - } -} - -func WithDir(d string) DbOption { - return func(opts *dbOptions) { - opts.Path = d - } -} - -func WithBatchMaxSize(size int) DbOption { - return func(opts *dbOptions) { - opts.BatchMaxSize = size - } -} - -func WithBatchMaxWait(t time.Duration) DbOption { - return func(opts *dbOptions) { - opts.BatchMaxWait = t - } -} - -func WithGcInterval(t time.Duration) DbOption { - return func(opts *dbOptions) { - opts.GcInterval = t - } -} - -func WithDatabase(db string) DbOption { - return func(opts *dbOptions) { - opts.Database = db - } -} - -func WithReadOnly(readOnly bool) DbOption { - return func(opts *dbOptions) { - opts.ReadOnly = readOnly - } -} diff --git a/internal/badgeridx/options.go b/internal/badgeridx/options.go new file mode 100644 index 0000000..f89935f --- /dev/null +++ b/internal/badgeridx/options.go @@ -0,0 +1,94 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badgeridx + +import ( + badgerOptions "github.com/dgraph-io/badger/v3/options" + "github.com/nlnwa/gowarcserver/index" + "time" +) + +func defaultDbOptions() *Options { + return &Options{ + Compression: badgerOptions.Snappy, + BatchMaxSize: 10000, + BatchMaxWait: 5 * time.Second, + GcInterval: 15 * time.Second, + Path: ".", + } +} + +type Options struct { + Compression badgerOptions.CompressionType + BatchMaxSize int + BatchMaxWait time.Duration + GcInterval time.Duration + Path string + ReadOnly bool + Database string + Index index.Indexer +} + +type Option func(opts *Options) + +func WithCompression(c badgerOptions.CompressionType) Option { + return func(opts *Options) { + opts.Compression = c + } +} + +func WithDir(d string) Option { + return func(opts *Options) { + opts.Path = d + } +} + +func WithBatchMaxSize(size int) Option { + return func(opts *Options) { + opts.BatchMaxSize = size + } +} + +func WithBatchMaxWait(t time.Duration) Option { + return func(opts *Options) { + opts.BatchMaxWait = t + } +} + +func WithGcInterval(t time.Duration) Option { + return func(opts *Options) { + opts.GcInterval = t + } +} + +func WithDatabase(db string) Option { + return func(opts *Options) { + opts.Database = db + } +} + +func WithReadOnly(readOnly bool) Option { + return func(opts *Options) { + opts.ReadOnly = readOnly + } +} + +func WithIndexer(indexer index.Indexer) Option { + return func(opts *Options) { + opts.Index = indexer + } +} diff --git a/internal/logger/initlog.go b/internal/logger/initlog.go index cbe7c41..adbf7bc 100644 --- a/internal/logger/initlog.go +++ b/internal/logger/initlog.go @@ -17,12 +17,13 @@ package logger import ( - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" stdlog "log" "os" "strings" "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) func InitLog(level string, format string, logCaller bool) { diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 448d89b..64379d2 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -1,9 +1,29 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx import ( "context" + "fmt" "strings" + "github.com/rs/zerolog/log" + "github.com/tikv/client-go/v2/rawkv" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" "google.golang.org/protobuf/proto" @@ -11,36 +31,27 @@ import ( // Closest returns the first closest cdx value(s). func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err - } - snapshot := db.client.GetSnapshot(ts) - it, err := NewIterClosest(ctx, snapshot, req.Key(), req.Closest()) + _, values, err := scanClosest(ctx, db.client, req.Key(), req.Closest()) if err != nil { return err } go func() { defer close(res) - defer it.Close() - for it.Valid() { - select { - case <-ctx.Done(): - return - default: - } - - cdx, err := cdxFromValue(it.Value()) + for _, v := range values { + var cdxResponse index.CdxResponse + cdx := new(schema.Cdx) + err := proto.Unmarshal(v, cdx) if err != nil { - res <- index.CdxResponse{Error: err} + cdxResponse = index.CdxResponse{Error: err} } else { - res <- index.CdxResponse{Cdx: cdx} + cdxResponse = index.CdxResponse{Cdx: cdx} } - - if err := it.Next(); err != nil { - res <- index.CdxResponse{Error: err} + select { + case <-ctx.Done(): + return + case res <- cdxResponse: } } }() @@ -48,22 +59,8 @@ func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- return nil } -func cdxFromValue(value []byte) (*schema.Cdx, error) { - result := new(schema.Cdx) - if err := proto.Unmarshal(value, result); err != nil { - return nil, err - } - return result, nil -} - func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err - } - snapshot := db.client.GetSnapshot(ts) - - it, err := newIter(ctx, snapshot, req) + it, err := newIter(ctx, db.client, req) if err != nil { return err } @@ -71,6 +68,7 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in close(res) return nil } + go func() { defer close(res) defer it.Close() @@ -80,10 +78,8 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in for it.Valid() && limit > 0 { select { case <-ctx.Done(): - res <- index.CdxResponse{Error: ctx.Err()} return default: - limit-- } func() { @@ -114,41 +110,29 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in } func (db *DB) List(ctx context.Context, limit int, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(cdxPrefix), []byte(cdxEOF)) + _, values, err := db.client.Scan(ctx, []byte(cdxPrefix), []byte(cdxEOF), limit) if err != nil { return err } go func() { - defer it.Close() defer close(res) - - for it.Valid() && limit > 0 { + for _, v := range values { select { case <-ctx.Done(): - res <- index.CdxResponse{Error: ctx.Err()} return default: } - limit-- cdx := new(schema.Cdx) - err := proto.Unmarshal(it.Value(), cdx) + err := proto.Unmarshal(v, cdx) if err != nil { res <- index.CdxResponse{Error: err} } else { res <- index.CdxResponse{Cdx: cdx} } - err = it.Next() - if err != nil { - res <- index.CdxResponse{Error: err} - break - } } }() @@ -159,82 +143,63 @@ func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, return db.getFileInfo(filename) } -func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err +func (db *DB) ListFileInfo(ctx context.Context, limit int, res chan<- index.FileResponse) error { + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(filePrefix), []byte(fileEOF)) + _, values, err := db.client.Scan(ctx, []byte(filePrefix), []byte(fileEOF), limit) if err != nil { return err } - go func() { - defer it.Close() // close iterator defer close(res) // close response channel + for _, v := range values { + select { + case <-ctx.Done(): + return + default: + } - for it.Valid() && limit > 0 { - limit-- fileInfo := new(schema.Fileinfo) - err := proto.Unmarshal(it.Value(), fileInfo) + err := proto.Unmarshal(v, fileInfo) if err != nil { res <- index.FileResponse{Error: err} } else { + log.Info().Msgf("GOOD: %s", fileInfo.Name) res <- index.FileResponse{Fileinfo: fileInfo} } - err = it.Next() - if err != nil { - res <- index.FileResponse{Error: err} - break - } } + log.Info().Msg("Hvordor kommer vi kkke hit") }() return nil } func (db *DB) GetStorageRef(ctx context.Context, id string) (string, error) { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return "", err - } - snapshot := db.client.GetSnapshot(ts) - - b, err := snapshot.Get(ctx, []byte(id)) + b, err := db.client.Get(ctx, []byte(id)) return string(b), err } -func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err +func (db *DB) ListStorageRef(ctx context.Context, limit int, res chan<- index.IdResponse) error { + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(idPrefix), []byte(idEOF)) + keys, values, err := db.client.Scan(ctx, []byte(idPrefix), []byte(idEOF), limit) if err != nil { return err } go func() { - defer it.Close() defer close(res) - for it.Valid() && limit > 0 { - limit-- - if err != nil { - res <- index.IdResponse{Error: err} - } else { - k := strings.TrimPrefix(string(it.Key()), idPrefix) - res <- index.IdResponse{Key: k, Value: string(it.Value())} - } - err = it.Next() - if err != nil { - res <- index.IdResponse{Error: err} - break + for i, k := range keys { + select { + case <-ctx.Done(): + return + default: } + k := strings.TrimPrefix(string(k), idPrefix) + res <- index.IdResponse{Key: k, Value: string(values[i])} } }() @@ -245,15 +210,21 @@ func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdRe func (db *DB) Resolve(ctx context.Context, warcId string) (string, error) { key := []byte(idPrefix + warcId) - kv, err := db.get(ctx, key) + val, err := db.client.Get(ctx, key) if err != nil { return "", err } - return string(kv.V), nil + return string(val), nil } // ResolvePath looks up filename in file index and returns the path field. func (db *DB) ResolvePath(filename string) (filePath string, err error) { fileInfo, err := db.getFileInfo(filename) + if err != nil { + return "", err + } + if fileInfo == nil { + return "", fmt.Errorf("file not found: %s", filename) + } return fileInfo.Path, err } diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 320512c..09de0a8 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -1,5 +1,5 @@ /* - * Copyright 2021 National Library of Norway. + * Copyright 2022 National Library of Norway. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import ( "github.com/nlnwa/gowarcserver/schema" "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" - "github.com/tikv/client-go/v2/txnkv" + "github.com/tikv/client-go/v2/rawkv" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -67,14 +67,12 @@ var ( ) type DB struct { - client *txnkv.Client + client *rawkv.Client batch chan index.Record done chan struct{} - opts *Options - wg sync.WaitGroup } @@ -84,7 +82,6 @@ func NewDB(options ...Option) (db *DB, err error) { opt(opts) } // prefix all keys with name of database - // TODO use keyspace when TiKV support it idPrefix = opts.Database + idPrefix idEOF = opts.Database + idEOF filePrefix = opts.Database + filePrefix @@ -92,7 +89,10 @@ func NewDB(options ...Option) (db *DB, err error) { cdxPrefix = opts.Database + cdxPrefix cdxEOF = opts.Database + cdxEOF - client, err := txnkv.NewClient(opts.PdAddr) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + client, err := rawkv.NewClientWithOpts(ctx, opts.PdAddr) if err != nil { return nil, err } @@ -102,7 +102,6 @@ func NewDB(options ...Option) (db *DB, err error) { db = &DB{ client: client, done: done, - opts: opts, } if opts.ReadOnly { @@ -135,6 +134,7 @@ func NewDB(options ...Option) (db *DB, err error) { func (db *DB) Close() { close(db.done) db.wg.Wait() + _ = db.client.Close() } // addFile checks if file referenced by filePath is indexed or has changed and adds/updates the index accordingly. @@ -147,7 +147,7 @@ func (db *DB) addFile(filePath string) error { fileSize := stat.Size() fileLastModified := stat.ModTime() fn := filepath.Base(filePath) - if fileInfo, err := db.getFileInfo(fn); err == nil { + if fileInfo, err := db.getFileInfo(fn); err == nil && fileInfo != nil { if err := fileInfo.GetLastModified().CheckValid(); err != nil { return err } @@ -187,69 +187,37 @@ func (db *DB) putFileInfo(fi *schema.Fileinfo) error { k := []byte(filePrefix + fi.Name) v, err := proto.Marshal(fi) if err != nil { + log.Error().Err(err).Msg("") + return err } - return db.puts(ctx, KV{K: k, V: v}) + err = db.client.Put(ctx, k, v) + if err != nil { + log.Error().Err(err).Msg("") + return err + } + return nil } func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() key := []byte(filePrefix + fileName) - val, err := db.get(ctx, key) + val, err := db.client.Get(ctx, key) if err != nil { return nil, err } + if val == nil { + return nil, nil + } fi := new(schema.Fileinfo) - err = proto.Unmarshal(val.V, fi) + err = proto.Unmarshal(val, fi) if err != nil { return nil, err } return fi, nil } -func (db *DB) get(ctx context.Context, k []byte) (*KV, error) { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return nil, err - } - snapshot := db.client.GetSnapshot(ts) - - if err != nil { - return nil, err - } - v, err := snapshot.Get(ctx, k) - if err != nil { - return nil, err - } - return &KV{K: k, V: v}, nil -} - -func (db *DB) puts(ctx context.Context, kvs ...KV) error { - retries := db.opts.BatchMaxRetries - for { - tx, err := db.client.Begin() - if err != nil { - return err - } - for _, kv := range kvs { - err := tx.Set(kv.K, kv.V) - if err != nil { - return err - } - } - err = tx.Commit(ctx) - if err == nil { - return nil - } else if retries > 0 { - retries-- - log.Warn().Err(err).Msgf("Commit failed, will try again %d times", retries) - } else { - return err - } - } -} - // write schedules a Record to be added to the DB via the batch channel. func (db *DB) write(rec index.Record) { select { @@ -264,44 +232,39 @@ func (db *DB) write(rec index.Record) { } } -// collectBatch returns a slice of all the records in the batch channel. -func (db *DB) collectBatch() (records []index.Record) { +func (db *DB) collectBatch() ([][]byte, [][]byte) { + var keys [][]byte + var values [][]byte for { select { - case record := <-db.batch: - records = append(records, record) + case r := <-db.batch: + id := idKV(r) + cdx, err := cdxKV(r) + if err != nil { + log.Error().Err(err).Msgf("failed to marshal record: %v", r) + continue + } + keys = append(keys, id.K, cdx.K) + values = append(values, id.V, cdx.V) default: - return + return keys, values } } } // flushBatch collects all records in the batch channel and updates the id and cdx indices. func (db *DB) flushBatch() { - records := db.collectBatch() - if len(records) == 0 { + keys, values := db.collectBatch() + if len(keys) == 0 { return } - var kvs []KV - for _, r := range records { - id := idKV(r) - cdx, err := cdxKV(r) - if err != nil { - log.Error().Err(err).Msgf("failed to marshal record: %v", r) - continue - } - kvs = append(kvs, id, cdx) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - err := db.puts(ctx, kvs...) + err := db.client.BatchPut(ctx, keys, values) if err != nil { - for _, r := range records { - log.Warn().Err(err).Msgf("failed to commit record: %s", r.Ref) - } + log.Error().Err(err).Msgf("Batch put failed") } } @@ -329,10 +292,6 @@ func (db *DB) Write(rec index.Record) error { return nil } -func (db *DB) Index(fileName string) error { - err := db.addFile(fileName) - if err != nil { - return err - } - return index.ReadFile(fileName, db) +func (db *DB) Index(path string) error { + return db.addFile(path) } diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index f5f1d8d..8a5d116 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -1,16 +1,34 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx import ( - "bytes" "context" - "fmt" "time" + "github.com/nlnwa/gowarcserver/server/api" + "github.com/tikv/client-go/v2/rawkv" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/timestamp" - "github.com/tikv/client-go/v2/txnkv" ) +type comparator func(KV, KV) bool + func CompareClosest(ts int64) func(int64, int64) bool { return func(ts1 int64, ts2 int64) bool { return timestamp.AbsInt64(ts-ts1) < timestamp.AbsInt64(ts-ts2) @@ -34,223 +52,207 @@ type iterator interface { Close() } -type iterClosest struct { - forward iterator - backward iterator - curr iterator - prefix []byte - key []byte - value []byte - valid bool - cmp func(int64, int64) bool +type closestScanner struct { + fKeys, bKeys [][]byte + fValues, bValues [][]byte + fIndex, bIndex int + cmp func(int64, int64) bool } -func NewIterClosest(_ context.Context, snapshot *txnkv.KVSnapshot, key string, closest string) (*iterClosest, error) { - ic := new(iterClosest) +const startDate = "19700101000000" +const endbyte = "\xff" + +func scanClosest(ctx context.Context, client *rawkv.Client, key string, closest string, options ...rawkv.RawOption) ([][]byte, [][]byte, error) { + ic := new(closestScanner) if t, err := time.Parse(timestamp.CDX, closest); err != nil { - return nil, err + return nil, nil, err } else { ic.cmp = CompareClosest(t.Unix()) } - ic.prefix = []byte(cdxPrefix + key) - k := []byte(cdxPrefix + key + " " + closest) + startKey := []byte(cdxPrefix + key + " " + closest) + limit := 10 + var err error - // initialize forward iterator - forward, err := snapshot.Iter(k, []byte(cdxEOF)) + ic.fKeys, ic.fValues, err = client.Scan(ctx, startKey, []byte(cdxPrefix+key+endbyte), limit, options...) if err != nil { - return nil, err + return nil, nil, err } - ic.forward = forward - // initialize backward iterator - backward, err := snapshot.IterReverse(k) + // scan backward + ic.bKeys, ic.bValues, err = client.ReverseScan(ctx, startKey, []byte(cdxPrefix+key+" "+startDate), limit, options...) if err != nil { - return nil, err + return nil, nil, err } - ic.backward = backward - - return ic, ic.next() -} -func (ic *iterClosest) Next() error { - err := ic.curr.Next() - if err != nil { - return err + var keys [][]byte + var values [][]byte + for { + k, v, valid := ic.next() + if !valid { + return keys, values, nil + } + keys = append(keys, k) + values = append(values, v) } - return ic.next() } -func (ic *iterClosest) next() error { +func (cs *closestScanner) next() ([]byte, []byte, bool) { var ft int64 var bt int64 // get forward ts - if ic.forward.Valid() && bytes.HasPrefix(ic.forward.Key(), ic.prefix) { - ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.forward.Key()).ts()) - ft = ts.Unix() + if len(cs.fKeys) > cs.fIndex { + fts, _ := time.Parse(timestamp.CDX, cdxKey(cs.fKeys[cs.fIndex]).ts()) + ft = fts.Unix() } + // get backward ts - if ic.backward.Valid() && bytes.HasPrefix(ic.backward.Key(), ic.prefix) { - ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.backward.Key()).ts()) - bt = ts.Unix() + if len(cs.bKeys) > cs.bIndex { + bts, _ := time.Parse(timestamp.CDX, cdxKey(cs.bKeys[cs.bIndex]).ts()) + bt = bts.Unix() } - var it iterator + var itKeys [][]byte + var itValues [][]byte + var i *int if ft != 0 && bt != 0 { // find closest of forward and backward - isForward := ic.cmp(ft, bt) + isForward := cs.cmp(ft, bt) if isForward { - it = ic.forward + itKeys = cs.fKeys + i = &cs.fIndex } else { - it = ic.backward + itKeys = cs.bKeys + i = &cs.bIndex } } else if ft != 0 { - it = ic.forward + itKeys = cs.fKeys + i = &cs.fIndex } else if bt != 0 { - it = ic.backward + itKeys = cs.bKeys + i = &cs.bIndex } else { - ic.valid = false - return nil + return nil, nil, false } + key := itKeys[*i] + value := itValues[*i] + *i++ - ic.curr = it - ic.key = it.Key() - ic.value = it.Value() - ic.valid = true - - return nil -} - -func (ic *iterClosest) Key() []byte { - return ic.key + return key, value, true } -func (ic *iterClosest) Value() []byte { - return ic.value +type maybeKV struct { + kv KV + error error } -func (ic *iterClosest) Valid() bool { - return ic.valid +func getComparator(req index.SearchRequest) (comparator, error) { + switch req.Sort() { + case index.SortDesc: + return func(a KV, b KV) bool { + return CompareDesc(a.ts(), b.ts()) + }, nil + case index.SortAsc: + fallthrough + case index.SortNone: + fallthrough + case index.SortClosest: + fallthrough + default: + return func(a KV, b KV) bool { + return CompareAsc(a.ts(), b.ts()) + }, nil + } } -func (ic *iterClosest) Close() { - ic.forward.Close() - ic.backward.Close() -} +type scan func(context.Context, []byte, []byte, int, ...rawkv.RawOption) ([][]byte, [][]byte, error) -type maybeKV struct { - kv KV - error error +type iter struct { + key []byte + value []byte + valid bool + next <-chan maybeKV } -type iterSort struct { - iterators []iterator - key []byte - value []byte - valid bool - next <-chan maybeKV -} +func newIter(ctx context.Context, client *rawkv.Client, req index.SearchRequest) (iterator, error) { + cmp, err := getComparator(req) + if err != nil { + return nil, err + } -func newIter(ctx context.Context, tx *txnkv.KVSnapshot, req index.SearchRequest) (iterator, error) { - is := new(iterSort) - var prefixes [][]byte - var results []chan *maybeKV - var it iterator - var err error - for _, key := range req.Keys() { - k := []byte(cdxPrefix + key) - switch req.Sort() { - case index.SortAsc: - it, err = tx.Iter(k, []byte(cdxEOF)) - case index.SortClosest: - it, err = NewIterClosest(ctx, tx, key, req.Closest()) + getScanner := func(sort index.Sort) scan { + switch sort { case index.SortDesc: if len(req.Keys()) == 1 { - return tx.IterReverse(k) + return client.ReverseScan + } else { + return client.Scan } - it, err = tx.Iter(k, []byte(cdxEOF)) + case index.SortAsc: + fallthrough case index.SortNone: fallthrough + case index.SortClosest: + fallthrough default: - it, err = tx.Iter(k, []byte(cdxEOF)) - } - if err != nil { - break - } - if !it.Valid() { - continue + return client.Scan } - is.iterators = append(is.iterators, it) - prefixes = append(prefixes, k) - results = append(results, make(chan *maybeKV)) - } - if err != nil { - is.Close() - return nil, err - } - if len(is.iterators) == 0 { - return nil, nil } - if len(is.iterators) == 1 { - return is.iterators[0], nil + + makeStartKey := func(endKey []byte) []byte { + return append([]byte(api.MatchType(string(endKey), req.MatchType())), 0) } - // initialize comparator - var cmp func(a KV, b KV) bool - switch req.Sort() { - case index.SortDesc: - cmp = func(a KV, b KV) bool { - return CompareDesc(a.ts(), b.ts()) - } - case index.SortClosest: - var t time.Time - t, err = time.Parse(timestamp.CDX, req.Closest()) - if err != nil { - return nil, fmt.Errorf("failed to parse closest timestamp: %w", err) - } - cmp = func(a KV, b KV) bool { - return CompareClosest(t.Unix())(a.ts(), b.ts()) - } - case index.SortAsc: - fallthrough - case index.SortNone: - fallthrough - default: - cmp = func(a KV, b KV) bool { - return CompareAsc(a.ts(), b.ts()) - } + makeEndkey := func(startKey []byte) []byte { + endKey := make([]byte, len(startKey)+1) + copy(endKey, startKey) + endKey[len(endKey)-1] = 0xff + return endKey } - is.next = mergeIter(ctx.Done(), cmp, results...) + var results []chan *maybeKV + for i, key := range req.Keys() { + results = append(results, make(chan *maybeKV)) - for i, iter := range is.iterators { - i := i - go func(iter iterator, prefix []byte, ch chan<- *maybeKV) { + go func(scan scan, key []byte, ch chan<- *maybeKV, done <-chan struct{}) { defer close(ch) - for iter.Valid() && bytes.HasPrefix(iter.Key(), prefix) { - select { - case <-ctx.Done(): - return - case ch <- &maybeKV{kv: KV{K: iter.Key(), V: iter.Value()}}: - } - err := iter.Next() + scanLimit := 256 + startKey := key + endKey := makeEndkey(startKey) + for { + keys, values, err := scan(ctx, startKey, endKey, scanLimit) if err != nil { select { - case <-ctx.Done(): + case <-done: + return case ch <- &maybeKV{error: err}: + return + } + } + for j, k := range keys { + select { + case <-done: + return + case ch <- &maybeKV{kv: KV{K: k, V: values[j]}}: } + } + if len(keys) < scanLimit { return } + startKey = makeStartKey(keys[len(keys)-1]) } - }(iter, prefixes[i], results[i]) + }(getScanner(req.Sort()), []byte(cdxPrefix+key), results[i], ctx.Done()) } + is := new(iter) + is.next = mergeIter(ctx.Done(), cmp, results...) + return is, is.Next() } // Next updates the next key, value and validity. -func (is *iterSort) Next() error { +func (is *iter) Next() error { mkv, ok := <-is.next if !ok { is.valid = false @@ -267,24 +269,20 @@ func (is *iterSort) Next() error { return nil } -func (is *iterSort) Key() []byte { +func (is *iter) Key() []byte { return is.key } -func (is *iterSort) Value() []byte { +func (is *iter) Value() []byte { return is.value } -func (is *iterSort) Valid() bool { +func (is *iter) Valid() bool { return is.valid } -func (is *iterSort) Close() { - for _, it := range is.iterators { - if it != nil { - it.Close() - } - } +func (is *iter) Close() { + // noop } // mergeIter merges sorted input channels into a sorted output channel diff --git a/internal/tikvidx/options.go b/internal/tikvidx/options.go index 164f04e..a19b343 100644 --- a/internal/tikvidx/options.go +++ b/internal/tikvidx/options.go @@ -1,6 +1,24 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx -import "time" +import ( + "time" +) func defaultOptions() *Options { return &Options{ diff --git a/main.go b/main.go index 4627334..be59448 100644 --- a/main.go +++ b/main.go @@ -1,18 +1,19 @@ /* -Copyright © 2019 National Library of Norway + * Copyright 2019 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ package main import ( diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index 4442f34..26bc3e0 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -54,7 +54,10 @@ func (h Handler) search(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - log.Debug().Msgf("%v", coreAPI) + log.Debug().Msgf("%+v", coreAPI) + if coreAPI.Limit == 0 { + coreAPI.Limit = 100 + } if err = h.CdxAPI.Search(ctx, api.SearchAPI{CoreAPI: coreAPI}, response); err != nil { log.Error().Err(err).Msg("failed to search") http.Error(w, err.Error(), http.StatusInternalServerError) From c75e613021561cdbead92e79b19578d85158a667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 14:02:01 +0100 Subject: [PATCH 32/38] Update protoc to version 21.10 --- schema/Makefile | 4 ++-- schema/cdx.pb.go | 4 ++-- schema/fileinfo.pb.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/schema/Makefile b/schema/Makefile index a649afe..c89b068 100644 --- a/schema/Makefile +++ b/schema/Makefile @@ -1,4 +1,4 @@ -PROTOC_VERSION=21.8 +PROTOC_VERSION=21.10 .PHONY: all .PHONY: clean @@ -7,7 +7,7 @@ go_files := $(patsubst %.proto,%.pb.go,$(wildcard *.proto)) all: tools $(go_files) clean: - rm *.pb.go + rm -f *.pb.go distclean: clean rm -rf tools diff --git a/schema/cdx.pb.go b/schema/cdx.pb.go index 37fb541..9fe67a6 100644 --- a/schema/cdx.pb.go +++ b/schema/cdx.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.21.8 +// protoc-gen-go v1.28.1 +// protoc v3.21.9 // source: cdx.proto package schema diff --git a/schema/fileinfo.pb.go b/schema/fileinfo.pb.go index 1215437..cf89692 100644 --- a/schema/fileinfo.pb.go +++ b/schema/fileinfo.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.21.8 +// protoc-gen-go v1.28.1 +// protoc v3.21.9 // source: fileinfo.proto package schema From 1e3b3d12c4fba5db182675c0103389095d8ecad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 14:02:37 +0100 Subject: [PATCH 33/38] Promote logger package out of internal --- cmd/root.go | 2 +- {internal/logger => logger}/initlog.go | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {internal/logger => logger}/initlog.go (100%) diff --git a/cmd/root.go b/cmd/root.go index 18c73a4..a08d855 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -25,7 +25,7 @@ import ( "github.com/nlnwa/gowarcserver/cmd/index" "github.com/nlnwa/gowarcserver/cmd/serve" "github.com/nlnwa/gowarcserver/cmd/version" - "github.com/nlnwa/gowarcserver/internal/logger" + "github.com/nlnwa/gowarcserver/logger" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" diff --git a/internal/logger/initlog.go b/logger/initlog.go similarity index 100% rename from internal/logger/initlog.go rename to logger/initlog.go From b48b83beec1408a45e41648884ee15af25d679e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 14:05:11 +0100 Subject: [PATCH 34/38] Close WarcFileReader independent of context --- loader/filestorageloader.go | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/loader/filestorageloader.go b/loader/filestorageloader.go index f11282c..af889aa 100644 --- a/loader/filestorageloader.go +++ b/loader/filestorageloader.go @@ -43,24 +43,16 @@ func (f FileStorageLoader) Load(ctx context.Context, storageRef string) (record wf, err := gowarc.NewWarcFileReader(filePath, offset, gowarc.WithSyntaxErrorPolicy(gowarc.ErrIgnore), - gowarc.WithSpecViolationPolicy(gowarc.ErrIgnore)) + gowarc.WithSpecViolationPolicy(gowarc.ErrIgnore), + ) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize warc reader: %s#%d, %w", filePath, offset, err) } + defer wf.Close() - go func() { - <-ctx.Done() - _ = wf.Close() - }() - - record, offset, validation, err := wf.Next() - if !validation.Valid() { - log.Warn().Msg(validation.String()) - return nil, fmt.Errorf("validation error in warcfile at offset %d", offset) - } + record, offset, _, err = wf.Next() if err != nil { - log.Error().Msgf("%s, offset %v\n", err, offset) - return nil, err + return nil, fmt.Errorf("failed to read record: %s#%d: %w", filePath, offset, err) } return } From b58a4ba8ee7bfb6143a1e44eb68c25e3e39bf955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 15:57:37 +0100 Subject: [PATCH 35/38] Close revisit record before merge --- loader/loader.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index fefc09f..d118b36 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -99,18 +99,17 @@ func (l *Loader) LoadByStorageRef(ctx context.Context, storageRef string) (gowar if err != nil { return nil, err } + defer revisitOf.Close() rtrRecord, err = record.Merge(revisitOf) if err != nil { return nil, err } + return rtrRecord, nil case gowarc.Continuation: - log.Warn().Msg("Not implemented: storage ref resolved to a continuation record") - // TODO continuation not implemented - fallthrough + // TODO implement continuation record + return nil, fmt.Errorf("%s resolves to a continuation record (not implemented)", storageRef) default: - rtrRecord = record + return record, nil } - - return rtrRecord, nil } From 419f8a528984dd5c94fa18525f64981c946b96a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 16:08:17 +0100 Subject: [PATCH 36/38] Consolidate error handling in handlers --- server/coreserver/handler.go | 64 ++++++++++++++++-------------------- server/warcserver/handler.go | 44 ++++++++++++++++--------- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index 26bc3e0..d6de219 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -50,17 +50,17 @@ func (h Handler) search(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + response := make(chan index.CdxResponse) ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - log.Debug().Msgf("%+v", coreAPI) if coreAPI.Limit == 0 { coreAPI.Limit = 100 } if err = h.CdxAPI.Search(ctx, api.SearchAPI{CoreAPI: coreAPI}, response); err != nil { - log.Error().Err(err).Msg("failed to search") http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msgf("Search failed: %+v", coreAPI) return } @@ -85,11 +85,10 @@ func (h Handler) search(w http.ResponseWriter, r *http.Request) { } _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Warn().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write result") + log.Warn().Err(err).Msg("failed to write result") return - } else { - count++ } + count++ } } @@ -101,12 +100,14 @@ type storageRef struct { func (h Handler) listIds(w http.ResponseWriter, r *http.Request) { limit := parseLimit(r) + response := make(chan index.IdResponse) ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() if err := h.IdAPI.ListStorageRef(ctx, limit, response); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to list ids") return } start := time.Now() @@ -132,18 +133,15 @@ func (h Handler) listIds(w http.ResponseWriter, r *http.Request) { } v, err := json.Marshal(ref) if err != nil { - log.Warn().Err(err).Msg("failed to marshal storage ref") + log.Warn().Err(err).Msgf("failed to marshal storage ref: %+v", ref) + continue } if count > 0 { _, _ = w.Write([]byte("\r\n")) } _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write storage ref") - if count == 0 { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + log.Warn().Err(err).Msgf("failed to write storage ref: %+v", ref) return } count++ @@ -156,13 +154,14 @@ func (h Handler) getStorageRefByURN(w http.ResponseWriter, r *http.Request) { storageRef, err := h.StorageRefResolver.Resolve(r.Context(), urn) if err != nil { - msg := fmt.Sprintf("failed to resolve storage reference of urn: %v", err) - http.Error(w, msg, http.StatusInternalServerError) + err := fmt.Errorf("failed to resolve storage ref: %s: %w", urn, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("") return } _, err = fmt.Fprintln(w, storageRef) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + log.Warn().Err(err).Msgf("Failed to write storage ref: %s", storageRef) } } @@ -175,6 +174,7 @@ func (h Handler) listFiles(w http.ResponseWriter, r *http.Request) { if err := h.FileAPI.ListFileInfo(ctx, limit, responses); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to list files") return } @@ -191,14 +191,15 @@ func (h Handler) listFiles(w http.ResponseWriter, r *http.Request) { } v, err := protojson.Marshal(res.Fileinfo) if err != nil { - log.Warn().Err(err).Msg("failed to marshal storage ref") + log.Warn().Err(err).Msg("failed to marshal file info") + continue } if count > 0 { _, _ = w.Write([]byte("\r\n")) } _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write storage ref") + log.Warn().Err(err).Msg("failed to write file info") return } count++ @@ -215,11 +216,12 @@ func (h Handler) getFileInfoByFilename(w http.ResponseWriter, r *http.Request) { fileInfo, err := h.FileAPI.GetFileInfo(ctx, filename) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msgf("Failed to get file info: %s", filename) return } _, err = fmt.Fprintln(w, protojson.Format(fileInfo)) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + log.Warn().Err(err).Msgf("Failed to write file info: %s", protojson.Format(fileInfo)) } } @@ -231,6 +233,7 @@ func (h Handler) listCdxs(w http.ResponseWriter, r *http.Request) { if err := h.CdxAPI.List(ctx, limit, responses); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to list cdx records") return } @@ -255,11 +258,7 @@ func (h Handler) listCdxs(w http.ResponseWriter, r *http.Request) { } _, err = io.Copy(w, bytes.NewReader(v)) if err != nil { - log.Error().Err(err).Int("status", http.StatusInternalServerError).Msg("failed to write result") - if count == 0 { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + log.Warn().Err(err).Msg("failed to write cdx record") return } count++ @@ -274,23 +273,18 @@ func (h Handler) loadRecordByUrn(w http.ResponseWriter, r *http.Request) { defer cancel() record, err := h.WarcLoader.LoadById(ctx, warcId) - if err != nil { - msg := fmt.Sprintf("failed to load record '%s': %v", warcId, err) - http.Error(w, msg, http.StatusInternalServerError) - return + if record != nil { + defer record.Close() } - defer func() { - if err := record.Close(); err != nil { - log.Warn().Err(err).Msgf("Closing record: %v", record) - } - }() - - n, err := handlers.RenderRecord(w, record) - if n == 0 { + if err != nil { + err := fmt.Errorf("failed to load record '%s': %w", warcId, err) http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("") + return } + _, err = handlers.RenderRecord(w, record) if err != nil { - log.Warn().Err(err).Msgf("Failed to write record: %v", record) + log.Warn().Err(err).Msgf("Failed to write record '%s': %v", warcId, record) } } diff --git a/server/warcserver/handler.go b/server/warcserver/handler.go index ac3b070..72f4130 100644 --- a/server/warcserver/handler.go +++ b/server/warcserver/handler.go @@ -38,19 +38,17 @@ func (h Handler) index(w http.ResponseWriter, r *http.Request) { start := time.Now() count := 0 defer func() { - if count > 0 { - log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) - } + log.Debug().Msgf("Found %d items in %s", count, time.Since(start)) }() ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() + response := make(chan index.CdxResponse) - searchApi := api.SearchAPI{CoreAPI: coreAPI} - if err = h.CdxAPI.Search(ctx, searchApi, response); err != nil { - log.Error().Err(err).Msg("search failed") + if err = h.CdxAPI.Search(ctx, api.SearchAPI{CoreAPI: coreAPI}, response); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msgf("Search failed: %+v", coreAPI) return } @@ -74,9 +72,9 @@ func (h Handler) index(w http.ResponseWriter, r *http.Request) { } if err != nil { log.Warn().Err(err).Msg("failed to write result") - } else { - count++ + return } + count++ } } @@ -95,15 +93,15 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { response := make(chan index.CdxResponse) err = h.CdxAPI.Closest(ctx, closestReq, response) if err != nil { - log.Error().Err(err).Msg("failed to query closest cdx") http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msgf("Failed search closest: %+v", closestReq) return } var res index.CdxResponse for res = range response { if res.Error != nil { - log.Warn().Err(err).Msg("failed cdx response") + log.Warn().Err(err).Msg("Failed cdx response") continue } if res.Cdx == nil { @@ -127,15 +125,19 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { var errResolveRevisit loader.ErrResolveRevisit if errors.As(err, &errResolveRevisit) { http.Error(w, errResolveRevisit.Error(), http.StatusNotImplemented) + log.Error().Err(errResolveRevisit).Msg("Failed to load record") return } http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load record") return } block, ok := warcRecord.Block().(gowarc.HttpResponseBlock) if !ok { - http.Error(w, fmt.Sprintf("Record not renderable: %s", warcRecord), http.StatusInternalServerError) + err := fmt.Errorf("record not renderable: %s", warcRecord) + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load resource") return } @@ -145,11 +147,12 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { p, err := block.PayloadBytes() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load resource") return } err = handlers.Render(w, *block.HttpHeader(), block.HttpStatusCode(), p) if err != nil { - log.Warn().Err(err).Msg("Failed to render response") + log.Warn().Err(err).Msg("Failed to load resource") } return } @@ -157,7 +160,9 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { // handle redirect location := block.HttpHeader().Get("Location") if location == "" { - http.Error(w, "Redirected to empty location", http.StatusInternalServerError) + err := errors.New("empty redirect location") + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load resource") return } @@ -165,12 +170,16 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { if urlErrors.Code(err) == urlErrors.FailRelativeUrlWithNoBase { locUrl, err = url.ParseRef(closestReq.rawUrl, location) if err != nil { - http.Error(w, fmt.Sprintf("Failed to parse relative location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) + err = fmt.Errorf("failed to parse relative location header as URL: %s: %s: %w", warcRecord, location, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load resource") return } } if err != nil { - http.Error(w, fmt.Sprintf("Failed to parse location header as URL: %s: %s: %v", warcRecord, location, err), http.StatusInternalServerError) + err = fmt.Errorf("failed to parse location header as URL: %s: %s: %w", warcRecord, location, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to load resource") return } @@ -183,6 +192,7 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { err = h.Closest(ctx, closestReq, response) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed to find closest redirect") return } @@ -192,7 +202,7 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { for res := range response { if res.Error != nil { - log.Warn().Err(err).Msg("") + log.Warn().Err(err).Msg("failed result") continue } sts = timestamp.TimeTo14(res.GetSts().AsTime()) @@ -210,7 +220,9 @@ func (h Handler) resource(w http.ResponseWriter, r *http.Request) { host := r.Host u, err := url.Parse(scheme + "://" + host) if err != nil { + err := fmt.Errorf("failed to construct redirect location: %s: %w", uri, err) http.Error(w, err.Error(), http.StatusInternalServerError) + log.Error().Err(err).Msg("Failed load resource") return } u.SetPathname(path) From 6ef0747a1b2d87bfa652868d758e9749945c7b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 16:21:19 +0100 Subject: [PATCH 37/38] Update dependencies --- go.mod | 45 +++-- go.sum | 536 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 532 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 91a573a..c992af3 100644 --- a/go.mod +++ b/go.mod @@ -3,44 +3,43 @@ module github.com/nlnwa/gowarcserver go 1.16 require ( - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.3.1 github.com/dgraph-io/badger/v3 v3.2103.4 + github.com/gorilla/handlers v1.5.1 + github.com/julienschmidt/httprouter v1.3.0 + github.com/mitchellh/mapstructure v1.5.0 + github.com/nlnwa/gowarc v1.0.0-beta.5 + github.com/nlnwa/whatwg-url v0.1.2 + github.com/rs/zerolog v1.28.0 + github.com/segmentio/kafka-go v0.4.38 + github.com/spf13/cobra v1.6.1 + github.com/spf13/viper v1.14.0 + github.com/tikv/client-go/v2 v2.0.2 + google.golang.org/protobuf v1.28.1 +) + +require ( + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v22.11.23+incompatible // indirect - github.com/gorilla/handlers v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/julienschmidt/httprouter v1.3.0 - github.com/klauspost/compress v1.15.12 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/mitchellh/mapstructure v1.5.0 - github.com/nlnwa/gowarc v1.0.0-beta.4 - github.com/nlnwa/whatwg-url v0.1.2 github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect - github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 // indirect + github.com/pingcap/kvproto v0.0.0-20221130051817-ca78592cde8c // indirect github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c // indirect github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/tsdb v0.10.0 // indirect - github.com/rs/zerolog v1.28.0 - github.com/segmentio/kafka-go v0.4.38 + github.com/prometheus/prometheus v0.40.5 // indirect github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cobra v1.6.1 - github.com/spf13/viper v1.14.0 - github.com/tikv/client-go/v2 v2.0.2 - github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b // indirect + github.com/tikv/pd/client v0.0.0-20221130112201-58fde4919361 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/zap v1.23.0 // indirect + go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.2.0 // indirect - google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/genproto v0.0.0-20221130183247-a2ec334bae6f // indirect google.golang.org/grpc v1.51.0 // indirect - google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index 8dd4858..be014c0 100644 --- a/go.sum +++ b/go.sum @@ -364,19 +364,46 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -384,6 +411,17 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.128/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -398,17 +436,26 @@ github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -419,29 +466,57 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/digitalocean/godo v1.88.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.21+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -451,43 +526,119 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.6.13/go.mod h1:qEySVqXrEugbHKvmhI8ZqtQi75/RHSSRNpffvB4I6Bw= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= +github.com/go-openapi/runtime v0.23.1/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -520,6 +671,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -527,11 +679,13 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v22.11.23+incompatible h1:334TygA7iuxt0hoamawsM36xoui01YiouEZnr0qeFMI= github.com/google/flatbuffers v22.11.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -548,7 +702,10 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -569,9 +726,11 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20221102093814-76f304f74e5e/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= @@ -588,20 +747,39 @@ github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqE github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gophercloud/gophercloud v1.0.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.15.3/go.mod h1:/g/qgcoBcEXALCNZgRRisyTW0nY86++L0KbeAMXYCeY= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= +github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -614,6 +792,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= @@ -621,34 +801,61 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/nomad/api v0.0.0-20221102143410-8a95f1239005/go.mod h1:vgJmrz4Bz9E1cR/uy70oP9udUJKFRkcEYHlHTp4nFwI= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.9.8/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hetznercloud/hcloud-go v1.35.3/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/ionos-cloud/sdk-go/v6 v6.1.3/go.mod h1:Ox3W0iiEz0GHnfY9e5LmAxwklsxguuNFEUSu0gVRTME= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -657,30 +864,49 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/linode/linodego v1.9.3/go.mod h1:h6AuFR/JpqwwM/vkj7s8KV3iGN8/jxn+zc437F8SZ8w= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -688,55 +914,111 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nlnwa/gowarc v1.0.0-beta.4 h1:XxGNxZnW6Stmebsw458hM9NPf0YYp9siOiNYU6ONww4= -github.com/nlnwa/gowarc v1.0.0-beta.4/go.mod h1:37HyH1JP9GpGS5WZsBDTbZ8rwvCCDdsSDOr+3gYGEE4= -github.com/nlnwa/whatwg-url v0.0.0-20200306110950-d1a95e2e8fc3/go.mod h1:v3hJLcAdjhIn7PA89dVhJ9GSWooX0z2/qPgwlhz0HD8= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nlnwa/gowarc v1.0.0-beta.5 h1:m1RK0Jt+ukckit1nzycFYCTnSl5NXGfzkRQ6wL8epcY= +github.com/nlnwa/gowarc v1.0.0-beta.5/go.mod h1:z//s7tRz7zyEz2rlcwSy4aJQdQWlxI3a4Au71YSMW/0= github.com/nlnwa/whatwg-url v0.1.2 h1:BqqsIVG6xv71wOoMAoFDmV6OK6/2sXn7BJdOsTkBl88= github.com/nlnwa/whatwg-url v0.1.2/go.mod h1:b0r+dEyM/KztLMDSVY6ApcO9Fmzgq+e9+Ugq20UBYck= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/ovh/go-ovh v1.1.0/go.mod h1:AxitLZ5HBRPyUd+Zl60Ajaag+rNTdVXWIkzfrVuTXWA= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -751,45 +1033,64 @@ github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 h1:IjRv6eg+44iApCtFKPA+POfq+6pU4xCG8pdrw7xUuD8= -github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20221130051817-ca78592cde8c h1:X+uoOUnXip0RVApv26K4PgCL9/SCZ8HExoPWZGRSJlc= +github.com/pingcap/kvproto v0.0.0-20221130051817-ca78592cde8c/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYhW5Piw+kYNKZqFQqfC2puUf6gMI= github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= +github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= +github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prometheus/exporter-toolkit v0.8.1/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= +github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -797,45 +1098,72 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= +github.com/prometheus/prometheus v0.40.3/go.mod h1:/UhsWkOXkO11wqTW2Bx5YDOwRweSDcaFBlTIzFe7P0Y= +github.com/prometheus/prometheus v0.40.5 h1:wmk5yNrQlkQ2OvZucMhUB4k78AVfG34szb1UtopS8Vc= +github.com/prometheus/prometheus v0.40.5/go.mod h1:bxgdmtoSNLmmIVPGmeTJ3OiP67VmuY4yalE4ZP6L/j8= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.8.0/go.mod h1:TmKwZAo97S4Fy4sfMH/HX/cQP5D+ijra2NyLpNNmttY= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.4.38 h1:iQdOBbUSdfuYlFpvjuALgj7N6DrdPA0HfB4AhREOdtg= github.com/segmentio/kafka-go v0.4.38/go.mod h1:ikyuGon/60MN/vXFgykf7Zm8P5Be49gJU6vezwjnnhU= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shoenig/test v0.4.3/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -844,6 +1172,10 @@ github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -861,28 +1193,43 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= -github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b h1:7BbO/1w3JUnU8ul270MhLXB/V8enVR2ugP45qvxCUSQ= -github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b/go.mod h1:V94ltDCVCbxtzkM3wK9ksXfppIlJ4TL043xU9mYZOUs= +github.com/tikv/pd/client v0.0.0-20221130112201-58fde4919361 h1:+VNn7DBfc/u7m2FsS1X6hBUjd5+3vQAIhGzdJ2JZfbg= +github.com/tikv/pd/client v0.0.0-20221130112201-58fde4919361/go.mod h1:V94ltDCVCbxtzkM3wK9ksXfppIlJ4TL043xU9mYZOUs= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= @@ -890,6 +1237,13 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4 go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= +go.mongodb.org/mongo-driver v1.10.2/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -899,43 +1253,74 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4/go.mod h1:l2MdsbKTocpPS5nQZscqTR9jd8u96VYZdcpF8Sye7mA= +go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1/go.mod h1:QrRRQiY3kzAoYPNLP0W/Ikg0gR6V3LMc+ODSxr7yyvg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1/go.mod h1:X620Jww3RajCJXw/unA+8IRTgxkdS7pi+ZwK9b7KUJk= +go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= +go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= +go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -946,6 +1331,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -971,12 +1357,20 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -987,6 +1381,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1014,10 +1410,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1031,9 +1430,12 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220921155015-db77216a4ee9/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1057,14 +1459,17 @@ golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1081,19 +1486,25 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1103,6 +1514,7 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1137,9 +1549,9 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1148,9 +1560,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1158,8 +1572,11 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1169,12 +1586,17 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1188,28 +1610,40 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1218,6 +1652,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1230,6 +1665,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1251,7 +1687,11 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1260,6 +1700,7 @@ golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1311,6 +1752,7 @@ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91 google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -1322,6 +1764,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1350,6 +1793,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1389,6 +1833,7 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= @@ -1404,6 +1849,7 @@ google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220728213248-dd149ef739b9/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= @@ -1424,12 +1870,17 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221130183247-a2ec334bae6f h1:G9JGt1JAvAkcv0+x6gGEWkgjEPxsI3fQhNq6L/DJTZ8= +google.golang.org/genproto v0.0.0-20221130183247-a2ec334bae6f/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1454,6 +1905,7 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= @@ -1487,15 +1939,25 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1506,9 +1968,14 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1516,9 +1983,26 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= +k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= From 9e7a577cb7bf9d54a2a1990f281e23255b08f06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Thu, 1 Dec 2022 16:34:52 +0100 Subject: [PATCH 38/38] Update README and example config --- README.md | 26 -------------------------- exampleconfig.yaml | 2 -- 2 files changed, 28 deletions(-) diff --git a/README.md b/README.md index d139720..1b54a4e 100644 --- a/README.md +++ b/README.md @@ -28,29 +28,3 @@ following the steps described in golangci-lint's [local installation](https://golangci-lint.run/usage/install/#local-installation) guide. Note that on **linux** the guide expects you to have `$GOPATH/bin` included in your `PATH` variable. golangci-lint run -E "bodyclose" -E "dogsled" -E "durationcheck" -E "errorlint" -E "forcetypeassert" -E "noctx" -E "exhaustive" -E "exportloopref" --timeout 3m0s - -## Configuration - -gowarcserver can be configured with a config file, environment variables and flags. Flags has precedence over -environment variables that has precedence over config file entries. An environment variable match the uppercased flag -name with underscore in place of dash. - -| Name | Type | Description | Default | Sub command | -| ------------- | ------------- | ----------- | ------- | ------- | -| config | string | Path to configuration file | ./config.yaml | global | -| log-level | string | Log level. Legal values are "trace" , "debug", "info", "warn" or "error" | "info" | global | -| port | int | Server port | 9999 | serve | -| index | bool | Enable indexing when running server | true | serve | -| watch | bool | Update index when files change | false | serve | -| log-requests | bool | Enable request logging | false | serve | -| dirs | list of paths | Comma separated list of directories to index | ["."] | index, serve | -| db-dir | path | Location of index database | "." | index, serve | -| max-depth | int | Maximum index recursion depth | 4 | index, serve | -| include | list of suffixes | Only index files that match one of these suffixes | [] | index, serve | -| workers | int | Number of index workers | 8 | index, serve | -| compression | string | Database compression type. Legal values are: 'none', 'snappy', 'zstd' | "snappy" | index, serve | -| bloom | bool | Enable bloom filter when indexing with "toc" format | true | index | -| bloom-capacity | uint | Estimated bloom filter capacity | 1000 | index | -| bloom-fp | float64 | Estimated bloom filter false positive rate | 0.01 | index | -| child-urls | []string | Urls pointing to other gowarcserver processes running a server | [] | proxy | -| child-query-timeout | Duration | Child query timeout a request to a child can take before resulting in timeout | 300ms | proxy | diff --git a/exampleconfig.yaml b/exampleconfig.yaml index 2518cc8..a4a09a3 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -12,8 +12,6 @@ port: 8880 path-prefix: "" # log server requests log-requests: true -# index storage engine: "badger" or "tikv" -storage-engine: badger # INDEX