From f77ad379e649a77f8fa8b5b2fb53d6d88dacdade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Andr=C3=A9=20Elsfjordstrand=20Beck?= Date: Tue, 29 Nov 2022 19:45:56 +0100 Subject: [PATCH] Refactor and consolidate Remove file watcher --- .github/workflows/release.yaml | 8 +- .github/workflows/test.yaml | 24 +++ cmd/index/index.go | 154 ++++++++------- cmd/index/index_test.go | 29 --- cmd/root.go | 18 +- cmd/root_test.go | 16 ++ cmd/serve/serve.go | 176 +++++++++-------- exampleconfig.yaml | 47 +++-- go.mod | 30 ++- go.sum | 214 ++++++++++++++------ index/api.go | 4 + index/autoindex.go | 148 +++++++------- index/index.go | 88 +++++++++ index/io.go | 60 ++---- index/io_test.go | 4 +- index/options.go | 32 +-- index/watcher.go | 82 -------- index/worker.go | 76 ++------ index/writers.go | 29 +-- internal/badgeridx/compression.go | 3 +- internal/badgeridx/db.go | 91 ++------- internal/badgeridx/options.go | 94 +++++++++ internal/logger/initlog.go | 5 +- internal/tikvidx/api.go | 171 +++++++--------- internal/tikvidx/db.go | 123 ++++-------- internal/tikvidx/iter.go | 312 +++++++++++++++--------------- internal/tikvidx/options.go | 20 +- main.go | 27 +-- server/coreserver/handler.go | 5 +- 29 files changed, 1060 insertions(+), 1030 deletions(-) delete mode 100644 cmd/index/index_test.go create mode 100644 index/index.go delete mode 100644 index/watcher.go create mode 100644 internal/badgeridx/options.go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b2d8cb3..443849b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,9 +1,6 @@ name: release on: - pull_request: - branches: - - main push: branches: - main @@ -17,11 +14,14 @@ env: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3df2546..e6eabb3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,6 +6,10 @@ on: permissions: contents: read +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: unit_test: name: Golang unit tests @@ -44,3 +48,23 @@ jobs: version: latest # Enable additional linters (see: https://golangci-lint.run/usage/linters/) args: -E "bodyclose" -E "dogsled" -E "durationcheck" -E "errorlint" -E "forcetypeassert" -E "noctx" -E "exhaustive" -E "exportloopref" --timeout 3m0s + build: + runs-on: ubuntu-latest + steps: + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + build-args: | + VERSION=${{ steps.meta.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/cmd/index/index.go b/cmd/index/index.go index 7816bf0..495b917 100644 --- a/cmd/index/index.go +++ b/cmd/index/index.go @@ -17,9 +17,13 @@ package index import ( + "context" "fmt" + "os" + "os/signal" "regexp" "runtime" + "syscall" "time" "github.com/bits-and-blooms/bloom/v3" @@ -34,7 +38,7 @@ import ( func NewCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "index [dir] ...", + Use: "index", Short: "Index warc file(s)", PreRunE: func(cmd *cobra.Command, args []string) error { if err := viper.BindPFlags(cmd.Flags()); err != nil { @@ -44,55 +48,51 @@ func NewCommand() *cobra.Command { }, RunE: indexCmd, } - // defaults - format := "cdxj" - indexDepth := 4 - indexWorkers := 8 - badgerDir := "./warcdb" - badgerBatchMaxSize := 1000 - badgerBatchMaxWait := 5 * time.Second - badgerCompression := "snappy" - badgerDatabase := "" - var tikvPdAddr []string - tikvBatchMaxSize := 1000 - tikvBatchMaxWait := 5 * time.Second - tikvDatabase := "" - bloomCapacity := uint(1000) - bloomFp := 0.01 - - cmd.Flags().StringP("format", "f", format, `index format: "cdxj", "cdxpb", "badger", "tikv" or "toc"`) - cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") - cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") - cmd.Flags().IntP("max-depth", "d", indexDepth, "maximum directory recursion") - cmd.Flags().Int("workers", indexWorkers, "number of files") - cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") - cmd.Flags().Uint("toc-bloom-capacity", bloomCapacity, "estimated bloom filter capacity") - cmd.Flags().Float64("toc-bloom-fp", bloomFp, "estimated bloom filter false positive rate") - cmd.Flags().String("badger-dir", badgerDir, "path to index database") - cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") - cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") + // index options + cmd.Flags().StringP("index-source", "s", "file", `index source: "file" or "kafka"`) + cmd.Flags().StringP("index-format", "o", "cdxj", `index format: "cdxj", "cdxpb", "toc", badger", "tikv"`) + cmd.Flags().StringSlice("index-include", nil, "only include files matching these regular expressions") + cmd.Flags().StringSlice("index-exclude", nil, "exclude files matching these regular expressions") + cmd.Flags().Int("index-workers", 8, "number of index workers") + + // auto indexer options + cmd.Flags().StringSliceP("file-paths", "f", []string{"./testdata"}, "directories to search for warc files in") + cmd.Flags().Int("file-max-depth", 4, "maximum directory recursion") + + // kafka indexer options + cmd.Flags().StringSlice("kafka-brokers", nil, "the list of broker addresses used to connect to the kafka cluster") + cmd.Flags().String("kafka-group-id", "", "optional consumer group id") + cmd.Flags().String("kafka-topic", "", "the topic to read messages from") + cmd.Flags().Int("kafka-min-bytes", 0, "indicates to the broker the minimum batch size that the consumer will accept") + cmd.Flags().Int("kafka-max-bytes", 0, "indicates to the broker the maximum batch size that the consumer will accept") + cmd.Flags().Duration("kafka-max-wait", 0, "maximum amount of time to wait for new data to come when fetching batches of messages from kafka") + + // toc indexer options + cmd.Flags().Uint("toc-bloom-capacity", uint(1000), "estimated bloom filter capacity") + cmd.Flags().Float64("toc-bloom-fp", 0.01, "estimated bloom filter false positive rate") + + // badger options + cmd.Flags().String("badger-dir", "./warcdb", "path to index database") + cmd.Flags().String("badger-database", "", "name of badger database") + cmd.Flags().Int("badger-batch-max-size", 1000, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", 5*time.Second, "max wait time before flushing batched records") + cmd.Flags().String("badger-compression", "snappy", "compression algorithm") + cmd.Flags().Bool("badger-read-only", false, "run badger in read-only mode") + + // tikv options + cmd.Flags().StringSlice("tikv-pd-addr", nil, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", 1000, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", 5*time.Second, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", "", "name of tikv database") + return cmd } -func indexCmd(_ *cobra.Command, args []string) error { - // collect paths from args or flag - var dirs []string - if len(args) > 0 { - dirs = append(dirs, args...) - } else { - dirs = viper.GetStringSlice("dirs") - } - - var w index.Indexer +func indexCmd(_ *cobra.Command, _ []string) error { + var w index.RecordWriter - format := viper.GetString("format") - switch format { + indexFormat := viper.GetString("index-format") + switch indexFormat { case "cdxj": w = index.CdxJ{} case "cdxpb": @@ -135,14 +135,11 @@ func indexCmd(_ *cobra.Command, args []string) error { BloomFilter: bloom.NewWithEstimates(viper.GetUint("toc-bloom-capacity"), viper.GetFloat64("toc-bloom-fp")), } default: - return fmt.Errorf("unsupported format %s", format) + return fmt.Errorf("unknown index format: %s", indexFormat) } - indexWorker := index.NewWorker(w, viper.GetInt("workers")) - defer indexWorker.Close() - var includes []*regexp.Regexp - for _, r := range viper.GetStringSlice("include") { + for _, r := range viper.GetStringSlice("index-include") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -151,7 +148,7 @@ func indexCmd(_ *cobra.Command, args []string) error { } var excludes []*regexp.Regexp - for _, r := range viper.GetStringSlice("exclude") { + for _, r := range viper.GetStringSlice("index-exclude") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -159,23 +156,50 @@ func indexCmd(_ *cobra.Command, args []string) error { } } - log.Info().Msg("Starting auto indexer") - indexer, err := index.NewAutoIndexer(indexWorker, - index.WithMaxDepth(viper.GetInt("max-depth")), + indexer := index.NewIndexer(w, index.WithIncludes(includes...), index.WithExcludes(excludes...), ) - if err != nil { - return err - } - defer indexer.Close() + queue := index.NewWorkQueue(indexer, + viper.GetInt("index-workers"), + ) + defer queue.Close() - for _, dir := range dirs { - err := indexer.Index(dir) - if err != nil { - log.Warn().Msgf(`Error indexing "%s": %v`, dir, err) - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + sig := <-sigs + log.Info().Msgf("Received %s signal, shutting down...", sig) + cancel() + }() + + var runner index.Runner + + indexSource := viper.GetString("index-source") + switch indexSource { + case "file": + runner = index.NewAutoIndexer(queue, + index.WithMaxDepth(viper.GetInt("file-max-depth")), + index.WithPaths(viper.GetStringSlice("file-paths")), + index.WithExcludeDirs(excludes...), + ) + case "kafka": + runner = index.NewKafkaIndexer(queue, + index.WithBrokers(viper.GetStringSlice("kafka-brokers")), + index.WithGroupID(viper.GetString("kafka-group-id")), + index.WithTopic(viper.GetString("kafka-topic")), + index.WithMinBytes(viper.GetInt("kafka-min-bytes")), + index.WithMaxBytes(viper.GetInt("kafka-max-bytes")), + index.WithMaxWait(viper.GetDuration("kafka-max-wait")), + ) + default: + return fmt.Errorf("unknown index source: %s", indexSource) } - return nil + log.Info().Msg("Starting indexer") + + return runner.Run(ctx) } diff --git a/cmd/index/index_test.go b/cmd/index/index_test.go deleted file mode 100644 index d02c51f..0000000 --- a/cmd/index/index_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package index - -import ( - "testing" -) - -func TestIndexCmd(t *testing.T) { - cmd := NewCommand() - err := cmd.Execute() - if err != nil { - t.Errorf("%v", err) - } -} diff --git a/cmd/root.go b/cmd/root.go index 535d78a..18c73a4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -33,7 +33,7 @@ import ( // NewCommand returns a new cobra.Command implementing the root command for warc func NewCommand() *cobra.Command { - cobra.OnInitialize(func() { initConfig() }) + cobra.OnInitialize(initConfig) cmd := &cobra.Command{ Use: "gowarcserver", @@ -41,7 +41,7 @@ func NewCommand() *cobra.Command { } // Global flags - _ = cmd.PersistentFlags().StringP("config", "c", "", `path to config file, default paths are "./config.yaml", "$HOME/.gowarcserver/config.yaml" or "/etc/gowarcserver/config.yaml"`) + _ = cmd.PersistentFlags().String("config", "", `path to config file, default paths are "./config.yaml", "$HOME/.gowarcserver/config.yaml" or "/etc/gowarcserver/config.yaml"`) _ = cmd.PersistentFlags().StringP("log-level", "l", "info", `set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace"`) _ = cmd.PersistentFlags().String("log-formatter", "logfmt", "log formatter, available values are logfmt and json") _ = cmd.PersistentFlags().Bool("log-method", false, "log method caller") @@ -76,14 +76,14 @@ func initConfig() { viper.AddConfigPath("/etc/gowarcserver/") // global configuration directory } - if err := viper.ReadInConfig(); err != nil { - if errors.As(err, new(viper.ConfigFileNotFoundError)) { - return - } + defer func() { + logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) + log.Debug().Msgf("Using config file: %s", viper.ConfigFileUsed()) + }() + + err := viper.ReadInConfig() + if err != nil && !errors.As(err, new(viper.ConfigFileNotFoundError)) { _, _ = fmt.Fprintf(os.Stderr, "Failed to read config file: %v", err) os.Exit(1) } - logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method")) - - log.Info().Msgf("Using config file: %s", viper.ConfigFileUsed()) } diff --git a/cmd/root_test.go b/cmd/root_test.go index a0cff70..88115d5 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -1,3 +1,19 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package cmd import ( diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 7ca0df6..aff41e4 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -54,62 +54,51 @@ func NewCommand() *cobra.Command { }, RunE: serveCmd, } - // defaults - port := 9999 - watch := false - storageEngine := "badger" - autoIndex := true - indexDepth := 4 - indexWorkers := 8 - badgerDir := "./warcdb" - badgerCompression := "snappy" - badgerDatabase := "" - badgerBatchMaxSize := 1000 - badgerBatchMaxWait := 5 * time.Second - badgerReadOnly := false - logRequests := false - tikvPdAddr := []string{} - tikvBatchMaxSize := 255 - tikvBatchMaxWait := 5 * time.Second - tikvDatabase := "" - pathPrefix := "" - cmd.Flags().IntP("port", "p", port, "server port") - cmd.Flags().String("path-prefix", pathPrefix, "prefix for all server endpoints") - cmd.Flags().StringSlice("include", nil, "only include files matching these regular expressions") - cmd.Flags().StringSlice("exclude", nil, "exclude files matching these regular expressions") - cmd.Flags().IntP("max-depth", "w", indexDepth, "maximum directory recursion depth") - cmd.Flags().Int("workers", indexWorkers, "number of index workers") - cmd.Flags().StringSlice("dirs", nil, "directories to search for warc files in") - cmd.Flags().Bool("index", autoIndex, "run indexing") - cmd.Flags().Bool("watch", watch, "watch files for changes") - cmd.Flags().Bool("log-requests", logRequests, "log http requests") - cmd.Flags().StringP("storage-engine", "b", storageEngine, `storage engine: "badger" or "tikv"`) - cmd.Flags().String("badger-dir", badgerDir, "path to index database") - cmd.Flags().String("badger-database", badgerDatabase, "name of badger database") - cmd.Flags().String("badger-compression", badgerCompression, "compression algorithm") - cmd.Flags().Int("badger-batch-max-size", badgerBatchMaxSize, "max transaction batch size in badger") - cmd.Flags().Duration("badger-batch-max-wait", badgerBatchMaxWait, "max wait time before flushing batched records") - cmd.Flags().Bool("badger-read-only", badgerReadOnly, "run badger read-only") - cmd.Flags().StringSlice("tikv-pd-addr", tikvPdAddr, "host:port of TiKV placement driver") - cmd.Flags().Int("tikv-batch-max-size", tikvBatchMaxSize, "max transaction batch size") - cmd.Flags().Duration("tikv-batch-max-wait", tikvBatchMaxWait, "max wait time before flushing batched records regardless of max batch size") - cmd.Flags().String("tikv-database", tikvDatabase, "name of tikv database") + // server options + cmd.Flags().IntP("port", "p", 9999, "server port") + cmd.Flags().String("path-prefix", "", "path prefix for all server endpoints") + cmd.Flags().Bool("log-requests", false, "log incoming http requests") + + // index options + cmd.Flags().StringP("index-source", "s", "file", `index source: "file" or "kafka"`) + cmd.Flags().StringP("index-format", "o", "badger", `index format: "badger", "tikv"`) + cmd.Flags().StringSlice("index-include", nil, "only include files matching these regular expressions") + cmd.Flags().StringSlice("index-exclude", nil, "exclude files matching these regular expressions") + cmd.Flags().Int("index-workers", 8, "number of index workers") + + // auto indexer options + cmd.Flags().StringSlice("file-paths", []string{"./testdata"}, "list of paths to warc files or directories containing warc files") + cmd.Flags().Int("file-max-depth", 4, "maximum directory recursion depth") + + // kafka indexer options + cmd.Flags().StringSlice("kafka-brokers", nil, "the list of broker addresses used to connect to the kafka cluster") + cmd.Flags().String("kafka-group-id", "", "optional consumer group id") + cmd.Flags().String("kafka-topic", "", "the topic to read messages from") + cmd.Flags().Int("kafka-min-bytes", 0, "indicates to the broker the minimum batch size that the consumer will accept") + cmd.Flags().Int("kafka-max-bytes", 0, "indicates to the broker the maximum batch size that the consumer will accept") + cmd.Flags().Duration("kafka-max-wait", 0, "maximum amount of time to wait for new data to come when fetching batches of messages from kafka") + + // badger options + cmd.Flags().String("badger-dir", "./warcdb", "path to index database") + cmd.Flags().String("badger-database", "", "name of badger database") + cmd.Flags().Int("badger-batch-max-size", 1000, "max transaction batch size in badger") + cmd.Flags().Duration("badger-batch-max-wait", 5*time.Second, "max wait time before flushing batched records") + cmd.Flags().String("badger-compression", "snappy", "compression algorithm") + + // tikv options + cmd.Flags().StringSlice("tikv-pd-addr", nil, "host:port of TiKV placement driver") + cmd.Flags().Int("tikv-batch-max-size", 1000, "max transaction batch size") + cmd.Flags().Duration("tikv-batch-max-wait", 5*time.Second, "max wait time before flushing batched records regardless of max batch size") + cmd.Flags().String("tikv-database", "", "name of tikv database") return cmd } -func serveCmd(cmd *cobra.Command, args []string) error { - // collect paths from args or flag - var dirs []string - if len(args) > 0 { - dirs = append(dirs, args...) - } else { - dirs = viper.GetStringSlice("dirs") - } +func serveCmd(_ *cobra.Command, _ []string) error { // parse include patterns var includes []*regexp.Regexp - for _, r := range viper.GetStringSlice("include") { + for _, r := range viper.GetStringSlice("index-include") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -118,7 +107,7 @@ func serveCmd(cmd *cobra.Command, args []string) error { } // parse exclude patterns var excludes []*regexp.Regexp - for _, r := range viper.GetStringSlice("exclude") { + for _, r := range viper.GetStringSlice("index-exclude") { if re, err := regexp.Compile(r); err != nil { return fmt.Errorf("%s: %w", r, err) } else { @@ -126,14 +115,15 @@ func serveCmd(cmd *cobra.Command, args []string) error { } } - var indexer index.Indexer + var writer index.RecordWriter var fileApi index.FileAPI var cdxApi index.CdxAPI var idApi index.IdAPI var storageRefResolver loader.StorageRefResolver var filePathResolver loader.FilePathResolver - switch viper.GetString("storage-engine") { + indexFormat := viper.GetString("index-format") + switch indexFormat { case "badger": // Increase GOMAXPROCS as recommended by badger // https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use @@ -150,14 +140,14 @@ func serveCmd(cmd *cobra.Command, args []string) error { badgeridx.WithDir(viper.GetString("badger-dir")), badgeridx.WithBatchMaxSize(viper.GetInt("badger-batch-max-size")), badgeridx.WithBatchMaxWait(viper.GetDuration("badger-batch-max-wait")), - badgeridx.WithReadOnly(viper.GetBool("badger-read-only")), + badgeridx.WithReadOnly(viper.GetString("index-source") == ""), badgeridx.WithDatabase(viper.GetString("badger-database")), ) if err != nil { return err } defer db.Close() - indexer = db + writer = db storageRefResolver = db filePathResolver = db cdxApi = db @@ -169,47 +159,63 @@ func serveCmd(cmd *cobra.Command, args []string) error { tikvidx.WithBatchMaxSize(viper.GetInt("tikv-batch-max-size")), tikvidx.WithBatchMaxWait(viper.GetDuration("tikv-batch-max-wait")), tikvidx.WithDatabase(viper.GetString("tikv-database")), + tikvidx.WithReadOnly(viper.GetString("index-source") == ""), ) if err != nil { return err } defer db.Close() - indexer = db + writer = db storageRefResolver = db filePathResolver = db cdxApi = db fileApi = db idApi = db default: - return fmt.Errorf("invalid storage engine") + return fmt.Errorf("unknown index format: %s", indexFormat) } - // optionally start autoindexer - if viper.GetBool("index") { - log.Info().Msg("Starting auto indexer") + ctx, cancelIndexer := context.WithCancel(context.Background()) + defer cancelIndexer() - indexWorker := index.NewWorker(indexer, viper.GetInt("workers")) - defer indexWorker.Close() - - indexer, err := index.NewAutoIndexer(indexWorker, - index.WithWatch(viper.GetBool("watch")), - index.WithMaxDepth(viper.GetInt("max-depth")), + indexSource := viper.GetString("index-source") + if indexSource != "" { + indexer := index.NewIndexer(writer, index.WithIncludes(includes...), index.WithExcludes(excludes...), ) - if err != nil { - return err - } - defer indexer.Close() + queue := index.NewWorkQueue(indexer, + viper.GetInt("index-workers"), + ) + defer queue.Close() - for _, dir := range dirs { - dir := dir - go func() { - if err := indexer.Index(dir); err != nil { - log.Warn().Msgf(`Error indexing "%s": %v`, dir, err) - } - }() + var runner index.Runner + switch indexSource { + case "file": + runner = index.NewAutoIndexer(queue, + index.WithMaxDepth(viper.GetInt("file-max-depth")), + index.WithPaths(viper.GetStringSlice("file-paths")), + ) + case "kafka": + runner = index.NewKafkaIndexer(queue, + index.WithBrokers(viper.GetStringSlice("kafka-brokers")), + index.WithGroupID(viper.GetString("kafka-group-id")), + index.WithTopic(viper.GetString("kafka-topic")), + index.WithMinBytes(viper.GetInt("kafka-min-bytes")), + index.WithMaxBytes(viper.GetInt("kafka-max-bytes")), + index.WithMaxWait(viper.GetDuration("kafka-max-wait")), + ) + default: + return fmt.Errorf("unknown index source: %s", indexSource) } + go func() { + log.Info().Msg("Starting indexer") + err := runner.Run(ctx) + if err != nil { + log.Error().Err(err).Msg("Indexer has stopped") + } + }() + } // create record loader @@ -255,24 +261,28 @@ func serveCmd(cmd *cobra.Command, args []string) error { Handler: handler, } - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) sig := <-sigs - log.Debug().Msgf("Received %s signal, shutting down server...", sig) + log.Info().Msgf("Received %s signal, shutting down...", sig) + + cancelIndexer() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + err := httpServer.Shutdown(ctx) if err != nil { log.Error().Msgf("Failed to shut down server: %v", err) } }() - log.Info().Msgf("Starting web server at :%v", port) + log.Info().Msgf("Starting server at :%v", port) err := httpServer.ListenAndServe() - if !errors.Is(err, http.ErrServerClosed) { - return err + if errors.Is(err, http.ErrServerClosed) { + return nil } - return nil + return err } diff --git a/exampleconfig.yaml b/exampleconfig.yaml index 07dca9b..2518cc8 100644 --- a/exampleconfig.yaml +++ b/exampleconfig.yaml @@ -10,10 +10,6 @@ log-method: false port: 8880 # prefix for server endpoint paths path-prefix: "" -# start autoindexer -index: true -# watch for file changes (NFS does not support this) -watch: false # log server requests log-requests: true # index storage engine: "badger" or "tikv" @@ -21,21 +17,42 @@ storage-engine: badger # INDEX -# index format: "cdxj", "cdxpb", "badger", "tikv" or "toc" -format: cdxj - -# directories to search for warc files in -dirs: - - "./testdata" +# index format (index): "cdxj", "cdxpb", "badger", "tikv" or "toc" +# index format (serve): "badger" or "tikv" +index-format: cdxj +# index source: "file" or "kafka" +index-source: file # include only files matching regular expressions -include: +index-include: - ".warc.gz$" # exclude files matching regular expressions -exclude: [] -# max number of directory recursions -max-depth: 4 +index-exclude: [] # number of index workers -workers: 8 +index-workers: 8 + +# FILE TRAVERSAL INDEX SOURCE + +# warc files or directories to search for warc files in +file-paths: + - "./testdata" +# max number of directory recursions +file-max-depth: 4 + +# KAFKA INDEX SOURCE + +# the list of broker addresses used to connect to the kafka cluster +kafka-brokers: + - 127.0.0.1:3939 +# optional consumer group id +kafka-group-id: "my-group-id" +# the topic to read messages from +kafka-topic: "my-topic" +# indicates to the broker the minimum batch size that the consumer will accept +kafka-min-bytes: 0 +# indicates to the broker the maximum batch size that the consumer will accept +kafka-max-bytes: 0 +# maximum amount of time to wait for new data to come when fetching batches of messages from kafka +kafka-max-wait: 0 # path to badger database badger-dir: ./badger diff --git a/go.mod b/go.mod index fc4bcec..91a573a 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,14 @@ go 1.16 require ( github.com/benbjohnson/clock v1.3.0 // indirect - github.com/bits-and-blooms/bitset v1.3.3 // indirect + github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.3.1 - github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/dgraph-io/badger/v3 v3.2103.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.6.0 github.com/golang/glog v1.0.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v22.10.26+incompatible // indirect + github.com/google/flatbuffers v22.11.23+incompatible // indirect github.com/gorilla/handlers v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/julienschmidt/httprouter v1.3.0 @@ -23,26 +21,26 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nlnwa/gowarc v1.0.0-beta.4 github.com/nlnwa/whatwg-url v0.1.2 + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect - github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 // indirect - github.com/prometheus/client_golang v1.13.1 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 // indirect + github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c // indirect + github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect github.com/rs/zerolog v1.28.0 - github.com/spf13/afero v1.9.2 // indirect + github.com/segmentio/kafka-go v0.4.38 + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cobra v1.6.1 - github.com/spf13/viper v1.13.0 + github.com/spf13/viper v1.14.0 github.com/tikv/client-go/v2 v2.0.2 - github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect + github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/sync v0.1.0 // indirect - google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect + golang.org/x/net v0.2.0 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index 226551f..8dd4858 100644 --- a/go.sum +++ b/go.sum @@ -34,33 +34,45 @@ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34h cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -68,19 +80,27 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -88,23 +108,34 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -112,81 +143,114 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.8.0/go.mod h1:r3KB8cAdRIe8znzoPWLw8S6gpDVd9treohhn8b09424= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -198,42 +262,59 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOj cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -244,30 +325,44 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -286,6 +381,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -298,8 +394,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg= -github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= +github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -326,16 +422,12 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -370,7 +462,6 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -439,8 +530,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v22.10.26+incompatible h1:z1QiaMyPu1x3Z6xf2u1dsLj1ZxicdGSeaLpCuIsQNZM= -github.com/google/flatbuffers v22.10.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v22.11.23+incompatible h1:334TygA7iuxt0hoamawsM36xoui01YiouEZnr0qeFMI= +github.com/google/flatbuffers v22.11.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -505,25 +596,31 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/consul/api v1.15.3/go.mod h1:/g/qgcoBcEXALCNZgRRisyTW0nY86++L0KbeAMXYCeY= +github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -532,8 +629,9 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/serf v0.9.8/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -556,6 +654,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -594,12 +693,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -634,8 +734,12 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -647,10 +751,12 @@ github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1 h1:iJXUNA0LoOYuuMJ6U0tJGg2gCo/8xGZVhKLvuUWNjzw= -github.com/pingcap/kvproto v0.0.0-20221104101942-09d82b914df1/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY= +github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7 h1:IjRv6eg+44iApCtFKPA+POfq+6pU4xCG8pdrw7xUuD8= +github.com/pingcap/kvproto v0.0.0-20221129094810-e53d558bc6d7/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYhW5Piw+kYNKZqFQqfC2puUf6gMI= +github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -667,8 +773,8 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -695,8 +801,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= -github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= @@ -707,8 +811,11 @@ github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6us github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.8.0/go.mod h1:TmKwZAo97S4Fy4sfMH/HX/cQP5D+ijra2NyLpNNmttY= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/kafka-go v0.4.38 h1:iQdOBbUSdfuYlFpvjuALgj7N6DrdPA0HfB4AhREOdtg= +github.com/segmentio/kafka-go v0.4.38/go.mod h1:ikyuGon/60MN/vXFgykf7Zm8P5Be49gJU6vezwjnnhU= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -717,9 +824,9 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -733,8 +840,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= -github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -757,14 +864,18 @@ github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG github.com/tikv/client-go/v2 v2.0.2 h1:2kBozATh8SEnZONEM0Eeib+5wZ1J8bfjj3wTypSbvtU= github.com/tikv/client-go/v2 v2.0.2/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk= github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= -github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a h1:YMq0aKR30gBQBlUCqwnB4oBGNDcl0ddYPXQ1KzU3TTw= -github.com/tikv/pd/client v0.0.0-20221104043400-ea9b1e93b71a/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A= +github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b h1:7BbO/1w3JUnU8ul270MhLXB/V8enVR2ugP45qvxCUSQ= +github.com/tikv/pd/client v0.0.0-20221129041159-ea61786d506b/go.mod h1:V94ltDCVCbxtzkM3wK9ksXfppIlJ4TL043xU9mYZOUs= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -773,17 +884,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= -go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -814,6 +919,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -827,7 +933,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -911,22 +1019,23 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -942,7 +1051,6 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= @@ -964,7 +1072,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1001,7 +1108,6 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1045,7 +1151,6 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1066,11 +1171,11 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1086,6 +1191,7 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1183,7 +1289,6 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6 google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= @@ -1194,7 +1299,6 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69 google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= @@ -1205,6 +1309,7 @@ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1273,8 +1378,6 @@ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1294,7 +1397,6 @@ google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= @@ -1321,8 +1423,9 @@ google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1360,8 +1463,9 @@ google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/index/api.go b/index/api.go index debb468..8fb20d2 100644 --- a/index/api.go +++ b/index/api.go @@ -7,6 +7,10 @@ import ( "github.com/nlnwa/gowarcserver/schema" ) +type Runner interface { + Run(context.Context) error +} + type indexError string const AlreadyIndexedError indexError = "already indexed" diff --git a/index/autoindex.go b/index/autoindex.go index 6a7d23a..b1c9093 100644 --- a/index/autoindex.go +++ b/index/autoindex.go @@ -17,130 +17,112 @@ package index import ( + "context" "fmt" "os" "path/filepath" - "time" + "regexp" + + "github.com/rs/zerolog/log" ) -type autoIndexer struct { - done chan struct{} - perFileFn func(string) - perDirFn func(string) bool - watcher *watcher - settings *Options +type AutoIndexOptions struct { + MaxDepth int + Paths []string + Options } +type AutoIndexOption func(*AutoIndexOptions) -type Scheduler interface { - Schedule(job string, batchWindow time.Duration) +func WithMaxDepth(depth int) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.MaxDepth = depth + } } -func NewAutoIndexer(s Scheduler, opts ...Option) (*autoIndexer, error) { - a := new(autoIndexer) +func WithPaths(paths []string) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.Paths = paths + } +} - settings := defaultOptions() - for _, opt := range opts { - opt(settings) +func WithExcludeDirs(res ...*regexp.Regexp) AutoIndexOption { + return func(opts *AutoIndexOptions) { + opts.Excludes = res } - a.settings = settings +} - done := make(chan struct{}) - a.done = done +type Queue interface { + Add(path string) +} - isDone := func() bool { - select { - case <-done: - return true - default: - return false - } - } +type AutoIndexer struct { + Queue + opts *AutoIndexOptions +} - perDirFn := func(name string) bool { - if isDone() { - return false - } - return !settings.isExcluded(name) +func NewAutoIndexer(s Queue, options ...AutoIndexOption) AutoIndexer { + opts := new(AutoIndexOptions) + for _, apply := range options { + apply(opts) } - perFileFn := func(name string) { - if isDone() { - return - } - if settings.filter(name) { - s.Schedule(name, 0) - } + return AutoIndexer{ + Queue: s, + opts: opts, } +} - if settings.Watch { - w, err := newWatcher() +func (a AutoIndexer) Run(ctx context.Context) error { + for _, path := range a.opts.Paths { + err := a.index(ctx.Done(), path) if err != nil { - return nil, err - } - a.watcher = w - - perDirFn = func(name string) bool { - if isDone() { - return false - } - if settings.isExcluded(name) { - return false - } - _ = w.Add(name) - return true + log.Warn().Msgf(`Error indexing "%s": %v`, path, err) } - - onFileChanged := func(name string) { - if isDone() { - return - } - if settings.filter(name) { - s.Schedule(name, 10*time.Second) - } - } - go w.Watch(onFileChanged) } - a.perFileFn = perFileFn - a.perDirFn = perDirFn - - return a, nil + return nil } -func (a *autoIndexer) Index(path string) error { +func (a AutoIndexer) index(done <-chan struct{}, path string) error { + select { + case <-done: + return nil + default: + } info, err := os.Stat(path) if err != nil { return fmt.Errorf("failed to get file info: %w", err) } if info.IsDir() { - if err := a.walk(path, 0); err != nil { + if err := a.walk(done, path, 0); err != nil { return err } } else { - a.perFileFn(path) + a.Add(path) } return nil } -func (a *autoIndexer) Close() { - close(a.done) - a.watcher.Close() -} - -func (a *autoIndexer) walk(dir string, currentDepth int) error { +func (a AutoIndexer) walk(done <-chan struct{}, dir string, currentDepth int) error { + if a.opts.isExcluded(dir) { + return nil + } entries, err := os.ReadDir(dir) if err != nil { return fmt.Errorf(`failed to read directory "%s": %w`, dir, err) } for _, entry := range entries { - name := filepath.Join(dir, entry.Name()) + select { + case <-done: + return nil + default: + } + path := filepath.Join(dir, entry.Name()) if !entry.IsDir() { - a.perFileFn(name) - } else if currentDepth < a.settings.MaxDepth { - if a.perDirFn(name) { - err = a.walk(name, currentDepth+1) - if err != nil { - return err - } + a.Queue.Add(path) + } else if currentDepth < a.opts.MaxDepth { + err = a.walk(done, path, currentDepth+1) + if err != nil { + return err } } } diff --git a/index/index.go b/index/index.go new file mode 100644 index 0000000..1dff00b --- /dev/null +++ b/index/index.go @@ -0,0 +1,88 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package index + +import ( + "errors" + "path/filepath" + "strings" + "time" + + "github.com/nlnwa/gowarc" + "github.com/rs/zerolog/log" +) + +type Indexer interface { + Index(string) error +} + +func NewIndexer(w RecordWriter, options ...Option) func(string) { + opts := new(Options) + for _, apply := range options { + apply(opts) + } + + indexer, ok := w.(Indexer) + + return func(path string) { + // file + dir := filepath.Dir(path) + base := filepath.Base(path) + if opts.isExcluded(dir) || !opts.filter(base) { + return + } + + // if writer implements Index interface we call its index interface + if ok { + if err := indexer.Index(path); err != nil { + if errors.Is(err, AlreadyIndexedError) { + log.Debug().Err(err).Msgf("%s", path) + } else if err != nil { + log.Error().Err(err).Msgf("%s", path) + } + return + } + } + + index(path, w, opts) + } +} + +func index(filename string, r RecordWriter, opts *Options) { + start := time.Now() + + filter := func(wr gowarc.WarcRecord, validation *gowarc.Validation) bool { + if !validation.Valid() { + log.Debug().Msg(validation.Error()) + } + // only index response and revisit records of type application/http + if wr.Type() == gowarc.Response || wr.Type() == gowarc.Revisit { + // of type application/http + if strings.HasPrefix(wr.WarcHeader().Get(gowarc.ContentType), gowarc.ApplicationHttp) { + return true + } + } + return false + } + + count, total, err := readFile(filename, r, filter, opts.warcRecordOption...) + if err != nil { + log.Error().Err(err).Msgf("Indexing failed: %s", filename) + } + + log.Info().Msgf("Indexed %5d of %5d records in %10v: %s\n", count, total, time.Since(start), filename) +} diff --git a/index/io.go b/index/io.go index 61b628e..d3992b6 100644 --- a/index/io.go +++ b/index/io.go @@ -21,17 +21,21 @@ import ( "fmt" "io" "path/filepath" - "strings" - "time" "github.com/nlnwa/gowarc" "github.com/rs/zerolog/log" ) -type recordFilter func(gowarc.WarcRecord) bool +type recordFilter func(gowarc.WarcRecord, *gowarc.Validation) bool + +type RecordWriter interface { + Write(Record) error +} + +// readFile reads, filters and writes records of a warc file to a record writer +func readFile(path string, writer RecordWriter, filter recordFilter, opts ...gowarc.WarcRecordOption) (int, int, error) { + filename := filepath.Base(path) -// readFile reads a file using the supplied config and writes with a IndexWriter. -func readFile(path string, writer recordWriter, filter recordFilter, opts ...gowarc.WarcRecordOption) (int, int, error) { wf, err := gowarc.NewWarcFileReader(path, 0, opts...) if err != nil { return 0, 0, err @@ -40,17 +44,8 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow _ = wf.Close() }() - filename := filepath.Base(path) var prevOffset int64 var prevWr gowarc.WarcRecord - // write record from the previous iteration because we need to calculate record length - write := func(offset int64) error { - r, err := newRecord(prevWr, filename, prevOffset, offset-prevOffset) - if err != nil { - return err - } - return writer.Write(r) - } count := 0 total := 0 @@ -58,8 +53,10 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow for { wr, offset, validation, err := wf.Next() if prevWr != nil { - if err := write(offset); err != nil { - log.Warn().Err(err).Msgf("Failed to index record: %s#%d", filename, prevOffset) + if r, err := newRecord(prevWr, filename, prevOffset, offset-prevOffset); err != nil { + log.Error().Err(err).Msgf("Failed to create index record %s#%d", filename, prevOffset) + } else if err = writer.Write(r); err != nil { + log.Error().Err(err).Msgf("Failed to index record: %s#%d", filename, prevOffset) } else { count++ } @@ -69,12 +66,9 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow break } if err != nil { - return count, total, fmt.Errorf("failed to get record number %d in %s at offset %d: %w", count, filename, offset, err) + return count, total, fmt.Errorf("failed to read record #%d at %s#%d: %w", total, filename, offset, err) } - if !validation.Valid() { - log.Warn().Err(validation).Str("filename", filename).Int64("offset", offset).Msgf("Invalid %s record: %s", wr.Type(), wr.RecordId()) - } - if filter(wr) { + if filter(wr, validation) { prevWr = wr } total++ @@ -82,27 +76,3 @@ func readFile(path string, writer recordWriter, filter recordFilter, opts ...gow } return count, total, err } - -type recordWriter interface { - Write(Record) error -} - -func warcRecordFilter(wr gowarc.WarcRecord) bool { - // only write response and revisit records - if wr.Type() == gowarc.Response || wr.Type() == gowarc.Revisit { - // of type application/http - if strings.HasPrefix(wr.WarcHeader().Get(gowarc.ContentType), gowarc.ApplicationHttp) { - return true - } - } - return false -} - -func ReadFile(fileName string, r recordWriter) error { - start := time.Now() - - count, total, err := readFile(fileName, r, warcRecordFilter) - - log.Info().Msgf("Indexed %5d of %5d records in %10v: %s\n", count, total, time.Since(start), fileName) - return err -} diff --git a/index/io_test.go b/index/io_test.go index 5984ed7..8da27e3 100644 --- a/index/io_test.go +++ b/index/io_test.go @@ -51,7 +51,7 @@ Content-Length: 0`) tests := []struct { format string - writer recordWriter + writer RecordWriter }{ { "cdxj", @@ -70,7 +70,7 @@ Content-Length: 0`) for _, tt := range tests { t.Run(tt.format, func(t *testing.T) { - _, _, err = readFile(filepath, tt.writer, func(gowarc.WarcRecord) bool { return true }) + _, _, err = readFile(filepath, tt.writer, func(gowarc.WarcRecord, *gowarc.Validation) bool { return true }) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/index/options.go b/index/options.go index 545723e..df30313 100644 --- a/index/options.go +++ b/index/options.go @@ -18,15 +18,18 @@ package index import ( "regexp" + + "github.com/nlnwa/gowarc" ) type Options struct { - Watch bool - Includes []*regexp.Regexp - Excludes []*regexp.Regexp - MaxDepth int + Includes []*regexp.Regexp + Excludes []*regexp.Regexp + warcRecordOption []gowarc.WarcRecordOption } +type Option func(*Options) + func (o *Options) filter(name string) bool { return o.isIncluded(name) && !o.isExcluded(name) } @@ -52,21 +55,6 @@ func (o *Options) isIncluded(name string) bool { return false } -func defaultOptions() *Options { - return &Options{ - Watch: false, - MaxDepth: 4, - } -} - -type Option func(*Options) - -func WithMaxDepth(depth int) Option { - return func(opts *Options) { - opts.MaxDepth = depth - } -} - func WithIncludes(res ...*regexp.Regexp) Option { return func(opts *Options) { opts.Includes = res @@ -78,9 +66,3 @@ func WithExcludes(res ...*regexp.Regexp) Option { opts.Excludes = res } } - -func WithWatch(watch bool) Option { - return func(opts *Options) { - opts.Watch = watch - } -} diff --git a/index/watcher.go b/index/watcher.go deleted file mode 100644 index a3355f7..0000000 --- a/index/watcher.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2021 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package index - -import ( - "os" - - "github.com/fsnotify/fsnotify" - "github.com/rs/zerolog/log" -) - -func newWatcher() (*watcher, error) { - w, err := fsnotify.NewWatcher() - if err != nil { - return nil, err - } - return &watcher{ - Watcher: w, - }, nil -} - -type watcher struct { - *fsnotify.Watcher -} - -func (w *watcher) Close() { - if w == nil { - return - } - _ = w.Watcher.Close() -} - -func (w *watcher) Add(dir string) error { - if w == nil { - return nil - } - return w.Watcher.Add(dir) -} - -func (w *watcher) Watch(perFileFn func(string)) { - for { - select { - case event, ok := <-w.Events: - if !ok { - return - } - if event.Op&fsnotify.Write == fsnotify.Write { - perFileFn(event.Name) - } else if event.Op&fsnotify.Create == fsnotify.Create { - if info, err := os.Stat(event.Name); err != nil { - log.Warn().Msgf("Watcher failed to stat new file event: %v: %v", event.Name, err) - continue - } else if !info.Mode().IsDir() { - continue - } - - if err := w.Add(event.Name); err != nil { - log.Warn().Msgf("Watcher failed to add new directory: %s, %v", event.Name, err) - } - } - case err, ok := <-w.Errors: - if !ok { - return - } - log.Warn().Msgf("Watcher error: %v", err) - } - } -} diff --git a/index/worker.go b/index/worker.go index 12b9260..9d502de 100644 --- a/index/worker.go +++ b/index/worker.go @@ -17,49 +17,31 @@ package index import ( - "errors" "sync" - "time" - - "github.com/rs/zerolog/log" ) -type indexWorker struct { - jobs chan string - done chan struct{} - jobMap map[string]*time.Timer - mx sync.Mutex - wg sync.WaitGroup +type WorkQueue struct { + queue chan string + done chan struct{} + wg sync.WaitGroup } -// Indexer is the interface that wraps the Index method -type Indexer interface { - Index(string) error -} +type Worker func(string) -func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { - iw := &indexWorker{ - jobs: make(chan string, nrOfWorkers), - done: make(chan struct{}), - jobMap: map[string]*time.Timer{}, +func NewWorkQueue(execute Worker, nrOfWorkers int) *WorkQueue { + iw := &WorkQueue{ + queue: make(chan string, nrOfWorkers), + done: make(chan struct{}), } for i := 0; i < nrOfWorkers; i++ { + iw.wg.Add(1) go func() { + defer iw.wg.Done() for { select { - case job := <-iw.jobs: - if err := a.Index(job); err != nil { - if errors.Is(err, AlreadyIndexedError) { - log.Debug().Err(err).Msg(job) - } else { - log.Warn().Err(err).Msgf(job) - } - } - iw.wg.Done() - iw.mx.Lock() - delete(iw.jobMap, job) - iw.mx.Unlock() + case job := <-iw.queue: + execute(job) case <-iw.done: return } @@ -70,32 +52,14 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker { return iw } -func (iw *indexWorker) Close() { - // Wait for all queued jobs to complete - iw.wg.Wait() - // before closing workers. +func (iw *WorkQueue) Close() { + // stop workers close(iw.done) + // and wait for them to complete + iw.wg.Wait() } -// Schedule schedules a job to be processed by a worker -func (iw *indexWorker) Schedule(job string, batchWindow time.Duration) { - if batchWindow == 0 { - iw.wg.Add(1) - iw.jobs <- job - return - } - - iw.mx.Lock() - defer iw.mx.Unlock() - timer, ok := iw.jobMap[job] - - if ok { - timer.Stop() - timer.Reset(batchWindow) - } else { - iw.wg.Add(1) - iw.jobMap[job] = time.AfterFunc(batchWindow, func() { - iw.jobs <- job - }) - } +// Add job to work queue +func (iw *WorkQueue) Add(job string) { + iw.queue <- job } diff --git a/index/writers.go b/index/writers.go index 9bfecad..c2733db 100644 --- a/index/writers.go +++ b/index/writers.go @@ -18,7 +18,6 @@ package index import ( "fmt" - "strconv" "sync" "github.com/bits-and-blooms/bloom/v3" @@ -27,8 +26,7 @@ import ( "google.golang.org/protobuf/proto" ) -type Cdx struct { -} +type Cdx struct{} func (c Cdx) Write(rec Record) error { cdxj := protojson.Format(rec) @@ -37,8 +35,7 @@ func (c Cdx) Write(rec Record) error { return nil } -type CdxJ struct { -} +type CdxJ struct{} func (c CdxJ) Write(rec Record) error { cdxj := protojson.Format(rec) @@ -47,12 +44,7 @@ func (c CdxJ) Write(rec Record) error { return nil } -func (c CdxJ) Index(fileName string) error { - return ReadFile(fileName, c) -} - -type CdxPb struct { -} +type CdxPb struct{} func (c CdxPb) Write(rec Record) error { cdxpb, err := proto.Marshal(rec) @@ -64,10 +56,6 @@ func (c CdxPb) Write(rec Record) error { return nil } -func (c CdxPb) Index(fileName string) error { - return ReadFile(fileName, c) -} - type Toc struct { m sync.Mutex *bloom.BloomFilter @@ -79,21 +67,14 @@ func (t *Toc) Write(rec Record) error { if err != nil { return nil } - ts := rec.GetSts().AsTime() - year := strconv.Itoa(ts.Year()) - key := surthost + " " + year t.m.Lock() - hasSurt := t.BloomFilter.TestOrAddString(key) + hasSurt := t.BloomFilter.TestOrAddString(surthost) t.m.Unlock() if !hasSurt { - fmt.Println(key) + fmt.Println(surthost) } return nil } - -func (t *Toc) Index(fileName string) error { - return ReadFile(fileName, t) -} diff --git a/internal/badgeridx/compression.go b/internal/badgeridx/compression.go index e40b7dc..99d9290 100644 --- a/internal/badgeridx/compression.go +++ b/internal/badgeridx/compression.go @@ -18,10 +18,11 @@ package badgeridx import ( "fmt" - "github.com/mitchellh/mapstructure" "reflect" "strings" + "github.com/mitchellh/mapstructure" + "github.com/dgraph-io/badger/v3/options" ) diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go index 011381f..9c33856 100644 --- a/internal/badgeridx/db.go +++ b/internal/badgeridx/db.go @@ -26,7 +26,6 @@ import ( "time" "github.com/dgraph-io/badger/v3" - "github.com/dgraph-io/badger/v3/options" "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" "github.com/nlnwa/gowarcserver/timestamp" @@ -52,7 +51,7 @@ type DB struct { wg sync.WaitGroup } -func NewDB(options ...DbOption) (db *DB, err error) { +func NewDB(options ...Option) (db *DB, err error) { opts := defaultDbOptions() for _, opt := range options { opt(opts) @@ -83,7 +82,7 @@ func NewDB(options ...DbOption) (db *DB, err error) { done: done, } - // If read-only return. We don't need to run batch and gc workers when read-only. + // We don't need to run batch and gc workers when operating in read-only mode. if opts.ReadOnly { return } @@ -154,15 +153,15 @@ func (db *DB) Close() { } // addFile checks if file is indexed or has not changed since indexing, and adds file to file index. -func (db *DB) addFile(filePath string) error { - stat, err := os.Stat(filePath) +func (db *DB) addFile(path string) error { + stat, err := os.Stat(path) if err != nil { - return fmt.Errorf("failed to get file info: %s: %w", filePath, err) + return fmt.Errorf("failed to get file info: %s: %w", path, err) } fileSize := stat.Size() fileLastModified := stat.ModTime() - fn := filepath.Base(filePath) + fn := filepath.Base(path) if fileInfo, err := db.getFileInfo(fn); err == nil { if err := fileInfo.GetLastModified().CheckValid(); err != nil { return err @@ -173,14 +172,14 @@ func (db *DB) addFile(filePath string) error { } } - return db.updateFilePath(filePath) + return db.updateFilePath(path) } -func (db *DB) updateFilePath(filePath string) error { +func (db *DB) updateFilePath(path string) error { var err error fileInfo := &schema.Fileinfo{} - fileInfo.Path, err = filepath.Abs(filePath) + fileInfo.Path, err = filepath.Abs(path) if err != nil { return err } @@ -281,12 +280,8 @@ func (db *DB) Write(rec index.Record) error { return nil } -func (db *DB) Index(fileName string) error { - err := db.addFile(fileName) - if err != nil { - return err - } - return index.ReadFile(fileName, db) +func (db *DB) Index(path string) error { + return db.addFile(path) } // Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found. @@ -383,67 +378,3 @@ func (db *DB) GetCdx(key string) (*schema.Cdx, error) { }) return val, err } - -func defaultDbOptions() *dbOptions { - return &dbOptions{ - Compression: options.Snappy, - BatchMaxSize: 10000, - BatchMaxWait: 5 * time.Second, - GcInterval: 15 * time.Second, - Path: ".", - } -} - -type dbOptions struct { - Compression options.CompressionType - BatchMaxSize int - BatchMaxWait time.Duration - GcInterval time.Duration - Path string - ReadOnly bool - Database string -} - -type DbOption func(opts *dbOptions) - -func WithCompression(c options.CompressionType) DbOption { - return func(opts *dbOptions) { - opts.Compression = c - } -} - -func WithDir(d string) DbOption { - return func(opts *dbOptions) { - opts.Path = d - } -} - -func WithBatchMaxSize(size int) DbOption { - return func(opts *dbOptions) { - opts.BatchMaxSize = size - } -} - -func WithBatchMaxWait(t time.Duration) DbOption { - return func(opts *dbOptions) { - opts.BatchMaxWait = t - } -} - -func WithGcInterval(t time.Duration) DbOption { - return func(opts *dbOptions) { - opts.GcInterval = t - } -} - -func WithDatabase(db string) DbOption { - return func(opts *dbOptions) { - opts.Database = db - } -} - -func WithReadOnly(readOnly bool) DbOption { - return func(opts *dbOptions) { - opts.ReadOnly = readOnly - } -} diff --git a/internal/badgeridx/options.go b/internal/badgeridx/options.go new file mode 100644 index 0000000..f89935f --- /dev/null +++ b/internal/badgeridx/options.go @@ -0,0 +1,94 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badgeridx + +import ( + badgerOptions "github.com/dgraph-io/badger/v3/options" + "github.com/nlnwa/gowarcserver/index" + "time" +) + +func defaultDbOptions() *Options { + return &Options{ + Compression: badgerOptions.Snappy, + BatchMaxSize: 10000, + BatchMaxWait: 5 * time.Second, + GcInterval: 15 * time.Second, + Path: ".", + } +} + +type Options struct { + Compression badgerOptions.CompressionType + BatchMaxSize int + BatchMaxWait time.Duration + GcInterval time.Duration + Path string + ReadOnly bool + Database string + Index index.Indexer +} + +type Option func(opts *Options) + +func WithCompression(c badgerOptions.CompressionType) Option { + return func(opts *Options) { + opts.Compression = c + } +} + +func WithDir(d string) Option { + return func(opts *Options) { + opts.Path = d + } +} + +func WithBatchMaxSize(size int) Option { + return func(opts *Options) { + opts.BatchMaxSize = size + } +} + +func WithBatchMaxWait(t time.Duration) Option { + return func(opts *Options) { + opts.BatchMaxWait = t + } +} + +func WithGcInterval(t time.Duration) Option { + return func(opts *Options) { + opts.GcInterval = t + } +} + +func WithDatabase(db string) Option { + return func(opts *Options) { + opts.Database = db + } +} + +func WithReadOnly(readOnly bool) Option { + return func(opts *Options) { + opts.ReadOnly = readOnly + } +} + +func WithIndexer(indexer index.Indexer) Option { + return func(opts *Options) { + opts.Index = indexer + } +} diff --git a/internal/logger/initlog.go b/internal/logger/initlog.go index cbe7c41..adbf7bc 100644 --- a/internal/logger/initlog.go +++ b/internal/logger/initlog.go @@ -17,12 +17,13 @@ package logger import ( - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" stdlog "log" "os" "strings" "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) func InitLog(level string, format string, logCaller bool) { diff --git a/internal/tikvidx/api.go b/internal/tikvidx/api.go index 448d89b..64379d2 100644 --- a/internal/tikvidx/api.go +++ b/internal/tikvidx/api.go @@ -1,9 +1,29 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx import ( "context" + "fmt" "strings" + "github.com/rs/zerolog/log" + "github.com/tikv/client-go/v2/rawkv" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/schema" "google.golang.org/protobuf/proto" @@ -11,36 +31,27 @@ import ( // Closest returns the first closest cdx value(s). func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err - } - snapshot := db.client.GetSnapshot(ts) - it, err := NewIterClosest(ctx, snapshot, req.Key(), req.Closest()) + _, values, err := scanClosest(ctx, db.client, req.Key(), req.Closest()) if err != nil { return err } go func() { defer close(res) - defer it.Close() - for it.Valid() { - select { - case <-ctx.Done(): - return - default: - } - - cdx, err := cdxFromValue(it.Value()) + for _, v := range values { + var cdxResponse index.CdxResponse + cdx := new(schema.Cdx) + err := proto.Unmarshal(v, cdx) if err != nil { - res <- index.CdxResponse{Error: err} + cdxResponse = index.CdxResponse{Error: err} } else { - res <- index.CdxResponse{Cdx: cdx} + cdxResponse = index.CdxResponse{Cdx: cdx} } - - if err := it.Next(); err != nil { - res <- index.CdxResponse{Error: err} + select { + case <-ctx.Done(): + return + case res <- cdxResponse: } } }() @@ -48,22 +59,8 @@ func (db *DB) Closest(ctx context.Context, req index.ClosestRequest, res chan<- return nil } -func cdxFromValue(value []byte) (*schema.Cdx, error) { - result := new(schema.Cdx) - if err := proto.Unmarshal(value, result); err != nil { - return nil, err - } - return result, nil -} - func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err - } - snapshot := db.client.GetSnapshot(ts) - - it, err := newIter(ctx, snapshot, req) + it, err := newIter(ctx, db.client, req) if err != nil { return err } @@ -71,6 +68,7 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in close(res) return nil } + go func() { defer close(res) defer it.Close() @@ -80,10 +78,8 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in for it.Valid() && limit > 0 { select { case <-ctx.Done(): - res <- index.CdxResponse{Error: ctx.Err()} return default: - limit-- } func() { @@ -114,41 +110,29 @@ func (db *DB) Search(ctx context.Context, req index.SearchRequest, res chan<- in } func (db *DB) List(ctx context.Context, limit int, res chan<- index.CdxResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(cdxPrefix), []byte(cdxEOF)) + _, values, err := db.client.Scan(ctx, []byte(cdxPrefix), []byte(cdxEOF), limit) if err != nil { return err } go func() { - defer it.Close() defer close(res) - - for it.Valid() && limit > 0 { + for _, v := range values { select { case <-ctx.Done(): - res <- index.CdxResponse{Error: ctx.Err()} return default: } - limit-- cdx := new(schema.Cdx) - err := proto.Unmarshal(it.Value(), cdx) + err := proto.Unmarshal(v, cdx) if err != nil { res <- index.CdxResponse{Error: err} } else { res <- index.CdxResponse{Cdx: cdx} } - err = it.Next() - if err != nil { - res <- index.CdxResponse{Error: err} - break - } } }() @@ -159,82 +143,63 @@ func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.Fileinfo, return db.getFileInfo(filename) } -func (db *DB) ListFileInfo(_ context.Context, limit int, res chan<- index.FileResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err +func (db *DB) ListFileInfo(ctx context.Context, limit int, res chan<- index.FileResponse) error { + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(filePrefix), []byte(fileEOF)) + _, values, err := db.client.Scan(ctx, []byte(filePrefix), []byte(fileEOF), limit) if err != nil { return err } - go func() { - defer it.Close() // close iterator defer close(res) // close response channel + for _, v := range values { + select { + case <-ctx.Done(): + return + default: + } - for it.Valid() && limit > 0 { - limit-- fileInfo := new(schema.Fileinfo) - err := proto.Unmarshal(it.Value(), fileInfo) + err := proto.Unmarshal(v, fileInfo) if err != nil { res <- index.FileResponse{Error: err} } else { + log.Info().Msgf("GOOD: %s", fileInfo.Name) res <- index.FileResponse{Fileinfo: fileInfo} } - err = it.Next() - if err != nil { - res <- index.FileResponse{Error: err} - break - } } + log.Info().Msg("Hvordor kommer vi kkke hit") }() return nil } func (db *DB) GetStorageRef(ctx context.Context, id string) (string, error) { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return "", err - } - snapshot := db.client.GetSnapshot(ts) - - b, err := snapshot.Get(ctx, []byte(id)) + b, err := db.client.Get(ctx, []byte(id)) return string(b), err } -func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdResponse) error { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return err +func (db *DB) ListStorageRef(ctx context.Context, limit int, res chan<- index.IdResponse) error { + if limit > rawkv.MaxRawKVScanLimit { + limit = rawkv.MaxRawKVScanLimit } - snapshot := db.client.GetSnapshot(ts) - - it, err := snapshot.Iter([]byte(idPrefix), []byte(idEOF)) + keys, values, err := db.client.Scan(ctx, []byte(idPrefix), []byte(idEOF), limit) if err != nil { return err } go func() { - defer it.Close() defer close(res) - for it.Valid() && limit > 0 { - limit-- - if err != nil { - res <- index.IdResponse{Error: err} - } else { - k := strings.TrimPrefix(string(it.Key()), idPrefix) - res <- index.IdResponse{Key: k, Value: string(it.Value())} - } - err = it.Next() - if err != nil { - res <- index.IdResponse{Error: err} - break + for i, k := range keys { + select { + case <-ctx.Done(): + return + default: } + k := strings.TrimPrefix(string(k), idPrefix) + res <- index.IdResponse{Key: k, Value: string(values[i])} } }() @@ -245,15 +210,21 @@ func (db *DB) ListStorageRef(_ context.Context, limit int, res chan<- index.IdRe func (db *DB) Resolve(ctx context.Context, warcId string) (string, error) { key := []byte(idPrefix + warcId) - kv, err := db.get(ctx, key) + val, err := db.client.Get(ctx, key) if err != nil { return "", err } - return string(kv.V), nil + return string(val), nil } // ResolvePath looks up filename in file index and returns the path field. func (db *DB) ResolvePath(filename string) (filePath string, err error) { fileInfo, err := db.getFileInfo(filename) + if err != nil { + return "", err + } + if fileInfo == nil { + return "", fmt.Errorf("file not found: %s", filename) + } return fileInfo.Path, err } diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go index 320512c..09de0a8 100644 --- a/internal/tikvidx/db.go +++ b/internal/tikvidx/db.go @@ -1,5 +1,5 @@ /* - * Copyright 2021 National Library of Norway. + * Copyright 2022 National Library of Norway. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import ( "github.com/nlnwa/gowarcserver/schema" "github.com/nlnwa/gowarcserver/timestamp" "github.com/rs/zerolog/log" - "github.com/tikv/client-go/v2/txnkv" + "github.com/tikv/client-go/v2/rawkv" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -67,14 +67,12 @@ var ( ) type DB struct { - client *txnkv.Client + client *rawkv.Client batch chan index.Record done chan struct{} - opts *Options - wg sync.WaitGroup } @@ -84,7 +82,6 @@ func NewDB(options ...Option) (db *DB, err error) { opt(opts) } // prefix all keys with name of database - // TODO use keyspace when TiKV support it idPrefix = opts.Database + idPrefix idEOF = opts.Database + idEOF filePrefix = opts.Database + filePrefix @@ -92,7 +89,10 @@ func NewDB(options ...Option) (db *DB, err error) { cdxPrefix = opts.Database + cdxPrefix cdxEOF = opts.Database + cdxEOF - client, err := txnkv.NewClient(opts.PdAddr) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + client, err := rawkv.NewClientWithOpts(ctx, opts.PdAddr) if err != nil { return nil, err } @@ -102,7 +102,6 @@ func NewDB(options ...Option) (db *DB, err error) { db = &DB{ client: client, done: done, - opts: opts, } if opts.ReadOnly { @@ -135,6 +134,7 @@ func NewDB(options ...Option) (db *DB, err error) { func (db *DB) Close() { close(db.done) db.wg.Wait() + _ = db.client.Close() } // addFile checks if file referenced by filePath is indexed or has changed and adds/updates the index accordingly. @@ -147,7 +147,7 @@ func (db *DB) addFile(filePath string) error { fileSize := stat.Size() fileLastModified := stat.ModTime() fn := filepath.Base(filePath) - if fileInfo, err := db.getFileInfo(fn); err == nil { + if fileInfo, err := db.getFileInfo(fn); err == nil && fileInfo != nil { if err := fileInfo.GetLastModified().CheckValid(); err != nil { return err } @@ -187,69 +187,37 @@ func (db *DB) putFileInfo(fi *schema.Fileinfo) error { k := []byte(filePrefix + fi.Name) v, err := proto.Marshal(fi) if err != nil { + log.Error().Err(err).Msg("") + return err } - return db.puts(ctx, KV{K: k, V: v}) + err = db.client.Put(ctx, k, v) + if err != nil { + log.Error().Err(err).Msg("") + return err + } + return nil } func (db *DB) getFileInfo(fileName string) (*schema.Fileinfo, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() key := []byte(filePrefix + fileName) - val, err := db.get(ctx, key) + val, err := db.client.Get(ctx, key) if err != nil { return nil, err } + if val == nil { + return nil, nil + } fi := new(schema.Fileinfo) - err = proto.Unmarshal(val.V, fi) + err = proto.Unmarshal(val, fi) if err != nil { return nil, err } return fi, nil } -func (db *DB) get(ctx context.Context, k []byte) (*KV, error) { - ts, err := db.client.CurrentTimestamp("global") - if err != nil { - return nil, err - } - snapshot := db.client.GetSnapshot(ts) - - if err != nil { - return nil, err - } - v, err := snapshot.Get(ctx, k) - if err != nil { - return nil, err - } - return &KV{K: k, V: v}, nil -} - -func (db *DB) puts(ctx context.Context, kvs ...KV) error { - retries := db.opts.BatchMaxRetries - for { - tx, err := db.client.Begin() - if err != nil { - return err - } - for _, kv := range kvs { - err := tx.Set(kv.K, kv.V) - if err != nil { - return err - } - } - err = tx.Commit(ctx) - if err == nil { - return nil - } else if retries > 0 { - retries-- - log.Warn().Err(err).Msgf("Commit failed, will try again %d times", retries) - } else { - return err - } - } -} - // write schedules a Record to be added to the DB via the batch channel. func (db *DB) write(rec index.Record) { select { @@ -264,44 +232,39 @@ func (db *DB) write(rec index.Record) { } } -// collectBatch returns a slice of all the records in the batch channel. -func (db *DB) collectBatch() (records []index.Record) { +func (db *DB) collectBatch() ([][]byte, [][]byte) { + var keys [][]byte + var values [][]byte for { select { - case record := <-db.batch: - records = append(records, record) + case r := <-db.batch: + id := idKV(r) + cdx, err := cdxKV(r) + if err != nil { + log.Error().Err(err).Msgf("failed to marshal record: %v", r) + continue + } + keys = append(keys, id.K, cdx.K) + values = append(values, id.V, cdx.V) default: - return + return keys, values } } } // flushBatch collects all records in the batch channel and updates the id and cdx indices. func (db *DB) flushBatch() { - records := db.collectBatch() - if len(records) == 0 { + keys, values := db.collectBatch() + if len(keys) == 0 { return } - var kvs []KV - for _, r := range records { - id := idKV(r) - cdx, err := cdxKV(r) - if err != nil { - log.Error().Err(err).Msgf("failed to marshal record: %v", r) - continue - } - kvs = append(kvs, id, cdx) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - err := db.puts(ctx, kvs...) + err := db.client.BatchPut(ctx, keys, values) if err != nil { - for _, r := range records { - log.Warn().Err(err).Msgf("failed to commit record: %s", r.Ref) - } + log.Error().Err(err).Msgf("Batch put failed") } } @@ -329,10 +292,6 @@ func (db *DB) Write(rec index.Record) error { return nil } -func (db *DB) Index(fileName string) error { - err := db.addFile(fileName) - if err != nil { - return err - } - return index.ReadFile(fileName, db) +func (db *DB) Index(path string) error { + return db.addFile(path) } diff --git a/internal/tikvidx/iter.go b/internal/tikvidx/iter.go index f5f1d8d..8a5d116 100644 --- a/internal/tikvidx/iter.go +++ b/internal/tikvidx/iter.go @@ -1,16 +1,34 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx import ( - "bytes" "context" - "fmt" "time" + "github.com/nlnwa/gowarcserver/server/api" + "github.com/tikv/client-go/v2/rawkv" + "github.com/nlnwa/gowarcserver/index" "github.com/nlnwa/gowarcserver/timestamp" - "github.com/tikv/client-go/v2/txnkv" ) +type comparator func(KV, KV) bool + func CompareClosest(ts int64) func(int64, int64) bool { return func(ts1 int64, ts2 int64) bool { return timestamp.AbsInt64(ts-ts1) < timestamp.AbsInt64(ts-ts2) @@ -34,223 +52,207 @@ type iterator interface { Close() } -type iterClosest struct { - forward iterator - backward iterator - curr iterator - prefix []byte - key []byte - value []byte - valid bool - cmp func(int64, int64) bool +type closestScanner struct { + fKeys, bKeys [][]byte + fValues, bValues [][]byte + fIndex, bIndex int + cmp func(int64, int64) bool } -func NewIterClosest(_ context.Context, snapshot *txnkv.KVSnapshot, key string, closest string) (*iterClosest, error) { - ic := new(iterClosest) +const startDate = "19700101000000" +const endbyte = "\xff" + +func scanClosest(ctx context.Context, client *rawkv.Client, key string, closest string, options ...rawkv.RawOption) ([][]byte, [][]byte, error) { + ic := new(closestScanner) if t, err := time.Parse(timestamp.CDX, closest); err != nil { - return nil, err + return nil, nil, err } else { ic.cmp = CompareClosest(t.Unix()) } - ic.prefix = []byte(cdxPrefix + key) - k := []byte(cdxPrefix + key + " " + closest) + startKey := []byte(cdxPrefix + key + " " + closest) + limit := 10 + var err error - // initialize forward iterator - forward, err := snapshot.Iter(k, []byte(cdxEOF)) + ic.fKeys, ic.fValues, err = client.Scan(ctx, startKey, []byte(cdxPrefix+key+endbyte), limit, options...) if err != nil { - return nil, err + return nil, nil, err } - ic.forward = forward - // initialize backward iterator - backward, err := snapshot.IterReverse(k) + // scan backward + ic.bKeys, ic.bValues, err = client.ReverseScan(ctx, startKey, []byte(cdxPrefix+key+" "+startDate), limit, options...) if err != nil { - return nil, err + return nil, nil, err } - ic.backward = backward - - return ic, ic.next() -} -func (ic *iterClosest) Next() error { - err := ic.curr.Next() - if err != nil { - return err + var keys [][]byte + var values [][]byte + for { + k, v, valid := ic.next() + if !valid { + return keys, values, nil + } + keys = append(keys, k) + values = append(values, v) } - return ic.next() } -func (ic *iterClosest) next() error { +func (cs *closestScanner) next() ([]byte, []byte, bool) { var ft int64 var bt int64 // get forward ts - if ic.forward.Valid() && bytes.HasPrefix(ic.forward.Key(), ic.prefix) { - ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.forward.Key()).ts()) - ft = ts.Unix() + if len(cs.fKeys) > cs.fIndex { + fts, _ := time.Parse(timestamp.CDX, cdxKey(cs.fKeys[cs.fIndex]).ts()) + ft = fts.Unix() } + // get backward ts - if ic.backward.Valid() && bytes.HasPrefix(ic.backward.Key(), ic.prefix) { - ts, _ := time.Parse(timestamp.CDX, cdxKey(ic.backward.Key()).ts()) - bt = ts.Unix() + if len(cs.bKeys) > cs.bIndex { + bts, _ := time.Parse(timestamp.CDX, cdxKey(cs.bKeys[cs.bIndex]).ts()) + bt = bts.Unix() } - var it iterator + var itKeys [][]byte + var itValues [][]byte + var i *int if ft != 0 && bt != 0 { // find closest of forward and backward - isForward := ic.cmp(ft, bt) + isForward := cs.cmp(ft, bt) if isForward { - it = ic.forward + itKeys = cs.fKeys + i = &cs.fIndex } else { - it = ic.backward + itKeys = cs.bKeys + i = &cs.bIndex } } else if ft != 0 { - it = ic.forward + itKeys = cs.fKeys + i = &cs.fIndex } else if bt != 0 { - it = ic.backward + itKeys = cs.bKeys + i = &cs.bIndex } else { - ic.valid = false - return nil + return nil, nil, false } + key := itKeys[*i] + value := itValues[*i] + *i++ - ic.curr = it - ic.key = it.Key() - ic.value = it.Value() - ic.valid = true - - return nil -} - -func (ic *iterClosest) Key() []byte { - return ic.key + return key, value, true } -func (ic *iterClosest) Value() []byte { - return ic.value +type maybeKV struct { + kv KV + error error } -func (ic *iterClosest) Valid() bool { - return ic.valid +func getComparator(req index.SearchRequest) (comparator, error) { + switch req.Sort() { + case index.SortDesc: + return func(a KV, b KV) bool { + return CompareDesc(a.ts(), b.ts()) + }, nil + case index.SortAsc: + fallthrough + case index.SortNone: + fallthrough + case index.SortClosest: + fallthrough + default: + return func(a KV, b KV) bool { + return CompareAsc(a.ts(), b.ts()) + }, nil + } } -func (ic *iterClosest) Close() { - ic.forward.Close() - ic.backward.Close() -} +type scan func(context.Context, []byte, []byte, int, ...rawkv.RawOption) ([][]byte, [][]byte, error) -type maybeKV struct { - kv KV - error error +type iter struct { + key []byte + value []byte + valid bool + next <-chan maybeKV } -type iterSort struct { - iterators []iterator - key []byte - value []byte - valid bool - next <-chan maybeKV -} +func newIter(ctx context.Context, client *rawkv.Client, req index.SearchRequest) (iterator, error) { + cmp, err := getComparator(req) + if err != nil { + return nil, err + } -func newIter(ctx context.Context, tx *txnkv.KVSnapshot, req index.SearchRequest) (iterator, error) { - is := new(iterSort) - var prefixes [][]byte - var results []chan *maybeKV - var it iterator - var err error - for _, key := range req.Keys() { - k := []byte(cdxPrefix + key) - switch req.Sort() { - case index.SortAsc: - it, err = tx.Iter(k, []byte(cdxEOF)) - case index.SortClosest: - it, err = NewIterClosest(ctx, tx, key, req.Closest()) + getScanner := func(sort index.Sort) scan { + switch sort { case index.SortDesc: if len(req.Keys()) == 1 { - return tx.IterReverse(k) + return client.ReverseScan + } else { + return client.Scan } - it, err = tx.Iter(k, []byte(cdxEOF)) + case index.SortAsc: + fallthrough case index.SortNone: fallthrough + case index.SortClosest: + fallthrough default: - it, err = tx.Iter(k, []byte(cdxEOF)) - } - if err != nil { - break - } - if !it.Valid() { - continue + return client.Scan } - is.iterators = append(is.iterators, it) - prefixes = append(prefixes, k) - results = append(results, make(chan *maybeKV)) - } - if err != nil { - is.Close() - return nil, err - } - if len(is.iterators) == 0 { - return nil, nil } - if len(is.iterators) == 1 { - return is.iterators[0], nil + + makeStartKey := func(endKey []byte) []byte { + return append([]byte(api.MatchType(string(endKey), req.MatchType())), 0) } - // initialize comparator - var cmp func(a KV, b KV) bool - switch req.Sort() { - case index.SortDesc: - cmp = func(a KV, b KV) bool { - return CompareDesc(a.ts(), b.ts()) - } - case index.SortClosest: - var t time.Time - t, err = time.Parse(timestamp.CDX, req.Closest()) - if err != nil { - return nil, fmt.Errorf("failed to parse closest timestamp: %w", err) - } - cmp = func(a KV, b KV) bool { - return CompareClosest(t.Unix())(a.ts(), b.ts()) - } - case index.SortAsc: - fallthrough - case index.SortNone: - fallthrough - default: - cmp = func(a KV, b KV) bool { - return CompareAsc(a.ts(), b.ts()) - } + makeEndkey := func(startKey []byte) []byte { + endKey := make([]byte, len(startKey)+1) + copy(endKey, startKey) + endKey[len(endKey)-1] = 0xff + return endKey } - is.next = mergeIter(ctx.Done(), cmp, results...) + var results []chan *maybeKV + for i, key := range req.Keys() { + results = append(results, make(chan *maybeKV)) - for i, iter := range is.iterators { - i := i - go func(iter iterator, prefix []byte, ch chan<- *maybeKV) { + go func(scan scan, key []byte, ch chan<- *maybeKV, done <-chan struct{}) { defer close(ch) - for iter.Valid() && bytes.HasPrefix(iter.Key(), prefix) { - select { - case <-ctx.Done(): - return - case ch <- &maybeKV{kv: KV{K: iter.Key(), V: iter.Value()}}: - } - err := iter.Next() + scanLimit := 256 + startKey := key + endKey := makeEndkey(startKey) + for { + keys, values, err := scan(ctx, startKey, endKey, scanLimit) if err != nil { select { - case <-ctx.Done(): + case <-done: + return case ch <- &maybeKV{error: err}: + return + } + } + for j, k := range keys { + select { + case <-done: + return + case ch <- &maybeKV{kv: KV{K: k, V: values[j]}}: } + } + if len(keys) < scanLimit { return } + startKey = makeStartKey(keys[len(keys)-1]) } - }(iter, prefixes[i], results[i]) + }(getScanner(req.Sort()), []byte(cdxPrefix+key), results[i], ctx.Done()) } + is := new(iter) + is.next = mergeIter(ctx.Done(), cmp, results...) + return is, is.Next() } // Next updates the next key, value and validity. -func (is *iterSort) Next() error { +func (is *iter) Next() error { mkv, ok := <-is.next if !ok { is.valid = false @@ -267,24 +269,20 @@ func (is *iterSort) Next() error { return nil } -func (is *iterSort) Key() []byte { +func (is *iter) Key() []byte { return is.key } -func (is *iterSort) Value() []byte { +func (is *iter) Value() []byte { return is.value } -func (is *iterSort) Valid() bool { +func (is *iter) Valid() bool { return is.valid } -func (is *iterSort) Close() { - for _, it := range is.iterators { - if it != nil { - it.Close() - } - } +func (is *iter) Close() { + // noop } // mergeIter merges sorted input channels into a sorted output channel diff --git a/internal/tikvidx/options.go b/internal/tikvidx/options.go index 164f04e..a19b343 100644 --- a/internal/tikvidx/options.go +++ b/internal/tikvidx/options.go @@ -1,6 +1,24 @@ +/* + * Copyright 2022 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package tikvidx -import "time" +import ( + "time" +) func defaultOptions() *Options { return &Options{ diff --git a/main.go b/main.go index 4627334..be59448 100644 --- a/main.go +++ b/main.go @@ -1,18 +1,19 @@ /* -Copyright © 2019 National Library of Norway + * Copyright 2019 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ package main import ( diff --git a/server/coreserver/handler.go b/server/coreserver/handler.go index 4442f34..26bc3e0 100644 --- a/server/coreserver/handler.go +++ b/server/coreserver/handler.go @@ -54,7 +54,10 @@ func (h Handler) search(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() - log.Debug().Msgf("%v", coreAPI) + log.Debug().Msgf("%+v", coreAPI) + if coreAPI.Limit == 0 { + coreAPI.Limit = 100 + } if err = h.CdxAPI.Search(ctx, api.SearchAPI{CoreAPI: coreAPI}, response); err != nil { log.Error().Err(err).Msg("failed to search") http.Error(w, err.Error(), http.StatusInternalServerError)