Differentiate between already indexed errors and other errors

nlnwa · Nov 6, 2022 · 99885a2 · 99885a2
1 parent 0d4d845
commit 99885a2
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 14 deletions.
diff --git a/cmd/index/index.go b/cmd/index/index.go
@@ -48,7 +48,7 @@ func NewCommand() *cobra.Command {
 	format := "cdxj"
 	indexDepth := 4
 	indexWorkers := 8
-	badgerDir := "."
+	badgerDir := "./warcdb"
 	badgerBatchMaxSize := 1000
 	badgerBatchMaxWait := 5 * time.Second
 	badgerCompression := "snappy"
@@ -159,6 +159,7 @@ func indexCmd(_ *cobra.Command, args []string) error {
 		}
 	}
 
+	log.Info().Msg("Starting auto indexer")
 	indexer, err := index.NewAutoIndexer(indexWorker,
 		index.WithMaxDepth(viper.GetInt("max-depth")),
 		index.WithIncludes(includes...),

diff --git a/cmd/root.go b/cmd/root.go
@@ -85,5 +85,5 @@ func initConfig() {
 	}
 	logger.InitLog(viper.GetString("log-level"), viper.GetString("log-formatter"), viper.GetBool("log-method"))
 
-	log.Debug().Msgf("Using config file: %s", viper.ConfigFileUsed())
+	log.Info().Msgf("Using config file: %s", viper.ConfigFileUsed())
 }
diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go
@@ -61,7 +61,7 @@ func NewCommand() *cobra.Command {
 	autoIndex := true
 	indexDepth := 4
 	indexWorkers := 8
-	badgerDir := "."
+	badgerDir := "./warcdb"
 	badgerCompression := "snappy"
 	badgerDatabase := ""
 	badgerBatchMaxSize := 1000

diff --git a/index/api.go b/index/api.go
@@ -7,6 +7,14 @@ import (
 	"github.com/nlnwa/gowarcserver/schema"
 )
 
+type indexError string
+
+const AlreadyIndexedError indexError = "already indexed"
+
+func (a indexError) Error() string {
+	return string(a)
+}
+
 type DateRange interface {
 	ContainsTime(time.Time) (bool, error)
 	ContainsStr(string) (bool, error)

diff --git a/index/worker.go b/index/worker.go
@@ -17,6 +17,7 @@
 package index
 
 import (
+	"errors"
 	"sync"
 	"time"
 
@@ -49,7 +50,11 @@ func NewWorker(a Indexer, nrOfWorkers int) *indexWorker {
 				select {
 				case job := <-iw.jobs:
 					if err := a.Index(job); err != nil {
-						log.Warn().Err(err).Msgf("Index: %s", job)
+						if errors.Is(err, AlreadyIndexedError) {
+							log.Debug().Err(err).Msg(job)
+						} else {
+							log.Warn().Err(err).Msgf(job)
+						}
 					}
 					iw.wg.Done()
 					iw.mx.Lock()

diff --git a/internal/badgeridx/badger.go b/internal/badgeridx/badger.go
@@ -38,11 +38,11 @@ func (l logger) Warningf(fmt string, args ...interface{}) {
 }
 
 func (l logger) Infof(fmt string, args ...interface{}) {
-	log.Debug().Msgf(l.prefix+fmt, args...)
+	log.Trace().Msgf(l.prefix+fmt, args...)
 }
 
 func (l logger) Debugf(fmt string, args ...interface{}) {
-	log.Debug().Msgf(l.prefix+fmt, args...)
+	log.Trace().Msgf(l.prefix+fmt, args...)
 }
 
 func newBadgerDB(dir string, compression options.CompressionType, readOnly bool) (*badger.DB, error) {

diff --git a/internal/badgeridx/db.go b/internal/badgeridx/db.go
@@ -18,7 +18,6 @@ package badgeridx
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -63,17 +62,16 @@ func NewDB(options ...DbOption) (db *DB, err error) {
 	var fileIndex *badger.DB
 	var cdxIndex *badger.DB
 
-	dir := path.Join(opts.Path, "warcdb")
 	batch := make(chan index.Record, opts.BatchMaxSize)
 	done := make(chan struct{})
 
-	if idIndex, err = newBadgerDB(path.Join(dir, opts.Database, "id-index"), opts.Compression, opts.ReadOnly); err != nil {
+	if idIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "id-index"), opts.Compression, opts.ReadOnly); err != nil {
 		return
 	}
-	if fileIndex, err = newBadgerDB(path.Join(dir, opts.Database, "file-index"), opts.Compression, opts.ReadOnly); err != nil {
+	if fileIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "file-index"), opts.Compression, opts.ReadOnly); err != nil {
 		return
 	}
-	if cdxIndex, err = newBadgerDB(path.Join(dir, opts.Database, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil {
+	if cdxIndex, err = newBadgerDB(path.Join(opts.Path, opts.Database, "cdx-index"), opts.Compression, opts.ReadOnly); err != nil {
 		return
 	}
 
@@ -171,7 +169,7 @@ func (db *DB) addFile(filePath string) error {
 		}
 		fileInfoLastModified := fileInfo.LastModified.AsTime()
 		if fileInfo.Size == fileSize && fileInfoLastModified.Equal(fileLastModified) {
-			return errors.New("already indexed")
+			return index.AlreadyIndexedError
 		}
 	}
 

diff --git a/internal/tikvidx/db.go b/internal/tikvidx/db.go
@@ -19,7 +19,6 @@ package tikvidx
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -152,7 +151,7 @@ func (db *DB) addFile(filePath string) error {
 		}
 		fileInfoLastModified := fileInfo.LastModified.AsTime()
 		if fileInfo.Size == fileSize && fileInfoLastModified.Equal(fileLastModified) {
-			return errors.New("already indexed")
+			return index.AlreadyIndexedError
 		}
 	}