Allow empty package paths on startup

categories_test.go

@@ -38,7 +38,11 @@ func TestCategoriesWithProxyMode(t *testing.T) {
 	}))
 	defer webServer.Close()
 
-	indexerProxy := packages.NewFileSystemIndexer(testLogger, "./testdata/second_package_path")
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
+	indexerProxy := packages.NewFileSystemIndexer(fsOpts, "./testdata/second_package_path")
 	defer indexerProxy.Close(t.Context())
 
 	err := indexerProxy.Init(t.Context())

go.mod

@@ -52,6 +52,7 @@ require (
 	github.com/elastic/go-windows v1.0.2 // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
+	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-jose/go-jose/v4 v4.1.2 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -77,6 +77,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fsouza/fake-gcs-server v1.52.3 h1:hXddOPMGDKq5ENmttw6xkodVJy0uVhf7HhWvQgAOH6g=
 github.com/fsouza/fake-gcs-server v1.52.3/go.mod h1:A0XtSRX+zz5pLRAt88j9+Of0omQQW+RMqipFbvdNclQ=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=

main.go

@@ -84,6 +84,8 @@ var (
 	proxyTo          string
 	serviceName      = getServiceName()
 
+	packagePathsEnableWatcher = false
+
 	defaultConfig = Config{
 		CacheTimeIndex:               10 * time.Second,
 		CacheTimeSearch:              10 * time.Minute,
@@ -128,6 +130,8 @@ func init() {
 	// The following proxy-indexer related flags are technical preview and might be removed in the future or renamed
 	flag.BoolVar(&featureProxyMode, "feature-proxy-mode", false, "Enable proxy mode to include packages from other endpoint (technical preview).")
 	flag.StringVar(&proxyTo, "proxy-to", "https://epr.elastic.co/", "Proxy-to endpoint")
+
+	flag.BoolVar(&packagePathsEnableWatcher, "package-paths-enable-watcher", false, "Enable file system watcher for package paths to automatically detect new packages.")
 }
 
 type Config struct {
@@ -138,9 +142,9 @@ type Config struct {
 	CacheTimeCatchAll            time.Duration `config:"cache_time.catch_all"`
 	SQLIndexerDatabaseFolderPath string        `config:"sql_indexer.database_folder_path"` // technical preview, used by the SQL storage indexer
 	SearchCacheSize              int           `config:"search.cache_size"`                // technical preview, used by the SQL storage indexer
-	SearchCacheTTL               time.Duration `config:"search.cache_ttl"`                 // technical preview, used by the SQL storage indexe^
+	SearchCacheTTL               time.Duration `config:"search.cache_ttl"`                 // technical preview, used by the SQL storage indexer
 	CategoriesCacheSize          int           `config:"categories.cache_size"`            // technical preview, used by the SQL storage indexer
-	CategoriesCacheTTL           time.Duration `config:"categories.cache_ttl"`             // technical preview, used by the SQL storage indexe^
+	CategoriesCacheTTL           time.Duration `config:"categories.cache_ttl"`             // technical preview, used by the SQL storage indexer
 }
 
 func main() {
@@ -179,6 +183,10 @@ func main() {
 
 	config := mustLoadConfig(logger)
 
+	if len(config.PackagePaths) == 0 && packagePathsEnableWatcher {
+		logger.Fatal("no package paths configured, cannot enable watcher")
+	}
+
 	options := serverOptions{
 		apmTracer: apmTracer,
 		config:    config,
@@ -388,9 +396,15 @@ func initIndexer(ctx context.Context, logger *zap.Logger, options serverOptions)
 		combined = append(combined, indexer)
 	}
 
+	fsOptions := packages.FSIndexerOptions{
+		Logger:             logger,
+		EnablePathsWatcher: packagePathsEnableWatcher,
+		APMTracer:          options.apmTracer,
+	}
+
 	combined = append(combined,
-		packages.NewZipFileSystemIndexer(logger, packagesBasePaths...),
-		packages.NewFileSystemIndexer(logger, packagesBasePaths...),
+		packages.NewZipFileSystemIndexer(fsOptions, packagesBasePaths...),
+		packages.NewFileSystemIndexer(fsOptions, packagesBasePaths...),
 	)
 	ensurePackagesAvailable(ctx, logger, combined)
 	return combined
@@ -574,6 +588,7 @@ func getConfig(logger *zap.Logger) (*Config, error) {
 		}
 		config.CategoriesCacheTTL = cacheTTL
 	}
+
 	return &config, nil
 }
 
@@ -619,6 +634,8 @@ func ensurePackagesAvailable(ctx context.Context, logger *zap.Logger, indexer In
 		logger.Info(fmt.Sprintf("%v local package manifests loaded.", len(packages)))
 	} else if featureProxyMode {
 		logger.Info("No local packages found, but the proxy mode can access remote ones.")
+	} else if packagePathsEnableWatcher {
+		logger.Warn("No packages found at startup. The registry is running but no content is available yet.")
 	} else {
 		logger.Fatal("No local packages found.")
 	}

main_test.go

@@ -60,11 +60,14 @@ func TestRouter(t *testing.T) {
 
 func TestEndpoints(t *testing.T) {
 	t.Parallel()
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
 
 	packagesBasePaths := []string{"./testdata/second_package_path", "./testdata/package"}
 	indexer := NewCombinedIndexer(
-		packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
-		packages.NewFileSystemIndexer(testLogger, packagesBasePaths...),
+		packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage"),
+		packages.NewFileSystemIndexer(fsOpts, packagesBasePaths...),
 	)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
@@ -171,8 +174,12 @@ func TestEndpoints(t *testing.T) {
 func TestArtifacts(t *testing.T) {
 	t.Parallel()
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
 	packagesBasePaths := []string{"./testdata/package"}
-	indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
+	indexer := packages.NewFileSystemIndexer(fsOpts, packagesBasePaths...)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -203,7 +210,10 @@ func TestArtifacts(t *testing.T) {
 func TestSignatures(t *testing.T) {
 	t.Parallel()
 
-	indexer := packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage")
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+	indexer := packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage")
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -232,8 +242,11 @@ func TestSignatures(t *testing.T) {
 func TestStatics(t *testing.T) {
 	t.Parallel()
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
 	packagesBasePaths := []string{"./testdata/package"}
-	indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
+	indexer := packages.NewFileSystemIndexer(fsOpts, packagesBasePaths...)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -324,9 +337,13 @@ func TestStaticsModifiedTime(t *testing.T) {
 		},
 	}
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
 	indexer := NewCombinedIndexer(
-		packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
-		packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
+		packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage"),
+		packages.NewFileSystemIndexer(fsOpts, "./testdata/package"),
 	)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
@@ -361,7 +378,11 @@ func TestStaticsModifiedTime(t *testing.T) {
 func TestZippedArtifacts(t *testing.T) {
 	t.Parallel()
 
-	indexer := packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage")
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
+	indexer := packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage")
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -395,10 +416,12 @@ func TestZippedArtifacts(t *testing.T) {
 
 func TestPackageIndex(t *testing.T) {
 	t.Parallel()
-
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
 	indexer := NewCombinedIndexer(
-		packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
-		packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
+		packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage"),
+		packages.NewFileSystemIndexer(fsOpts, "./testdata/package"),
 	)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
@@ -435,8 +458,11 @@ func TestPackageIndex(t *testing.T) {
 func TestZippedPackageIndex(t *testing.T) {
 	t.Parallel()
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
 	packagesBasePaths := []string{"./testdata/local-storage"}
-	indexer := packages.NewZipFileSystemIndexer(testLogger, packagesBasePaths...)
+	indexer := packages.NewZipFileSystemIndexer(fsOpts, packagesBasePaths...)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -468,10 +494,14 @@ func TestZippedPackageIndex(t *testing.T) {
 func TestAllPackageIndex(t *testing.T) {
 	t.Parallel()
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
 	testPackagePath := filepath.Join("testdata", "package")
 	secondPackagePath := filepath.Join("testdata", "second_package_path")
 	packagesBasePaths := []string{secondPackagePath, testPackagePath}
-	indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
+	indexer := packages.NewFileSystemIndexer(fsOpts, packagesBasePaths...)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
 	err := indexer.Init(t.Context())
@@ -528,9 +558,13 @@ func TestContentTypes(t *testing.T) {
 		{"/package/example/1.0.1/img/kibana-envoyproxy.jpg", "image/jpeg"},
 	}
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
 	indexer := NewCombinedIndexer(
-		packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
-		packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
+		packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage"),
+		packages.NewFileSystemIndexer(fsOpts, "./testdata/package"),
 	)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 
@@ -562,9 +596,13 @@ func TestContentTypes(t *testing.T) {
 func TestRangeDownloads(t *testing.T) {
 	t.Parallel()
 
+	fsOpts := packages.FSIndexerOptions{
+		Logger: testLogger,
+	}
+
 	indexer := NewCombinedIndexer(
-		packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
-		packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
+		packages.NewZipFileSystemIndexer(fsOpts, "./testdata/local-storage"),
+		packages.NewFileSystemIndexer(fsOpts, "./testdata/package"),
 	)
 	t.Cleanup(func() { indexer.Close(context.Background()) })
 

packages/marshaler_test.go

@@ -23,7 +23,10 @@ var generateFlag = flag.Bool("generate", false, "Write golden files")
 func TestMarshalJSON(t *testing.T) {
 	// given
 	packagesBasePaths := []string{"../testdata/second_package_path", "../testdata/package"}
-	indexer := NewFileSystemIndexer(util.NewTestLogger(), packagesBasePaths...)
+	fsOpts := FSIndexerOptions{
+		Logger: util.NewTestLogger(),
+	}
+	indexer := NewFileSystemIndexer(fsOpts, packagesBasePaths...)
 	defer indexer.Close(t.Context())
 
 	err := indexer.Init(t.Context())
@@ -39,8 +42,12 @@ func TestMarshalJSON(t *testing.T) {
 
 func TestUnmarshalJSON(t *testing.T) {
 	// given
+
+	fsOpts := FSIndexerOptions{
+		Logger: util.NewTestLogger(),
+	}
 	packagesBasePaths := []string{"../testdata/second_package_path", "../testdata/package"}
-	indexer := NewFileSystemIndexer(util.NewTestLogger(), packagesBasePaths...)
+	indexer := NewFileSystemIndexer(fsOpts, packagesBasePaths...)
 	defer indexer.Close(t.Context())
 
 	err := indexer.Init(t.Context())