Skip to content

Allow empty package paths on startup #1482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
teresaromero merged 32 commits into from
Nov 27, 2025
Merged

Allow empty package paths on startup #1482

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
597e822
refactor: Log warn when no package found
teresaromero Oct 7, 2025
160ee72
Merge branch 'main' into 1351-empty-registry
teresaromero Nov 17, 2025
ee5afd8
Add flag to allow empty package paths on startup
teresaromero Nov 17, 2025
60a967d
Update indexer constructors to include watch interval parameter
teresaromero Nov 17, 2025
e4cb9b1
Update ensurePackagesAvailable function to accept serverOptions param…
teresaromero Nov 17, 2025
9844fc5
Add watchInterval parameter to FileSystemIndexer constructors
teresaromero Nov 17, 2025
624a848
Add packagePathsWatchInterval flag to control watch interval for pack…
teresaromero Nov 17, 2025
39e6423
lock read and write packages on fs watcher
teresaromero Nov 18, 2025
b65fd7a
Change log level from Info to Debug for package indexer watcher messages
teresaromero Nov 18, 2025
1ca31cf
Validate package paths configuration and log fatal error if none are set
teresaromero Nov 18, 2025
08e5d81
Use fsnotify for path watch
teresaromero Nov 18, 2025
ffa5cf5
Revert "Validate package paths configuration and log fatal error if n…
teresaromero Nov 18, 2025
165038d
Log fatal error if no package paths are configured when enabling watcher
teresaromero Nov 18, 2025
9fae69c
Fix typo in comment for SearchCacheTTL in Config struct
teresaromero Nov 18, 2025
95f239f
refactor debouncer at watcher and filter events by indexer
teresaromero Nov 21, 2025
c8fb821
Merge branch 'main' of github.com:elastic/package-registry into 1351-…
teresaromero Nov 21, 2025
9a5b55d
Add APM tracer support to FileSystemIndexer and ZipFileSystemIndexer
teresaromero Nov 21, 2025
dba6d8a
Update indexer initialization to use nil for the second parameter in …
teresaromero Nov 21, 2025
bcb985c
remove redundant log
teresaromero Nov 24, 2025
17c9f70
Refactor FileSystemIndexer initialization to use FSIndexerOptions struct
teresaromero Nov 24, 2025
f20c036
Add tests for FileSystemIndexer and ZipFileSystemIndexer functionality
teresaromero Nov 24, 2025
0231b20
Enhance event filtering in FileSystemIndexer to watch only relevant f…
teresaromero Nov 24, 2025
1afb4bc
Add TODO comment for issue #1488 in FileSystemIndexer watchPackageFil…
teresaromero Nov 24, 2025
a8b8a63
remove defer file close at createMockZipPackage
teresaromero Nov 25, 2025
6597ea9
Add missing file close in createMockZipPackage function
teresaromero Nov 25, 2025
b1f30cc
Improve Windows compatibility in FileSystemIndexer tests by adding de…
teresaromero Nov 25, 2025
f4b85c0
Revert "Improve Windows compatibility in FileSystemIndexer tests by a…
teresaromero Nov 25, 2025
7bb4530
Fix file closing in createMockZipPackage function to ensure proper re…
teresaromero Nov 25, 2025
2b5e470
Refactor createMockZipPackage to return the zip file path and ensure …
teresaromero Nov 25, 2025
c5353d0
Refactor TestFileSystemIndexer to initialize packageList directly and…
teresaromero Nov 25, 2025
718572e
Merge branch 'main' of github.com:elastic/package-registry into 1351-…
teresaromero Nov 26, 2025
4280300
Remove current transaction before starting new one in package file sy…
teresaromero Nov 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion categories_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func TestCategoriesWithProxyMode(t *testing.T) {
}))
defer webServer.Close()

indexerProxy := packages.NewFileSystemIndexer(testLogger, "./testdata/second_package_path")
indexerProxy := packages.NewFileSystemIndexer(testLogger, 0, "./testdata/second_package_path")
defer indexerProxy.Close(t.Context())

err := indexerProxy.Init(t.Context())
Expand Down
21 changes: 18 additions & 3 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ var (
proxyTo string
serviceName = getServiceName()

packagePathsAllowEmpty = false
packagePathsWatchInterval time.Duration

defaultConfig = Config{
CacheTimeIndex: 10 * time.Second,
CacheTimeSearch: 10 * time.Minute,
Expand Down Expand Up @@ -128,6 +131,9 @@ func init() {
// The following proxy-indexer related flags are technical preview and might be removed in the future or renamed
flag.BoolVar(&featureProxyMode, "feature-proxy-mode", false, "Enable proxy mode to include packages from other endpoint (technical preview).")
flag.StringVar(&proxyTo, "proxy-to", "https://epr.elastic.co/", "Proxy-to endpoint")

flag.BoolVar(&packagePathsAllowEmpty, "package-paths-allow-empty", false, "Allow empty package paths on startup")
Comment thread
teresaromero marked this conversation as resolved.
Outdated
flag.DurationVar(&packagePathsWatchInterval, "package-paths-watch-interval", 1*time.Minute, "Watch interval for package paths")
Comment thread
teresaromero marked this conversation as resolved.
Outdated
}

type Config struct {
Expand All @@ -140,7 +146,7 @@ type Config struct {
SearchCacheSize int `config:"search.cache_size"` // technical preview, used by the SQL storage indexer
SearchCacheTTL time.Duration `config:"search.cache_ttl"` // technical preview, used by the SQL storage indexe^
Comment thread
teresaromero marked this conversation as resolved.
Outdated
CategoriesCacheSize int `config:"categories.cache_size"` // technical preview, used by the SQL storage indexer
CategoriesCacheTTL time.Duration `config:"categories.cache_ttl"` // technical preview, used by the SQL storage indexe^
CategoriesCacheTTL time.Duration `config:"categories.cache_ttl"` // technical preview, used by the SQL storage indexer
}

func main() {
Expand Down Expand Up @@ -388,9 +394,15 @@ func initIndexer(ctx context.Context, logger *zap.Logger, options serverOptions)
combined = append(combined, indexer)
}

var fsWatchInterval time.Duration
if packagePathsAllowEmpty {
Comment thread
teresaromero marked this conversation as resolved.
Outdated
// Only allow watch when empty paths are allowed
fsWatchInterval = packagePathsWatchInterval
}

combined = append(combined,
packages.NewZipFileSystemIndexer(logger, packagesBasePaths...),
packages.NewFileSystemIndexer(logger, packagesBasePaths...),
packages.NewZipFileSystemIndexer(logger, fsWatchInterval, packagesBasePaths...),
packages.NewFileSystemIndexer(logger, fsWatchInterval, packagesBasePaths...),
)
ensurePackagesAvailable(ctx, logger, combined)
return combined
Expand Down Expand Up @@ -574,6 +586,7 @@ func getConfig(logger *zap.Logger) (*Config, error) {
}
config.CategoriesCacheTTL = cacheTTL
}

return &config, nil
}

Expand Down Expand Up @@ -619,6 +632,8 @@ func ensurePackagesAvailable(ctx context.Context, logger *zap.Logger, indexer In
logger.Info(fmt.Sprintf("%v local package manifests loaded.", len(packages)))
} else if featureProxyMode {
logger.Info("No local packages found, but the proxy mode can access remote ones.")
} else if packagePathsAllowEmpty {
logger.Warn("No packages found at startup. The registry is running but no content is available yet.")
} else {
logger.Fatal("No local packages found.")
}
Expand Down
32 changes: 16 additions & 16 deletions main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ func TestEndpoints(t *testing.T) {

packagesBasePaths := []string{"./testdata/second_package_path", "./testdata/package"}
indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, packagesBasePaths...),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, packagesBasePaths...),
)
t.Cleanup(func() { indexer.Close(context.Background()) })

Expand Down Expand Up @@ -172,7 +172,7 @@ func TestArtifacts(t *testing.T) {
t.Parallel()

packagesBasePaths := []string{"./testdata/package"}
indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
indexer := packages.NewFileSystemIndexer(testLogger, 0, packagesBasePaths...)
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -203,7 +203,7 @@ func TestArtifacts(t *testing.T) {
func TestSignatures(t *testing.T) {
t.Parallel()

indexer := packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage")
indexer := packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage")
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -233,7 +233,7 @@ func TestStatics(t *testing.T) {
t.Parallel()

packagesBasePaths := []string{"./testdata/package"}
indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
indexer := packages.NewFileSystemIndexer(testLogger, 0, packagesBasePaths...)
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -325,8 +325,8 @@ func TestStaticsModifiedTime(t *testing.T) {
}

indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, "./testdata/package"),
)
t.Cleanup(func() { indexer.Close(context.Background()) })

Expand Down Expand Up @@ -361,7 +361,7 @@ func TestStaticsModifiedTime(t *testing.T) {
func TestZippedArtifacts(t *testing.T) {
t.Parallel()

indexer := packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage")
indexer := packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage")
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -397,8 +397,8 @@ func TestPackageIndex(t *testing.T) {
t.Parallel()

indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, "./testdata/package"),
)
t.Cleanup(func() { indexer.Close(context.Background()) })

Expand Down Expand Up @@ -436,7 +436,7 @@ func TestZippedPackageIndex(t *testing.T) {
t.Parallel()

packagesBasePaths := []string{"./testdata/local-storage"}
indexer := packages.NewZipFileSystemIndexer(testLogger, packagesBasePaths...)
indexer := packages.NewZipFileSystemIndexer(testLogger, 0, packagesBasePaths...)
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -471,7 +471,7 @@ func TestAllPackageIndex(t *testing.T) {
testPackagePath := filepath.Join("testdata", "package")
secondPackagePath := filepath.Join("testdata", "second_package_path")
packagesBasePaths := []string{secondPackagePath, testPackagePath}
indexer := packages.NewFileSystemIndexer(testLogger, packagesBasePaths...)
indexer := packages.NewFileSystemIndexer(testLogger, 0, packagesBasePaths...)
t.Cleanup(func() { indexer.Close(context.Background()) })

err := indexer.Init(t.Context())
Expand Down Expand Up @@ -529,8 +529,8 @@ func TestContentTypes(t *testing.T) {
}

indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, "./testdata/package"),
)
t.Cleanup(func() { indexer.Close(context.Background()) })

Expand Down Expand Up @@ -563,8 +563,8 @@ func TestRangeDownloads(t *testing.T) {
t.Parallel()

indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, "./testdata/package"),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, "./testdata/package"),
)
t.Cleanup(func() { indexer.Close(context.Background()) })

Expand Down
4 changes: 2 additions & 2 deletions packages/marshaler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ var generateFlag = flag.Bool("generate", false, "Write golden files")
func TestMarshalJSON(t *testing.T) {
// given
packagesBasePaths := []string{"../testdata/second_package_path", "../testdata/package"}
indexer := NewFileSystemIndexer(util.NewTestLogger(), packagesBasePaths...)
indexer := NewFileSystemIndexer(util.NewTestLogger(), 0, packagesBasePaths...)
defer indexer.Close(t.Context())

err := indexer.Init(t.Context())
Expand All @@ -40,7 +40,7 @@ func TestMarshalJSON(t *testing.T) {
func TestUnmarshalJSON(t *testing.T) {
// given
packagesBasePaths := []string{"../testdata/second_package_path", "../testdata/package"}
indexer := NewFileSystemIndexer(util.NewTestLogger(), packagesBasePaths...)
indexer := NewFileSystemIndexer(util.NewTestLogger(), 0, packagesBasePaths...)
defer indexer.Close(t.Context())

err := indexer.Init(t.Context())
Expand Down
45 changes: 33 additions & 12 deletions packages/packages.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,12 @@ type FileSystemIndexer struct {
fsBuilder FileSystemBuilder

logger *zap.Logger

watchInterval time.Duration
}

// NewFileSystemIndexer creates a new FileSystemIndexer for the given paths.
func NewFileSystemIndexer(logger *zap.Logger, paths ...string) *FileSystemIndexer {
func NewFileSystemIndexer(logger *zap.Logger, watchInterval time.Duration, paths ...string) *FileSystemIndexer {
walkerFn := func(basePath, path string, info os.DirEntry) (bool, error) {
relativePath, err := filepath.Rel(basePath, path)
if err != nil {
Expand Down Expand Up @@ -141,11 +143,12 @@ func NewFileSystemIndexer(logger *zap.Logger, paths ...string) *FileSystemIndexe
return false, nil
}
return &FileSystemIndexer{
paths: paths,
label: "FileSystemIndexer",
walkerFn: walkerFn,
fsBuilder: ExtractedFileSystemBuilder,
logger: logger,
paths: paths,
label: "FileSystemIndexer",
walkerFn: walkerFn,
fsBuilder: ExtractedFileSystemBuilder,
logger: logger,
watchInterval: watchInterval,
}
}

Expand All @@ -154,7 +157,7 @@ var ExtractedFileSystemBuilder = func(p *Package) (PackageFileSystem, error) {
}

// NewZipFileSystemIndexer creates a new ZipFileSystemIndexer for the given paths.
func NewZipFileSystemIndexer(logger *zap.Logger, paths ...string) *FileSystemIndexer {
func NewZipFileSystemIndexer(logger *zap.Logger, watchInterval time.Duration, paths ...string) *FileSystemIndexer {
walkerFn := func(basePath, path string, info os.DirEntry) (bool, error) {
if info.IsDir() {
return false, nil
Expand All @@ -175,11 +178,12 @@ func NewZipFileSystemIndexer(logger *zap.Logger, paths ...string) *FileSystemInd
return true, nil
}
return &FileSystemIndexer{
paths: paths,
label: "ZipFileSystemIndexer",
walkerFn: walkerFn,
fsBuilder: ZipFileSystemBuilder,
logger: logger,
paths: paths,
label: "ZipFileSystemIndexer",
walkerFn: walkerFn,
fsBuilder: ZipFileSystemBuilder,
logger: logger,
watchInterval: watchInterval,
}
}

Expand All @@ -193,6 +197,23 @@ func (i *FileSystemIndexer) Init(ctx context.Context) (err error) {
if err != nil {
return fmt.Errorf("reading packages from filesystem failed: %w", err)
}
if i.watchInterval > 0 {
i.logger.Info("Watching package paths for changes",
zap.Duration("interval", i.watchInterval))
go func() {
ticker := time.NewTicker(i.watchInterval)
defer ticker.Stop()
for range ticker.C {
i.logger.Debug("Refreshing packages from filesystem")
newPackageList, err := i.getPackagesFromFileSystem(ctx)
if err != nil {
i.logger.Error("reading packages from filesystem failed", zap.Error(err))
continue
}
i.packageList = newPackageList
Comment thread
teresaromero marked this conversation as resolved.
Outdated
}
}()
}
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions search_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ func TestSearchWithProxyMode(t *testing.T) {

packagesBasePaths := []string{"./testdata/second_package_path", "./testdata/package"}
indexer := NewCombinedIndexer(
packages.NewZipFileSystemIndexer(testLogger, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, packagesBasePaths...),
packages.NewZipFileSystemIndexer(testLogger, 0, "./testdata/local-storage"),
packages.NewFileSystemIndexer(testLogger, 0, packagesBasePaths...),
)
defer indexer.Close(t.Context())

Expand Down