diff --git a/cmd/hiveview/gc.go b/cmd/hiveview/gc.go new file mode 100644 index 0000000000..74bdc3096e --- /dev/null +++ b/cmd/hiveview/gc.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/ethereum/hive/internal/libhive" +) + +func logdirGC(dir string, cutoff time.Time, keepMin int) error { + var ( + fsys = os.DirFS(dir) + usedFiles = make(map[string]struct{}) + keptSuites = 0 + oldest time.Time + ) + + // Walk all suite files and pouplate the usedFiles set. + err := walkSummaryFiles(fsys, ".", func(suite *libhive.TestSuite, fi fs.FileInfo) error { + // Skip when too old and when above the minimum. + // Note we rely on getting called in descending time order here. + if suiteStart(suite).Before(cutoff) && keptSuites >= keepMin { + return nil + } + if oldest.IsZero() || suiteStart(suite).Before(oldest) { + oldest = suiteStart(suite) + } + + // Add suite files and client logs. + keptSuites++ + usedFiles[fi.Name()] = struct{}{} + usedFiles[suite.SimulatorLog] = struct{}{} + for _, test := range suite.TestCases { + for _, client := range test.ClientInfo { + usedFiles[client.LogFile] = struct{}{} + } + } + return nil + }) + if err != nil { + return err + } + + fmt.Printf("keeping %d suites (%d files)\n", keptSuites, len(usedFiles)) + fmt.Println("oldest suite date:", oldest) + + // Delete all files which aren't in usedFiles. + return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // Ignore scan errors. + } + if d.IsDir() { + return nil // Don't delete directories. + } + if _, used := usedFiles[path]; !used { + file := filepath.Join(dir, filepath.FromSlash(path)) + // fmt.Println("rm", file) + err := os.Remove(file) + if err != nil { + fmt.Println("error:", err) + } + } + return nil + }) +} + +func suiteStart(suite *libhive.TestSuite) time.Time { + for _, test := range suite.TestCases { + return test.Start + } + return time.Time{} +} diff --git a/cmd/hiveview/listing.go b/cmd/hiveview/listing.go index b5721b15d2..ed96d7c5d3 100644 --- a/cmd/hiveview/listing.go +++ b/cmd/hiveview/listing.go @@ -2,49 +2,44 @@ package main import ( "encoding/json" + "errors" "io" - "io/ioutil" + "io/fs" "log" - "os" - "path/filepath" + "path" "sort" "strings" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/hive/internal/libhive" ) const listLimit = 200 // number of runs reported // generateListing processes hive simulation output files and generates a listing file. -func generateListing(output io.Writer, logdir string) error { - logfiles, err := ioutil.ReadDir(logdir) - if err != nil { - return err - } - // The files are prefixed by timestamp, so to get the latest 200 items, - // we just need to read the listing in reverse until we have 200 - var entries []listingEntry - for i := len(logfiles) - 1; i > 0; i-- { - finfo := logfiles[i] - if !strings.HasSuffix(finfo.Name(), ".json") || skipFile(finfo.Name()) { - continue - } - entry, err := convertSummaryFile(logdir, finfo) - if err != nil { - continue - } +func generateListing(fsys fs.FS, dir string, output io.Writer) error { + var ( + stop = errors.New("stop") + entries []listingEntry + ) + // The files are walked in name order high->low. So to get the latest 200 items, we + // just need to keep going until we have 200. + err := walkSummaryFiles(fsys, dir, func(suite *libhive.TestSuite, fi fs.FileInfo) error { + entry := suiteToEntry(suite, fi) entries = append(entries, entry) if len(entries) >= listLimit { - break + return stop } - } - sort.Slice(entries, func(i, j int) bool { return entries[i].SimLog > entries[j].SimLog }) - if len(entries) > listLimit { - entries = entries[:listLimit] + return nil + }) + if err != nil && err != stop { + return err } + // Write listing JSON lines to output. + sort.Slice(entries, func(i, j int) bool { + return entries[i].SimLog > entries[j].SimLog + }) enc := json.NewEncoder(output) for _, e := range entries { if err := enc.Encode(e); err != nil { @@ -71,29 +66,7 @@ type listingEntry struct { SimLog string `json:"simLog"` // simulator log file } -func convertSummaryFile(logdir string, file os.FileInfo) (listingEntry, error) { - info := new(libhive.TestSuite) - err := common.LoadJSON(filepath.Join(logdir, file.Name()), info) - if err != nil { - log.Printf("Skipping invalid summary file: %v", err) - return listingEntry{}, err - } - if !suiteValid(info) { - log.Printf("Skipping invalid summary file: %s", file.Name()) - return listingEntry{}, err - } - return suiteToEntry(file, info), nil -} - -func suiteValid(s *libhive.TestSuite) bool { - return s.SimulatorLog != "" -} - -func skipFile(f string) bool { - return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".") -} - -func suiteToEntry(file os.FileInfo, s *libhive.TestSuite) listingEntry { +func suiteToEntry(s *libhive.TestSuite, file fs.FileInfo) listingEntry { e := listingEntry{ Name: s.Name, FileName: file.Name(), @@ -128,3 +101,64 @@ func contains(list []string, s string) bool { } return false } + +type suiteCB func(*libhive.TestSuite, fs.FileInfo) error + +func walkSummaryFiles(fsys fs.FS, dir string, proc suiteCB) error { + logfiles, err := fs.ReadDir(fsys, dir) + if err != nil { + return err + } + // Sort by name newest-first. + sort.Slice(logfiles, func(i, j int) bool { + return logfiles[i].Name() > logfiles[j].Name() + }) + + for _, entry := range logfiles { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".json") || skipFile(name) { + continue + } + suite, fileInfo := parseSuite(fsys, path.Join(dir, name)) + if suite != nil { + if err := proc(suite, fileInfo); err != nil { + return err + } + } + } + return nil +} + +func parseSuite(fsys fs.FS, path string) (*libhive.TestSuite, fs.FileInfo) { + file, err := fsys.Open(path) + if err != nil { + log.Printf("Can't access summary file: %s", err) + return nil, nil + } + defer file.Close() + + fileInfo, err := file.Stat() + if err != nil { + log.Printf("Can't access summary file: %s", err) + return nil, nil + } + + var info libhive.TestSuite + if err := json.NewDecoder(file).Decode(&info); err != nil { + log.Printf("Skipping invalid summary file %s: %v", fileInfo.Name(), err) + return nil, nil + } + if !suiteValid(&info) { + log.Printf("Skipping invalid summary file %s", fileInfo.Name()) + return nil, nil + } + return &info, fileInfo +} + +func suiteValid(s *libhive.TestSuite) bool { + return s.SimulatorLog != "" +} + +func skipFile(f string) bool { + return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".") +} diff --git a/cmd/hiveview/main.go b/cmd/hiveview/main.go index 402fb58b10..66a33e6237 100644 --- a/cmd/hiveview/main.go +++ b/cmd/hiveview/main.go @@ -3,28 +3,28 @@ package main import ( - "embed" "flag" - "io/fs" "log" - "net" - "net/http" "os" - - "github.com/gorilla/mux" + "time" ) -//go:embed assets -var embeddedAssets embed.FS +const ( + durationDays = 24 * time.Hour + durationMonth = 31 * durationDays +) func main() { var ( - serve = flag.Bool("serve", false, "Enables the HTTP server") - listing = flag.Bool("listing", false, "Generates listing JSON to stdout") - config serverConfig + serve = flag.Bool("serve", false, "Enables the HTTP server") + listing = flag.Bool("listing", false, "Generates listing JSON to stdout") + gc = flag.Bool("gc", false, "Deletes old log files") + gcKeepInterval = flag.Duration("keep", 5*durationMonth, "Time interval of past log files to keep (for -gc)") + gcKeepMin = flag.Int("keep-min", 10, "Minmum number of suite outputs to keep (for -gc)") + config serverConfig ) flag.StringVar(&config.listenAddr, "addr", "0.0.0.0:8080", "HTTP server listen address") - flag.StringVar(&config.logdir, "logdir", "workspace/logs", "Path to hive simulator log directory") + flag.StringVar(&config.logDir, "logdir", "workspace/logs", "Path to hive simulator log directory") flag.StringVar(&config.assetsDir, "assets", "", "Path to static files directory. Serves baked-in assets when not set.") flag.Parse() @@ -33,56 +33,12 @@ func main() { case *serve: runServer(config) case *listing: - generateListing(os.Stdout, config.logdir) + fsys := os.DirFS(config.logDir) + generateListing(fsys, ".", os.Stdout) + case *gc: + cutoff := time.Now().Add(-*gcKeepInterval) + logdirGC(config.logDir, cutoff, *gcKeepMin) default: log.Fatalf("Use -serve or -listing to select mode") } } - -type serverConfig struct { - listenAddr string - logdir string - assetsDir string -} - -func runServer(config serverConfig) { - var assetFS fs.FS - if config.assetsDir != "" { - if stat, _ := os.Stat(config.assetsDir); stat == nil || !stat.IsDir() { - log.Fatalf("-assets: %q is not a directory", config.assetsDir) - } - assetFS = os.DirFS(config.assetsDir) - } else { - sub, err := fs.Sub(embeddedAssets, "assets") - if err != nil { - panic(err) - } - assetFS = sub - } - - // Create handlers. - logHandler := http.FileServer(http.Dir(config.logdir)) - listingHandler := serveListing{dir: config.logdir} - mux := mux.NewRouter() - mux.Handle("/listing.jsonl", listingHandler).Methods("GET") - mux.PathPrefix("/results").Handler(http.StripPrefix("/results/", logHandler)) - mux.PathPrefix("/").Handler(http.FileServer(http.FS(assetFS))) - - // Start the server. - l, err := net.Listen("tcp", config.listenAddr) - if err != nil { - log.Fatalf("Can't listen: %v", err) - } - log.Printf("Serving at http://%v/", l.Addr()) - http.Serve(l, mux) -} - -type serveListing struct{ dir string } - -func (h serveListing) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log.Printf("Generating listing...") - err := generateListing(w, h.dir) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } -} diff --git a/cmd/hiveview/serve.go b/cmd/hiveview/serve.go new file mode 100644 index 0000000000..e6674961f6 --- /dev/null +++ b/cmd/hiveview/serve.go @@ -0,0 +1,64 @@ +package main + +import ( + "embed" + "io/fs" + "log" + "net" + "net/http" + "os" + + "github.com/gorilla/mux" +) + +//go:embed assets +var embeddedAssets embed.FS + +type serverConfig struct { + listenAddr string + logDir string + assetsDir string +} + +func runServer(config serverConfig) { + var assetFS fs.FS + if config.assetsDir != "" { + if stat, _ := os.Stat(config.assetsDir); stat == nil || !stat.IsDir() { + log.Fatalf("-assets: %q is not a directory", config.assetsDir) + } + assetFS = os.DirFS(config.assetsDir) + } else { + sub, err := fs.Sub(embeddedAssets, "assets") + if err != nil { + panic(err) + } + assetFS = sub + } + + // Create handlers. + logDirFS := os.DirFS(config.logDir) + logHandler := http.FileServer(http.FS(logDirFS)) + listingHandler := serveListing{fsys: logDirFS} + mux := mux.NewRouter() + mux.Handle("/listing.jsonl", listingHandler).Methods("GET") + mux.PathPrefix("/results").Handler(http.StripPrefix("/results/", logHandler)) + mux.PathPrefix("/").Handler(http.FileServer(http.FS(assetFS))) + + // Start the server. + l, err := net.Listen("tcp", config.listenAddr) + if err != nil { + log.Fatalf("Can't listen: %v", err) + } + log.Printf("Serving at http://%v/", l.Addr()) + http.Serve(l, mux) +} + +type serveListing struct{ fsys fs.FS } + +func (h serveListing) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log.Printf("Generating listing...") + err := generateListing(h.fsys, ".", w) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } +}