cmd/hiveview: add -gc for deleting old test outputs

cmd/hiveview/gc.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ethereum/hive/internal/libhive"
+)
+
+func logdirGC(dir string, cutoff time.Time, keepMin int) error {
+	var (
+		fsys       = os.DirFS(dir)
+		usedFiles  = make(map[string]struct{})
+		keptSuites = 0
+		oldest     time.Time
+	)
+
+	// Walk all suite files and pouplate the usedFiles set.
+	err := walkSummaryFiles(fsys, ".", func(suite *libhive.TestSuite, fi fs.FileInfo) error {
+		// Skip when too old and when above the minimum.
+		// Note we rely on getting called in descending time order here.
+		if suiteStart(suite).Before(cutoff) && keptSuites >= keepMin {
+			return nil
+		}
+		if oldest.IsZero() || suiteStart(suite).Before(oldest) {
+			oldest = suiteStart(suite)
+		}
+
+		// Add suite files and client logs.
+		keptSuites++
+		usedFiles[fi.Name()] = struct{}{}
+		usedFiles[suite.SimulatorLog] = struct{}{}
+		for _, test := range suite.TestCases {
+			for _, client := range test.ClientInfo {
+				usedFiles[client.LogFile] = struct{}{}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("keeping %d suites (%d files)\n", keptSuites, len(usedFiles))
+	fmt.Println("oldest suite date:", oldest)
+
+	// Delete all files which aren't in usedFiles.
+	return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil // Ignore scan errors.
+		}
+		if d.IsDir() {
+			return nil // Don't delete directories.
+		}
+		if _, used := usedFiles[path]; !used {
+			file := filepath.Join(dir, filepath.FromSlash(path))
+			// fmt.Println("rm", file)
+			err := os.Remove(file)
+			if err != nil {
+				fmt.Println("error:", err)
+			}
+		}
+		return nil
+	})
+}
+
+func suiteStart(suite *libhive.TestSuite) time.Time {
+	for _, test := range suite.TestCases {
+		return test.Start
+	}
+	return time.Time{}
+}

cmd/hiveview/listing.go

@@ -2,49 +2,44 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"log"
-	"os"
-	"path/filepath"
+	"path"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/hive/internal/libhive"
 )
 
 const listLimit = 200 // number of runs reported
 
 // generateListing processes hive simulation output files and generates a listing file.
-func generateListing(output io.Writer, logdir string) error {
-	logfiles, err := ioutil.ReadDir(logdir)
-	if err != nil {
-		return err
-	}
-	// The files are prefixed by timestamp, so to get the latest 200 items,
-	// we just need to read the listing in reverse until we have 200
-	var entries []listingEntry
-	for i := len(logfiles) - 1; i > 0; i-- {
-		finfo := logfiles[i]
-		if !strings.HasSuffix(finfo.Name(), ".json") || skipFile(finfo.Name()) {
-			continue
-		}
-		entry, err := convertSummaryFile(logdir, finfo)
-		if err != nil {
-			continue
-		}
+func generateListing(fsys fs.FS, dir string, output io.Writer) error {
+	var (
+		stop    = errors.New("stop")
+		entries []listingEntry
+	)
+	// The files are walked in name order high->low. So to get the latest 200 items, we
+	// just need to keep going until we have 200.
+	err := walkSummaryFiles(fsys, dir, func(suite *libhive.TestSuite, fi fs.FileInfo) error {
+		entry := suiteToEntry(suite, fi)
 		entries = append(entries, entry)
 		if len(entries) >= listLimit {
-			break
+			return stop
 		}
-	}
-	sort.Slice(entries, func(i, j int) bool { return entries[i].SimLog > entries[j].SimLog })
-	if len(entries) > listLimit {
-		entries = entries[:listLimit]
+		return nil
+	})
+	if err != nil && err != stop {
+		return err
 	}
 
+	// Write listing JSON lines to output.
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].SimLog > entries[j].SimLog
+	})
 	enc := json.NewEncoder(output)
 	for _, e := range entries {
 		if err := enc.Encode(e); err != nil {
@@ -71,29 +66,7 @@ type listingEntry struct {
 	SimLog   string    `json:"simLog"`   // simulator log file
 }
 
-func convertSummaryFile(logdir string, file os.FileInfo) (listingEntry, error) {
-	info := new(libhive.TestSuite)
-	err := common.LoadJSON(filepath.Join(logdir, file.Name()), info)
-	if err != nil {
-		log.Printf("Skipping invalid summary file: %v", err)
-		return listingEntry{}, err
-	}
-	if !suiteValid(info) {
-		log.Printf("Skipping invalid summary file: %s", file.Name())
-		return listingEntry{}, err
-	}
-	return suiteToEntry(file, info), nil
-}
-
-func suiteValid(s *libhive.TestSuite) bool {
-	return s.SimulatorLog != ""
-}
-
-func skipFile(f string) bool {
-	return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".")
-}
-
-func suiteToEntry(file os.FileInfo, s *libhive.TestSuite) listingEntry {
+func suiteToEntry(s *libhive.TestSuite, file fs.FileInfo) listingEntry {
 	e := listingEntry{
 		Name:     s.Name,
 		FileName: file.Name(),
@@ -128,3 +101,64 @@ func contains(list []string, s string) bool {
 	}
 	return false
 }
+
+type suiteCB func(*libhive.TestSuite, fs.FileInfo) error
+
+func walkSummaryFiles(fsys fs.FS, dir string, proc suiteCB) error {
+	logfiles, err := fs.ReadDir(fsys, dir)
+	if err != nil {
+		return err
+	}
+	// Sort by name newest-first.
+	sort.Slice(logfiles, func(i, j int) bool {
+		return logfiles[i].Name() > logfiles[j].Name()
+	})
+
+	for _, entry := range logfiles {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasSuffix(name, ".json") || skipFile(name) {
+			continue
+		}
+		suite, fileInfo := parseSuite(fsys, path.Join(dir, name))
+		if suite != nil {
+			if err := proc(suite, fileInfo); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func parseSuite(fsys fs.FS, path string) (*libhive.TestSuite, fs.FileInfo) {
+	file, err := fsys.Open(path)
+	if err != nil {
+		log.Printf("Can't access summary file: %s", err)
+		return nil, nil
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		log.Printf("Can't access summary file: %s", err)
+		return nil, nil
+	}
+
+	var info libhive.TestSuite
+	if err := json.NewDecoder(file).Decode(&info); err != nil {
+		log.Printf("Skipping invalid summary file %s: %v", fileInfo.Name(), err)
+		return nil, nil
+	}
+	if !suiteValid(&info) {
+		log.Printf("Skipping invalid summary file %s", fileInfo.Name())
+		return nil, nil
+	}
+	return &info, fileInfo
+}
+
+func suiteValid(s *libhive.TestSuite) bool {
+	return s.SimulatorLog != ""
+}
+
+func skipFile(f string) bool {
+	return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".")
+}

cmd/hiveview/main.go

@@ -3,28 +3,28 @@
 package main
 
 import (
-	"embed"
 	"flag"
-	"io/fs"
 	"log"
-	"net"
-	"net/http"
 	"os"
-
-	"github.com/gorilla/mux"
+	"time"
 )
 
-//go:embed assets
-var embeddedAssets embed.FS
+const (
+	durationDays  = 24 * time.Hour
+	durationMonth = 31 * durationDays
+)
 
 func main() {
 	var (
-		serve   = flag.Bool("serve", false, "Enables the HTTP server")
-		listing = flag.Bool("listing", false, "Generates listing JSON to stdout")
-		config  serverConfig
+		serve          = flag.Bool("serve", false, "Enables the HTTP server")
+		listing        = flag.Bool("listing", false, "Generates listing JSON to stdout")
+		gc             = flag.Bool("gc", false, "Deletes old log files")
+		gcKeepInterval = flag.Duration("keep", 5*durationMonth, "Time interval of past log files to keep (for -gc)")
+		gcKeepMin      = flag.Int("keep-min", 10, "Minmum number of suite outputs to keep (for -gc)")
+		config         serverConfig
 	)
 	flag.StringVar(&config.listenAddr, "addr", "0.0.0.0:8080", "HTTP server listen address")
-	flag.StringVar(&config.logdir, "logdir", "workspace/logs", "Path to hive simulator log directory")
+	flag.StringVar(&config.logDir, "logdir", "workspace/logs", "Path to hive simulator log directory")
 	flag.StringVar(&config.assetsDir, "assets", "", "Path to static files directory. Serves baked-in assets when not set.")
 	flag.Parse()
 
@@ -33,56 +33,12 @@ func main() {
 	case *serve:
 		runServer(config)
 	case *listing:
-		generateListing(os.Stdout, config.logdir)
+		fsys := os.DirFS(config.logDir)
+		generateListing(fsys, ".", os.Stdout)
+	case *gc:
+		cutoff := time.Now().Add(-*gcKeepInterval)
+		logdirGC(config.logDir, cutoff, *gcKeepMin)
 	default:
 		log.Fatalf("Use -serve or -listing to select mode")
 	}
 }
-
-type serverConfig struct {
-	listenAddr string
-	logdir     string
-	assetsDir  string
-}
-
-func runServer(config serverConfig) {
-	var assetFS fs.FS
-	if config.assetsDir != "" {
-		if stat, _ := os.Stat(config.assetsDir); stat == nil || !stat.IsDir() {
-			log.Fatalf("-assets: %q is not a directory", config.assetsDir)
-		}
-		assetFS = os.DirFS(config.assetsDir)
-	} else {
-		sub, err := fs.Sub(embeddedAssets, "assets")
-		if err != nil {
-			panic(err)
-		}
-		assetFS = sub
-	}
-
-	// Create handlers.
-	logHandler := http.FileServer(http.Dir(config.logdir))
-	listingHandler := serveListing{dir: config.logdir}
-	mux := mux.NewRouter()
-	mux.Handle("/listing.jsonl", listingHandler).Methods("GET")
-	mux.PathPrefix("/results").Handler(http.StripPrefix("/results/", logHandler))
-	mux.PathPrefix("/").Handler(http.FileServer(http.FS(assetFS)))
-
-	// Start the server.
-	l, err := net.Listen("tcp", config.listenAddr)
-	if err != nil {
-		log.Fatalf("Can't listen: %v", err)
-	}
-	log.Printf("Serving at http://%v/", l.Addr())
-	http.Serve(l, mux)
-}
-
-type serveListing struct{ dir string }
-
-func (h serveListing) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log.Printf("Generating listing...")
-	err := generateListing(w, h.dir)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-	}
-}