feat: ES-198 enhance performance when loading big projects

app.go

@@ -132,7 +132,9 @@ func (a *App) initializeMenu() {
 
 	// View menu
 	ViewMenu := AppMenu.AddSubmenu("View")
-	ViewMenu.AddText("Sync Scroll Position", keys.Combo("e", keys.ShiftKey, keys.CmdOrCtrlKey), func(cd *menu.CallbackData) {})
+	ViewMenu.AddText("Sync Scroll Position", keys.Combo("e", keys.ShiftKey, keys.CmdOrCtrlKey), func(cd *menu.CallbackData) {
+		runtime.EventsEmit(a.ctx, string(entities.ActionToggleSyncScrollPosition))
+	})
 
 	// Scan menu
 	ScanMenu := AppMenu.AddSubmenu("Scan")

backend/entities/keyboard.go

@@ -53,7 +53,7 @@ const (
 	ActionReplaceComponentWithComments    Action = "replaceComponentWithComments"
 
 	// View
-	ActionSyncScrollPosition         Action = "toggleSyncScrollPosition"
+	ActionToggleSyncScrollPosition   Action = "toggleSyncScrollPosition"
 	ActionShowKeyboardShortcutsModal Action = "showKeyboardShortcutsModal"
 
 	// Scan
@@ -105,7 +105,7 @@ var AllShortcutActions = []struct {
 	{ActionReplaceFileWithComments, "ReplaceFileWithComments"},
 	{ActionReplaceComponentWithoutComments, "ReplaceComponentWithoutComments"},
 	{ActionReplaceComponentWithComments, "ReplaceComponentWithComments"},
-	{ActionSyncScrollPosition, "ToggleSyncScrollPosition"},
+	{ActionToggleSyncScrollPosition, "ToggleSyncScrollPosition"},
 	{ActionShowKeyboardShortcutsModal, "ShowKeyboardShortcutsModal"},
 	{ActionScanWithOptions, "ScanWithOptions"},
 }
@@ -296,8 +296,8 @@ var DefaultShortcuts = []Shortcut{
 		Description: "Sync the scroll position of the editors",
 		Accelerator: keys.Combo("e", keys.ShiftKey, keys.CmdOrCtrlKey),
 		Keys:        "shift+mod+e",
-		Group:       GroupActions,
-		Action:      ActionReplaceComponentWithComments,
+		Group:       GroupView,
+		Action:      ActionToggleSyncScrollPosition,
 	},
 
 	// Scan

backend/mappers/result_mapper_impl.go

@@ -25,7 +25,9 @@ package mappers
 
 import (
 	"fmt"
+	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/rs/zerolog/log"
 	purlutils "github.com/scanoss/go-purl-helper/pkg"
@@ -36,29 +38,52 @@ type ResultMapperImpl struct {
 	scanossSettings *entities.ScanossSettings
 }
 
+var (
+	resultDTOCache sync.Map
+	purlCache      sync.Map
+)
+
 func NewResultMapper(scanossSettings *entities.ScanossSettings) ResultMapper {
 	return &ResultMapperImpl{
 		scanossSettings: scanossSettings,
 	}
 }
 
+func (m ResultMapperImpl) generateCacheKey(result entities.Result, bomEntry entities.ComponentFilter) string {
+	return fmt.Sprintf("%s-%s-%s-%s-%s-%s-%s",
+		result.Path,
+		strings.Join(*result.Purl, ","),
+		result.MatchType,
+		bomEntry.ReplaceWith,
+		bomEntry.Comment,
+		m.scanossSettings.SettingsFile.GetResultWorkflowState(result),
+		m.scanossSettings.SettingsFile.GetResultFilterConfig(result),
+	)
+}
+
 func (m ResultMapperImpl) MapToResultDTO(result entities.Result) entities.ResultDTO {
-	return entities.ResultDTO{
+	bomEntry := m.scanossSettings.SettingsFile.GetBomEntryFromResult(result)
+	cacheKey := m.generateCacheKey(result, bomEntry)
+
+	if cached, ok := resultDTOCache.Load(cacheKey); ok {
+		return cached.(entities.ResultDTO)
+	}
+
+	dto := entities.ResultDTO{
 		MatchType:        entities.MatchType(result.MatchType),
 		Path:             result.Path,
 		DetectedPurl:     (*result.Purl)[0],
 		DetectedPurlUrl:  m.mapDetectedPurlUrl(result),
-		ConcludedPurl:    m.mapConcludedPurl(result),
+		ConcludedPurl:    bomEntry.ReplaceWith,
 		ConcludedPurlUrl: m.mapConcludedPurlUrl(result),
 		ConcludedName:    m.mapConcludedName(result),
 		WorkflowState:    m.mapWorkflowState(result),
 		FilterConfig:     m.mapFilterConfig(result),
-		Comment:          m.mapComment(result),
+		Comment:          bomEntry.Comment,
 	}
-}
 
-func (m ResultMapperImpl) mapComment(result entities.Result) string {
-	return m.scanossSettings.SettingsFile.GetBomEntryFromResult(result).Comment
+	resultDTOCache.Store(cacheKey, dto)
+	return dto
 }
 
 func (m ResultMapperImpl) mapConcludedPurl(result entities.Result) string {
@@ -67,11 +92,26 @@ func (m ResultMapperImpl) mapConcludedPurl(result entities.Result) string {
 
 func (m ResultMapperImpl) MapToResultDTOList(results []entities.Result) []entities.ResultDTO {
 	output := make([]entities.ResultDTO, len(results))
+	numWorkers := runtime.NumCPU()
+	jobChan := make(chan int, len(results))
+	var wg sync.WaitGroup
 
-	for i, v := range results {
-		output[i] = m.MapToResultDTO(v)
+	for i := 0; i < len(results); i++ {
+		jobChan <- i
+	}
+	close(jobChan)
+
+	wg.Add(numWorkers)
+	for w := 0; w < numWorkers; w++ {
+		go func() {
+			defer wg.Done()
+			for idx := range jobChan {
+				output[idx] = m.MapToResultDTO(results[idx])
+			}
+		}()
 	}
 
+	wg.Wait()
 	return output
 }
 
@@ -101,38 +141,40 @@ func (m *ResultMapperImpl) mapConcludedPurlUrl(result entities.Result) string {
 		purlName = fmt.Sprintf("%s/%s", purlObject.Namespace, purlObject.Name)
 	}
 
-	purlUrl, err := purlutils.ProjectUrl(purlName, purlObject.Type)
+	return m.getProjectURL(concludedPurl, func() (string, error) {
+		return purlutils.ProjectUrl(purlName, purlObject.Type)
+	})
+}
+
+func (m ResultMapperImpl) getProjectURL(purl string, compute func() (string, error)) string {
+	if cached, ok := purlCache.Load(purl); ok {
+		return cached.(string)
+	}
+	url, err := compute()
 	if err != nil {
-		log.Error().Err(err).Msg("Error getting project url")
+		log.Error().Err(err).Msg("Error computing project URL")
 		return ""
 	}
-
-	return purlUrl
+	purlCache.Store(purl, url)
+	return url
 }
 
 func (m ResultMapperImpl) mapDetectedPurlUrl(result entities.Result) string {
 	detectedPurl := (*result.Purl)[0]
 
 	purlObject, err := purlutils.PurlFromString(detectedPurl)
-
 	if err != nil {
 		log.Error().Err(err).Msg("Error parsing detected purl")
 		return ""
 	}
-
-	// Workaround for github purls until purlutils is updated
 	purlName := purlObject.Name
 	if purlObject.Type == "github" && purlObject.Namespace != "" {
 		purlName = fmt.Sprintf("%s/%s", purlObject.Namespace, purlObject.Name)
 	}
 
-	purlUrl, err := purlutils.ProjectUrl(purlName, purlObject.Type)
-	if err != nil {
-		log.Error().Err(err).Msg("Error getting detected purl url")
-		return ""
-	}
-
-	return purlUrl
+	return m.getProjectURL(detectedPurl, func() (string, error) {
+		return purlutils.ProjectUrl(purlName, purlObject.Type)
+	})
 }
 
 func (m ResultMapperImpl) mapConcludedName(result entities.Result) string {

backend/repository/result_repository_json_impl.go

@@ -26,6 +26,8 @@ package repository
 import (
 	"encoding/json"
 	"os"
+	"sync"
+	"time"
 
 	"github.com/rs/zerolog/log"
 	"github.com/scanoss/scanoss.cc/backend/entities"
@@ -34,7 +36,10 @@ import (
 )
 
 type ResultRepositoryJsonImpl struct {
-	fr utils.FileReader
+	fr           utils.FileReader
+	cache        []entities.Result
+	lastModified time.Time
+	mutex        sync.RWMutex
 }
 
 func NewResultRepositoryJsonImpl(fr utils.FileReader) *ResultRepositoryJsonImpl {
@@ -44,39 +49,78 @@ func NewResultRepositoryJsonImpl(fr utils.FileReader) *ResultRepositoryJsonImpl
 }
 
 func (r *ResultRepositoryJsonImpl) GetResults(filter entities.ResultFilter) ([]entities.Result, error) {
-	resultFilePath := config.GetInstance().GetResultFilePath()
-	resultByte, err := r.fr.ReadFile(resultFilePath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return []entities.Result{}, nil
-		}
-		log.Error().Err(err).Msg("Error reading result file")
-		return []entities.Result{}, entities.ErrReadingResultFile
-	}
+	r.mutex.RLock()
 
-	scanResults, err := r.parseScanResults(resultByte)
-	if err != nil {
-		return []entities.Result{}, entities.ErrParsingResultFile
+	if r.shouldRefreshCache() {
+		r.mutex.RUnlock()
+		r.mutex.Lock()
+		defer r.mutex.Unlock()
+
+		if r.shouldRefreshCache() {
+			if err := r.refreshCache(); err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		defer r.mutex.RUnlock()
 	}
 
 	if filter == nil {
-		return scanResults, nil
+		return r.cache, nil
 	}
 
-	// Filter scan results
 	var filteredResults []entities.Result
-	for _, result := range scanResults {
+	for _, result := range r.cache {
 		if result.IsEmpty() {
 			continue
 		}
-
 		if filter.IsValid(result) {
 			filteredResults = append(filteredResults, result)
 		}
 	}
+
 	return filteredResults, nil
 }
 
+func (r *ResultRepositoryJsonImpl) shouldRefreshCache() bool {
+	if r.cache == nil {
+		return true
+	}
+
+	resultFilePath := config.GetInstance().GetResultFilePath()
+	fileInfo, err := os.Stat(resultFilePath)
+	if err != nil {
+		return true
+	}
+
+	return fileInfo.ModTime().After(r.lastModified)
+}
+
+func (r *ResultRepositoryJsonImpl) refreshCache() error {
+	resultFilePath := config.GetInstance().GetResultFilePath()
+	resultByte, err := r.fr.ReadFile(resultFilePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			r.cache = []entities.Result{}
+			return nil
+		}
+		return entities.ErrReadingResultFile
+	}
+
+	scanResults, err := r.parseScanResults(resultByte)
+	if err != nil {
+		return entities.ErrParsingResultFile
+	}
+
+	fileInfo, err := os.Stat(resultFilePath)
+	if err == nil {
+		r.lastModified = fileInfo.ModTime()
+	}
+
+	r.cache = scanResults
+	return nil
+}
+
 func (r *ResultRepositoryJsonImpl) parseScanResults(resultByte []byte) ([]entities.Result, error) {
 	var intermediateMap map[string][]entities.Match
 	if err := json.Unmarshal(resultByte, &intermediateMap); err != nil {

frontend/package.json

@@ -29,6 +29,7 @@
     "@radix-ui/react-toast": "^1.2.1",
     "@radix-ui/react-tooltip": "^1.1.2",
     "@tanstack/react-query": "^5.51.23",
+    "@tanstack/react-virtual": "^3.13.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "^1.0.0",

frontend/package.json.md5

@@ -1 +1 @@
-78cb6f2d69974bb7aae419f9b4017b9a
+bfd1d83aa6b54a74af67412f356a7733