[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 2) (#2191)

* Movec ctags-related structs in ctags package

* Unified GCCPreprocRunner* functions

There is no need to duplicate a function that basically does the same
thing.

* Added implementation of functional algorithms

(until they are available in the golang standard library...)

* legacy: use generic algos where possibile

* Removed useless call to FilterSketchSource

* Converted AddPrototypes into a function

Author: cmaglie

internal/algorithms/slices.go

@@ -0,0 +1,73 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package f
+
+// Matcher is a function that tests if a given value match a certain criteria.
+type Matcher[T any] func(T) bool
+
+// Reducer is a function that combines two values of the same type and return
+// the combined value.
+type Reducer[T any] func(T, T) T
+
+// Mapper is a function that converts a value of one type to another type.
+type Mapper[T, U any] func(T) U
+
+// Filter takes a slice of type []T and a Matcher[T]. It returns a newly
+// allocated slice containing only those elements of the input slice that
+// satisfy the matcher.
+func Filter[T any](values []T, matcher Matcher[T]) []T {
+	res := []T{}
+	for _, x := range values {
+		if matcher(x) {
+			res = append(res, x)
+		}
+	}
+	return res
+}
+
+// Map applies the Mapper function to each element of the slice and returns
+// a new slice with the results in the same order.
+func Map[T, U any](values []T, mapper Mapper[T, U]) []U {
+	res := []U{}
+	for _, x := range values {
+		res = append(res, mapper(x))
+	}
+	return res
+}
+
+// Reduce applies the Reducer function to all elements of the input values
+// and returns the result.
+func Reduce[T any](values []T, reducer Reducer[T]) T {
+	var result T
+	for _, v := range values {
+		result = reducer(result, v)
+	}
+	return result
+}
+
+// Equals return a Matcher that matches the given value
+func Equals[T comparable](value T) Matcher[T] {
+	return func(x T) bool {
+		return x == value
+	}
+}
+
+// NotEquals return a Matcher that does not match the given value
+func NotEquals[T comparable](value T) Matcher[T] {
+	return func(x T) bool {
+		return x != value
+	}
+}

internal/algorithms/slices_test.go

@@ -0,0 +1,53 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package f_test
+
+import (
+	"strings"
+	"testing"
+
+	f "github.com/arduino/arduino-cli/internal/algorithms"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilter(t *testing.T) {
+	a := []string{"aaa", "bbb", "ccc"}
+	require.Equal(t, []string{"bbb", "ccc"}, f.Filter(a, func(x string) bool { return x > "b" }))
+	b := []int{5, 9, 15, 2, 4, -2}
+	require.Equal(t, []int{5, 9, 15}, f.Filter(b, func(x int) bool { return x > 4 }))
+}
+
+func TestIsEmpty(t *testing.T) {
+	require.True(t, f.Equals(int(0))(0))
+	require.False(t, f.Equals(int(1))(0))
+	require.True(t, f.Equals("")(""))
+	require.False(t, f.Equals("abc")(""))
+	require.False(t, f.NotEquals(int(0))(0))
+	require.True(t, f.NotEquals(int(1))(0))
+	require.False(t, f.NotEquals("")(""))
+	require.True(t, f.NotEquals("abc")(""))
+}
+
+func TestMap(t *testing.T) {
+	value := "hello, world , how are,you? "
+	parts := f.Map(strings.Split(value, ","), strings.TrimSpace)
+
+	require.Equal(t, 4, len(parts))
+	require.Equal(t, "hello", parts[0])
+	require.Equal(t, "world", parts[1])
+	require.Equal(t, "how are", parts[2])
+	require.Equal(t, "you?", parts[3])
+}

legacy/builder/add_additional_entries_to_context.go

@@ -53,8 +53,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 		ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL
 	}
 
-	ctx.CollectedSourceFiles = &types.UniqueSourceFileQueue{}
-
 	ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{}
 
 	return nil

legacy/builder/builder_utils/utils.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/globals"
 	"github.com/arduino/arduino-cli/i18n"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
@@ -262,10 +263,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool
 		return false, errors.WithStack(err)
 	}
 
-	rows = utils.Map(rows, removeEndingBackSlash)
-	rows = utils.Map(rows, strings.TrimSpace)
-	rows = utils.Map(rows, unescapeDep)
-	rows = utils.Filter(rows, nonEmptyString)
+	rows = f.Map(rows, removeEndingBackSlash)
+	rows = f.Map(rows, strings.TrimSpace)
+	rows = f.Map(rows, unescapeDep)
+	rows = f.Filter(rows, f.NotEquals(""))
 
 	if len(rows) == 0 {
 		return true, nil
@@ -327,10 +328,6 @@ func removeEndingBackSlash(s string) string {
 	return strings.TrimSuffix(s, "\\")
 }
 
-func nonEmptyString(s string) bool {
-	return s != constants.EMPTY_STRING
-}
-
 func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool {
 
 	targetFileStat, err := targetFile.Stat()

legacy/builder/container_add_prototypes.go

@@ -39,7 +39,9 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 
 	// Run preprocessor
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	if err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders); err != nil {
+	stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders)
+	ctx.WriteStderr(stderr)
+	if err != nil {
 		if !ctx.OnlyUpdateCompilationDatabase {
 			return errors.WithStack(err)
 		}
@@ -60,18 +62,10 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
-	commands := []types.Command{
-		&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"},
-		&PrototypesAdder{},
-	}
-
-	for _, command := range commands {
-		PrintRingNameIfDebug(ctx, command)
-		err := command.Run(ctx)
-		if err != nil {
-			return errors.WithStack(err)
-		}
+	if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil {
+		return errors.WithStack(err)
 	}
+	ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor)
 
 	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)

legacy/builder/container_find_includes.go

@@ -143,23 +143,24 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error {
 		appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path"))
 	}
 
+	sourceFileQueue := &types.UniqueSourceFileQueue{}
+
 	if !ctx.UseCachedLibrariesResolution {
 		sketch := ctx.Sketch
 		mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp"))
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		ctx.CollectedSourceFiles.Push(mergedfile)
+		sourceFileQueue.Push(mergedfile)
 
-		sourceFilePaths := ctx.CollectedSourceFiles
-		queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, ctx.SketchBuildPath, false /* recurse */)
+		queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */)
 		srcSubfolderPath := ctx.SketchBuildPath.Join("src")
 		if srcSubfolderPath.IsDir() {
-			queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */)
+			queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */)
 		}
 
-		for !sourceFilePaths.Empty() {
-			err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop())
+		for !sourceFileQueue.Empty() {
+			err := findIncludesUntilDone(ctx, cache, sourceFileQueue)
 			if err != nil {
 				cachePath.Remove()
 				return errors.WithStack(err)
@@ -314,7 +315,8 @@ func writeCache(cache *includeCache, path *paths.Path) error {
 	return nil
 }
 
-func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error {
+func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error {
+	sourceFile := sourceFileQueue.Pop()
 	sourcePath := sourceFile.SourcePath(ctx)
 	targetFilePath := paths.NullPath()
 	depPath := sourceFile.DepfilePath(ctx)
@@ -367,7 +369,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 				ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
 		} else {
-			preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
+			preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
 			// Unwrap error and see if it is an ExitError.
 			_, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError)
 			if preproc_err == nil {
@@ -399,7 +401,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 			// return errors.WithStack(err)
 			if preproc_err == nil || preproc_stderr == nil {
 				// Filename came from cache, so run preprocessor to obtain error to show
-				preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes)
+				preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
 				if preproc_err == nil {
 					// If there is a missing #include in the cache, but running
 					// gcc does not reproduce that, there is something wrong.
@@ -419,13 +421,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 		appendIncludeFolder(ctx, cache, sourcePath, include, library.SourceDir)
 		sourceDirs := library.SourceDirs()
 		for _, sourceDir := range sourceDirs {
-			queueSourceFilesFromFolder(ctx, ctx.CollectedSourceFiles, library, sourceDir.Dir, sourceDir.Recurse)
+			queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse)
 		}
 		first = false
 	}
 }
 
-func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error {
+func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error {
 	sourceFileExtensions := []string{}
 	for k := range globals.SourceFilesValidExtensions {
 		sourceFileExtensions = append(sourceFileExtensions, k)
@@ -440,7 +442,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFil
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		queue.Push(sourceFile)
+		sourceFileQueue.Push(sourceFile)
 	}
 
 	return nil

legacy/builder/create_cmake_rule.go

@@ -119,9 +119,6 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		return err
 	}
 
-	// Use old ctags method to generate export file
-	ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true)
-
 	err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions)
 	if err != nil {
 		fmt.Println(err)

legacy/builder/ctags/ctags_has_issues.go

@@ -19,8 +19,6 @@ import (
 	"bufio"
 	"os"
 	"strings"
-
-	"github.com/arduino/arduino-cli/legacy/builder/types"
 )
 
 func (p *CTagsParser) FixCLinkageTagsDeclarations() {
@@ -42,7 +40,7 @@ func sliceContainsInt(s []int, e int) bool {
 	return false
 }
 
-func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool {
+func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool {
 	if tag.SkipMe {
 		return true
 	}
@@ -107,7 +105,7 @@ func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool {
 	return ret == -1
 }
 
-func findTemplateMultiline(tag *types.CTag) string {
+func findTemplateMultiline(tag *CTag) string {
 	code, _ := getFunctionProtoUntilTemplateToken(tag, tag.Code)
 	return removeEverythingAfterClosingRoundBracket(code)
 }
@@ -117,7 +115,7 @@ func removeEverythingAfterClosingRoundBracket(s string) string {
 	return s[0 : n+1]
 }
 
-func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, int) {
+func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) {
 
 	/* FIXME I'm ugly */
 	line := 0
@@ -150,7 +148,7 @@ func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, i
 	return code, line
 }
 
-func getFunctionProtoWithNPreviousCharacters(tag *types.CTag, code string, n int) (string, int) {
+func getFunctionProtoWithNPreviousCharacters(tag *CTag, code string, n int) (string, int) {
 
 	/* FIXME I'm ugly */
 	expectedPrototypeLen := len(code) + n
@@ -216,7 +214,7 @@ func removeComments(text string, multilinecomment bool) (string, bool) {
 /* This function scans the source files searching for "extern C" context
  * It save the line numbers in a map filename -> {lines...}
  */
-func (p *CTagsParser) FindCLinkageLines(tags []*types.CTag) map[string][]int {
+func (p *CTagsParser) FindCLinkageLines(tags []*CTag) map[string][]int {
 
 	lines := make(map[string][]int)