feat: Test writing remote Actions locally

cmd/input.go

@@ -59,6 +59,7 @@ type Input struct {
 	logPrefixJobID                     bool
 	networkName                        string
 	useNewActionCache                  bool
+	localRepository                    []string
 }
 
 func (i *Input) resolve(path string) string {

cmd/root.go

@@ -100,6 +100,7 @@ func Execute(ctx context.Context, version string) {
 	rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull")
 	rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.")
 	rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally")
+	rootCmd.PersistentFlags().StringArrayVarP(&input.localRepository, "local-repository", "", []string{}, "Replaces the specified repository and ref with a local folder")
 	rootCmd.SetArgs(args())
 
 	if err := rootCmd.Execute(); err != nil {
@@ -618,10 +619,22 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
 			Matrix:                             matrixes,
 			ContainerNetworkMode:               docker_container.NetworkMode(input.networkName),
 		}
-		if input.useNewActionCache {
+		if input.useNewActionCache || len(input.localRepository) > 0 {
 			config.ActionCache = &runner.GoGitActionCache{
 				Path: config.ActionCacheDir,
 			}
+			if len(input.localRepository) > 0 {
+				localRepositories := map[string]string{}
+				for _, l := range input.localRepository {
+					k, v, _ := strings.Cut(l, "=")
+					localRepositories[k] = v
+				}
+				config.ActionCache = &runner.LocalRepositoryCache{
+					Parent:            config.ActionCache,
+					LocalRepositories: localRepositories,
+					CacheDirCache:     map[string]string{},
+				}
+			}
 		}
 		r, err := runner.New(config)
 		if err != nil {

pkg/runner/file_collector.go

@@ -0,0 +1,187 @@
+package runner
+
+import (
+	"archive/tar"
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	git "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/filemode"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
+	"github.com/go-git/go-git/v5/plumbing/format/index"
+)
+
+type fileCollectorHandler interface {
+	WriteFile(path string, fi fs.FileInfo, linkName string, f io.Reader) error
+}
+
+type tarCollector struct {
+	TarWriter *tar.Writer
+	UID       int
+	GID       int
+	DstDir    string
+}
+
+func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error {
+	// create a new dir/file header
+	header, err := tar.FileInfoHeader(fi, linkName)
+	if err != nil {
+		return err
+	}
+
+	// update the name to correctly reflect the desired destination when untaring
+	header.Name = path.Join(tc.DstDir, fpath)
+	header.Mode = int64(fi.Mode())
+	header.ModTime = fi.ModTime()
+	header.Uid = tc.UID
+	header.Gid = tc.GID
+
+	// write the header
+	if err := tc.TarWriter.WriteHeader(header); err != nil {
+		return err
+	}
+
+	// this is a symlink no reader provided
+	if f == nil {
+		return nil
+	}
+
+	// copy file data into tar writer
+	if _, err := io.Copy(tc.TarWriter, f); err != nil {
+		return err
+	}
+	return nil
+}
+
+type fileCollector struct {
+	Ignorer   gitignore.Matcher
+	SrcPath   string
+	SrcPrefix string
+	Fs        fileCollectorFs
+	Handler   fileCollectorHandler
+}
+
+type fileCollectorFs interface {
+	Walk(root string, fn filepath.WalkFunc) error
+	OpenGitIndex(path string) (*index.Index, error)
+	Open(path string) (io.ReadCloser, error)
+	Readlink(path string) (string, error)
+}
+
+type defaultFs struct {
+}
+
+func (*defaultFs) Walk(root string, fn filepath.WalkFunc) error {
+	return filepath.Walk(root, fn)
+}
+
+func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) {
+	r, err := git.PlainOpen(path)
+	if err != nil {
+		return nil, err
+	}
+	i, err := r.Storer.Index()
+	if err != nil {
+		return nil, err
+	}
+	return i, nil
+}
+
+func (*defaultFs) Open(path string) (io.ReadCloser, error) {
+	return os.Open(path)
+}
+
+func (*defaultFs) Readlink(path string) (string, error) {
+	return os.Readlink(path)
+}
+
+//nolint:gocyclo
+func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc {
+	i, _ := fc.Fs.OpenGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...)))
+	return func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if ctx != nil {
+			select {
+			case <-ctx.Done():
+				return fmt.Errorf("copy cancelled")
+			default:
+			}
+		}
+
+		sansPrefix := strings.TrimPrefix(file, fc.SrcPrefix)
+		split := strings.Split(sansPrefix, string(filepath.Separator))
+		// The root folders should be skipped, submodules only have the last path component set to "." by filepath.Walk
+		if fi.IsDir() && len(split) > 0 && split[len(split)-1] == "." {
+			return nil
+		}
+		var entry *index.Entry
+		if i != nil {
+			entry, err = i.Entry(strings.Join(split[len(submodulePath):], "/"))
+		} else {
+			err = index.ErrEntryNotFound
+		}
+		if err != nil && fc.Ignorer != nil && fc.Ignorer.Match(split, fi.IsDir()) {
+			if fi.IsDir() {
+				if i != nil {
+					ms, err := i.Glob(strings.Join(append(split[len(submodulePath):], "**"), "/"))
+					if err != nil || len(ms) == 0 {
+						return filepath.SkipDir
+					}
+				} else {
+					return filepath.SkipDir
+				}
+			} else {
+				return nil
+			}
+		}
+		if err == nil && entry.Mode == filemode.Submodule {
+			err = fc.Fs.Walk(file, fc.collectFiles(ctx, split))
+			if err != nil {
+				return err
+			}
+			return filepath.SkipDir
+		}
+		path := filepath.ToSlash(sansPrefix)
+
+		// return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update)
+		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			linkName, err := fc.Fs.Readlink(file)
+			if err != nil {
+				return fmt.Errorf("unable to readlink '%s': %w", file, err)
+			}
+			return fc.Handler.WriteFile(path, fi, linkName, nil)
+		} else if !fi.Mode().IsRegular() {
+			return nil
+		}
+
+		// open file
+		f, err := fc.Fs.Open(file)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		if ctx != nil {
+			// make io.Copy cancellable by closing the file
+			cpctx, cpfinish := context.WithCancel(ctx)
+			defer cpfinish()
+			go func() {
+				select {
+				case <-cpctx.Done():
+				case <-ctx.Done():
+					f.Close()
+				}
+			}()
+		}
+
+		return fc.Handler.WriteFile(path, fi, "", f)
+	}
+}

pkg/runner/local_repository_cache.go

@@ -0,0 +1,81 @@
+package runner
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type LocalRepositoryCache struct {
+	Parent            ActionCache
+	LocalRepositories map[string]string
+	CacheDirCache     map[string]string
+}
+
+func (l *LocalRepositoryCache) Fetch(ctx context.Context, cacheDir, url, ref, token string) (string, error) {
+	if dest, ok := l.LocalRepositories[fmt.Sprintf("%s@%s", url, ref)]; ok {
+		l.CacheDirCache[cacheDir] = dest
+		return "local-repository", nil
+	}
+	return l.Parent.Fetch(ctx, cacheDir, url, ref, token)
+}
+
+func (l *LocalRepositoryCache) GetTarArchive(ctx context.Context, cacheDir, sha, includePrefix string) (io.ReadCloser, error) {
+	if dest, ok := l.CacheDirCache[cacheDir]; ok {
+		srcPath := filepath.Join(dest, includePrefix)
+		buf := &bytes.Buffer{}
+		tw := tar.NewWriter(buf)
+		defer tw.Close()
+		srcPath = filepath.Clean(srcPath)
+		fi, err := os.Lstat(srcPath)
+		if err != nil {
+			return nil, err
+		}
+		tc := &tarCollector{
+			TarWriter: tw,
+		}
+		if fi.IsDir() {
+			srcPrefix := filepath.Dir(srcPath)
+			if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
+				srcPrefix += string(filepath.Separator)
+			}
+			fc := &fileCollector{
+				Fs:        &defaultFs{},
+				SrcPath:   srcPath,
+				SrcPrefix: srcPrefix,
+				Handler:   tc,
+			}
+			err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{}))
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			var f io.ReadCloser
+			var linkname string
+			if fi.Mode()&fs.ModeSymlink != 0 {
+				linkname, err = os.Readlink(srcPath)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				f, err = os.Open(srcPath)
+				if err != nil {
+					return nil, err
+				}
+				defer f.Close()
+			}
+			err := tc.WriteFile(fi.Name(), fi, linkname, f)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return io.NopCloser(buf), nil
+	}
+	return l.Parent.GetTarArchive(ctx, cacheDir, sha, includePrefix)
+}