Changed the LoadPaths function.

Squashing all the commits for following issue:
Fix for the issue when OPA doesnot load tarball on cmd line as a bundle.

Fixes #5879

Signed-off-by: Yogesh Sinha <[email protected]>

Author: yogisinha

internal/runtime/init/init.go

@@ -127,10 +127,16 @@ func LoadPaths(paths []string,
 		caps = ast.CapabilitiesForThisVersion()
 	}
 
+	// tar.gz files are automatically loaded as bundles
+	var likelyBundles, nonBundlePaths []string
+	if !asBundle {
+		likelyBundles, nonBundlePaths = splitByTarGzExt(paths)
+		paths = likelyBundles
+	}
+
 	var result LoadPathsResult
 	var err error
-
-	if asBundle {
+	if asBundle || len(likelyBundles) > 0 {
 		result.Bundles = make(map[string]*bundle.Bundle, len(paths))
 		for _, path := range paths {
 			result.Bundles[path], err = loader.NewFileLoader().
@@ -145,14 +151,18 @@ func LoadPaths(paths []string,
 				return nil, err
 			}
 		}
+	}
+
+	if len(nonBundlePaths) == 0 {
 		return &result, nil
 	}
 
 	files, err := loader.NewFileLoader().
 		WithFS(fsys).
 		WithProcessAnnotation(processAnnotations).
 		WithCapabilities(caps).
-		Filtered(paths, filter)
+		Filtered(nonBundlePaths, filter)
+
 	if err != nil {
 		return nil, err
 	}
@@ -162,6 +172,19 @@ func LoadPaths(paths []string,
 	return &result, nil
 }
 
+// splitByTarGzExt splits the paths in 2 groups. Ones with .tar.gz and another with
+// non .tar.gz extensions.
+func splitByTarGzExt(paths []string) (targzs []string, nonTargzs []string) {
+	for _, path := range paths {
+		if strings.HasSuffix(path, ".tar.gz") {
+			targzs = append(targzs, path)
+		} else {
+			nonTargzs = append(nonTargzs, path)
+		}
+	}
+	return
+}
+
 // WalkPaths reads data and policy from the given paths and returns a set of bundle directory loaders
 // or descriptors that contain information about files.
 func WalkPaths(paths []string, filter loader.Filter, asBundle bool) (*WalkPathsResult, error) {

internal/runtime/init/init_test.go

@@ -5,7 +5,9 @@
 package init
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"io"
 	"io/fs"
 	"os"
@@ -14,6 +16,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/open-policy-agent/opa/ast"
+	"github.com/open-policy-agent/opa/bundle"
+	"github.com/open-policy-agent/opa/internal/file/archive"
 	"github.com/open-policy-agent/opa/loader"
 	"github.com/open-policy-agent/opa/storage"
 	inmem "github.com/open-policy-agent/opa/storage/inmem/test"
@@ -183,6 +188,239 @@ p = true { 1 = 2 }`
 	}
 }
 
+func TestLoadTarGzsInBundleAndNonBundleMode(t *testing.T) {
+
+	type bundleInfo struct {
+		fileName  string
+		files     [][2]string
+		expBundle bundle.Bundle
+	}
+
+	bundle1TarGz := bundleInfo{
+		fileName: "bundle1.tar.gz",
+		files: [][2]string{
+			{"/a/data.json", `{"foo": "bar1", "x": {"y": {"z": [1]}}}`},
+			{"/a/.manifest", `{"roots": ["a"]}`},
+		},
+		expBundle: bundle.Bundle{
+			Manifest: bundle.Manifest{
+				Roots: &[]string{"a"},
+			},
+			Data: map[string]interface{}{
+				"a": map[string]interface{}{
+					"foo": "bar1",
+					"x": map[string]interface{}{
+						"y": map[string]interface{}{
+							"z": []interface{}{json.Number("1")},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundle2TarGz := bundleInfo{
+		fileName: "bundle2.tar.gz",
+		files: [][2]string{
+			{"/b/data.json", `{"foo": "bar2", "x": {"y": {"z": [1]}}}`},
+			{"/b/.manifest", `{"roots": ["b"]}`},
+		},
+		expBundle: bundle.Bundle{
+			Manifest: bundle.Manifest{
+				Roots: &[]string{"b"},
+			},
+			Data: map[string]interface{}{
+				"b": map[string]interface{}{
+					"foo": "bar2",
+					"x": map[string]interface{}{
+						"y": map[string]interface{}{
+							"z": []interface{}{json.Number("1")},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundle1Folder := map[string]string{
+		"/bundle1/a/data.json": `{"foo1": "bar2", "x": {"y": {"z": [2]}}}`,
+		"/bundle1/a/.manifest": `{"roots": ["a"]}`,
+		"/bundle1/a/foo.rego":  `package a.b.y`,
+	}
+
+	modulePath := "/bundle1/a/foo.rego"
+	module := `package a.b.y`
+
+	bundle1FolderInfo := bundleInfo{
+		fileName: "bundle1",
+		expBundle: bundle.Bundle{
+			Manifest: bundle.Manifest{
+				Roots: &[]string{"a"},
+			},
+			Data: map[string]interface{}{
+				"a": map[string]interface{}{
+					"foo1": "bar2",
+					"x": map[string]interface{}{
+						"y": map[string]interface{}{
+							"z": []interface{}{json.Number("2")},
+						},
+					},
+				},
+			},
+			Modules: []bundle.ModuleFile{
+				{
+					URL:    modulePath,
+					Path:   modulePath,
+					Parsed: ast.MustParseModule(module),
+					Raw:    []byte(module),
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		note            string
+		bundleInfoTC    []bundleInfo
+		folderContent   map[string]string
+		expectedBundles int
+		expectedModules int
+		asBundle        bool
+	}{
+		{
+			note:          "load multiple bundles. one tar.gz and one folder. Bundle mode is true",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1TarGz,
+				bundle1FolderInfo,
+			},
+			expectedBundles: 2,
+			expectedModules: 0,
+			asBundle:        true,
+		},
+		{
+			note:          "load multiple bundles. one tar.gz and one folder. Bundle mode is false",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1TarGz,
+				bundle1FolderInfo,
+			},
+			expectedBundles: 1,
+			expectedModules: 1,
+			asBundle:        false,
+		},
+		{
+			note:          "load multiple bundles. two tar.gz and one folder. Bundle mode is true",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1TarGz,
+				bundle2TarGz,
+				bundle1FolderInfo,
+			},
+			expectedBundles: 3,
+			expectedModules: 0,
+			asBundle:        true,
+		},
+		{
+			note:          "load multiple bundles. two tar.gz and one folder. Bundle mode is false",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1TarGz,
+				bundle2TarGz,
+				bundle1FolderInfo,
+			},
+			expectedBundles: 2,
+			expectedModules: 1,
+			asBundle:        false,
+		},
+		{
+			note:          "load just one folder. Bundle mode is true",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1FolderInfo,
+			},
+			expectedBundles: 1,
+			expectedModules: 0,
+			asBundle:        true,
+		},
+		{
+			note:          "load just one folder. Bundle mode is false",
+			folderContent: bundle1Folder,
+			bundleInfoTC: []bundleInfo{
+				bundle1FolderInfo,
+			},
+			expectedBundles: 0,
+			expectedModules: 1,
+			asBundle:        false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.note, func(t *testing.T) {
+
+			test.WithTempFS(tc.folderContent, func(rootDir string) {
+				paths := []string{}
+				for _, bdlInfo := range tc.bundleInfoTC {
+					if strings.HasSuffix(bdlInfo.fileName, ".tar.gz") {
+
+						// Create the tar gz files temporarily
+						buf := archive.MustWriteTarGz(bdlInfo.files)
+						bundleFile := filepath.Join(rootDir, bdlInfo.fileName)
+						out, err := os.Create(bundleFile)
+						if err != nil {
+							t.Fatalf("Unexpected error: %v", err)
+						}
+						_, err = out.Write(buf.Bytes())
+						if err != nil {
+							t.Fatalf("Unexpected error: %v", err)
+						}
+					}
+
+					paths = append(paths, filepath.Join(rootDir, bdlInfo.fileName))
+				}
+
+				loaded, err := LoadPaths(paths, nil, tc.asBundle, nil, true, false, nil, nil)
+				if err != nil {
+					t.Fatal("Failed LoadPaths ", err)
+				}
+				if tc.expectedBundles != len(loaded.Bundles) {
+					t.Fatalf("Expected %d bundles, got %d", tc.expectedBundles, len(loaded.Bundles))
+				}
+				if tc.expectedModules != len(loaded.Files.Modules) {
+					t.Fatalf("Expected %d modules, got %d", tc.expectedModules, len(loaded.Files.Modules))
+				}
+
+				// Testing the content
+				for path, actual := range loaded.Bundles {
+					for _, bdlInfo := range tc.bundleInfoTC {
+						if strings.HasSuffix(path, bdlInfo.fileName) {
+							var buf bytes.Buffer
+							if err := bundle.NewWriter(&buf).Write(bdlInfo.expBundle); err != nil {
+								t.Fatal(err)
+							}
+
+							expected, err := bundle.NewReader(&buf).Read()
+							if err != nil {
+								t.Fatal(err)
+							}
+
+							// adjusting the URL and Path due to /tmp/ path
+							if len(bdlInfo.expBundle.Modules) > 0 {
+								expected.Modules[0].URL = rootDir + expected.Modules[0].URL
+								expected.Modules[0].Path = rootDir + expected.Modules[0].Path
+							}
+
+							if !expected.Equal(*actual) {
+								t.Fatalf("\nExpected: %+v\nGot: %+v", expected, actual)
+							}
+						}
+					}
+				}
+			})
+		})
+	}
+
+}
+
 func TestWalkPaths(t *testing.T) {
 	files := map[string]string{
 		"/bundle1/a/data.json":   `{"foo": "bar1", "x": {"y": {"z": [1]}}}`,