Provide package indexes from file paths using "additional-paths" flag (#1093)

* Initial commit

* Bind new flag --additional-paths and add new function LoadIndexNoSign

* Use absolute file path to package_index.json

* Add documentation for additional-paths feature

* Update comment for LoadIndexNoSign

* Update variable names to pass checks + simplify code

* Add new config for additional_paths

Author: giulcioffi

arduino/cores/packageindex/index.go

@@ -312,3 +312,20 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) {
 	}
 	return &index, nil
 }
+
+// LoadIndexNoSign reads a package_index.json from a file and returns the corresponding Index structure.
+func LoadIndexNoSign(jsonIndexFile *paths.Path) (*Index, error) {
+	buff, err := jsonIndexFile.ReadFile()
+	if err != nil {
+		return nil, err
+	}
+	var index Index
+	err = json.Unmarshal(buff, &index)
+	if err != nil {
+		return nil, err
+	}
+
+	index.IsTrusted = true
+
+	return &index, nil
+}

cli/cli.go

@@ -104,6 +104,7 @@ func createCliCommandTree(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", "The output format, can be {text|json}.")
 	cmd.PersistentFlags().StringVar(&configFile, "config-file", "", "The custom config file (if not specified the default will be used).")
 	cmd.PersistentFlags().StringSlice("additional-urls", []string{}, "Comma-separated list of additional URLs for the Boards Manager.")
+	cmd.PersistentFlags().StringSlice("additional-paths", []string{}, "Comma-separated list of additional file paths for the Boards Manager.")
 	configuration.BindFlags(cmd, configuration.Settings)
 }
 

cli/config/validate.go

@@ -21,18 +21,19 @@ import (
 )
 
 var validMap = map[string]reflect.Kind{
-	"board_manager.additional_urls": reflect.Slice,
-	"daemon.port":                   reflect.String,
-	"directories.data":              reflect.String,
-	"directories.downloads":         reflect.String,
-	"directories.user":              reflect.String,
-	"library.enable_unsafe_install": reflect.Bool,
-	"logging.file":                  reflect.String,
-	"logging.format":                reflect.String,
-	"logging.level":                 reflect.String,
-	"sketch.always_export_binaries": reflect.Bool,
-	"telemetry.addr":                reflect.String,
-	"telemetry.enabled":             reflect.Bool,
+	"board_manager.additional_urls":  reflect.Slice,
+	"board_manager.additional_paths": reflect.Slice,
+	"daemon.port":                    reflect.String,
+	"directories.data":               reflect.String,
+	"directories.downloads":          reflect.String,
+	"directories.user":               reflect.String,
+	"library.enable_unsafe_install":  reflect.Bool,
+	"logging.file":                   reflect.String,
+	"logging.format":                 reflect.String,
+	"logging.level":                  reflect.String,
+	"sketch.always_export_binaries":  reflect.Bool,
+	"telemetry.addr":                 reflect.String,
+	"telemetry.enabled":              reflect.Bool,
 }
 
 func typeOf(key string) (reflect.Kind, error) {

commands/daemon/testdata/arduino-cli.yml

@@ -2,6 +2,7 @@ board_manager:
   additional_urls:
     - http://foobar.com
     - http://example.com
+  additional_paths: []
 
 daemon:
   port: "50051"

commands/instances.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"os"
 	"path"
 
 	"github.com/arduino/arduino-cli/arduino/builder"
@@ -200,9 +201,29 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo
 	}
 
 	indexpath := paths.New(configuration.Settings.GetString("directories.Data"))
+
+	for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
+		logrus.Info("JSON PATH: ", x)
+
+		pathJSON, _ := paths.New(x).Abs()
+
+		if _, err := packageindex.LoadIndexNoSign(pathJSON); err != nil {
+			return nil, fmt.Errorf("invalid package index in %s: %s", pathJSON, err)
+		}
+
+		fi, _ := os.Stat(x)
+		downloadCB(&rpc.DownloadProgress{
+			File:      "Updating index: " + pathJSON.Base(),
+			TotalSize: fi.Size(),
+		})
+		downloadCB(&rpc.DownloadProgress{Completed: true})
+
+	}
+
 	urls := []string{globals.DefaultIndexURL}
 	urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
 	for _, u := range urls {
+		logrus.Info("URL: ", u)
 		URL, err := url.Parse(u)
 		if err != nil {
 			logrus.Warnf("unable to parse additional URL: %s", u)
@@ -649,6 +670,15 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult
 			}
 		}
 
+		for _, x := range configuration.Settings.GetStringSlice("board_manager.additional_paths") {
+			pathJSON, _ := paths.New(x).Abs()
+
+			_, err := res.Pm.LoadPackageIndexFromFile(pathJSON)
+			if err != nil {
+				res.PlatformIndexErrors = append(res.PlatformIndexErrors, err.Error())
+			}
+		}
+
 		if err := res.Pm.LoadHardware(); err != nil {
 			return res, fmt.Errorf("error loading hardware packages: %s", err)
 		}

configuration/configuration.go

@@ -76,6 +76,7 @@ func BindFlags(cmd *cobra.Command, settings *viper.Viper) {
 	settings.BindPFlag("logging.file", cmd.Flag("log-file"))
 	settings.BindPFlag("logging.format", cmd.Flag("log-format"))
 	settings.BindPFlag("board_manager.additional_urls", cmd.Flag("additional-urls"))
+	settings.BindPFlag("board_manager.additional_paths", cmd.Flag("additional-paths"))
 }
 
 // getDefaultArduinoDataDir returns the full path to the default arduino folder

configuration/configuration_test.go

@@ -89,6 +89,7 @@ func TestInit(t *testing.T) {
 	require.Equal(t, "text", settings.GetString("logging.format"))
 
 	require.Empty(t, settings.GetStringSlice("board_manager.additional_urls"))
+	require.Empty(t, settings.GetStringSlice("board_manager.additional_paths"))
 
 	require.NotEmpty(t, settings.GetString("directories.Data"))
 	require.NotEmpty(t, settings.GetString("directories.Downloads"))

configuration/defaults.go

@@ -33,6 +33,7 @@ func SetDefaults(settings *viper.Viper) {
 
 	// Boards Manager
 	settings.SetDefault("board_manager.additional_urls", []string{})
+	settings.SetDefault("board_manager.additional_paths", []string{})
 
 	// arduino directories
 	settings.SetDefault("directories.Data", getDefaultArduinoDataDir())

docs/configuration.md

@@ -2,6 +2,8 @@
 
 - `board_manager`
   - `additional_urls` - the URLs to any additional Boards Manager package index files needed for your boards platforms.
+  - `additional_paths` - the absolute file paths to any additional Boards Manager package index files needed for your
+    boards platforms.
 - `daemon` - options related to running Arduino CLI as a [gRPC] server.
   - `port` - TCP port used for gRPC client connections.
 - `directories` - directories used by Arduino CLI.
@@ -66,6 +68,12 @@ Setting an additional Boards Manager URL using the `ARDUINO_BOARD_MANAGER_ADDITI
 $ export ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://downloads.arduino.cc/packages/package_staging_index.json
 ```
 
+Setting an additional Boards Manager file using the `ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS` environment variable:
+
+```sh
+$ export ARDUINO_BOARD_MANAGER_ADDITIONAL_PATHS=/path/to/your/package_staging_index.json
+```
+
 ### Configuration file
 
 [`arduino-cli config init`][arduino-cli config init] creates or updates a configuration file with the current
@@ -128,6 +136,21 @@ Doing the same using a TOML format file:
 additional_urls = [ "https://downloads.arduino.cc/packages/package_staging_index.json" ]
 ```
 
+Setting an additional Boards Manager File using a YAML format configuration file:
+
+```yaml
+board_manager:
+  additional_paths:
+    - /path/to/your/package_staging_index.json
+```
+
+Doing the same using a TOML format file:
+
+```toml
+[board_manager]
+additional_paths = [ "/path/to/your/package_staging_index.json" ]
+```
+
 [grpc]: https://grpc.io
 [sketchbook directory]: sketch-specification.md#sketchbook
 [arduino cli lib install]: commands/arduino-cli_lib_install.md

docs/getting-started.md

@@ -177,12 +177,24 @@ board_manager:
     - https://arduino.esp8266.com/stable/package_esp8266com_index.json
 ```
 
-From now on, commands supporting custom cores will automatically use the additional URL from the configuration file:
+If you have your package indexes locally installed, you can list their file path in the Arduino CLI configuration file.
+
+For example, to add the NRF52832 core, edit the configuration file and change the `board_manager` settings as follows:
+
+```yaml
+board_manager:
+  additional_paths:
+    - /absolute/path/to/your/package_nrf52832_index.json
+```
+
+From now on, commands supporting custom cores will automatically use the additional URL and additional paths from the
+configuration file:
 
 ```sh
 $ arduino-cli core update-index
 Updating index: package_index.json downloaded
 Updating index: package_esp8266com_index.json downloaded
+Updating index: package_nrf52832_index.json
 Updating index: package_index.json downloaded
 
 $ arduino-cli core search esp8266
@@ -202,6 +214,18 @@ ID              Version Name
 esp8266:esp8266 2.5.2   esp8266
 ```
 
+The same applies to the additional package index file provided by file paths. Use the `--additional-paths` option, that
+has to be specified every time and for every command that operates on a 3rd party platform core, for example:
+
+```sh
+$ arduino-cli  core update-index --additional-paths /absolute/path/to/your/package_esp8266com_index.json
+Updating index: package_esp8266com_index.json downloaded
+
+$ arduino-cli core search esp8266 --additional-paths /absolute/path/to/your/package_esp8266com_index.json
+ID              Version Name
+esp8266:esp8266 2.5.2   esp8266
+```
+
 ## Compile and upload the sketch
 
 To compile the sketch you run the `compile` command, passing the proper FQBN string: