Introducing plugins system (#173)

* Removed useless 'required' tag

* Added fields for plugin system

* Made LatestFirmware a method of BoardIndex

* Fixed some linter warnings

* Tools required for plugins are now downloaded

* Created infrastructure for plugin based get-version

Equivalent code changes, no changes in behaviour

* Added FwPlugin object to ease access to fwuploader-plugins

* get-version now supports fwuploader-plugins

* Added license cache

* Better recording of errors from plugins

* Extracted function to exec fwuploader plugins

* Small cosmetic changes

- perform arg checks early
- removed unneeded err variable in function scope

* Another cosmetic change

* Prepared scaffolding for flash-firmware command using plugins

* Implemented plugin-based firmware upload

* Using packagemanager to handle package_index downloads

* Added additional index URLx flags to ease plugin development

* Added missing license data

* Allow file paths as additional 'urls'

* Verify signarure only on offical indexes

* Improve error messages and avoid panic in case of missing tool

* Avoid double buffering of command output in getFirmwareVersion

* Allow merging with overwrite

This means that additional indexex are allowed to overwrite official boards.

* Added plugin-based firmware index

* add support for arduino/fwuploader-plugin-helper#9 (#174)

* Created scaffolding to implement certificate upload via plugin

* Removed useless variable

* Avoid globals arguemnts in flash-certificate

* Factored function to scrape TLS certs from webserver

* Factored function to read certificates

* Small refactoring in cert building subroutines

* Factored function to encode certs as PEM

* Added certificate flash support for plugins

* fix licensed

* Make board override less strict

* Do not consider LICENSE files in plugins archive

* Fixed PEM decoding... 🤦

* pass `-v` and `--log-level` to plugin (#175)

* move vars to global

* add support for arduino/fwuploader-plugin-helper#10

* Update cli/common/common.go

Co-authored-by: Umberto Baldi <[email protected]>

* Renamed variable for clarity

---------

Co-authored-by: Umberto Baldi <[email protected]>
Co-authored-by: Umberto Baldi <[email protected]>

Author: cmaglie

.licensed.yml

@@ -2,6 +2,11 @@
 sources:
   go: true
 
+reviewed:
+  go:
+    - golang.org/x/exp/constraints
+    - golang.org/x/exp/slices
+
 apps:
   - source_path: ./
 
@@ -11,6 +16,7 @@ allowed:
   - gpl-1.0-or-later
   - gpl-1.0+ # Deprecated ID for `gpl-1.0-or-later`
   - gpl-2.0-or-later
+  - gpl-2.0
   - gpl-2.0+ # Deprecated ID for `gpl-2.0-or-later`
   - gpl-3.0-only
   - gpl-3.0 # Deprecated ID for `gpl-3.0-only`

.licenses/arduino-fwuploader/go/github.com/arduino/arduino-cli/arduino/cores/packagemanager.dep.yml

@@ -0,0 +1,63 @@
+---
+name: golang.org/x/exp/constraints
+version: v0.0.0-20230321023759-10a507213a29
+type: go
+summary: Package constraints defines a set of useful constraints to be used with type
+  parameters.
+homepage: https://pkg.go.dev/golang.org/x/exp/constraints
+license: other
+licenses:
+- sources: [email protected]/LICENSE
+  text: |
+    Copyright (c) 2009 The Go Authors. All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+       * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+       * Neither the name of Google Inc. nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+- sources: [email protected]/PATENTS
+  text: |
+    Additional IP Rights Grant (Patents)
+
+    "This implementation" means the copyrightable works distributed by
+    Google as part of the Go project.
+
+    Google hereby grants to You a perpetual, worldwide, non-exclusive,
+    no-charge, royalty-free, irrevocable (except as stated in this section)
+    patent license to make, have made, use, offer to sell, sell, import,
+    transfer and otherwise run, modify and propagate the contents of this
+    implementation of Go, where such license applies only to those patent
+    claims, both currently owned or controlled by Google and acquired in
+    the future, licensable by Google that are necessarily infringed by this
+    implementation of Go.  This grant does not include claims that would be
+    infringed only as a consequence of further modification of this
+    implementation.  If you or your agent or exclusive licensee institute or
+    order or agree to the institution of patent litigation against any
+    entity (including a cross-claim or counterclaim in a lawsuit) alleging
+    that this implementation of Go or any code incorporated within this
+    implementation of Go constitutes direct or contributory patent
+    infringement, or inducement of patent infringement, then any patent
+    rights granted to you under this License for this implementation of Go
+    shall terminate as of the date such litigation is filed.
+notices: []

.licenses/arduino-fwuploader/go/github.com/arduino/arduino-cli/arduino/discovery.dep.yml

@@ -0,0 +1,62 @@
+---
+name: golang.org/x/exp/slices
+version: v0.0.0-20230321023759-10a507213a29
+type: go
+summary: Package slices defines various functions useful with slices of any type.
+homepage: https://pkg.go.dev/golang.org/x/exp/slices
+license: other
+licenses:
+- sources: [email protected]/LICENSE
+  text: |
+    Copyright (c) 2009 The Go Authors. All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+       * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+       * Neither the name of Google Inc. nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+- sources: [email protected]/PATENTS
+  text: |
+    Additional IP Rights Grant (Patents)
+
+    "This implementation" means the copyrightable works distributed by
+    Google as part of the Go project.
+
+    Google hereby grants to You a perpetual, worldwide, non-exclusive,
+    no-charge, royalty-free, irrevocable (except as stated in this section)
+    patent license to make, have made, use, offer to sell, sell, import,
+    transfer and otherwise run, modify and propagate the contents of this
+    implementation of Go, where such license applies only to those patent
+    claims, both currently owned or controlled by Google and acquired in
+    the future, licensable by Google that are necessarily infringed by this
+    implementation of Go.  This grant does not include claims that would be
+    infringed only as a consequence of further modification of this
+    implementation.  If you or your agent or exclusive licensee institute or
+    order or agree to the institution of patent litigation against any
+    entity (including a cross-claim or counterclaim in a lawsuit) alleging
+    that this implementation of Go or any code incorporated within this
+    implementation of Go constitutes direct or contributory patent
+    infringement, or inducement of patent infringement, then any patent
+    rights granted to you under this License for this implementation of Go
+    shall terminate as of the date such litigation is filed.
+notices: []

.licenses/arduino-fwuploader/go/github.com/arduino/arduino-cli/arduino/discovery/discoverymanager.dep.yml

@@ -0,0 +1,80 @@
+---
+name: gopkg.in/yaml.v3
+version: v3.0.1
+type: go
+summary: Package yaml implements YAML support for the Go language.
+homepage: https://pkg.go.dev/gopkg.in/yaml.v3
+license: other
+licenses:
+- sources: LICENSE
+  text: |2
+
+    This project is covered by two different licenses: MIT and Apache.
+
+    #### MIT License ####
+
+    The following files were ported to Go from C files of libyaml, and thus
+    are still covered by their original MIT license, with the additional
+    copyright staring in 2011 when the project was ported over:
+
+        apic.go emitterc.go parserc.go readerc.go scannerc.go
+        writerc.go yamlh.go yamlprivateh.go
+
+    Copyright (c) 2006-2010 Kirill Simonov
+    Copyright (c) 2006-2011 Kirill Simonov
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is furnished to do
+    so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+
+    ### Apache License ###
+
+    All the remaining project files are covered by the Apache license:
+
+    Copyright (c) 2011-2019 Canonical Ltd
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+- sources: README.md
+  text: |-
+    The yaml package is licensed under the MIT and Apache License 2.0 licenses.
+    Please see the LICENSE file for details.
+notices:
+- sources: NOTICE
+  text: |-
+    Copyright 2011-2016 Canonical Ltd.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

.licenses/arduino-fwuploader/go/github.com/arduino/arduino-cli/arduino/sketch.dep.yml

@@ -0,0 +1,108 @@
+/*
+	arduino-fwuploader
+	Copyright (c) 2023 Arduino LLC.  All right reserved.
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published
+	by the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+package certificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/sirupsen/logrus"
+)
+
+// ScrapeRootCertificatesFromURL downloads from a webserver the root certificate
+// required to connect to that server from the TLS handshake response.
+func ScrapeRootCertificatesFromURL(URL string) (*x509.Certificate, error) {
+	conn, err := tls.Dial("tcp", URL, &tls.Config{
+		InsecureSkipVerify: true,
+	})
+	if err != nil {
+		logrus.Error(err)
+		return nil, err
+	}
+	defer conn.Close()
+
+	if err := conn.Handshake(); err != nil {
+		logrus.Error(err)
+		return nil, err
+	}
+
+	peerCertificates := conn.ConnectionState().PeerCertificates
+	if len(peerCertificates) == 0 {
+		err = fmt.Errorf("no peer certificates found at %s", URL)
+		logrus.Error(err)
+		return nil, err
+	}
+
+	rootCertificate := peerCertificates[len(peerCertificates)-1]
+	return rootCertificate, nil
+}
+
+// LoadCertificatesFromFile read certificates from the given file. PEM and CER formats
+// are supported.
+func LoadCertificatesFromFile(certificateFile *paths.Path) ([]*x509.Certificate, error) {
+	data, err := certificateFile.ReadFile()
+	if err != nil {
+		logrus.Error(err)
+		return nil, err
+	}
+	var res []*x509.Certificate
+	switch certificateFile.Ext() {
+	case ".cer":
+		cert, err := x509.ParseCertificate(data)
+		if err != nil {
+			logrus.Error(err)
+		}
+		res = append(res, cert)
+		return res, err
+
+	case ".pem":
+		for {
+			block, rest := pem.Decode(data)
+			data = rest
+			if block == nil && len(rest) > 0 {
+				return nil, fmt.Errorf("invalid .pem data")
+			}
+			if block == nil {
+				return res, nil
+			}
+			cert, err := x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse certificate: %w", err)
+			}
+			res = append(res, cert)
+			if len(rest) == 0 {
+				return res, nil
+			}
+		}
+	default:
+		return nil, fmt.Errorf("cert format %s not supported, please use .pem or .cer", certificateFile.Ext())
+	}
+}
+
+// EncodeCertificateAsPEM returns the PEM encoding of the given certificate
+func EncodeCertificateAsPEM(cert *x509.Certificate) []byte {
+	pemBlock := &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(pemBlock)
+}