Arduino-ESP32 release management scripted (#1515)

* Calculate an absolute path for a custom partitions table (#1452)

* * Arduino-ESP32 release management scripted
(ready-to-merge)

* * secure env for espressif/arduino-esp32

* * build tests enabled
* gitter webhook enabled

* * gitter room link fixed
* better comment

* * filepaths fixed

Author: pacucha42

.travis.yml

@@ -7,52 +7,34 @@ python:
 os:
   - linux
 
+env:
+    global:
+        - secure: "l/4Dt+KQ/mACtGAHDUsPr66fUte840PZoQ4xpPikqWZI0uARu4l+Ym7+sHinnT6fBqrj8AJeBYGz4nFa8NK4LutZn9mSD40w+sxl0wSV4oHV8rzKe3Cd8+sMG3+o33yWoikMNjSvqa73Q0rm+SgrlInNdZbuAyixL+a2alaWSnGPm4F2xwUGj+S33TOy5P/Xp77CYtCV5S8vzyk/eEdNhoF0GYePJVdfuzCOUjXMyT5OWxORkzzQ7Hnn/Ka/RDfV8Si4HgujLQBrK5q6iPnNBFqBSqilYBepSMn4opnOBpIm0SCgePz7XQEFC83buA7GUcnCnfg38bf+dCwHaODf1d1PmqVRYt2QmfinexXtM4afAtL0iBUDtvrfnXHzwW9w82VeZhpbJSVh9DUQvB0IlsZeCz9J9PUBAi3N+SMX+9l+BomYwRUlPuKY+Ef2JKk9q6mxtUkky5R0daAlVxEhpVdQks1rT+T+NMoDMemxQ3SKEiqAHh6EgHecruszffmZ71uLX9MpERpew0qN+UFiafws+jkTjx+3yF9yut0Hf9sMbeAYzzkGzRqJTUEBJ6B29Cql8M0yRXCNN/8wuuTHhG8esstozga4ZQoIVrq7mEAgup376PTcNfr1+imbbWVQ7lJdYIuDe6OS5V3OX6np11vgK/DbhfyzvQv9Z1zAGnM="
+        - REMOTE_URL=https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG
+    
 script:
-  #- set -e
-  - echo -e "travis_fold:start:sketch_test_env_prepare"
-  - pip install pyserial
-  - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
-  - tar xf arduino.tar.xz
-  - mv arduino-nightly $HOME/arduino_ide
-  - mkdir -p $HOME/Arduino/libraries
-  - cd $HOME/arduino_ide/hardware
-  - mkdir espressif
-  - cd espressif
-  - ln -s $TRAVIS_BUILD_DIR esp32
-  - cd esp32
-  - git submodule update --init --recursive
-  - cd tools
-  - python get.py
-  - export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-esp32-elf/bin:$PATH"
-  - which arduino
-  - cd $TRAVIS_BUILD_DIR
-  - source tools/common.sh
-  - echo -e "travis_fold:end:sketch_test_env_prepare"
-  - echo -e "travis_fold:start:sketch_test"
-  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
-  - echo -e "travis_fold:end:sketch_test"
-  - echo -e "travis_fold:start:size_report"
-  - cat size.log
-  - echo -e "travis_fold:end:size_report"
+  # run sketch tests
+  - bash $TRAVIS_BUILD_DIR/tools/build-tests.sh
+  
+  # zip the package if tagged build, otherwise finish here
+  - bash $TRAVIS_BUILD_DIR/tools/build-release.sh -a$ESP32_GITHUB_TOKEN
+
+deploy:
+  provider: script
+  skip_cleanup: true
+  script: bash $TRAVIS_BUILD_DIR/tools/deploy.sh -t$TRAVIS_TAG -a$ESP32_GITHUB_TOKEN -s$TRAVIS_REPO_SLUG -drelease
+
+  on:
+    tags: true
 
-  # test library examples with PlatformIO
-  - echo -e "travis_fold:start:platformio_test_env_prepare"
-  - pip install -U https://github.com/platformio/platformio/archive/develop.zip
-  - platformio platform install https://github.com/platformio/platform-espressif32.git#feature/stage
-  - sed -i 's/https:\/\/github\.com\/espressif\/arduino-esp32\.git/*/' ~/.platformio/platforms/espressif32/platform.json
-  - ln -s $TRAVIS_BUILD_DIR ~/.platformio/packages/framework-arduinoespressif32
-  - echo -e "travis_fold:end:platformio_test_env_prepare"
-  - echo -e "travis_fold:start:platformio_test"
-  - "python -c \"import glob,os,subprocess,sys; map(lambda p: (sys.stdout.write('Library example: %s\\n' % p), subprocess.call(['pio', 'ci', p, '--board', 'esp32dev'])), set([os.path.dirname(p) for p in glob.glob('libraries/*/examples/*/*.ino') + glob.glob('libraries/*/examples/*/*/*.ino')]))\""
-  - echo -e "travis_fold:end:platformio_test"
 
 notifications:
   email:
     on_success: change
     on_failure: change
   webhooks:
     urls:
-      - https://webhooks.gitter.im/e/cb057279c430d91a47a8
+        - https://webhooks.gitter.im/e/cb057279c430d91a47a8
     on_success: change  # options: [always|never|change] default: always
     on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false
+    on_start: never     # options: [always|never|change] default: always

package/merge_packages.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# This script merges two Arduino Board Manager package json files.
+# Usage:
+#   python merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json
+# Written by Ivan Grokhotkov, 2015
+#
+from __future__ import print_function
+from distutils.version import LooseVersion
+import re
+import json
+import sys
+
+def load_package(filename):
+    pkg = json.load(open(filename))['packages'][0]
+    print("Loaded package {0} from {1}".format(pkg['name'], filename), file=sys.stderr)
+    print("{0} platform(s), {1} tools".format(len(pkg['platforms']), len(pkg['tools'])), file=sys.stderr)
+    return pkg
+
+def merge_objects(versions, obj):
+    for o in obj:
+        name = o['name'].encode('ascii')
+        ver = o['version'].encode('ascii')
+        if not name in versions:
+            print("found new object, {0}".format(name), file=sys.stderr)
+            versions[name] = {}
+        if not ver in versions[name]:
+            print("found new version {0} for object {1}".format(ver, name), file=sys.stderr)
+            versions[name][ver] = o
+    return versions
+
+# Normalize ESP release version string (x.x.x) by adding '-rc<MAXINT>' (x.x.x-rc9223372036854775807) to ensure having REL above any RC
+# Dummy approach, functional anyway for current ESP package versioning (unlike NormalizedVersion/LooseVersion/StrictVersion & similar crap)
+def pkgVersionNormalized(versionString):
+
+    verStr = str(versionString)
+    verParts = re.split('\.|-rc', verStr, flags=re.IGNORECASE)
+    
+    if len(verParts) == 3:
+        verStr = str(versionString) + '-rc' + str(sys.maxint)
+    elif len(verParts) != 4:
+        print("pkgVersionNormalized WARNING: unexpected version format: {0})".format(verStr), file=sys.stderr)
+        
+    return verStr
+
+
+def main(args):
+    if len(args) < 3:
+        print("Usage: {0} <package1> <package2>".format(args[0]), file=sys.stderr)
+        return 1
+
+    tools = {}
+    platforms = {} 
+    pkg1 = load_package(args[1])
+    tools = merge_objects(tools, pkg1['tools']);
+    platforms = merge_objects(platforms, pkg1['platforms']);
+    pkg2 = load_package(args[2])
+    tools = merge_objects(tools, pkg2['tools']);
+    platforms = merge_objects(platforms, pkg2['platforms']);
+
+    pkg1['tools'] = []
+    pkg1['platforms'] = []
+
+    for name in tools:
+        for version in tools[name]:
+            print("Adding tool {0}-{1}".format(name, version), file=sys.stderr)
+            pkg1['tools'].append(tools[name][version])
+
+    for name in platforms:
+        for version in platforms[name]:
+            print("Adding platform {0}-{1}".format(name, version), file=sys.stderr)
+            pkg1['platforms'].append(platforms[name][version])
+                
+    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+
+    json.dump({'packages':[pkg1]}, sys.stdout, indent=2)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

package/package_esp32_index.template.json

@@ -43,7 +43,7 @@
             {
               "packager": "esp32",
               "name": "mkspiffs",
-              "version": "0.2.2"
+              "version": "0.2.3"
             }
           ]
         }
@@ -105,49 +105,49 @@
         },
         {
           "name": "mkspiffs",
-          "version": "0.2.2",
+          "version": "0.2.3",
           "systems": [
             {
               "host": "i686-mingw32",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-win32.zip",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-win32.zip",
-              "checksum": "SHA-256:988baa2827005a20a7c7028f0c2d45d19df2e0a7d42319f4a7a5776a3f0dff2e",
-              "size": "347207"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-win32.zip",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-win32.zip",
+              "checksum": "SHA-256:b647f2c2efe6949819c85ea9404271b55c7c9c25bcb98d3b98a1d0ba771adf56",
+              "size": "249809"
             },
             {
               "host": "x86_64-apple-darwin",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-osx.tar.gz",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-osx.tar.gz",
-              "checksum": "SHA-256:7aee138be9a73fe7fd1f75cf3f3695f0afae812d04fcbf74b17da330f66ae4cd",
-              "size": "130211"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-osx.tar.gz",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-osx.tar.gz",
+              "checksum": "SHA-256:9f43fc74a858cf564966b5035322c3e5e61c31a647c5a1d71b388ed6efc48423",
+              "size": "130270"
             },
             {
               "host": "i386-apple-darwin",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-osx.tar.gz",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-osx.tar.gz",
-              "checksum": "SHA-256:7aee138be9a73fe7fd1f75cf3f3695f0afae812d04fcbf74b17da330f66ae4cd",
-              "size": "130211"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-osx.tar.gz",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-osx.tar.gz",
+              "checksum": "SHA-256:9f43fc74a858cf564966b5035322c3e5e61c31a647c5a1d71b388ed6efc48423",
+              "size": "130270"
             },
             {
               "host": "x86_64-pc-linux-gnu",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux64.tar.gz",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux64.tar.gz",
-              "checksum": "SHA-256:17f89d9b38d4f68f2f03f7561b951d1d3b6d6f5b74d35b6d3eb8da3440be3400",
-              "size": "50611"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux64.tar.gz",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux64.tar.gz",
+              "checksum": "SHA-256:5e1a4ff41385e842f389f6b5254102a547e566a06b49babeffa93ef37115cb5d",
+              "size": "50646"
             },
             {
               "host": "i686-pc-linux-gnu",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux32.tar.gz",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux32.tar.gz",
-              "checksum": "SHA-256:181fca76210de04a23eb7af028d9886de5a73e638c63d351a691a24cfb9f03d3",
-              "size": "48730"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux32.tar.gz",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux32.tar.gz",
+              "checksum": "SHA-256:464463a93e8833209cdc29ba65e1a12fec31718dc10075c195a2445b2c3f6cb0",
+              "size": "48751"
             },
             {
               "host": "arm-linux-gnueabihf",
-              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux-armhf.tar.gz",
-              "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux-armhf.tar.gz",
-              "checksum": "SHA-256:2e99cbdf5ee60b27d6ade096d4caf03a90edfd5f4edf4da2a8674d770aa4ca1b",
-              "size": "40658"
+              "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux-armhf.tar.gz",
+              "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux-armhf.tar.gz",
+              "checksum": "SHA-256:ade3dc00117912ac08a1bdbfbfe76b12d21a34bc5fa1de0cfc45fe7a8d0a0185",
+              "size": "40665"
             }
           ]
         }