add exe (#1471)

.github/workflows/pyinstaller.yml

@@ -0,0 +1,58 @@
+name: docker
+
+on:
+  # Trigger this workflow when the "deploy" workflow has completed successfully
+  # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#workflow_run
+  workflow_run:
+    workflows:
+      - deploy
+    branches:
+      - master
+    types:
+      - completed
+
+jobs:
+
+  # runs on x86-64 (xeon-d) host: matterhorn
+  image_amd64:
+    runs-on: [self-hosted, linux, X64]
+
+    # docker images are only built and published (to DockerHub) when merging
+    # to master (not a yet unmerged PR!)
+    if: github.ref == 'refs/heads/master'
+
+    env:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set environment
+      run: |
+        echo AUTOBAHN_BUILD_DATE=`date -u +"%Y-%m-%d"` >> $GITHUB_ENV
+        echo AUTOBAHN_BUILD_ID=$(date --utc +%Y%m%d)-$(git rev-parse --short ${GITHUB_SHA}) >> $GITHUB_ENV
+        echo AUTOBAHN_VCS_REF=`git rev-parse --short ${GITHUB_SHA}` >> $GITHUB_ENV
+        echo AUTOBAHN_VERSION=$(grep -E '^(__version__)' ./autobahn/_version.py | cut -d ' ' -f3 | sed -e 's|[u"'\'']||g') >> $GITHUB_ENV
+        echo XBRNETWORK_EXE_FILENAME="xbrnetwork-linux-amd64-$(date --utc +%Y%m%d)-$(git rev-parse --short ${GITHUB_SHA})" >> $GITHUB_ENV
+
+    - name: Print environment
+      run: |
+        echo ""
+        echo "Build environment configured:"
+        echo ""
+        echo "  AUTOBAHN_BUILD_DATE     = ${AUTOBAHN_BUILD_DATE}"
+        echo "  AUTOBAHN_BUILD_ID       = ${AUTOBAHN_BUILD_ID}"
+        echo "  AUTOBAHN_VCS_REF        = ${AUTOBAHN_VCS_REF}"
+        echo "  AUTOBAHN_VERSION        = ${AUTOBAHN_VERSION}"
+        echo ""
+        echo "  XBRNETWORK_EXE_FILENAME = ${XBRNETWORK_EXE_FILENAME}"
+        echo ""
+        echo "  AWS_DEFAULT_REGION  = ${AWS_DEFAULT_REGION}"
+        echo "  AWS_S3_BUCKET_NAME  = ${AWS_S3_BUCKET_NAME}"
+        echo ""
+
+    - name: Build & publish EXE
+      run: |
+        make build_exe
+        make upload_exe

.gitignore

@@ -44,3 +44,4 @@ coverage.xml
 .coverage.*
 autobahn/xbr/contracts
 .wheels
+get-pip.py

Makefile

@@ -1,5 +1,7 @@
 .PHONY: test docs pep8 build
 
+XBRNETWORK=${HOME}/.local/bin/xbrnetwork
+
 all:
 	@echo "Targets:"
 	@echo ""
@@ -56,6 +58,7 @@ clean:
 	-rm -rf ~/coverage
 	-rm -f  ./twisted/plugins/dropin.cache
 	-find . -name "*dropin.cache.new" -type f -exec rm -f {} \;
+	-find . -name ".pytest_cache" -type d -exec rm -rf {} \;
 	-find . -name "*.tar.gz" -type f -exec rm -f {} \;
 	-find . -name "*.egg" -type f -exec rm -f {} \;
 	-find . -name "*.pyc" -type f -exec rm -f {} \;
@@ -80,6 +83,30 @@ spelling:
 run_docs:
 	twistd --nodaemon web --port=tcp:8090 --path=./docs/build/html/
 
+
+download_exe:
+	curl -o $(XBRNETWORK) \
+		https://download.crossbario.com/xbrnetwork/linux-amd64/xbrnetwork-latest
+	chmod +x $(XBRNETWORK)
+	$(XBRNETWORK) version
+
+build_exe:
+	tox -e buildexe
+
+upload_exe:
+	aws s3 cp --acl public-read \
+		./dist/xbrnetwork \
+		s3://download.crossbario.com/xbrnetwork/linux-amd64/${XBRNETWORK_EXE_FILENAME}
+
+	aws s3api copy-object --acl public-read --copy-source \
+		download.crossbario.com/xbrnetwork/linux-amd64/${XBRNETWORK_EXE_FILENAME} \
+		--bucket download.crossbario.com \
+		--key xbrnetwork/linux-amd64/xbrnetwork-latest
+
+	aws cloudfront create-invalidation \
+		--distribution-id E2QIG9LNGCJSP9 --paths "/xbrnetwork/linux-amd64/*"
+
+
 test_xbr_cli:
 	xbrnetwork
 	xbrnetwork version

autobahn/_version.py

@@ -24,4 +24,6 @@
 #
 ###############################################################################
 
-__version__ = '21.3.1'
+__version__ = '21.3.2.dev1'
+
+__build__ = u'00000000-0000000'

inject-build-id.py

@@ -0,0 +1,47 @@
+#!python
+
+import os
+
+if __name__ == '__main__':
+    _EVAR = "AUTOBAHN_BUILD_ID"
+    _SEARCH = "__build__ = u'00000'"
+    _REPLACE = "__build__ = u'{}'"
+
+    if _EVAR in os.environ:
+        files = []
+        try:
+            from autobahn import _version
+        except ImportError:
+            pass
+        else:
+            files.append(os.path.abspath(_version.__file__))
+
+        fn = 'autobahn/_version.py'
+        if os.path.exists(fn):
+            files.append(os.path.abspath(fn))
+
+        done = []
+
+        for fn in files:
+            if fn in done:
+                print('Skipping file "{}": already processed'.format(fn))
+            else:
+                with open(fn) as f:
+                    contents = f.read()
+                build_id_stmt = _REPLACE.format(os.environ[_EVAR])
+                if contents.find(_SEARCH):
+                    contents = contents.replace(_SEARCH, build_id_stmt)
+                    print(contents)
+                    with open(fn, 'w') as f:
+                        f.write(contents)
+                        f.flush()
+                    print('Ok: replaced placeholder build ID in file "{}" with "{}"'.format(fn, build_id_stmt))
+                    done.append(fn)
+                else:
+                    if contents.find(build_id_stmt):
+                        print('Skipping file "{}": build ID already correct')
+                    else:
+                        error_msg = 'Error: could not find search string "{}" to inject build ID in file "{}"'.format(_SEARCH, _version.__file__)
+                        raise Exception(error_msg)
+    else:
+        print('Skipping injection of build ID: AUTOBAHN_BUILD_ID not set')

pyinstaller/hook-autobahn.py

@@ -0,0 +1,34 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = collect_data_files('autobahn')
+
+# Most of the following are found automatically by PyInstaller - but some are not!
+# To be on the safe side, we list _all_ modules here, which needs to be kept in
+# sync manually
+[
+    'autobahn',
+    'autobahn.asyncio',
+    'autobahn.asyncio.test',
+    'autobahn.asyncio.xbr',
+    'autobahn.nvx',
+    'autobahn.nvx.test',
+    'autobahn.rawsocket',
+    'autobahn.rawsocket.test',
+    'autobahn.test',
+    'autobahn.twisted',
+    'autobahn.twisted.test',
+    'autobahn.twisted.testing',
+    'autobahn.twisted.xbr',
+    'autobahn.wamp',
+    'autobahn.wamp.flatbuffers',
+    'autobahn.wamp.gen',
+    'autobahn.wamp.gen.schema',
+    'autobahn.wamp.gen.wamp',
+    'autobahn.wamp.gen.wamp.proto',
+    'autobahn.wamp.test',
+    'autobahn.websocket',
+    'autobahn.websocket.test',
+    'autobahn.xbr',
+    'autobahn.xbr.templates',
+    'autobahn.xbr.test'
+]

pyinstaller/hook-twisted.py

@@ -0,0 +1,11 @@
+# https://github.com/pyinstaller/pyinstaller/issues/3390
+
+import sys
+
+# this creates module: sys.modules['twisted.internet.reactor']
+if sys.platform in ['win32']:
+    from twisted.internet.iocpreactor import reactor as _iocpreactor
+    _iocpreactor.install()
+else:
+    from twisted.internet import default
+    default.install()

pyinstaller/hook-xbr.py

@@ -0,0 +1,18 @@
+from pprint import pprint
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = collect_data_files('xbr')
+
+pprint(datas)
+
+hiddenimports = [
+    'Crypto.Cipher',
+    'Crypto.Signature',
+    'Crypto.Hash',
+    'Crypto.PublicKey',
+    'Crypto.Protocol',
+    'Crypto.IO',
+    'Crypto.Random',
+    'Crypto.Util',
+    'Crypto.IO',
+]

pyinstaller/xbrnetwork.spec

@@ -0,0 +1,33 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['../autobahn/xbr/_cli.py'],
+             pathex=['/home/oberstet/scm/crossbario/autobahn-python/pyinstaller'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=['/home/oberstet/scm/crossbario/autobahn-python/pyinstaller'],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [],
+          name='xbrnetwork',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True )

tox.ini

@@ -67,6 +67,13 @@ whitelist_externals =
     curl
     unzip
     find
+    echo
+    file
+    du
+    xbrnetwork
+    chmod
+    ldd
+    which
 
 setenv =
     # NaCl/Sodium: use and build bundled sources (don't use any system library)
@@ -182,3 +189,60 @@ commands =
 
     # generate HTML output
     sphinx-build -b html ./docs ./docs/_build
+
+
+[testenv:buildexe]
+description =
+    Build one-file executable using PyInstaller.
+skip_install = True
+passenv =
+    AUTOBAHN_BUILD_DATE
+    AUTOBAHN_BUILD_ID
+    AUTOBAHN_VCS_REF
+    AUTOBAHN_VERSION
+    XBRNETWORK_EXE_FILENAME
+deps =
+    pyinstaller==4.2
+commands =
+    echo "Building with AUTOBAHN_BUILD_DATE={env:AUTOBAHN_BUILD_DATE}"
+    echo "Building with AUTOBAHN_BUILD_ID={env:AUTOBAHN_BUILD_ID}"
+    echo "Building with AUTOBAHN_VCS_REF={env:AUTOBAHN_VCS_REF}"
+    echo "Building with AUTOBAHN_VERSION={env:AUTOBAHN_VERSION}"
+    echo "Building with XBRNETWORK_EXE_FILENAME={env:XBRNETWORK_EXE_FILENAME}"
+
+    # Install pip v19.3.1 (yes, absolutely, we do not want the "new resolver" shit)
+    curl -sSL -o get-pip.py https://bootstrap.pypa.io/get-pip.py
+    python get-pip.py "pip<20"
+    pip show pip
+
+    # Install development dependencies
+    pip install -r{toxinidir}/requirements-dev.txt
+
+    # inject build ID into source code: modifies autobahn/_version.py!
+    echo "Building with AUTOBAHN_BUILD_ID={env:AUTOBAHN_BUILD_ID}"
+    python inject-build-id.py
+
+    # Install the package itself
+    pip install .[all]
+
+    # print effective version/build
+    python -c "from autobahn._version import __version__, __build__; print('autobahn==', __version__, __build__)"
+
+    # run CLI and print versions
+    xbrnetwork version
+
+    # bundle up dist/xbrnetwork
+    which pyinstaller
+    pyinstaller --version
+	pyinstaller --additional-hooks-dir={toxinidir}/pyinstaller \
+        --specpath={toxinidir}/pyinstaller --onefile --clean \
+        --name xbrnetwork {toxinidir}/autobahn/xbr/_cli.py
+
+    chmod +x {toxinidir}/dist/xbrnetwork
+    cp {toxinidir}/dist/xbrnetwork {homedir}/xbrnetwork
+
+    # base check of the produced executable
+	file {homedir}/xbrnetwork
+	ldd {homedir}/xbrnetwork
+    du -h {homedir}/xbrnetwork
+	{homedir}/xbrnetwork version

versions.sh

@@ -3,14 +3,16 @@
 export AUTOBAHN_BUILD_DATE=`date -u +"%Y-%m-%d"`
 export AUTOBAHN_BUILD_ID=$(date --utc +%Y%m%d)-$(git rev-parse --short HEAD)
 export AUTOBAHN_VCS_REF=`git rev-parse --short HEAD`
-# export AUTOBAHN_VCS_REF=`git --git-dir="./.git" rev-list -n 1 v${AUTOBAHN_VERSION} --abbrev-commit`
 export AUTOBAHN_VERSION=$(grep -E '^(__version__)' ./autobahn/_version.py | cut -d ' ' -f3 | sed -e 's|[u"'\'']||g')
+export XBRNETWORK_EXE_FILENAME="xbrnetwork-linux-amd64-${AUTOBAHN_BUILD_ID}"
 
 echo ""
 echo "Build environment configured:"
 echo ""
-echo "  AUTOBAHN_BUILD_DATE  = ${AUTOBAHN_BUILD_DATE}"
-echo "  AUTOBAHN_BUILD_ID    = ${AUTOBAHN_BUILD_ID}"
-echo "  AUTOBAHN_VCS_REF     = ${AUTOBAHN_VCS_REF}"
-echo "  AUTOBAHN_VERSION     = ${AUTOBAHN_VERSION}"
+echo "  AUTOBAHN_BUILD_DATE     = ${AUTOBAHN_BUILD_DATE}"
+echo "  AUTOBAHN_BUILD_ID       = ${AUTOBAHN_BUILD_ID}"
+echo "  AUTOBAHN_VCS_REF        = ${AUTOBAHN_VCS_REF}"
+echo "  AUTOBAHN_VERSION        = ${AUTOBAHN_VERSION}"
+echo ""
+echo "  XBRNETWORK_EXE_FILENAME = ${XBRNETWORK_EXE_FILENAME}"
 echo ""