diff --git a/.github/workflows/pyinstaller.yml b/.github/workflows/pyinstaller.yml new file mode 100644 index 000000000..497fb6835 --- /dev/null +++ b/.github/workflows/pyinstaller.yml @@ -0,0 +1,58 @@ +name: docker + +on: + # Trigger this workflow when the "deploy" workflow has completed successfully + # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#workflow_run + workflow_run: + workflows: + - deploy + branches: + - master + types: + - completed + +jobs: + + # runs on x86-64 (xeon-d) host: matterhorn + image_amd64: + runs-on: [self-hosted, linux, X64] + + # docker images are only built and published (to DockerHub) when merging + # to master (not a yet unmerged PR!) + if: github.ref == 'refs/heads/master' + + env: + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} + + steps: + - uses: actions/checkout@v2 + + - name: Set environment + run: | + echo AUTOBAHN_BUILD_DATE=`date -u +"%Y-%m-%d"` >> $GITHUB_ENV + echo AUTOBAHN_BUILD_ID=$(date --utc +%Y%m%d)-$(git rev-parse --short ${GITHUB_SHA}) >> $GITHUB_ENV + echo AUTOBAHN_VCS_REF=`git rev-parse --short ${GITHUB_SHA}` >> $GITHUB_ENV + echo AUTOBAHN_VERSION=$(grep -E '^(__version__)' ./autobahn/_version.py | cut -d ' ' -f3 | sed -e 's|[u"'\'']||g') >> $GITHUB_ENV + echo XBRNETWORK_EXE_FILENAME="xbrnetwork-linux-amd64-$(date --utc +%Y%m%d)-$(git rev-parse --short ${GITHUB_SHA})" >> $GITHUB_ENV + + - name: Print environment + run: | + echo "" + echo "Build environment configured:" + echo "" + echo " AUTOBAHN_BUILD_DATE = ${AUTOBAHN_BUILD_DATE}" + echo " AUTOBAHN_BUILD_ID = ${AUTOBAHN_BUILD_ID}" + echo " AUTOBAHN_VCS_REF = ${AUTOBAHN_VCS_REF}" + echo " AUTOBAHN_VERSION = ${AUTOBAHN_VERSION}" + echo "" + echo " XBRNETWORK_EXE_FILENAME = ${XBRNETWORK_EXE_FILENAME}" + echo "" + echo " AWS_DEFAULT_REGION = ${AWS_DEFAULT_REGION}" + echo " AWS_S3_BUCKET_NAME = ${AWS_S3_BUCKET_NAME}" + echo "" + + - name: Build & publish EXE + run: | + make build_exe + make upload_exe diff --git a/.gitignore b/.gitignore index 058f5495e..f59b7cddf 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ coverage.xml .coverage.* autobahn/xbr/contracts .wheels +get-pip.py diff --git a/Makefile b/Makefile index e6525bfdf..07601e861 100755 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ .PHONY: test docs pep8 build +XBRNETWORK=${HOME}/.local/bin/xbrnetwork + all: @echo "Targets:" @echo "" @@ -56,6 +58,7 @@ clean: -rm -rf ~/coverage -rm -f ./twisted/plugins/dropin.cache -find . -name "*dropin.cache.new" -type f -exec rm -f {} \; + -find . -name ".pytest_cache" -type d -exec rm -rf {} \; -find . -name "*.tar.gz" -type f -exec rm -f {} \; -find . -name "*.egg" -type f -exec rm -f {} \; -find . -name "*.pyc" -type f -exec rm -f {} \; @@ -80,6 +83,30 @@ spelling: run_docs: twistd --nodaemon web --port=tcp:8090 --path=./docs/build/html/ + +download_exe: + curl -o $(XBRNETWORK) \ + https://download.crossbario.com/xbrnetwork/linux-amd64/xbrnetwork-latest + chmod +x $(XBRNETWORK) + $(XBRNETWORK) version + +build_exe: + tox -e buildexe + +upload_exe: + aws s3 cp --acl public-read \ + ./dist/xbrnetwork \ + s3://download.crossbario.com/xbrnetwork/linux-amd64/${XBRNETWORK_EXE_FILENAME} + + aws s3api copy-object --acl public-read --copy-source \ + download.crossbario.com/xbrnetwork/linux-amd64/${XBRNETWORK_EXE_FILENAME} \ + --bucket download.crossbario.com \ + --key xbrnetwork/linux-amd64/xbrnetwork-latest + + aws cloudfront create-invalidation \ + --distribution-id E2QIG9LNGCJSP9 --paths "/xbrnetwork/linux-amd64/*" + + test_xbr_cli: xbrnetwork xbrnetwork version diff --git a/autobahn/_version.py b/autobahn/_version.py index 31a301137..f36478175 100644 --- a/autobahn/_version.py +++ b/autobahn/_version.py @@ -24,4 +24,6 @@ # ############################################################################### -__version__ = '21.3.1' +__version__ = '21.3.2.dev1' + +__build__ = u'00000000-0000000' diff --git a/inject-build-id.py b/inject-build-id.py new file mode 100644 index 000000000..127289599 --- /dev/null +++ b/inject-build-id.py @@ -0,0 +1,47 @@ +#!python + +import os + +if __name__ == '__main__': + _EVAR = "AUTOBAHN_BUILD_ID" + _SEARCH = "__build__ = u'00000'" + _REPLACE = "__build__ = u'{}'" + + if _EVAR in os.environ: + files = [] + try: + from autobahn import _version + except ImportError: + pass + else: + files.append(os.path.abspath(_version.__file__)) + + fn = 'autobahn/_version.py' + if os.path.exists(fn): + files.append(os.path.abspath(fn)) + + done = [] + + for fn in files: + if fn in done: + print('Skipping file "{}": already processed'.format(fn)) + else: + with open(fn) as f: + contents = f.read() + build_id_stmt = _REPLACE.format(os.environ[_EVAR]) + if contents.find(_SEARCH): + contents = contents.replace(_SEARCH, build_id_stmt) + print(contents) + with open(fn, 'w') as f: + f.write(contents) + f.flush() + print('Ok: replaced placeholder build ID in file "{}" with "{}"'.format(fn, build_id_stmt)) + done.append(fn) + else: + if contents.find(build_id_stmt): + print('Skipping file "{}": build ID already correct') + else: + error_msg = 'Error: could not find search string "{}" to inject build ID in file "{}"'.format(_SEARCH, _version.__file__) + raise Exception(error_msg) + else: + print('Skipping injection of build ID: AUTOBAHN_BUILD_ID not set') diff --git a/pyinstaller/hook-autobahn.py b/pyinstaller/hook-autobahn.py new file mode 100644 index 000000000..f58b49972 --- /dev/null +++ b/pyinstaller/hook-autobahn.py @@ -0,0 +1,34 @@ +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('autobahn') + +# Most of the following are found automatically by PyInstaller - but some are not! +# To be on the safe side, we list _all_ modules here, which needs to be kept in +# sync manually +[ + 'autobahn', + 'autobahn.asyncio', + 'autobahn.asyncio.test', + 'autobahn.asyncio.xbr', + 'autobahn.nvx', + 'autobahn.nvx.test', + 'autobahn.rawsocket', + 'autobahn.rawsocket.test', + 'autobahn.test', + 'autobahn.twisted', + 'autobahn.twisted.test', + 'autobahn.twisted.testing', + 'autobahn.twisted.xbr', + 'autobahn.wamp', + 'autobahn.wamp.flatbuffers', + 'autobahn.wamp.gen', + 'autobahn.wamp.gen.schema', + 'autobahn.wamp.gen.wamp', + 'autobahn.wamp.gen.wamp.proto', + 'autobahn.wamp.test', + 'autobahn.websocket', + 'autobahn.websocket.test', + 'autobahn.xbr', + 'autobahn.xbr.templates', + 'autobahn.xbr.test' +] diff --git a/pyinstaller/hook-twisted.py b/pyinstaller/hook-twisted.py new file mode 100644 index 000000000..138fc3896 --- /dev/null +++ b/pyinstaller/hook-twisted.py @@ -0,0 +1,11 @@ +# https://github.com/pyinstaller/pyinstaller/issues/3390 + +import sys + +# this creates module: sys.modules['twisted.internet.reactor'] +if sys.platform in ['win32']: + from twisted.internet.iocpreactor import reactor as _iocpreactor + _iocpreactor.install() +else: + from twisted.internet import default + default.install() diff --git a/pyinstaller/hook-xbr.py b/pyinstaller/hook-xbr.py new file mode 100644 index 000000000..555d75906 --- /dev/null +++ b/pyinstaller/hook-xbr.py @@ -0,0 +1,18 @@ +from pprint import pprint +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('xbr') + +pprint(datas) + +hiddenimports = [ + 'Crypto.Cipher', + 'Crypto.Signature', + 'Crypto.Hash', + 'Crypto.PublicKey', + 'Crypto.Protocol', + 'Crypto.IO', + 'Crypto.Random', + 'Crypto.Util', + 'Crypto.IO', +] diff --git a/pyinstaller/xbrnetwork.spec b/pyinstaller/xbrnetwork.spec new file mode 100644 index 000000000..b1cb3f97a --- /dev/null +++ b/pyinstaller/xbrnetwork.spec @@ -0,0 +1,33 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['../autobahn/xbr/_cli.py'], + pathex=['/home/oberstet/scm/crossbario/autobahn-python/pyinstaller'], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=['/home/oberstet/scm/crossbario/autobahn-python/pyinstaller'], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='xbrnetwork', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True ) diff --git a/tox.ini b/tox.ini index 2ebd9a0e8..cf9c43e7d 100644 --- a/tox.ini +++ b/tox.ini @@ -67,6 +67,13 @@ whitelist_externals = curl unzip find + echo + file + du + xbrnetwork + chmod + ldd + which setenv = # NaCl/Sodium: use and build bundled sources (don't use any system library) @@ -182,3 +189,60 @@ commands = # generate HTML output sphinx-build -b html ./docs ./docs/_build + + +[testenv:buildexe] +description = + Build one-file executable using PyInstaller. +skip_install = True +passenv = + AUTOBAHN_BUILD_DATE + AUTOBAHN_BUILD_ID + AUTOBAHN_VCS_REF + AUTOBAHN_VERSION + XBRNETWORK_EXE_FILENAME +deps = + pyinstaller==4.2 +commands = + echo "Building with AUTOBAHN_BUILD_DATE={env:AUTOBAHN_BUILD_DATE}" + echo "Building with AUTOBAHN_BUILD_ID={env:AUTOBAHN_BUILD_ID}" + echo "Building with AUTOBAHN_VCS_REF={env:AUTOBAHN_VCS_REF}" + echo "Building with AUTOBAHN_VERSION={env:AUTOBAHN_VERSION}" + echo "Building with XBRNETWORK_EXE_FILENAME={env:XBRNETWORK_EXE_FILENAME}" + + # Install pip v19.3.1 (yes, absolutely, we do not want the "new resolver" shit) + curl -sSL -o get-pip.py https://bootstrap.pypa.io/get-pip.py + python get-pip.py "pip<20" + pip show pip + + # Install development dependencies + pip install -r{toxinidir}/requirements-dev.txt + + # inject build ID into source code: modifies autobahn/_version.py! + echo "Building with AUTOBAHN_BUILD_ID={env:AUTOBAHN_BUILD_ID}" + python inject-build-id.py + + # Install the package itself + pip install .[all] + + # print effective version/build + python -c "from autobahn._version import __version__, __build__; print('autobahn==', __version__, __build__)" + + # run CLI and print versions + xbrnetwork version + + # bundle up dist/xbrnetwork + which pyinstaller + pyinstaller --version + pyinstaller --additional-hooks-dir={toxinidir}/pyinstaller \ + --specpath={toxinidir}/pyinstaller --onefile --clean \ + --name xbrnetwork {toxinidir}/autobahn/xbr/_cli.py + + chmod +x {toxinidir}/dist/xbrnetwork + cp {toxinidir}/dist/xbrnetwork {homedir}/xbrnetwork + + # base check of the produced executable + file {homedir}/xbrnetwork + ldd {homedir}/xbrnetwork + du -h {homedir}/xbrnetwork + {homedir}/xbrnetwork version diff --git a/versions.sh b/versions.sh index a746b2f87..fbf17b2b2 100755 --- a/versions.sh +++ b/versions.sh @@ -3,14 +3,16 @@ export AUTOBAHN_BUILD_DATE=`date -u +"%Y-%m-%d"` export AUTOBAHN_BUILD_ID=$(date --utc +%Y%m%d)-$(git rev-parse --short HEAD) export AUTOBAHN_VCS_REF=`git rev-parse --short HEAD` -# export AUTOBAHN_VCS_REF=`git --git-dir="./.git" rev-list -n 1 v${AUTOBAHN_VERSION} --abbrev-commit` export AUTOBAHN_VERSION=$(grep -E '^(__version__)' ./autobahn/_version.py | cut -d ' ' -f3 | sed -e 's|[u"'\'']||g') +export XBRNETWORK_EXE_FILENAME="xbrnetwork-linux-amd64-${AUTOBAHN_BUILD_ID}" echo "" echo "Build environment configured:" echo "" -echo " AUTOBAHN_BUILD_DATE = ${AUTOBAHN_BUILD_DATE}" -echo " AUTOBAHN_BUILD_ID = ${AUTOBAHN_BUILD_ID}" -echo " AUTOBAHN_VCS_REF = ${AUTOBAHN_VCS_REF}" -echo " AUTOBAHN_VERSION = ${AUTOBAHN_VERSION}" +echo " AUTOBAHN_BUILD_DATE = ${AUTOBAHN_BUILD_DATE}" +echo " AUTOBAHN_BUILD_ID = ${AUTOBAHN_BUILD_ID}" +echo " AUTOBAHN_VCS_REF = ${AUTOBAHN_VCS_REF}" +echo " AUTOBAHN_VERSION = ${AUTOBAHN_VERSION}" +echo "" +echo " XBRNETWORK_EXE_FILENAME = ${XBRNETWORK_EXE_FILENAME}" echo ""