Chore: Upgrade to python 3.10 (#1688)

* changed class name * Working requirements.in * Reordered lines to see changes better * Working minimal upgrades * Added hashes * - Upgraded python to 3.10 and bitcoin to v23.0 - Added libzmq to build process, to enable future use cases such as #778 - added --without-gui to remove configure warning that gui will not be built. * Added line-break * - Updated the embit version to 0.4.13 - pushed python-bitcoind image * - Build registry.gitlab.com/c8527/specter/cirrus-jammy:20220504 - updated docker references * - added cypress-base-ubuntu-jammy * - added cypress-python * - fixed unchanged registry path * Fixed wrong path * readme change * Fixed path * Changed github-changelog * changes registrar links * Fixed bitcoin version * Set reset bitcoin version to 22 * Set bitcoin version to 22.0 * Fixed black issue via psf/black#2964 (comment) * Revert for changelog * Upgraded pytest because otherwise I get the error pytest-dev/pytest#9195 * Added temporary fix for upgraded hwi version. Should be fixed in #1693 * more verbosity for page generation * doc about python 3.10 * docs: update TOC * yet another library necessary * black * Corrected tag * Fixed wrong python version introduced in the merge commit * Use same babel version across requirements and test_requirements * Created the reuqirements.txt with python 3.8 such, that that netlify can create the python docs in python 3.8, while everything elese works also with python 3.10 * testing https://peps.python.org/pep-0508/#environment-markers * CHanged just the txt * Added comments * electron docker build successful * rename image to cypress-python:v9.7.0 * doc * docker file rename * fix pyinstaller pip error by changing requirements.in * reference jammy electron dockerfile See electron-userland/electron-builder#6922 (comment) * updated all references to dockerimages * fix release-build * fix the image-version to something else than latest. pin all the stuff! * revert to old image to avoid glibc issues Co-authored-by: Kim Neunert <[email protected]> Co-authored-by: k9ert <[email protected]>
cryptoadvance · Sep 13, 2022 · a8f1b80 · a8f1b80
1 parent c528e90
commit a8f1b80
Show file tree

Hide file tree

Showing 24 changed files with 605 additions and 296 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -1,8 +1,8 @@
 container:
 #  image: python:slim
-#  image: ubuntu:focal
-#  image: python:3.8-buster
-  image: registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:20210831
+#  image: ubuntu:jammy
+#  image: python:3.10
+  image: registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-jammy:20220908
 
 # We assume here that we're having a proper python3 system including virtualenv and pip
 prep_stuff_template: &PREP_STUFF_TEMPLATE
@@ -70,7 +70,7 @@ test_task:
 
 cypress_test_task:
   container:
-    image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python:v9.7.0
+    image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0
     cpu: 6
     memory: 6G
   pre_prep_script: 
@@ -96,4 +96,4 @@ cypress_test_task:
     cypress_screenshots_artifacts:
       path: "./cypress/screenshots/**"
     cypress_videos_artifacts:
-      path: "./cypress/videos/**"
+      path: "./cypress/videos/**"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,6 +1,6 @@
 # this image contains python, bitcoind and docker
 # check docker/python-bitcoind on how it's built
-image: registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1
+image: registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v22.0
 
 variables:
   # Cache documentation: https://docs.gitlab.com/ee/ci/caching/
@@ -64,7 +64,7 @@ check:
    - py.test --cov-report term --cov cryptoadvance --docker
 
 .test-cypress:
-  image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python:20210424
+  image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0
   stage: testing
   script:
     # start the server in the background
@@ -158,7 +158,7 @@ release_electron_linux_windows:
   only: 
     - tags
   before_script:
-    - python -V  # Print out python version for debugging
+    - python3 -V  # Print out python version for debugging
     - apt update
     - apt install -y unzip libusb-1.0-0-dev libudev-dev # usb-support in hidapi
     - pip3 install virtualenv
@@ -234,4 +234,4 @@ release_docker:
   before_script:
     - echo "Triggering Docker Release"
   script:
-    - ./utils/trigger_docker_build.sh
+    - ./utils/trigger_docker_build.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,4 +3,4 @@ repos:
     rev: 22.3.0
     hooks:
     - id: black
-      language_version: python3.8
+      language_version: python3.10
diff --git a/docker/cirrus-focal/Dockerfile → docker/cirrus-jammy/Dockerfile b/docker/cirrus-focal/Dockerfile → docker/cirrus-jammy/Dockerfile
@@ -1,4 +1,4 @@
-from ubuntu:focal
+from ubuntu:jammy
 
 # 1. python-stuff and HWI dependencies
 # 2. capability to build bitcoind
@@ -7,7 +7,7 @@ RUN apt update && DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-
     libusb-1.0-0-dev libudev-dev python3 python3-virtualenv \
     build-essential libtool autotools-dev automake autoconf pkg-config bsdmainutils libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev bc  \
     nodejs npm libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb \
-    wget
+    wget libzmq5-dev  
 
 # Stuff needed for Elements (compilation)
-RUN DEBIAN_FRONTEND="noninteractive" apt-get install  --no-install-recommends -y  libboost-thread-dev libsqlite3-dev git
+RUN DEBIAN_FRONTEND="noninteractive" apt-get install  --no-install-recommends -y  libboost-thread-dev libsqlite3-dev git
diff --git a/docker/cirrus-focal/README.md → docker/cirrus-jammy/README.md b/docker/cirrus-focal/README.md → docker/cirrus-jammy/README.md
@@ -3,10 +3,8 @@ An image used to run the build on cirrus (tests only, not cypress-tests).
 Create it like this:
 
 ```
-current_date=$(date +"%Y%m%d")
-docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:${current_date}
-docker push registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:${current_date}
-# Do not forget to update the $current_date in .cirrus.yml
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-jammy:20220908
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-jammy:20220908
 ```
 
-Check the `.cirrus.yml` on how this is used and update the $current_date there.
+Check the `.cirrus.yml` on how this is used and update the $current_date there.
diff --git a/docker/cypress-base-ubuntu-focal/Dockerfile → docker/cypress-base-ubuntu-jammy/Dockerfile b/docker/cypress-base-ubuntu-focal/Dockerfile → docker/cypress-base-ubuntu-jammy/Dockerfile
@@ -1,11 +1,8 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 RUN apt-get update && \
-  apt-get install -y apt-transport-https curl
+  apt-get install -y apt-transport-https curl  nodejs npm
 
-RUN curl -sL https://deb.nodesource.com/setup_12.x -o nodesource_setup.sh
-RUN bash nodesource_setup.sh
-RUN apt-get install -y nodejs
 
 # Install latest NPM and Yarn
 RUN npm install -g npm@latest
@@ -59,4 +56,4 @@ RUN echo  " node version:    $(node -v) \n" \
 
 RUN echo "More version info"
 RUN cat /etc/lsb-release
-RUN cat /etc/os-release
+RUN cat /etc/os-release
diff --git a/docker/cypress-base-ubuntu-focal/README.md → docker/cypress-base-ubuntu-jammy/README.md b/docker/cypress-base-ubuntu-focal/README.md → docker/cypress-base-ubuntu-jammy/README.md
@@ -1,10 +1,10 @@
-A base-image used in cypress-python to use a newer ubuntu focal rather than a buster.
+A base-image used in cypress-python to use a newer ubuntu jammy rather than a buster.
 
 Create it like this:
 
 ``` 
-docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-focal:latest
-docker push registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-focal:latest
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-jammy:20220908
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-jammy:20220908
 ```
 
 used in cypress-python

diff --git a/docker/cypress-python/Dockerfile → docker/cypress-python-jammy/Dockerfile b/docker/cypress-python/Dockerfile → docker/cypress-python-jammy/Dockerfile
@@ -1,12 +1,15 @@
-FROM registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-focal:latest
+FROM registry.gitlab.com/cryptoadvance/specter-desktop/cypress-base-ubuntu-jammy:20220908
 
 RUN apt-get update && apt-get install -y \
     python3-pip python3-virtualenv zip unzip file apt libusb-1.0-0-dev libudev-dev \
-    bc libevent-2.1-7 jq wget curl
+    bc libevent-2.1-7 jq wget curl  libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev \
+    libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
+	# https://docs.cypress.io/guides/continuous-integration/introduction#Dependencies
 
 # Stuff needed for Elements (compilation)
-RUN DEBIAN_FRONTEND="noninteractive" apt-get install  --no-install-recommends -y bsdmainutils libboost-test-dev libboost-filesystem-dev libboost-thread-dev libsqlite3-dev git libevent-pthreads-2.1-7
-
+RUN DEBIAN_FRONTEND="noninteractive" apt-get install  --no-install-recommends -y bsdmainutils \
+			libboost-test-dev libboost-filesystem-dev libboost-thread-dev libsqlite3-dev git \
+			libevent-pthreads-2.1-7  
 
 WORKDIR /test
 RUN rm -rf node_modules package-lock.json ~/.cache/Cypress

diff --git a/docker/cypress-python/Readme.md → docker/cypress-python-jammy/Readme.md b/docker/cypress-python/Readme.md → docker/cypress-python-jammy/Readme.md
@@ -3,8 +3,8 @@ An image, ready to be used with cypress but also provides all the dependencies w
 Use versions of cypress as the version part of the tag. So e.g.:
 
 ```
-docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python:v9.7.0
-docker push registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python:v9.7.0
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0
 ```
 
-Search for `cypress-python` on where this is used in the project.
+Search for `cypress-python` on where this is used in the project.
diff --git a/docker/electron-builder/README.md b/docker/electron-builder/README.md
@@ -1,6 +1,12 @@
 Used for building the electron-app. In short it's the /pyinstaller/build-unix.sh script which is running in this image.
 
-manually do it something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)):
+By intention, this is using an older OS-version in order to avoid glibc-issues. For details, see:
+* https://github.com/cryptoadvance/specter-desktop/pull/1688#issuecomment-1242796681
+* https://github.com/cryptoadvance/specter-desktop/issues/373#issuecomment-695068924
+
+
+If you want to run the image manually, do something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)):
+
 
 ```
 docker run --rm -ti \
@@ -19,4 +25,4 @@ build the image like:
 ```
 docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest .
 docker push registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest
-```
+```
diff --git a/docker/github-changelog/Dockerfile b/docker/github-changelog/Dockerfile
@@ -15,4 +15,4 @@ RUN python3 setup.py install
 
 ENV PYTHONUNBUFFERED="1"
 
-ENTRYPOINT ["changelog"]
+ENTRYPOINT ["changelog"]
diff --git a/docker/python-bitcoind/Dockerfile b/docker/python-bitcoind/Dockerfile
@@ -1,5 +1,5 @@
 
-FROM python:3.8
+FROM python:3.10
 
 # Installing BerkleyDB 4.8.30
 # Ubuntu specific buw works on debian Buster as well
@@ -12,14 +12,16 @@ RUN wget https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin/+files/libdb4.8+
 	wget https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin/+files/libdb4.8++_4.8.30-artful3_amd64.deb
 RUN dpkg -i *.deb && rm *.deb
 
-RUN apt update && apt install git build-essential autoconf libboost-all-dev libssl-dev libprotobuf-dev protobuf-compiler libqt4-dev libqrencode-dev libtool libevent-dev pkg-config bsdmainutils -y 
+RUN apt update && apt install git build-essential autoconf libboost-all-dev libssl-dev libprotobuf-dev \
+				protobuf-compiler libqrencode-dev libtool libevent-dev pkg-config bsdmainutils  \
+				libzmq5-dev  -y  
 
 RUN git clone https://github.com/bitcoin/bitcoin.git
-RUN cd bitcoin && git checkout v0.20.1
-RUN cd bitcoin && ./autogen.sh && ./configure && make
+RUN cd bitcoin && git checkout v22.0
+RUN cd bitcoin && ./autogen.sh && ./configure --without-gui && make -j8
 RUN cd bitcoin && make install
 
-FROM python:3.8
+FROM python:3.10
 
 # Installing BerkleyDB 4.8.30
 # Works on debian Buster as well
@@ -32,7 +34,8 @@ RUN wget https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin/+files/libdb4.8+
         wget https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin/+files/libdb4.8++_4.8.30-artful3_amd64.deb
 RUN dpkg -i *.deb && rm *.deb
 
-RUN apt update && apt install libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev jq -y
+RUN apt update && apt install libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev \
+				libboost-test-dev libboost-thread-dev jq  libzmq5-dev   -y
 
 COPY --from=0 /usr/local/bin/bitcoind /usr/local/bin
 COPY --from=0 /usr/local/bin/bitcoin-cli /usr/local/bin

diff --git a/docker/python-bitcoind/Readme.md b/docker/python-bitcoind/Readme.md
@@ -1,22 +1,20 @@
 This Dockerimage is manually created and uploaded:
 
 ```
-docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1 .
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v22.0
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v22.0
 ```
-```
-docker push registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1
-```
-Here the version is v0.20.1 but that's just an example. This folder just explains how the image is created. Which image is USED is specified:
+Here the version is v23.0 but that's just an example. This folder just explains how the image is created. Which image is USED is specified:
 
 * In the case of tests in pytest.ini in the addopts-line (MIGHT be different in different branches)
 * in the case of running 
 
 ```
 python3 -m cryptoadvance.specter bitcoind #it's using "latest"
-``` 
+```
 
 but you can override like this:
 
 ```
-python3 -m cryptoadvance.specter bitcoind --docker-tag v0.19.1
-```
+python3 -m cryptoadvance.specter bitcoind --docker-tag v22.0
+```
diff --git a/docs/development.md b/docs/development.md
@@ -34,6 +34,7 @@
     - [Some words about dependencies](#some-words-about-dependencies)
     - [Some words specific to the frontend](#some-words-specific-to-the-frontend)
     - [Some words about style](#some-words-about-style)
+    - [Troubleshooting and migration to python3.10](#troubleshooting-and-migration-to-python310)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -159,7 +160,7 @@ pytest
 OR run against bitcoind in Docker (deprecated):
 ```
 # Pull the bitcoind image if you haven't already:
-docker pull registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.20.1
+docker pull registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v0.22.0
 
 # install prerequisites
 pip3 install docker
@@ -459,3 +460,29 @@ We're aware that currently the app is not very compatible on different browsers
   * nice blue #4A90E2
 * A designer would probably rant about all these bad choices. Professional help, especially in the frontend, is very much appreciated.
 
+### Troubleshooting and migration to python3.10
+We're currently migrating to python3.10 while alos supporting older versions. This is creating some extra challenges for those that want to run 3.10 but don't have 3.10 available in their standard-package. So here are some hints on how to get that going.
+
+`pyenv` is a great tool to granually install. The [installation](https://github.com/pyenv/pyenv#basic-github-checkout) worked great. However, i don't want to pyenv to screw up my existing python setup. So i only use pyenv if i explicitely do (put that in a script):
+```
+export PYENV_ROOT="$HOME/.pyenv"
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+
+echo "    --> now do:omething like:"
+echo "        pyenv shell 3.10.4"
+```
+So i'm using `pyenv shell` but before you install now a 3.10 version via `pyenv install 3.10.4` make sure to install sqlite3:
+```
+sudo apt-get install sqlite3 libbz2-dev 
+```
+If you miss that, you might later have issues while pre-commit-hooks kick in, something like `No module named '_sqlite3`.
+
+Now you can switch your shell to use python 3.10 via `pyenv shell 3.10.4` and after that create your extra virtualenv which uses 3.10:
+```
+python3 -m virtualenv --python=python3.10 .env310
+```
+
+
+
+
diff --git a/pyinstaller/requirements.in b/pyinstaller/requirements.in
@@ -2,4 +2,5 @@ pyinstaller==5.2
 pefile==2022.5.30
 macholib
 pywin32-ctypes
-babel
+babel==2.10.1
+pytz==2022.1
diff --git a/pyinstaller/requirements.txt b/pyinstaller/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile --generate-hashes requirements.in
@@ -10,9 +10,9 @@ altgraph==0.17 \
     # via
     #   macholib
     #   pyinstaller
-babel==2.9.1 \
-    --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \
-    --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0
+babel==2.10.1 \
+    --hash=sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2 \
+    --hash=sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13
     # via -r requirements.in
 future==0.18.2 \
     --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
@@ -24,10 +24,6 @@ macholib==1.14 \
 pefile==2022.5.30 \
     --hash=sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b
     # via -r requirements.in
-pyinstaller-hooks-contrib==2022.8 \
-    --hash=sha256:c4210fc50282c9c6a918e485e0bfae9405592390508e3be9fde19acc2213da56 \
-    --hash=sha256:e46f099934dd4577fb1ddcf37a99fa04027c92f8f5291c8802f326345988d001
-    # via pyinstaller
 pyinstaller==5.2 \
     --hash=sha256:03984eed0baa252ea9854eb0785a1c40ac033c5c28d3abdae7d820da734aed5a \
     --hash=sha256:1061f7a6de055007949ec9ad1c6a080b93e102b2a288c8ff88f65e5d7716d4aa \
@@ -41,10 +37,16 @@ pyinstaller==5.2 \
     --hash=sha256:dda3a4787fa4498bb9e688f81bed918f061bd583c8ff0e47881a5422a4b2093b \
     --hash=sha256:f10b19ad4f66ccad16574ff1979cc15e1ea010f8577292500125dd45abcd8303
     # via -r requirements.in
-pytz==2021.1 \
-    --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \
-    --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798
-    # via babel
+pyinstaller-hooks-contrib==2022.8 \
+    --hash=sha256:c4210fc50282c9c6a918e485e0bfae9405592390508e3be9fde19acc2213da56 \
+    --hash=sha256:e46f099934dd4577fb1ddcf37a99fa04027c92f8f5291c8802f326345988d001
+    # via pyinstaller
+pytz==2022.1 \
+    --hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \
+    --hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c
+    # via
+    #   -r requirements.in
+    #   babel
 pywin32-ctypes==0.2.0 \
     --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
     --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98

diff --git a/pytest.ini b/pytest.ini
@@ -8,4 +8,4 @@ markers =
 #log_cli = 1
 
 filterwarnings =
-    ignore::DeprecationWarning:bitbox02[.*]
+    ignore::DeprecationWarning:bitbox02[.*]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,4 +15,4 @@ RUN python3 setup.py install

		ENV PYTHONUNBUFFERED="1"

		ENTRYPOINT ["changelog"]
		ENTRYPOINT ["changelog"]