Release v0.53.0

.github/workflows/TESTING.md

@@ -0,0 +1,39 @@
+# Testing Github Action workflows
+
+It's possible to test the Github Action workflows locally by using https://github.com/nektos/act
+
+You can start by listing all available jobs in our workflows:
+
+```bash
+act -l
+```
+
+We have prepared examples on how to test some types of triggers in the `docker.yml` workflow, but it shouldn't be hard to adapt these examples to test other combinations of jobs and triggers.
+
+## Testing a Tag Push
+
+To simulate the workflow being trigger by the push of a tag, first generate an event file like this:
+
+```bash
+cat <<EOF > event.json
+{
+  "ref": "refs/tags/v0.53.0-rc.1"
+}
+EOF
+```
+
+You can change the tag in this event to simulate different types of tags.
+
+Then, run the `buildx` job with a `push` event providing the event context and a secret called `DOCKERHUB_IMAGE`:
+
+```bash
+act push -e event.json -j buildx -s DOCKERHUB_IMAGE=testing_locally
+```
+
+## Testing a Scheduled run
+
+Simulating a scheduled run is similar, just change the type of event:
+
+```bash
+act schedule -j buildx -s DOCKERHUB_IMAGE=testing_locally
+```

.github/workflows/docker.yml

@@ -7,9 +7,6 @@ on:
     - 'experimental/**'
     tags:
     - v*
-  pull_request:
-    branches:
-    - release
   schedule:
   - cron: '0 4 * * *'  # nightlies at 4am UTC
 env:
@@ -29,81 +26,34 @@ jobs:
         - '3.8'
         - '3.9'
         - '3.10'
+        - '3.11'
         exclude:
-        # XXX: pypy-3.10 does not exist yet
+        # XXX: neither pypy-3.10 nor pypy-3.11 exist yet, maybe pypy-3.10 will be out on PyPy v7.3.10
         - python-impl: pypy
           python-version: '3.10'
+        - python-impl: pypy
+          python-version: '3.11'
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - name: Prepare tags
+    - name: Prepare base version
       id: prep
-      shell: python
       run: |
-        import datetime
-        import re
-        def extract_pyver(filename):
-            for line in open(filename).readlines():
-                if line.startswith('ARG PYTHON'):
-                    return line.split('=')[1].strip()
-        ref = '${{ github.ref }}'
-        dockerfile_cpython = 'Dockerfile'
-        dockerfile_pypy = 'Dockerfile.pypy'
-        default_python = 'python' + extract_pyver(dockerfile_cpython)
-        default_pypy = 'pypy' + extract_pyver(dockerfile_pypy)
-        if '${{ github.event_name }}' == 'schedule':
-            base_version = 'nightly'
-        elif ref.startswith('refs/tags/'):
-            base_version = ref[10:].split('-', 1)[0]
-        elif ref.startswith('refs/heads/'):
-            base_version = ref[11:].replace('/', '-')
-            if base_version == '${{ github.event.repository.default_branch }}':
-              base_version = 'stable'
-        elif ref.startswith('refs/pull/'):
-            base_version = 'pr-${{ github.event.number }}'
-        else:
-            base_version = 'noop'
-        if '${{ matrix.python-impl }}' == 'pypy':
-            dockerfile = dockerfile_pypy
-            suffix = 'pypy${{ matrix.python-version }}'
-        else:
-            dockerfile = dockerfile_cpython
-            suffix = 'python${{ matrix.python-version }}'
-        version = base_version + '-' + suffix
-        tags = {version}
-        if suffix == default_python:
-            tags.add(base_version)
-        elif suffix == default_pypy:
-            tags.add(base_version + '-pypy')
-        if re.match(r'^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$', base_version):
-            minor = base_version.rpartition('.')[0]
-            tags.add(minor + '-' + suffix)
-            if suffix == default_python:
-                tags.add('latest')
-        elif '${{ github.event_name }}' == 'push':
-            tags.add('sha-' + '${{ github.sha }}'[:8])
-        print('::set-output name=version::' + version)
-        images = []
-        docker_image = '${{ secrets.DOCKERHUB_IMAGE }}'
-        if docker_image:
-            images.append(docker_image)
-            print('::set-output name=login-dockerhub::true')
-        else:
-            print('::set-output name=login-dockerhub::false')
-        ghcr_image = '${{ secrets.GHCR_IMAGE }}'
-        if ghcr_image:
-            images.append(ghcr_image)
-            print('::set-output name=login-ghcr::true')
-        else:
-            print('::set-output name=login-ghcr::false')
-        if images:
-            print('::set-output name=tags::' + ','.join(f'{i}:{t}' for i in images for t in tags))
-            print('::set-output name=push::true')
-        else:
-            print('::set-output name=tags::dont-push--local-only')
-            print('::set-output name=push::false')
-        print('::set-output name=created::' + datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'))
-        print('::set-output name=dockerfile::' + dockerfile)
+        export GITHUB_REF='${{ github.ref }}'
+        export GITHUB_EVENT_NAME='${{ github.event_name }}'
+        export GITHUB_SHA='${{ github.sha }}'
+        export GITHUB_EVENT_DEFAULT_BRANCH='${{ github.event.repository.default_branch }}'
+        export GITHUB_EVENT_NUMBER='${{ github.event.number }}'
+        export MATRIX_PYTHON_IMPL='${{ matrix.python-impl }}'
+        export MATRIX_PYTHON_VERSION='${{ matrix.python-version }}'
+        export SECRETS_DOCKERHUB_IMAGE='${{ secrets.DOCKERHUB_IMAGE }}'
+        export SECRETS_GHCR_IMAGE='${{ secrets.GHCR_IMAGE }}'
+
+        python extras/github/docker.py
+    - name: Check version
+      if: steps.prep.outputs.check-version
+      run: |
+        make check-version VERSION='${{ steps.prep.outputs.check-version }}'
     - name: Set up QEMU  # arm64 is not available natively
       uses: docker/setup-qemu-action@v2
     - name: Set up Docker Buildx
@@ -114,22 +64,23 @@ jobs:
         driver-opts: network=host
     - name: Login to DockerHub
       uses: docker/login-action@v2
-      if: steps.prep.outputs.login-dockerhub
+      if: steps.prep.outputs.login-dockerhub == 'true'
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v2
-      if: steps.prep.outputs.login-ghcr
+      if: steps.prep.outputs.login-ghcr == 'true'
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Cache Docker layers
       uses: actions/cache@v3
+      if: steps.prep_base_version.outputs.is-nightly == 'false'
       with:
         path: /tmp/.buildx-cache
-        # this key is setup such that every branch has its cache and new branches can reuse dev's cache, but not the other way around
+        # this key is setup such that every branch has its cache and new branches can reuse master's cache, but not the other way around
         key: ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-${{ github.head_ref || github.ref }}-${{ github.sha }}
         restore-keys: |
           ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-refs/heads/master-
@@ -149,6 +100,7 @@ jobs:
     - name: Build and push
       uses: docker/build-push-action@v3
       continue-on-error: ${{ matrix.python-impl == 'pypy' }}  # PyPy is not first-class and has been causing some build failures
+      if: ${{ !env.ACT }}  # Skip this step when testing locally with https://github.com/nektos/act
       with:
         context: .
         file: ${{ steps.prep.outputs.dockerfile }}
@@ -169,3 +121,13 @@ jobs:
           org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }}
         cache-from: type=local,src=/tmp/.buildx-cache
         cache-to: type=local,dest=/tmp/.buildx-cache
+    - name: Slack Notification
+      if: ${{ steps.prep.outputs.slack-notification-version && steps.prep_base_version.outputs.disable-slack-notification == 'false' && job.status == 'success' }}
+      uses: rtCamp/action-slack-notify@28e8b353eabda5998a2e1203aed33c5999944779
+      env:
+        SLACK_COLOR: ${{ job.status }} # It can turn the job status into a color. Success will be green.
+        SLACK_MESSAGE: 'We will be deploying this new image soon. Get in touch with the hathor-core team if you want to talk about this deployment.'
+        SLACK_TITLE: 'Hathor Core - new ${{ steps.prep.outputs.slack-notification-version }} Docker image pushed :rocket:'
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        SLACK_FOOTER: ''
+        MSG_MINIMAL: actions url

.github/workflows/main.yml

@@ -7,9 +7,6 @@ on:
     tags:
     - v*
   pull_request:
-    branches:
-    - dev
-    - release
 jobs:
   matrix:
     runs-on: ubuntu-latest
@@ -22,7 +19,7 @@ jobs:
         import os
         import json
         full_matrix = {
-          'python': ['3.8', '3.9', '3.10'],
+          'python': ['3.8', '3.9', '3.10', '3.11'],
           # available OS's: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on
           'os': ['ubuntu-22.04', 'macos-12', 'windows-2022'],
           'include': [
@@ -82,8 +79,8 @@ jobs:
       if: startsWith(matrix.os, 'macos')
       run: |
         brew cleanup -q
-        brew update -q
-        brew install -q graphviz rocksdb
+        # brew update -q
+        brew install -q graphviz rocksdb pkg-config
     - name: Install Poetry dependencies
       run: poetry install -n --no-root
     - name: Cache mypy

.gitignore

@@ -12,6 +12,7 @@ cover/
 /coverage*
 
 .mypy_cache
+.dmypy.json
 .pytest_cache
 
 extras/docker/_build/

Dockerfile

@@ -29,5 +29,8 @@ ARG PYTHON
 RUN apt-get -qy update
 RUN apt-get -qy install libssl1.1 graphviz librocksdb6.11
 COPY --from=stage-0 /app/.venv/lib/python${PYTHON}/site-packages/ /usr/local/lib/python${PYTHON}/site-packages/
+# XXX: copy optional BUILD_VERSION file using ...VERSIO[N] instead of ...VERSION* to ensure only one file will be copied
+# XXX: also copying the README.md because we need at least one existing file
+COPY README.md BUILD_VERSIO[N] /
 EXPOSE 40403 8080
 ENTRYPOINT ["python", "-m", "hathor"]

Dockerfile.pypy

@@ -29,5 +29,8 @@ ARG PYTHON
 RUN apt-get -qy update && apt-get -qy upgrade
 RUN apt-get -qy install libssl1.1 graphviz librocksdb6.11
 COPY --from=stage-0 /app/.venv/lib/pypy${PYTHON}/site-packages/ /opt/pypy/lib/pypy${PYTHON}/site-packages/
+# XXX: copy optional BUILD_VERSION file using ...VERSIO[N] instead of ...VERSION* to ensure only one file will be copied
+# XXX: also copying the README.md because we need at least one existing file
+COPY README.md BUILD_VERSIO[N] /
 EXPOSE 40403 8080
 ENTRYPOINT ["pypy", "-m", "hathor"]

Makefile

@@ -7,6 +7,7 @@ all: check tests
 
 tests_cli = tests/cli/
 tests_lib = $(filter-out ${tests_cli} tests/__pycache__/, $(dir $(wildcard tests/*/.)))
+tests_ci = extras/github/
 
 pytest_flags = -p no:warnings --cov-report=term --cov-report=html --cov-report=xml --cov=hathor
 
@@ -44,8 +45,12 @@ tests-genesis:
 	HATHOR_TEST_CONFIG_FILE=hathor.conf.mainnet pytest tests/tx/test_genesis.py
 	HATHOR_TEST_CONFIG_FILE=hathor.conf.testnet pytest tests/tx/test_genesis.py
 
+.PHONY: tests-ci
+tests-ci:
+	pytest $(tests_ci)
+
 .PHONY: tests
-tests: tests-cli tests-lib tests-genesis
+tests: tests-cli tests-lib tests-genesis tests-ci
 
 .PHONY: tests-full
 tests-full:
@@ -57,6 +62,10 @@ tests-full:
 mypy:
 	mypy -p hathor -p tests
 
+.PHONY: dmypy
+dmypy:
+	dmypy run --timeout 86400 -- -p hathor -p tests
+
 .PHONY: flake8
 flake8:
 	flake8 $(py_sources)
@@ -67,11 +76,14 @@ isort-check:
 
 .PHONY: check-version
 check-version:
-	bash ./extras/check_version.sh
+	bash ./extras/check_version.sh $(VERSION)
 
 .PHONY: check
 check: check-version flake8 isort-check mypy
 
+.PHONY: dcheck
+dcheck: check-version flake8 isort-check dmypy
+
 # formatting:
 
 .PHONY: fmt

README.md

@@ -2,8 +2,8 @@
 
 [![Mainnet](https://img.shields.io/badge/mainnet-live-success)](https://explorer.hathor.network/)
 [![Version](https://img.shields.io/github/v/release/HathorNetwork/hathor-core)](https://github.com/HathorNetwork/hathor-core/releases/latest)
-[![Testing](https://img.shields.io/github/workflow/status/HathorNetwork/hathor-core/tests?label=tests&logo=github)](https://github.com/HathorNetwork/hathor-core/actions?query=workflow%3Atests+branch%3Amaster)
-[![Ｄocker](https://img.shields.io/github/workflow/status/HathorNetwork/hathor-core/docker?label=build&logo=docker)](https://hub.docker.com/repository/docker/hathornetwork/hathor-core)
+[![Testing](https://img.shields.io/github/actions/workflow/status/HathorNetwork/hathor-core/main.yml?branch=master&label=tests&logo=github)](https://github.com/HathorNetwork/hathor-core/actions?query=workflow%3Atests+branch%3Amaster)
+[![Docker](https://img.shields.io/github/actions/workflow/status/HathorNetwork/hathor-core/docker.yml?branch=master&label=build&logo=docker)](https://hub.docker.com/repository/docker/hathornetwork/hathor-core)
 [![Codecov](https://img.shields.io/codecov/c/github/HathorNetwork/hathor-core?logo=codecov)](https://codecov.io/gh/hathornetwork/hathor-core)
 [![Discord](https://img.shields.io/discord/566500848570466316?logo=discord)](https://discord.com/invite/35mFEhk)
 [![License](https://img.shields.io/github/license/HathorNetwork/hathor-core)](./LICENSE.txt)
@@ -59,11 +59,11 @@ First, you need to have Python >=3.8 installed. If you don't, we recommend you t
   brew install pyenv
   ```
 
-  then Python 3.10 (you could check the latest 3.10.x version with `pyenv install --list`):
+  then Python 3.11 (you could check the latest 3.11.x version with `pyenv install --list`):
 
   ```
-  pyenv install 3.10.6
-  pyenv local 3.10.6
+  pyenv install 3.11.0
+  pyenv local 3.11.0
   pip install -U poetry
   ```
 
@@ -75,7 +75,7 @@ First, you need to have Python >=3.8 installed. If you don't, we recommend you t
 - on Windows 10 (using [winget](https://github.com/microsoft/winget-cli)):
 
   ```
-  winget install python-3.10
+  winget install python-3.11
   pip install -U poetry
   ```
 

extras/check_version.sh

@@ -1,11 +1,22 @@
 #!/bin/bash
 
+###
+# This script will check all source files containing the project version and exit with an error code -1 in case
+# they don't match.
+#
+# usage: ./extras/check_version.sh [version]
+#
+# example: ./extras/check_version.sh 0.52.1
+#
+# When a version is provided, it is checked against the package version.
+###
+
 OPENAPI_FILE="hathor/cli/openapi_files/openapi_base.json"
 SRC_FILE="hathor/version.py"
 PACKAGE_FILE="pyproject.toml"
 
 OPENAPI_VERSION=`grep "version\":" ${OPENAPI_FILE} | cut -d'"' -f4`
-SRC_VERSION=`grep "__version__" ${SRC_FILE} | cut -d "'" -f2`
+SRC_VERSION=`grep "BASE_VERSION =" ${SRC_FILE} | cut -d "'" -f2`
 PACKAGE_VERSION=`grep '^version' ${PACKAGE_FILE} | cut -d '"' -f2`
 
 # For debugging:
@@ -25,4 +36,12 @@ if [[ x${PACKAGE_VERSION}x != x${OPENAPI_VERSION}x ]]; then
 	EXITCODE=-1
 fi
 
+# We expect an optional argument containing a version string to be checked against the others
+if [[ $# -eq 1 ]]; then
+    if [[ x${PACKAGE_VERSION}x != x$1x ]]; then
+        echo "Version different in ${PACKAGE_FILE} and passed argument"
+        EXITCODE=-1
+    fi
+fi
+
 exit $EXITCODE