Skip to content

Commit 5dee333

Browse files
olethanholethanh
committed
Merge remote-tracking branch 'origin/main' into bjrint-use-jwcrypto
2 parents d5e3e7a + cb0a9f9 commit 5dee333

File tree

22 files changed

+639
-698
lines changed

22 files changed

+639
-698
lines changed

‎.github/workflows/deploy-main-on-staging.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# This workflow automatically deploys main on staging
2+
name: "Deploy `main` automatically on staging"
3+
4+
on:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
deploy_staging_servers:
11+
name: "Deploying on ${{ matrix.staging_servers.hostname }}"
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
staging_servers:
16+
- hostname: "ovh.staging.aleph.sh"
17+
# Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
18+
host_keys: "fDF8b3JHVkxyOU83Qnh0QmkvWjd4N0lVTkRDSHFRPXwxZEdZSnNjNlFyejA5QkR6cGROR1BLYjNES009IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHZwSmNpV2dscTNCbEsxY2xOUmNETnVQeFVCeGF3bE5qVElHZFV2MmoyTVo4KzliVVpDSkI1aXFIKzNvbkc5Vklla1RQdW1ybFlXbFMvZkkvSzM3dTh5UXJuQ3JkNi9XRm9XWWJOaTJ4NWxSOUhzaTViRXQ4MFAwRkFyVVpaSkdzbnRQdVhGeFJGR3dHeFNENTN2emVMbmU4VjRlaUxrQ3BjMDU5YzZVVHBublcvSjdRRnlzZURDUXIwVzFsMzBNcjlnTm1LbmpBd2VLWXdCS0hYaG42VGdSd1RYT1E3VXJCc3c2Q1d0OHI2N2g4QkJ2UHQ5OWt5OHl4dUw2Z25TRlhqeWhyKzVhd1lTY3VhVU5JS3B0Y2JFOWpISHhEY1FLSHN0akZZRHRsM0JWN29rUEkvUWJablpSMDVTdDgvZldoK2p5K3ZtR3BTWmtFckJ2NkUwdFhHMDhmdkdheVRXNWFVcWxRQmlLMzJpNmJlUWordjI3b0pUWndvcndBOVJCY1QramlCWVRNVUFpTTJrblhXMGlqT2ViWDNackpITm5QbXJkbjBTd1JldGlLRzg2SGdRK3d3a0dwd3UxVk01eTFlbTVwZ0VUdnU5SHg1RTFKeEJLcXJ3ZkdhTVVRWFZEWG8yNDg5bW1XZzA1aUFHejZIaXNTVWRESFlRKzhnWnA4PQp8MXxvUzkyc1NEb3RxU1hSb0F6MUpFS1V2RDhVTGM9fDVtSHZBSVdqbk1CN2IwcllRQlo0SXBpaFlqQT0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQkZNanZFOEFsQmYxbkp1Y0ZlaEJjSUY2RE8wdGJOdU96OEx5QlFUdC82RlEwaWYyWVAxQUJ1TjBmYXVIT3R4WEx6b25vSGVhTDZaV0JoakhmRGV4NlY4PQp8MXxMc2lPc3RhVGk5bEhYSlFsWDJYQ3c3Q0lTU1k9fGk1RzlFTHJydHpaYkUrR2JjbWF1SDIxOG1ZND0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUp1QVNEMWY1d2dXM3pnd3FGalBXYzhPRi9BZ1pmSFFVa3lRMDE2c1MrRmoK"
19+
os: "debian-12"
20+
make_target: "all-podman-debian-12"
21+
artifact_name: "aleph-vm.debian-12.deb"
22+
23+
- hostname: "hetzner.staging.aleph.sh"
24+
# Use `ssh-keyscan -H host | base64 --wrap=0` to obtain the host keys
25+
host_keys: "fDF8WUlKd0FYWnYxZ24vNkRCU0tkYjg0TC9sUngwPXwrRk96RzdoSTJ5Y3JzUW1uSEwrdEFBQkR4YUU9IHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRRHBKcHF5ajUxWUluRkNyZjZUWjE5eUF3cHlXNTNHaFAxNXQ0Wm56cHBwOUVnNTNnZmVWdmk5WUV1bVV6cnVUN01LdFpobjNsb0U5YVFtRUYzSElpb3c5ZmlCWVA3aUMzUUFGdUJCandPUmQwV1RVWDZQQUN2c2p0b1JLWjJpTWZ2YXdITHdrWHErWnkrc2hHNU44L2pwQlJ4MC9paXJta2xPS0F5QWw0QTYzZ2MxMndsVGQzcS9IcDVxd1dSYVV3M1JVUTFTVVJSN2RGRW81VWxqeUZVYS9zdWV1STBYOXdLd0tPZ09iOEo3ZFZDMEdDT3VibkJXL3Jmb3N0YVV5eStaSzdQdzBrM251M2szTFZuUVlPTGlNOG1NMnJub2ZWZ2RSWXpiM3RTUVVrbk9wektBVzBXK3llWmpSOXp1UG4yNXF4bWxsRmRaNmt3QTFDcWY2MmQyQ0dtQ2NDU3dUSUl4ZHJ3M29oOEZOclpROTI4OGQvcmF4djZXZi9oZDI0Y1JqeDdFSEJxOUFWMW02UTZWeGxnSWl0WjIzODlsYjRmODNGclRrNUtib3J3Zm5oM3NaZFRSSkJqRjRhdHZ5NktsWFYxenROc05BeDhFN1RnTDMzVFlnRGc4RWlldGN1TVlzUlcwSnREdldBNGxsZDFQS3JrbDJ1LzZvLzNUb0xVPQp8MXxmQ3FnTjB2WHpMTnAzdklnZXdkSFRQRTA0ZUk9fDhnSituTC9hUGpEQlRMcUNJak1sZFpVbFRpST0gZWNkc2Etc2hhMi1uaXN0cDI1NiBBQUFBRTJWalpITmhMWE5vWVRJdGJtbHpkSEF5TlRZQUFBQUlibWx6ZEhBeU5UWUFBQUJCQktWbnE5aWsvcHZFaDdXbHFydUtWZmdZeTlwOVpNQnVKV2IrZkVvS0hZY0ZSYld5c0lYRjJlalBnaUMyOFEvZExqeUhXd2RVZlMySFBMbGNxRVFEZlpvPQp8MXxtVzA4T3ZqUnh0bmRjYVNyc0poWXBQcXp2akk9fFlDcktMeUg4ZnJJR0lRV05RS3hiUnArNlIvTT0gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUl5ZGNhTXF1dkZFTEpNUDBlRmhNUGJWZVBSVjlSUEhVRzhIZGZIQmRvaTEK"
26+
os: "debian-12"
27+
make_target: "all-podman-debian-12"
28+
artifact_name: "aleph-vm.debian-12.deb"
29+
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v4
33+
with:
34+
# Fetch the whole history for all tags and branches (required for aleph.__version__)
35+
fetch-depth: 0
36+
37+
- run: |
38+
cd packaging && make ${{ matrix.staging_servers.make_target }} && cd ..
39+
ls packaging/target
40+
41+
- name: Setup SSH private key
42+
run: |
43+
mkdir ~/.ssh
44+
echo $STAGING_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519
45+
chmod 0700 ~/.ssh
46+
chmod 0600 ~/.ssh/id_ed25519
47+
env:
48+
# Create using:
49+
# ssh-keygen -t ed25519 -f ./id_ed25519
50+
# cat ./id_ed25519 | base64 --wrap=0
51+
STAGING_SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
52+
53+
- name: Install Aleph-VM on the Staging servers
54+
run: |
55+
echo ${{ matrix.staging_servers.host_keys }} | base64 --decode > ~/.ssh/known_hosts
56+
57+
# Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH.
58+
while ssh root@${{ matrix.staging_servers.hostname }} lsof /var/lib/apt/lists/lock; do sleep 1; done
59+
60+
scp packaging/target/${{ matrix.staging_servers.artifact_name }} root@${{ matrix.staging_servers.hostname }}:/opt
61+
ssh root@${{ matrix.staging_servers.hostname }} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y --allow-downgrades /opt/${{ matrix.staging_servers.artifact_name }}"
62+

‎.github/workflows/test-on-droplets-matrix.yml‎

Lines changed: 25 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,107 +1,30 @@
1-
name: "Test on DigitalOcean Droplets"
2-
1+
# These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines
2+
# with the different operating systems that are supported.
3+
#
4+
# The main focus of these tests is to ensure that the packaging works on all supported platforms
5+
# and to ensure the compatibility of dependencies (system and vendored) across these platforms.
6+
name: "Testing on DigitalOcean Droplets"
7+
8+
# Run automatically on main branches, Pull Request updates and allow manual execution using `workflow_dispatch`.
39
on:
4-
push
10+
push:
11+
branches:
12+
- main
13+
pull_request:
14+
types:
15+
- "opened"
16+
- "reopened"
17+
- "synchronize"
18+
- "ready_for_review"
19+
workflow_dispatch:
520

621
jobs:
7-
tests-python:
8-
name: "Test Python code"
9-
runs-on: ubuntu-22.04
10-
11-
steps:
12-
- uses: actions/checkout@v4
13-
14-
- name: Workaround github issue https://github.com/actions/runner-images/issues/7192
15-
run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
16-
17-
- name: Install required system packages only for Ubuntu Linux
18-
run: |
19-
sudo apt-get update
20-
sudo apt-get -y upgrade
21-
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
22-
pip install --upgrade typing-extensions types-PyYAML
23-
24-
- name: Install required Python packages
25-
run: |
26-
python3 -m pip install hatch hatch-vcs coverage
27-
28-
- name: Test style wth ruff, black and isort
29-
run: |
30-
hatch run lint:style
31-
32-
- name: Test typing with Mypy
33-
run: |
34-
hatch run lint:typing
35-
36-
- name: Install required system packages for installing and running tests
37-
run: |
38-
sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
39-
40-
- name: Download and build required files for running tests. Copied from packaging/Makefile.
41-
run: |
42-
sudo mkdir --parents /opt/firecracker/
43-
sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
44-
45-
rm -fr /tmp/firecracker-release
46-
mkdir --parents /tmp/firecracker-release /opt/firecracker
47-
curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
48-
# Copy binaries:
49-
cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
50-
cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
51-
chmod +x /opt/firecracker/firecracker
52-
chmod +x /opt/firecracker/jailer
53-
54-
find /opt
55-
56-
- name: "Build custom runtime"
57-
run: |
58-
sudo apt update
59-
sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
60-
cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
61-
62-
- name: "Build example volume"
63-
run: |
64-
cd examples/volumes && bash build_squashfs.sh
65-
66-
# Unit tests create and delete network interfaces, and therefore require to run as root
67-
- name: Run unit tests
68-
run: |
69-
sudo python3 -m pip install hatch hatch-vcs coverage
70-
sudo hatch run testing:cov
71-
72-
- name: Upload coverage reports to Codecov
73-
uses: codecov/[email protected]
74-
with:
75-
token: ${{ secrets.CODECOV_TOKEN }}
76-
slug: aleph-im/aleph-vm
77-
78-
code-quality-shell:
79-
runs-on: ubuntu-22.04
80-
81-
steps:
82-
- uses: actions/checkout@v4
83-
84-
- name: Workaround github issue https://github.com/actions/runner-images/issues/7192
85-
run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
86-
87-
- name: Install required system packages only for Ubuntu Linux
88-
run: |
89-
sudo apt-get update
90-
sudo apt-get install -y shellcheck
91-
92-
- name: Run Shellcheck on all shell scripts
93-
run: |
94-
find ./ -type f -name "*.sh" -exec shellcheck {} \;
95-
9622

9723
run_on_droplet:
9824
name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
9925
runs-on: ubuntu-latest
10026
concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
10127
timeout-minutes: 10
102-
needs:
103-
- tests-python
104-
- code-quality-shell
10528

10629
strategy:
10730
matrix:
@@ -238,6 +161,13 @@ jobs:
238161
-d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \
239162
"http://${DROPLET_IPV4}:4020/control/allocations"
240163
164+
- name: Fetch system usage endpoint
165+
run: |
166+
export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)"
167+
curl -X GET -H "Content-Type: application/json" \
168+
"http://${DROPLET_IPV4}:4020/about/usage/system"
169+
170+
241171
- name: Export aleph logs
242172
if: always()
243173
run: |

‎.github/workflows/test-using-pytest.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: "Test on DigitalOcean Droplets"
2+
3+
on:
4+
push
5+
6+
jobs:
7+
tests-python:
8+
name: "Test Python code"
9+
runs-on: ubuntu-22.04
10+
11+
steps:
12+
- uses: actions/checkout@v4
13+
14+
- name: Workaround github issue https://github.com/actions/runner-images/issues/7192
15+
run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
16+
17+
- name: Install required system packages only for Ubuntu Linux
18+
run: |
19+
sudo apt-get update
20+
sudo apt-get -y upgrade
21+
sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
22+
pip install --upgrade typing-extensions types-PyYAML
23+
24+
- name: Install required Python packages
25+
run: |
26+
python3 -m pip install hatch hatch-vcs coverage
27+
28+
- name: Test style wth ruff, black and isort
29+
run: |
30+
hatch run lint:style
31+
32+
- name: Test typing with Mypy
33+
run: |
34+
hatch run lint:typing
35+
36+
- name: Install required system packages for installing and running tests
37+
run: |
38+
sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
39+
40+
- name: Download and build required files for running tests. Copied from packaging/Makefile.
41+
run: |
42+
sudo mkdir --parents /opt/firecracker/
43+
sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
44+
45+
rm -fr /tmp/firecracker-release
46+
mkdir --parents /tmp/firecracker-release /opt/firecracker
47+
curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
48+
# Copy binaries:
49+
cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
50+
cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
51+
chmod +x /opt/firecracker/firecracker
52+
chmod +x /opt/firecracker/jailer
53+
54+
find /opt
55+
56+
- name: "Build custom runtimes"
57+
run: |
58+
sudo apt update
59+
sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
60+
cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
61+
cd runtimes/instance-rootfs && sudo ./create-ubuntu-22-04-qemu-disk.sh && cd ../..
62+
63+
- name: "Build example volume"
64+
run: |
65+
cd examples/volumes && bash build_squashfs.sh
66+
67+
# Unit tests create and delete network interfaces, and therefore require to run as root
68+
- name: Run unit tests
69+
run: |
70+
sudo python3 -m pip install hatch hatch-vcs coverage
71+
sudo hatch run testing:cov
72+
73+
- name: Upload coverage reports to Codecov
74+
uses: codecov/[email protected]
75+
with:
76+
token: ${{ secrets.CODECOV_TOKEN }}
77+
slug: aleph-im/aleph-vm
78+
79+
code-quality-shell:
80+
runs-on: ubuntu-22.04
81+
82+
steps:
83+
- uses: actions/checkout@v4
84+
85+
- name: Workaround github issue https://github.com/actions/runner-images/issues/7192
86+
run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
87+
88+
- name: Install required system packages only for Ubuntu Linux
89+
run: |
90+
sudo apt-get update
91+
sudo apt-get install -y shellcheck
92+
93+
- name: Run Shellcheck on all shell scripts
94+
run: |
95+
find ./ -type f -name "*.sh" -exec shellcheck {} \;
96+

‎README.md‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@ Writing programs in Python using ASGI compatible frameworks (
1717

1818
Install Aleph-VM to run an Aleph.im Compute Resource Node easily from official pre-built packages.
1919

20-
- [On Debian 11](./doc/INSTALL-Debian-11.md)
21-
- [On Debian 12](./doc/INSTALL-Debian-12.md)
22-
- [On Ubuntu 22.04](./doc/INSTALL-Ubuntu-22.04.md)
20+
See the official user doc https://docs.aleph.im/nodes/compute/
2321

2422
## 2. Install Aleph-VM from source
2523

0 commit comments

Comments
 (0)