Skip to content

Commit c89f486

Browse files
hussein-aitlahcenhussein-aitlahcen
committed
devnet release
1 parent 40c44c1 commit c89f486

File tree

9 files changed

+5673
-0
lines changed

9 files changed

+5673
-0
lines changed

‎.github/workflows/devnet-release.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
name: "Devnet Community Release"
2+
3+
on:
4+
push:
5+
branches:
6+
- nix-devnet
7+
8+
jobs:
9+
deploy-devnet:
10+
runs-on:
11+
- ubuntu-latest
12+
concurrency:
13+
group: deploy-devnet
14+
cancel-in-progress: false
15+
steps:
16+
- uses: actions/checkout@v2
17+
18+
- uses: google-github-actions/setup-gcloud@master
19+
with:
20+
service_account_key: ${{ secrets.GCP_CREDENTIALS }}
21+
export_default_credentials: true
22+
23+
- uses: actions/setup-python@v2
24+
25+
- uses: cachix/install-nix-action@v16
26+
with:
27+
nix_path: nixpkgs=channel:nixos-unstable
28+
29+
- uses: cachix/cachix-action@v10
30+
with:
31+
name: composable-community
32+
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
33+
extraPullNames: composable-community
34+
35+
- uses: actions-rs/toolchain@v1
36+
with:
37+
profile: minimal
38+
toolchain: nightly-2021-11-30
39+
override: true
40+
41+
- uses: Swatinem/rust-cache@v1
42+
43+
- name: Install wasm
44+
run: |
45+
rustup target add wasm32-unknown-unknown
46+
47+
- name: Set env
48+
run: |
49+
echo "RELEASE_VERSION=$GITHUB_SHA" >> $GITHUB_ENV
50+
51+
- name: Push artifact
52+
run: |
53+
cargo build --release -p composable -p picasso-runtime --features=develop
54+
tar -czvf composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz target/release/composable
55+
gsutil mv *.tar.gz gs://composable-binaries/community-releases/picasso/
56+
57+
- name: Load state
58+
run: |
59+
cd nix
60+
61+
echo $(cat devnet.json | jq --arg version "${{ env.RELEASE_VERSION }}" '.composable.version = $version' | jq --arg hash "$(nix-prefetch-url https://storage.googleapis.com/composable-binaries/community-releases/picasso/composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz)" '.composable.hash = $hash') > devnet.json
62+
63+
jq --null-input --arg client_email "$GCP_DEVNET_SERVICE_ACCOUNT" --arg project_id "$GCP_PROJECT_ID" --arg key "\"$GCP_DEVNET_SERVICE_ACCOUNT_KEY\"" '{ "project_id": $project_id, "private_key": ($key | fromjson), "client_email": $client_email }' > ops.json
64+
65+
if gsutil -q stat $NIXOPS_STATE_URL/$NIXOPS_STATE;
66+
then
67+
gsutil cp $NIXOPS_STATE_URL/$NIXOPS_STATE $NIXOPS_STATE
68+
else
69+
nix develop .#deploy --impure --command nixops create -d devnet-gce
70+
fi
71+
env:
72+
NIXOPS_STATE_URL: "gs://composable-state"
73+
NIXOPS_STATE: "deployment.nixops"
74+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
75+
GCP_DEVNET_SERVICE_ACCOUNT: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT }}
76+
GCP_DEVNET_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT_KEY }}
77+
78+
- name: Deploy
79+
run: |
80+
cd nix
81+
nix develop .#deploy --impure --command nixops deploy --check --confirm -d devnet-gce
82+
env:
83+
NIXOPS_STATE: "deployment.nixops"
84+
85+
- name: Store state
86+
if: always()
87+
run: |
88+
cd nix
89+
gsutil cp $NIXOPS_STATE $NIXOPS_STATE_URL/
90+
env:
91+
NIXOPS_STATE: "deployment.nixops"
92+
NIXOPS_STATE_URL: "gs://composable-state"
93+

‎nix/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ops.json

‎nix/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Install Nix + Flakes
2+
3+
1. https://nixos.org/download.html
4+
2. https://nixos.wiki/wiki/Flakes
5+
6+
# Run locally
7+
8+
1. `nix develop`
9+
2. `launch-devnet`
10+
3. Reach alice at `https://polkadot.js.org/apps/?rpc=ws://localhost:9944#/explorer`
11+
12+
# Deploy to GCE
13+
14+
1. Download your GCE service account key and save it as `ops.json`
15+
2. `nix develop .#deploy`
16+
3. `nixops create -d devnet-gce`
17+
4. `nixops deploy -d devnet-gce`

‎nix/devnet-gce.nix‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
{ composable,
2+
polkadot,
3+
credentials,
4+
localtunnel,
5+
}:
6+
let
7+
gcefy-version = version:
8+
builtins.replaceStrings [ "." ] [ "-" ] version;
9+
domain = "${composable.name}-${composable.spec}-${gcefy-version composable.version}";
10+
domain-latest = "${composable.name}-${composable.spec}-latest";
11+
in {
12+
resources.gceNetworks.composable-devnet = credentials // {
13+
name = "composable-devnet-network";
14+
firewall = {
15+
allow-http = {
16+
targetTags = [ "http" ];
17+
allowed.tcp = [ 80 ];
18+
};
19+
allow-https = {
20+
targetTags = [ "https" ];
21+
allowed.tcp = [ 443 ];
22+
};
23+
};
24+
};
25+
devnet-machine = { pkgs, resources, ... }:
26+
let
27+
devnet = pkgs.callPackage ./devnet.nix {
28+
inherit composable;
29+
inherit polkadot;
30+
};
31+
in {
32+
deployment = {
33+
targetEnv = "gce";
34+
gce = credentials // {
35+
machineName = "composable-devnet";
36+
network = resources.gceNetworks.composable-devnet;
37+
region = "europe-central2-c";
38+
instanceType = "n2-standard-4";
39+
rootDiskSize = 50;
40+
tags = [
41+
"http"
42+
"https"
43+
];
44+
};
45+
};
46+
networking.firewall.allowedTCPPorts = [ 80 443 ];
47+
systemd.services.composable-devnet = {
48+
wantedBy = [ "multi-user.target" ];
49+
after = [ "network.target" ];
50+
description = "Composable Devnet";
51+
serviceConfig = {
52+
Type = "simple";
53+
User = "root";
54+
ExecStart = "${devnet}/bin/launch-devnet";
55+
Restart = "always";
56+
RuntimeMaxSec = "86400"; # 1 day lease period for rococo, restart it
57+
};
58+
};
59+
systemd.services.localtunnel-commit = {
60+
wantedBy = [ "multi-user.target" ];
61+
after = [ "network.target" ];
62+
description = "Local Tunnel Server";
63+
serviceConfig = {
64+
Type = "simple";
65+
User = "root";
66+
Restart = "always";
67+
ExecStart = "${localtunnel}/bin/lt --port 80 --subdomain ${domain}";
68+
};
69+
};
70+
systemd.services.localtunnel-latest = {
71+
wantedBy = [ "multi-user.target" ];
72+
after = [ "network.target" ];
73+
description = "Local Tunnel Server";
74+
serviceConfig = {
75+
Type = "simple";
76+
User = "root";
77+
Restart = "always";
78+
ExecStart = "${localtunnel}/bin/lt --port 80 --subdomain ${domain-latest}";
79+
};
80+
};
81+
services.nginx =
82+
let virtualConfig =
83+
let
84+
routify-nodes = prefix:
85+
map (node: (node // {
86+
name = prefix + node.name;
87+
}));
88+
routified-composable-nodes =
89+
routify-nodes "parachain/" composable.nodes;
90+
routified-polkadot-nodes =
91+
routify-nodes "relaychain/" polkadot.nodes;
92+
routified-nodes =
93+
routified-composable-nodes ++ routified-polkadot-nodes;
94+
in
95+
{
96+
locations = builtins.foldl' (x: y: x // y) {} (map (node: {
97+
"/${node.name}" = {
98+
proxyPass = "http://127.0.0.1:${builtins.toString node.wsPort}";
99+
proxyWebsockets = true;
100+
};
101+
}) routified-nodes);
102+
};
103+
in {
104+
enable = true;
105+
virtualHosts."${domain}.loca.lt" = virtualConfig;
106+
virtualHosts."${domain-latest}.loca.lt" = virtualConfig;
107+
};
108+
};
109+
}

‎nix/devnet.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"composable": {
3+
"name": "picasso",
4+
"version": "1.1.4",
5+
"spec": "picasso-dev",
6+
"hash": "sha256:0lh8v5m9nmyffgrqxpp2a395ia2m4ly91m8cnnkabx9kx868fx0r"
7+
},
8+
"polkadot": {
9+
"version": "0.9.13",
10+
"spec": "rococo-local",
11+
"hash": "sha256:1iwhb1sgi8yk1nmmix7jr8rzjdl9kh50jx8s8a2fllzpllbbwdkw"
12+
}
13+
}

‎nix/devnet.nix‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
{ pkgs,
2+
fetchFromGitHub,
3+
fetchurl,
4+
composable,
5+
polkadot,
6+
}:
7+
let
8+
polkalaunch = pkgs.callPackage (pkgs.stdenv.mkDerivation {
9+
name = "polkadot-launch";
10+
version = "1.0.0";
11+
src = fetchFromGitHub {
12+
owner = "paritytech";
13+
repo = "polkadot-launch";
14+
rev = "99c395b9e7dc7468a4b755440d67e317370974c4";
15+
hash = "sha256:0is74ad9khbqivnnqfarm8012jvbpg5mcs2p9gl9bz1p7sz1f97d";
16+
};
17+
patches = [ ./polkadot-launch.patch ];
18+
installPhase = ''
19+
mkdir $out
20+
cp -r * $out
21+
'';
22+
}) {};
23+
24+
polkadot-bin = pkgs.stdenv.mkDerivation {
25+
name = "polkadot-${polkadot.version}";
26+
version = polkadot.version;
27+
src = fetchurl {
28+
url = "https://github.com/paritytech/polkadot/releases/download/v${polkadot.version}/polkadot";
29+
sha256 = polkadot.hash;
30+
};
31+
nativeBuildInputs = [
32+
pkgs.autoPatchelfHook
33+
];
34+
buildInputs = [ pkgs.stdenv.cc.cc ];
35+
dontUnpack = true;
36+
installPhase = ''
37+
mkdir -p $out/bin
38+
cp $src $out/bin/polkadot
39+
chmod +x $out/bin/polkadot
40+
'';
41+
};
42+
43+
composable-bin = pkgs.stdenv.mkDerivation rec {
44+
name = "composable-${composable.name}-${composable.version}";
45+
version = composable.version;
46+
src = fetchurl {
47+
url = "https://storage.googleapis.com/composable-binaries/community-releases/${composable.name}/${name}.tar.gz";
48+
sha256 = composable.hash;
49+
};
50+
nativeBuildInputs = [
51+
pkgs.autoPatchelfHook
52+
];
53+
buildInputs = [ pkgs.stdenv.cc.cc pkgs.zlib ];
54+
installPhase = ''
55+
tar -xvf $src
56+
mkdir -p $out/bin
57+
mv release/composable $out/bin
58+
'';
59+
};
60+
61+
make-node = tmp-directory: node-type: { name, wsPort, port }: {
62+
inherit name;
63+
inherit wsPort;
64+
inherit port;
65+
basePath = "${tmp-directory}/${node-type}/${name}";
66+
};
67+
68+
make-polkalaunch-config =
69+
{ tmp-directory, relaychain-spec, relaychain-bin, parachain-spec, parachain-bin }: {
70+
relaychain = {
71+
bin = relaychain-bin;
72+
chain = relaychain-spec;
73+
nodes = map (make-node tmp-directory "relaychain") polkadot.nodes;
74+
genesis = {
75+
runtime = {
76+
runtime_genesis_config = {
77+
configuration = {
78+
config = {
79+
validation_upgrade_frequency = 1;
80+
validation_upgrade_delay = 1;
81+
};
82+
};
83+
};
84+
};
85+
};
86+
};
87+
parachains = [
88+
{
89+
bin = parachain-bin;
90+
balance = "1000000000000000000000";
91+
chain = parachain-spec;
92+
nodes =
93+
map (node:
94+
(make-node tmp-directory "parachain" node) // {
95+
flags = ["--" "--execution=wasm"];
96+
}) composable.nodes;
97+
}
98+
];
99+
types = {};
100+
finalization = false;
101+
simpleParachains = [];
102+
};
103+
104+
tmp-directory = "/tmp/polkadot-launch";
105+
106+
devnet-config =
107+
pkgs.writeTextFile {
108+
name = "devnet.json";
109+
text = builtins.toJSON (
110+
make-polkalaunch-config
111+
{ inherit tmp-directory;
112+
relaychain-spec = polkadot.spec;
113+
relaychain-bin = "${polkadot-bin}/bin/polkadot";
114+
parachain-spec = composable.spec;
115+
parachain-bin = "${composable-bin}/bin/composable";
116+
}
117+
);
118+
};
119+
in
120+
pkgs.writeScriptBin "launch-devnet" ''
121+
#!${pkgs.bash}/bin/bash -e
122+
rm -rf ${tmp-directory}
123+
${polkalaunch}/bin/polkadot-launch ${devnet-config}
124+
''

0 commit comments

Comments
 (0)