Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions ci/ci-pr.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# This file is used to govern CI jobs for GitHub PRs

args@{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ], src ? null, ... }:
import ./ci.nix (args // { inherit supportedSystems src; isMaster = false; })
args@{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ], src ? builtins.fetchGit ../., ... }:
import ./ci.nix (args // { inherit supportedSystems src; })
7 changes: 2 additions & 5 deletions ci/ci.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,12 @@
# The version of the release. Will be set to the right value in ./release.nix.
, releaseVersion ? "latest"

# TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149)
, isMaster ? true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: we may re-introduce isMaster in the future for longer-running tests

Copy link
Contributor Author

@basvandijk basvandijk Apr 14, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know, but let's add it back when we need it again. That way we'll only add it where we need it. Now it's passed all over the place.


, pkgs ? import ../nix { inherit system isMaster RustSec-advisory-db; }
, pkgs ? import ../nix { inherit system RustSec-advisory-db; }
}:
pkgs.lib.mk-jobset {
inherit supportedSystems;
inherit (src) rev;
mkJobsetSpec = { system, pkgs, jobset }: import ../. {
inherit system pkgs jobset RustSec-advisory-db releaseVersion isMaster src;
inherit system pkgs jobset RustSec-advisory-db releaseVersion src;
};
}
5 changes: 0 additions & 5 deletions ci/release.nix
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,5 @@ let
ci = import ./ci.nix { inherit src releaseVersion; };
in
if !doRelease then {} else {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are theses release.nix and doRelease things still used?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes they're used to release the SDK.

# TODO: remove these jobs when the `publish.dfx` job below
# is working successfully and the CloudFront CDN is online.
# See: https://dfinity.atlassian.net/browse/INF-1143
inherit (ci) dfx-release install-sh-release;

publish.dfx.x86_64-linux = ci.publish.dfx.x86_64-linux;
}
9 changes: 3 additions & 6 deletions default.nix
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
{ system ? builtins.currentSystem
, src ? null
, releaseVersion ? "latest"
# TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149)
, isMaster ? false
, RustSec-advisory-db ? null
, pkgs ? import ./nix { inherit system RustSec-advisory-db; }
, jobset ? import ./ci/ci.nix { inherit system releaseVersion RustSec-advisory-db pkgs isMaster src; }
, jobset ? import ./ci/ci.nix { inherit system releaseVersion RustSec-advisory-db pkgs src; }
}:
rec {
dfx = import ./dfx.nix { inherit pkgs userlib-js; };
Expand All @@ -19,8 +17,7 @@ rec {

inherit (pkgs) nix-fmt nix-fmt-check;

public = import ./public { inherit pkgs src releaseVersion isMaster; };
inherit (public) install-sh-release install-sh;
install = import ./public { inherit pkgs src; };

# This is to make sure CI evaluates shell derivations, builds their
# dependencies and populates the hydra cache with them. We also use this in
Expand All @@ -39,6 +36,6 @@ rec {

publish = import ./publish.nix {
inherit pkgs releaseVersion;
inherit (jobset) dfx-release install-sh-release;
inherit (jobset) dfx-release install;
};
}
8 changes: 3 additions & 5 deletions nix/default.nix
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# Returns the nixpkgs set overridden and extended with DFINITY specific
# packages.
{ system ? builtins.currentSystem
# TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149)
, isMaster ? false
, RustSec-advisory-db ? null
}:
let
Expand Down Expand Up @@ -47,7 +45,7 @@ let
then RustSec-advisory-db
else self.sources.advisory-db;

motoko = import self.sources.motoko { system = self.system; };
motoko = import self.sources.motoko { inherit (self) system; };
dfinity = (import self.sources.dfinity { inherit (self) system; }).dfinity.rs;
napalm = self.callPackage self.sources.napalm {
pkgs = self // { nodejs = self.nodejs-12_x; };
Expand All @@ -59,7 +57,7 @@ let
lib = super.lib // {
mk-jobset = import ./mk-jobset.nix self;

mkRelease = super.callPackage ./mk-release.nix { inherit isMaster; };
mkRelease = super.callPackage ./mk-release.nix {};
};

# An attribute set mapping every supported system to a nixpkgs evaluated for
Expand All @@ -70,7 +68,7 @@ let
if supportedSystem == system
then self
else import ./. {
inherit isMaster RustSec-advisory-db;
inherit RustSec-advisory-db;
system = supportedSystem;
}
);
Expand Down
34 changes: 4 additions & 30 deletions nix/mk-release.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,21 @@
, gzip
, jo
, patchelf
# TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149)
, isMaster ? false
}:
rname: version: from: what:
stdenv.mkDerivation {
name = "${rname}-release";
inherit version;
inherit rname version from what;
phases = [ "buildPhase" ];
buildInputs = [ gzip jo patchelf ];
allowedRequisites = [];
inherit isMaster;
buildPhase = ''
# Building the artifacts
mkdir -p $out
# we embed the system into the name of the archive
the_release="${rname}-$version.tar.gz"
the_release="$rname-$version.tar.gz"
# Assemble the fully standalone archive
collection=$(mktemp -d)
cp ${from}/bin/${what} $collection/${what}
chmod 0755 $collection/${what}

cp $from/bin/$what $collection/$what
chmod 0755 $collection/$what
tar -cvzf "$out/$the_release" -C $collection/ .

# Creating the manifest
manifest_file=$out/manifest.json

sha256hash=($(sha256sum "$out/$the_release")) # using this to autosplit on space
sha1hash=($(sha1sum "$out/$the_release")) # using this to autosplit on space

jo -pa \
$(jo package="${rname}" \
version="$version" \
system="${stdenv.system}" \
name="${stdenv.system}/$the_release" \
file="$out/$the_release" \
sha256hash="$sha256hash" \
sha1hash="$sha1hash") >$manifest_file

# Marking the manifest for publishing
mkdir -p $out/nix-support
echo "upload manifest $manifest_file" >> \
$out/nix-support/hydra-build-products
'';
}
162 changes: 32 additions & 130 deletions public/default.nix
Original file line number Diff line number Diff line change
@@ -1,147 +1,49 @@
{ pkgs ? import ../nix { inherit system; }
, system ? builtins.currentSystem
, src ? null
, releaseVersion ? "latest"
# TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149)
, isMaster ? false
, src ? builtins.fetchGit ../.
}:

let
public = pkgs.lib.noNixFiles (pkgs.lib.gitOnlySource ../. "public");
version = releaseVersion;
gitDir = pkgs.lib.gitDir ../.;
in

rec {
install-sh =
pkgs.runCommandNoCC "install-sh" {
inherit public;
install-template =
pkgs.runCommandNoCC "install.sh.template" {
src = pkgs.lib.gitOnlySource ../. ./install;
preferLocalBuild = true;
allowSubstitutes = false;
nativeBuildInputs = [ pkgs.shfmt pkgs.shellcheck ];
allowedReferences = [];
} ''
# git describe --abbrev=7 --tags
mkdir -p $out
echo "Running shellcheck..."
shellcheck --shell=sh $src/*.sh \
--exclude SC2039 \
--exclude SC2154 \
--exclude SC2034

cat $public/install/*.sh > $out/install.sh
echo "Running shfmt..."
shfmt -d -p -i 4 -ci -bn -s $src/*.sh

echo "Creating output..."
cat $src/*.sh > $out

echo "Fixing up..."
# Get rid of comments that don't start with '##' or '#!'.
sed -i "
/#!.*/p
/##.*/p
/^ *$/d
/^ *#/d
s/ *#.*//
" $out/install.sh
" $out
'';

# Check if the install.sh script is well formatted and
# has no shellcheck issues. We ignore 'local' warning by shellcheck, because
# any existing sh implementation supports it.
# TODO: streamline mkRelease and this
install-sh-lint =
let
shfmtOpts = "-p -i 4 -ci -bn -s";
shellcheckOpts = "-s sh -S warning";
in
pkgs.runCommandNoCC "install-sh-lint" {
inherit version public;
inherit isMaster;
buildInputs = [ install-sh pkgs.shfmt pkgs.shellcheck ];
preferLocalBuild = true;
allowSubstitutes = false;
} ''
set -Eeuo pipefail
# Check if we have an sh compatible script
shckResult="$(shellcheck -Cnever -f gcc ${shellcheckOpts} "$public/install.sh" | \
grep -v "warning: In POSIX sh, 'local' is undefined. \[SC2039\]" | \
sed -e "s%^${install-sh}/?%%g" || true)"

if [ -n "$shckResult" ] ; then
echo "There are some shellcheck warnings:"
echo
echo "$shckResult"
echo
exit 1
fi

# Check if the file is properly formatted
if ! shfmt ${shfmtOpts} -d "${install-sh}/install.sh"; then
echo "Formatting error. Please run:"
echo
echo "shfmt ${shfmtOpts} -w public/install.sh"
exit 1
fi

if grep source "${install-sh}/install.sh"; then
echo "Found a source above in the output. There should be none remaining (inlined)."
exit 1
fi

# Make sure Nix sees the output.
touch $out
'';

# The following prepares a manifest for copying install.sh
install-sh-release =
pkgs.lib.linuxOnly (
pkgs.runCommandNoCC "install-sh-release" {
inherit version;
inherit isMaster;

# `revision` will be printed by `install.sh` as follows:
#
# log "Executing DFINITY SDK install script, commit: @revision@"
revision =
if src != null
then src.rev
else pkgs.lib.commitIdFromGitRepo (pkgs.lib.gitDir ../.);

manifest = ./manifest.json;
buildInputs = [ pkgs.jo install-sh-lint install-sh ];
preferLocalBuild = true;
allowSubstitutes = false;
} ''
set -Eeuo pipefail

mkdir -p $out

version_manifest_file=$out/manifest.json

cp $manifest $version_manifest_file
# we stamp the file with the revision
substitute "${install-sh}/install.sh" $out/install.sh \
--subst-var revision

# Creating the manifest
# We name it "_manifest.json" as opposed to "manifest.json" because we
# also export a "manifest.json" (which has nothing to do with the
# release)
hydra_manifest_file=$out/_manifest.json

sha256hashinstall=($(sha256sum "$out/install.sh")) # using this to autosplit on space
sha1hashinstall=($(sha1sum "$out/install.sh")) # using this to autosplit on space

sha256manifest=($(sha256sum "$version_manifest_file")) # using this to autosplit on space
sha1manifest=($(sha1sum "$version_manifest_file")) # using this to autosplit on space

jo -pa \
$(jo package="public" \
version="$version" \
name="installer" \
file="$out/install.sh" \
sha256hash="$sha256hashinstall" \
sha1hash="$sha1hashinstall") \
$(jo package="public" \
version="$version" \
name="manifest.json" \
file="$version_manifest_file" \
sha256hash="$sha256manifest" \
sha1hash="$sha1manifest") >$hydra_manifest_file

# Marking the manifest for publishing
mkdir -p $out/nix-support
echo "upload manifest $hydra_manifest_file" >> \
$out/nix-support/hydra-build-products
''
);
}
in
pkgs.lib.linuxOnly (
pkgs.runCommandNoCC "install.sh" {
# `revision` will be printed by `install.sh` as follows:
#
# log "Executing DFINITY SDK install script, commit: @revision@"
revision = src.rev;
preferLocalBuild = true;
allowSubstitutes = false;
} ''
# we stamp the file with the revision
substitute "${install-template}" $out --subst-var revision
''
)
17 changes: 10 additions & 7 deletions public/install/020_flags.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,20 @@ DFX_BOOL_FLAGS=""
# $2 - A description of the flag. This is not currently used but will be when we have enough
# flags to implement help.
define_flag_BOOL() {
local VARNAME="flag_$(echo $1 | tr /a-z/ /A-Z)"
eval $VARNAME="\${$VARNAME:-}"
local VARNAME
VARNAME="flag_$(echo "$1" | tr /a-z/ /A-Z)"
eval "$VARNAME=\${$VARNAME:-}"
DFX_BOOL_FLAGS="${DFX_BOOL_FLAGS}--${1} $VARNAME $2"
}

# Get the flag name of a line in the flag description.
get_flag_name() {
echo $1
echo "$1"
}

# Get the variable name of a line in the flag description.
get_var_name() {
echo $2
echo "$2"
}

# Read all the command line flags and set the flag_XXXX environment variables.
Expand All @@ -35,6 +36,8 @@ get_var_name() {
# Environment variables are set according to flags defined and flags.
read_flags() {
# Set values from command line.
# shellcheck disable=SC2199
# https://github.com/koalaman/shellcheck/wiki/SC2199
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@knl do I understand correctly and is the above URL saying I should rewrite the below while loop into a for loop like:

for ARG in "$@"; do

?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Written this way implies there will be only a single iteration of while.

while [[ "$@" ]]; do
local ARG=$1
shift
Expand All @@ -45,11 +48,11 @@ read_flags() {
[ "$line" ] || break

IFS="$OLD_IFS"
FLAG=$(get_flag_name $line)
VARNAME=$(get_var_name $line)
FLAG="$(get_flag_name "$line")"
VARNAME="$(get_var_name "$line")"

if [ "$ARG" == "$FLAG" ]; then
eval $VARNAME="1"
eval "$VARNAME=1"
fi
done
done
Expand Down
2 changes: 1 addition & 1 deletion public/install/100_log.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
## 100_log.sh

log() {
if $_ansi_escapes_are_valid; then
if "$_ansi_escapes_are_valid"; then
printf "\33[1minfo:\33[0m %s\n" "$1" 1>&2
else
printf '%s\n' "$1" 1>&2
Expand Down
Loading