-
Notifications
You must be signed in to change notification settings - Fork 98
refactor: fixes install.sh lint errors and remove blobules publishing code #554
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,3 @@ | ||
| # This file is used to govern CI jobs for GitHub PRs | ||
|
|
||
| args@{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ], src ? null, ... }: | ||
| import ./ci.nix (args // { inherit supportedSystems src; isMaster = false; }) | ||
| args@{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ], src ? builtins.fetchGit ../., ... }: | ||
| import ./ci.nix (args // { inherit supportedSystems src; }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,10 +11,5 @@ let | |
| ci = import ./ci.nix { inherit src releaseVersion; }; | ||
| in | ||
| if !doRelease then {} else { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are theses
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes they're used to release the SDK. |
||
| # TODO: remove these jobs when the `publish.dfx` job below | ||
| # is working successfully and the CloudFront CDN is online. | ||
| # See: https://dfinity.atlassian.net/browse/INF-1143 | ||
| inherit (ci) dfx-release install-sh-release; | ||
|
|
||
| publish.dfx.x86_64-linux = ci.publish.dfx.x86_64-linux; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,147 +1,49 @@ | ||
| { pkgs ? import ../nix { inherit system; } | ||
| , system ? builtins.currentSystem | ||
| , src ? null | ||
| , releaseVersion ? "latest" | ||
| # TODO: Remove isMaster once switched to new CD system (https://dfinity.atlassian.net/browse/INF-1149) | ||
| , isMaster ? false | ||
| , src ? builtins.fetchGit ../. | ||
| }: | ||
|
|
||
| let | ||
| public = pkgs.lib.noNixFiles (pkgs.lib.gitOnlySource ../. "public"); | ||
| version = releaseVersion; | ||
| gitDir = pkgs.lib.gitDir ../.; | ||
| in | ||
|
|
||
| rec { | ||
| install-sh = | ||
| pkgs.runCommandNoCC "install-sh" { | ||
| inherit public; | ||
| install-template = | ||
| pkgs.runCommandNoCC "install.sh.template" { | ||
| src = pkgs.lib.gitOnlySource ../. ./install; | ||
| preferLocalBuild = true; | ||
| allowSubstitutes = false; | ||
| nativeBuildInputs = [ pkgs.shfmt pkgs.shellcheck ]; | ||
| allowedReferences = []; | ||
| } '' | ||
| # git describe --abbrev=7 --tags | ||
| mkdir -p $out | ||
| echo "Running shellcheck..." | ||
| shellcheck --shell=sh $src/*.sh \ | ||
| --exclude SC2039 \ | ||
| --exclude SC2154 \ | ||
| --exclude SC2034 | ||
|
|
||
| cat $public/install/*.sh > $out/install.sh | ||
| echo "Running shfmt..." | ||
| shfmt -d -p -i 4 -ci -bn -s $src/*.sh | ||
|
|
||
| echo "Creating output..." | ||
| cat $src/*.sh > $out | ||
|
|
||
| echo "Fixing up..." | ||
| # Get rid of comments that don't start with '##' or '#!'. | ||
| sed -i " | ||
| /#!.*/p | ||
| /##.*/p | ||
| /^ *$/d | ||
| /^ *#/d | ||
| s/ *#.*// | ||
| " $out/install.sh | ||
| " $out | ||
| ''; | ||
|
|
||
| # Check if the install.sh script is well formatted and | ||
| # has no shellcheck issues. We ignore 'local' warning by shellcheck, because | ||
| # any existing sh implementation supports it. | ||
| # TODO: streamline mkRelease and this | ||
| install-sh-lint = | ||
| let | ||
| shfmtOpts = "-p -i 4 -ci -bn -s"; | ||
| shellcheckOpts = "-s sh -S warning"; | ||
| in | ||
| pkgs.runCommandNoCC "install-sh-lint" { | ||
| inherit version public; | ||
| inherit isMaster; | ||
| buildInputs = [ install-sh pkgs.shfmt pkgs.shellcheck ]; | ||
| preferLocalBuild = true; | ||
| allowSubstitutes = false; | ||
| } '' | ||
| set -Eeuo pipefail | ||
| # Check if we have an sh compatible script | ||
| shckResult="$(shellcheck -Cnever -f gcc ${shellcheckOpts} "$public/install.sh" | \ | ||
| grep -v "warning: In POSIX sh, 'local' is undefined. \[SC2039\]" | \ | ||
| sed -e "s%^${install-sh}/?%%g" || true)" | ||
|
|
||
| if [ -n "$shckResult" ] ; then | ||
| echo "There are some shellcheck warnings:" | ||
| echo | ||
| echo "$shckResult" | ||
| echo | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Check if the file is properly formatted | ||
| if ! shfmt ${shfmtOpts} -d "${install-sh}/install.sh"; then | ||
| echo "Formatting error. Please run:" | ||
| echo | ||
| echo "shfmt ${shfmtOpts} -w public/install.sh" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if grep source "${install-sh}/install.sh"; then | ||
| echo "Found a source above in the output. There should be none remaining (inlined)." | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Make sure Nix sees the output. | ||
| touch $out | ||
| ''; | ||
|
|
||
| # The following prepares a manifest for copying install.sh | ||
| install-sh-release = | ||
| pkgs.lib.linuxOnly ( | ||
| pkgs.runCommandNoCC "install-sh-release" { | ||
| inherit version; | ||
| inherit isMaster; | ||
|
|
||
| # `revision` will be printed by `install.sh` as follows: | ||
| # | ||
| # log "Executing DFINITY SDK install script, commit: @revision@" | ||
| revision = | ||
| if src != null | ||
| then src.rev | ||
| else pkgs.lib.commitIdFromGitRepo (pkgs.lib.gitDir ../.); | ||
|
|
||
| manifest = ./manifest.json; | ||
| buildInputs = [ pkgs.jo install-sh-lint install-sh ]; | ||
| preferLocalBuild = true; | ||
| allowSubstitutes = false; | ||
| } '' | ||
| set -Eeuo pipefail | ||
|
|
||
| mkdir -p $out | ||
|
|
||
| version_manifest_file=$out/manifest.json | ||
|
|
||
| cp $manifest $version_manifest_file | ||
| # we stamp the file with the revision | ||
| substitute "${install-sh}/install.sh" $out/install.sh \ | ||
| --subst-var revision | ||
|
|
||
| # Creating the manifest | ||
| # We name it "_manifest.json" as opposed to "manifest.json" because we | ||
| # also export a "manifest.json" (which has nothing to do with the | ||
| # release) | ||
| hydra_manifest_file=$out/_manifest.json | ||
|
|
||
| sha256hashinstall=($(sha256sum "$out/install.sh")) # using this to autosplit on space | ||
| sha1hashinstall=($(sha1sum "$out/install.sh")) # using this to autosplit on space | ||
|
|
||
| sha256manifest=($(sha256sum "$version_manifest_file")) # using this to autosplit on space | ||
| sha1manifest=($(sha1sum "$version_manifest_file")) # using this to autosplit on space | ||
|
|
||
| jo -pa \ | ||
| $(jo package="public" \ | ||
| version="$version" \ | ||
| name="installer" \ | ||
| file="$out/install.sh" \ | ||
| sha256hash="$sha256hashinstall" \ | ||
| sha1hash="$sha1hashinstall") \ | ||
| $(jo package="public" \ | ||
| version="$version" \ | ||
| name="manifest.json" \ | ||
| file="$version_manifest_file" \ | ||
| sha256hash="$sha256manifest" \ | ||
| sha1hash="$sha1manifest") >$hydra_manifest_file | ||
|
|
||
| # Marking the manifest for publishing | ||
| mkdir -p $out/nix-support | ||
| echo "upload manifest $hydra_manifest_file" >> \ | ||
| $out/nix-support/hydra-build-products | ||
| '' | ||
| ); | ||
| } | ||
| in | ||
| pkgs.lib.linuxOnly ( | ||
| pkgs.runCommandNoCC "install.sh" { | ||
| # `revision` will be printed by `install.sh` as follows: | ||
| # | ||
| # log "Executing DFINITY SDK install script, commit: @revision@" | ||
| revision = src.rev; | ||
| preferLocalBuild = true; | ||
| allowSubstitutes = false; | ||
| } '' | ||
| # we stamp the file with the revision | ||
| substitute "${install-template}" $out --subst-var revision | ||
| '' | ||
| ) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,19 +12,20 @@ DFX_BOOL_FLAGS="" | |
| # $2 - A description of the flag. This is not currently used but will be when we have enough | ||
| # flags to implement help. | ||
| define_flag_BOOL() { | ||
| local VARNAME="flag_$(echo $1 | tr /a-z/ /A-Z)" | ||
| eval $VARNAME="\${$VARNAME:-}" | ||
| local VARNAME | ||
| VARNAME="flag_$(echo "$1" | tr /a-z/ /A-Z)" | ||
| eval "$VARNAME=\${$VARNAME:-}" | ||
| DFX_BOOL_FLAGS="${DFX_BOOL_FLAGS}--${1} $VARNAME $2" | ||
| } | ||
|
|
||
| # Get the flag name of a line in the flag description. | ||
| get_flag_name() { | ||
| echo $1 | ||
| echo "$1" | ||
| } | ||
|
|
||
| # Get the variable name of a line in the flag description. | ||
| get_var_name() { | ||
| echo $2 | ||
| echo "$2" | ||
| } | ||
|
|
||
| # Read all the command line flags and set the flag_XXXX environment variables. | ||
|
|
@@ -35,6 +36,8 @@ get_var_name() { | |
| # Environment variables are set according to flags defined and flags. | ||
| read_flags() { | ||
| # Set values from command line. | ||
| # shellcheck disable=SC2199 | ||
| # https://github.com/koalaman/shellcheck/wiki/SC2199 | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @knl do I understand correctly and is the above URL saying I should rewrite the below for ARG in "$@"; do?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes. Written this way implies there will be only a single iteration of |
||
| while [[ "$@" ]]; do | ||
| local ARG=$1 | ||
| shift | ||
|
|
@@ -45,11 +48,11 @@ read_flags() { | |
| [ "$line" ] || break | ||
|
|
||
| IFS="$OLD_IFS" | ||
| FLAG=$(get_flag_name $line) | ||
| VARNAME=$(get_var_name $line) | ||
| FLAG="$(get_flag_name "$line")" | ||
| VARNAME="$(get_var_name "$line")" | ||
|
|
||
| if [ "$ARG" == "$FLAG" ]; then | ||
| eval $VARNAME="1" | ||
| eval "$VARNAME=1" | ||
| fi | ||
| done | ||
| done | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
note: we may re-introduce
isMasterin the future for longer-running testsUh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I know, but let's add it back when we need it again. That way we'll only add it where we need it. Now it's passed all over the place.