Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions Building.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,20 +81,12 @@ https://github.com/ocaml/dune/issues/57 to see when a coverage report is viable
## Updating Haskell Packages

When the `.cabal` file of a Haskell package is changed you need to make sure the
corresponding `default.nix` file (stored in the same directory as the `.cabal`
file) is kept in sync with it.
corresponding `default.nix` file (stored in `nix/generated/`) is kept in sync
with it.

As mentioned in the `default.nix` files, these files are automatically generated:
As mentioned in the `nix/generate.nix` files, these files are automatically
generated. See `nix/generate.nix` for the command to update them.

```
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
```

Note that they also contain the instructions on how to update them.

Don't worry if you forget to update the `default.nix` file. There are CI jobs
(like [`haskellSrc2nix.lsp-int`](https://hydra.dfinity.systems/job/dfinity-ci-build/motoko/haskellSrc2nix.lsp-int)
that check if the files are in sync and fail with a diff if they aren't.
Don't worry if you forget to update the `default.nix` file, the CI job
`check-generated` checks if these files are in sync and fail with a diff if
they aren't.
27 changes: 9 additions & 18 deletions default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -411,24 +411,14 @@ rec {
};
stdlib-adocs = stdlib-doc.adocs;

haskellSrc2nix =
let
check = dir: nixpkgs.runCommandNoCC "check-${builtins.baseNameOf dir}" {
preferLocalBuild = true;
allowSubstitutes = false;
nativeBuildInputs = [ nixpkgs.diffutils ];
expected = import (dir + "/generate.nix") { pkgs = nixpkgs; };
inherit dir;
} ''
diff -U 3 $expected/default.nix $dir/default.nix
touch $out
'';
in {
lsp-int = check ./test/lsp-int;
qc-motoko = check ./test/random;
ic-stub = check ./ic-stub;
winter = check ./winter;
};
check-generated = nixpkgs.runCommandNoCC "check-generated" {
nativeBuildInputs = [ nixpkgs.diffutils ];
Copy link
Copy Markdown
Contributor

@basvandijk basvandijk Feb 21, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For inexpensive builds like this I would use:

        preferLocalBuild = true;
        allowSubstitutes = false;

To let CI build this locally instead of shipping inputs and outputs to and from remote builders / cache.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But is it inexpensive? It still depends on cabal2nix, so you’d have to build that on hydra, rather than the builders… Maybe it is not a concern.

The other reason is that we want cabal2nix etc. to be available on the cache, and this is the derivation that uses it, and the cache is only fed by the builders, isn’t it?

expected = import ./nix/generate.nix { pkgs = nixpkgs; };
dir = ./nix/generated;
} ''
diff -r -U 3 $expected $dir
touch $out
'';

all-systems-go = nixpkgs.releaseTools.aggregate {
name = "all-systems-go";
Expand All @@ -447,6 +437,7 @@ rec {
users-guide
ic-stub
shell
check-generated
] ++ builtins.attrValues tests
++ builtins.attrValues examples;
};
Expand Down
1 change: 0 additions & 1 deletion ic-stub/generate.nix

This file was deleted.

40 changes: 0 additions & 40 deletions nix/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -25,46 +25,6 @@ let
};
}
)
(self: super: {
# A variant of `haskellSrc2nixWithDoc` for local Haskell packages.
localHaskellSrc2nixWithDoc = name: path: extraCabal2nixOptions:
self.haskellSrc2nixWithDoc {
inherit name extraCabal2nixOptions;
src = import ./gitSource.nix path;
src_subst = "./.";
};

# `haskellSrc2nixWithDoc` is used to generate `default.nix` files for
# Haskell packages which are intended to be stored in the repository.
#
# The function generates a directory containing a `default.nix` which
# is the result of running `cabal2nix` with the `extraCabal2nixOptions`
# on the provided `src`.
#
# A header is added to `default.nix` which contains instructions on
# how to regenerate that file.
#
# Finally the `src` attribute in the `default.nix` will be defined as
# `src_subst` such that it can be pointed to local or niv-managed
# sources.
haskellSrc2nixWithDoc = {name, src, src_subst, extraCabal2nixOptions}:
let
drv = pkgs.haskellPackages.haskellSrc2nix {
inherit name extraCabal2nixOptions src;
};
in drv.overrideAttrs (oldAttrs: {
message = ''
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!\
# To regenerate this file execute the following command in this directory:\
#\
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
'';
inherit src_subst;
installPhase = oldAttrs.installPhase + ''
sed -i "1i$message;s|src = .*|src = $src_subst;|" $out/default.nix
'';
});
})
];
};
in
Expand Down
75 changes: 75 additions & 0 deletions nix/generate.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# This file generates the contents of nix/generated/. Use
#
# cp -fv $(nix-build generate.nix --no-link)/ generated/
#
# to update

{ pkgs ? import ../nix {} }:

let

# `haskellSrc2nixWithDoc` is used to generate `default.nix` files for
# Haskell packages which are intended to be stored in the repository.
#
# The function generates a directory containing a `default.nix` which
# is the result of running `cabal2nix` with the `extraCabal2nixOptions`
# on the provided `src`.
#
# A header is added to `default.nix` which contains instructions on
# how to regenerate that file.
#
# Finally the `src` attribute in the `default.nix` will be defined as
# `src_subst` such that it can be pointed to local or niv-managed
# sources.
haskellSrc2nixWithDoc = {name, src, src_subst, extraCabal2nixOptions}:
let
drv = pkgs.haskellPackages.haskellSrc2nix {
inherit name extraCabal2nixOptions src;
};
in drv.overrideAttrs (oldAttrs: {
message = ''
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!\
# See ./nix/generate.nix for instructions.\

'';
inherit src_subst;
installPhase = oldAttrs.installPhase + ''
sed -i "1i$message;s|src = .*|src = $src_subst;|" $out/default.nix
# Accept `pkgs` as an argument in case the `src_subst` depends on it.
sed -i "s|{ mkDerivation|{ mkDerivation, pkgs|" $out/default.nix
'';
});

# A variant of `haskellSrc2nixWithDoc` for local Haskell packages.
localHaskellSrc2nixWithDoc = name: path: extraCabal2nixOptions:
haskellSrc2nixWithDoc {
inherit name extraCabal2nixOptions;
src = import ./gitSource.nix path;
src_subst = "import ../gitSource.nix \"${path}\"";
};


winter = (haskellSrc2nixWithDoc {
name = "winter";
src = pkgs.sources.winter;
src_subst = "pkgs.sources.winter";
extraCabal2nixOptions = "--no-check";
});

ic-stub = localHaskellSrc2nixWithDoc "ic-stub" "ic-stub" "--no-check";
random = localHaskellSrc2nixWithDoc "qc-motoko" "test/random" "";
lsp-int = localHaskellSrc2nixWithDoc "lsp-int" "test/lsp-int" "";

allGenerated = pkgs.runCommandNoCC "generated" {} ''
mkdir -p $out
cp ${winter}/default.nix $out/winter.nix
cp ${ic-stub}/default.nix $out/ic-stub.nix
cp ${random}/default.nix $out/random.nix
cp ${lsp-int}/default.nix $out/lsp-int.nix
'';
in
allGenerated




9 changes: 4 additions & 5 deletions ic-stub/default.nix → nix/generated/ic-stub.nix
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
{ mkDerivation, base, binary, bytestring, containers
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, binary, bytestring, containers
, data-default-class, filepath, hex-text, mtl, optparse-applicative
, primitive, stdenv, text, transformers, utf8-string, vector
, winter
}:
mkDerivation {
pname = "ic-stub";
version = "0.1.0.0";
src = ./.;
src = import ../gitSource.nix "ic-stub";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
Expand Down
20 changes: 20 additions & 0 deletions nix/generated/lsp-int.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, data-default, directory, filepath
, haskell-lsp-types, hspec, HUnit, lens, lsp-test, stdenv, text
}:
mkDerivation {
pname = "lsp-int";
version = "0";
src = import ../gitSource.nix "test/lsp-int";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
base data-default directory filepath haskell-lsp-types hspec HUnit
lens lsp-test text
];
description = "Integration tests for the language server";
license = "unknown";
hydraPlatforms = stdenv.lib.platforms.none;
}
9 changes: 4 additions & 5 deletions test/random/default.nix → nix/generated/random.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
{ mkDerivation, base, exceptions, managed, process, QuickCheck
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, exceptions, managed, process, QuickCheck
, quickcheck-text, quickcheck-unicode, stdenv, tasty
, tasty-quickcheck, text, turtle
}:
mkDerivation {
pname = "qc-motoko";
version = "1";
src = ./.;
src = import ../gitSource.nix "test/random";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
Expand Down
5 changes: 2 additions & 3 deletions winter/default.nix → nix/generated/winter.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, array, base, binary, byte-order, bytestring
, cmdargs, containers, data-default-class, data-fix, deepseq
, directory, filepath, FloatingHex, lifted-base, microlens-platform
Expand Down
8 changes: 4 additions & 4 deletions nix/haskell-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ nix: subpath:

lsp-test = nix.haskell.lib.dontCheck self.lsp-test_0_10_1_0;

lsp-int = super.callPackage ../test/lsp-int {};
lsp-int = super.callPackage generated/lsp-int.nix {};

qc-motoko = super.callPackage ../test/random {};
qc-motoko = super.callPackage generated/random.nix {};

winter = super.callPackage ../winter {};
winter = super.callPackage generated/winter.nix {};

ic-stub = super.callPackage ../ic-stub {};
ic-stub = super.callPackage generated/ic-stub.nix {};
}
1 change: 0 additions & 1 deletion test/random/generate.nix

This file was deleted.

15 changes: 0 additions & 15 deletions winter/generate.nix

This file was deleted.