From d3f05e692c27f728e800f537606d6fdb12924eda Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 5 Oct 2023 14:24:56 +0200 Subject: [PATCH 1/7] model: Distinguish fetched artifacts --- plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala | 4 ++-- plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala b/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala index 5526db1..4cec8f0 100644 --- a/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala +++ b/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala @@ -63,7 +63,7 @@ class FindArtifactsOfRepo(repoName: String, root: String) { targetArtifacts.map { artifactLocalFile => val calcUrl = ga.calculateURI(artifactLocalFile).toURL - NixArtifact( + NixFetchedArtifact( repoName, calcUrl.toString.replace(authedRootURI.toString, "").stripPrefix("/"), calcUrl.toString, @@ -80,7 +80,7 @@ class FindArtifactsOfRepo(repoName: String, root: String) { metaArtifacts.filter(f => """.*(\.jar|\.pom|ivy.xml)$""".r.findFirstIn(f.artifactUrl).isDefined) targetMetaArtifacts.map { meta => - NixArtifact( + NixFetchedArtifact( repoName = repoName, relative = meta.artifactUrl.replace(root, "").stripPrefix("/"), url = meta.artifactUrl, diff --git a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala index d901d06..62555f8 100644 --- a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala +++ b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala @@ -71,7 +71,10 @@ case class NixArtifactCollection(artifacts: Seq[NixArtifact]) extends NixBuilder } -case class NixArtifact(repoName: String, relative: String, url: String, sha256: String) extends NixBuilder { +trait NixArtifact extends NixBuilder { +} + +case class NixFetchedArtifact(repoName: String, relative: String, url: String, sha256: String) extends NixArtifact { val toNixRef = s"${quote(repoName + "/" + relative)}" def toNixValue = From 4d630f820df746c21b777e7c305a966623a7aa6c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 2 Feb 2023 15:59:15 +0100 Subject: [PATCH 2/7] Improve testing (cherry picked from commit 3c43b5effc83a295a99361b7a62b3477c9a2c56d) --- CONTRIBUTING.md | 7 +++++++ flake.nix | 2 ++ 2 files changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 842b2e2..14c238a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,13 @@ The `tests` directory contains further integration tests. Move to the `tests` directory and run the appropriate `run.sh` from there. +```console +nix develop # load the NIX_PATH and other dependencies +nix shell . # build sbtix and add it to PATH +./tests/multi-build/run.sh +./tests/template-generation/run.sh +``` + #### CI preview The integration tests aren't Nix builds because they require network access and build access to the Nix store. diff --git a/flake.nix b/flake.nix index a6229e3..b25b538 100644 --- a/flake.nix +++ b/flake.nix @@ -23,6 +23,8 @@ # See CONTRIBUTING.md pkgs.hci ]; + # TODO: Don't rely on NIX_PATH in tests. + NIX_PATH = "nixpkgs=${inputs.nixpkgs}"; }; }; flake = { From 7ac1563e8eed764787a97bedc3f81410a8344153 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 5 Oct 2023 17:09:43 +0200 Subject: [PATCH 3/7] Support nix-built dependencies regardless of "nix" extra attribute Extra attribute as in https://www.scala-sbt.org/1.x/docs/Library-Management.html#Extra+Attributes --- README.md | 7 ++++ plugin/nix-exprs/sbtix.nix | 37 +++++++++++++++---- .../se/nullable/sbtix/FindArtifacts.scala | 30 +++++++++++++-- .../scala/se/nullable/sbtix/NixWriter.scala | 11 ++++++ src/sbtix.sh | 1 + tests/multi-build/three/build.sbt | 2 +- tests/multi-build/two/build.sbt | 3 +- 7 files changed, 77 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fcadf5b..30dae8d 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,13 @@ sbtix.buildSbtProgram { } ``` +#### `nix` Extra Attribute + +Source dependencies do not need locking. Originally this was implemented by completely ignoring dependencies with `nix` [extra attribute](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Extra+Attributes), but this is not necessary anymore. + +Additionally, `sbtix` will generate a `type = "built";` entry for JARs it finds in the `sbtix-build-inputs.nix` repository, and handle them appropriately during the build. + + ### Authentication In order to use a private repository, add your credentials to `coursierCredentials`. Note that the key should be the name of the repository, see diff --git a/plugin/nix-exprs/sbtix.nix b/plugin/nix-exprs/sbtix.nix index 980c2cd..110673f 100644 --- a/plugin/nix-exprs/sbtix.nix +++ b/plugin/nix-exprs/sbtix.nix @@ -102,6 +102,11 @@ let concatStringsSep "\n" (["mkdir -p $out"] ++ concatLists (map copyTemplate templates)) ); + copyFile = name: input: + runCommand name { inherit input; } '' + cp -a $input $out + ''; + in rec { mkRepo = name: artifacts: localBuildsRepo: runCommand name {} (let @@ -110,23 +115,28 @@ in rec { linkArtifact = outputPath: urlAttrs: let artifact = - (if builtins.substring 0 5 urlAttrs.url == "file:" then + if urlAttrs.type or null == "built" then if localBuildsRepo == "" then abort "'sbtixBuildInputs' parameter missing from 'buildSbtProgram'/'buildSbtLibrary', but local dependencies found." else - # replace the nix store path prefix to make sure - # a repo.nix generated with a different version of - # nixpkgs but the same dependencies will still work: - localBuildsRepo + "/" + (lib.strings.concatStringsSep "/" (lib.lists.drop 4 (lib.strings.splitString "/" urlAttrs.url))) + # Unlike fetched JARs which are content addressed derivations by virtue of being fixed-output, + # these copies are not content addressed, because ca-derivations is still experimental. + # This is unfortunate, because it will create duplication in the store during builds and development, + # but at least these can be GC-ed and allow the end result to only reference the single JARs that + # result from this copying operation. + copyFile (baseNameOf urlAttrs.path) (localBuildsRepo + "/" + urlAttrs.path) else - fetchurl urlAttrs - ); + fetchurl urlAttrs; + hashBash = + if urlAttrs.type or null == "built" + then ''$(sha256sum "${artifact}" | cut -c -64)'' + else ''$(echo ${toLower urlAttrs.sha256} | tr / _)''; in [ ''mkdir -p "$out/${parentDirs outputPath}"'' ''ln -fsn "${artifact}" "$out/${outputPath}"'' # TODO: include the executable bit as a suffix of the hash. # Shouldn't matter in our use case though. - ''ln -fsn "${artifact}" "$out/cas/$(echo ${toLower urlAttrs.sha256} | tr / _)"'' + ''ln -fsn "${artifact}" "$out/cas/${hashBash}"'' ]; in '' @@ -232,6 +242,17 @@ in rec { runHook postDist ''; + # These inputs are only meant for the build process. If they stick + # around in the outputs, they'd just bloat the user package for no + # good reason. + disallowedReferences = [ + combinedCas + sbtixRepos + nixrepo + ] + ++ lib.optional (localBuildsRepo != "") [ + localBuildsRepo + ]; } // args // { repo = null; buildInputs = [ makeWrapper jdk sbt ] ++ buildInputs; diff --git a/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala b/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala index 4cec8f0..94372c7 100644 --- a/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala +++ b/plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala @@ -44,7 +44,15 @@ object FindArtifactsOfRepo { } } +object NixBuiltReposSetting { + val builtRepos = sys.env.get("SBTIX_NIX_BUILT_REPOS").map(_.split(",").toSet).getOrElse(Set.empty) +} + class FindArtifactsOfRepo(repoName: String, root: String) { + /** + * Whether this repo is a nix-built repo. + */ + val isNixBuiltRepo: Boolean = NixBuiltReposSetting.builtRepos.contains(repoName) def findArtifacts(logger: Logger, modules: Set[GenericModule]): Set[NixArtifact] = modules.flatMap { ga => @@ -63,14 +71,14 @@ class FindArtifactsOfRepo(repoName: String, root: String) { targetArtifacts.map { artifactLocalFile => val calcUrl = ga.calculateURI(artifactLocalFile).toURL - NixFetchedArtifact( + improveArtifact(NixFetchedArtifact( repoName, calcUrl.toString.replace(authedRootURI.toString, "").stripPrefix("/"), calcUrl.toString, FindArtifactsOfRepo .fetchChecksum(calcUrl.toString, "Artifact", artifactLocalFile.toURI.toURL) .get - ) + )) } } @@ -80,12 +88,26 @@ class FindArtifactsOfRepo(repoName: String, root: String) { metaArtifacts.filter(f => """.*(\.jar|\.pom|ivy.xml)$""".r.findFirstIn(f.artifactUrl).isDefined) targetMetaArtifacts.map { meta => - NixFetchedArtifact( + improveArtifact(NixFetchedArtifact( repoName = repoName, relative = meta.artifactUrl.replace(root, "").stripPrefix("/"), url = meta.artifactUrl, sha256 = meta.checkSum - ) + )) + } + } + + /** + * Artifacts start out as fetched artifacts, because that's all SBT knows. + * This function picks out the ones that are built artifacts, and returns + * either a NixBuiltArtifact or a NixFetchedArtifact. + */ + def improveArtifact(artifact: NixFetchedArtifact): NixArtifact = { + if (isNixBuiltRepo) { + NixBuiltArtifact(repoName, artifact.relative) + } + else { + artifact } } diff --git a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala index 62555f8..41fd8a4 100644 --- a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala +++ b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala @@ -83,3 +83,14 @@ case class NixFetchedArtifact(repoName: String, relative: String, url: String, s | sha256 = ${quote(sha256)}; |}""".stripMargin } + +case class NixBuiltArtifact(repoName: String, path: String) extends NixArtifact { + val toNixRef = s"${quote(repoName + "/" + path)}" + + def toNixValue = + s"""{ + | type = "built"; + | repo = ${quote(repoName)}; + | path = ${quote(path)}; + |}""".stripMargin +} diff --git a/src/sbtix.sh b/src/sbtix.sh index 9bd1558..75c10d8 100644 --- a/src/sbtix.sh +++ b/src/sbtix.sh @@ -26,6 +26,7 @@ new MavenCache("sbtix-local-dependencies", file("$(nix-build sbtix-build-inputs. Resolver.file("sbtix-local-dependencies-ivy", file("$(nix-build sbtix-build-inputs.nix)"))(Resolver.ivyStylePatterns), ) EOF + export SBTIX_NIX_BUILT_REPOS="nix-sbtix-local-dependencies" fi # if sbtix_plugin.sbt is a link or does not exist then update the link. If it is a regular file do not replace it. diff --git a/tests/multi-build/three/build.sbt b/tests/multi-build/three/build.sbt index 17e05d6..d6094f1 100644 --- a/tests/multi-build/three/build.sbt +++ b/tests/multi-build/three/build.sbt @@ -8,7 +8,7 @@ name := "mb-three" version := ver -libraryDependencies += org %% "mb-two" % ver extra ("nix" -> "") +libraryDependencies += org %% "mb-two" % ver addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.4") enablePlugins(JavaAppPackaging) diff --git a/tests/multi-build/two/build.sbt b/tests/multi-build/two/build.sbt index d8fffdc..c1804e5 100644 --- a/tests/multi-build/two/build.sbt +++ b/tests/multi-build/two/build.sbt @@ -10,4 +10,5 @@ version := ver libraryDependencies += org %% "mb-one" % ver extra ("nix" -> "") -projectID := projectID.value.extra("nix" -> "") +// For this one, we don't set the "nix" extra attribute, to test that we can detect nix-built dependencies without it. +// projectID := projectID.value.extra("nix" -> "") From 630c28fd57c6211e47e471d0df838949a9ac1f5e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 6 Oct 2023 17:31:34 +0200 Subject: [PATCH 4/7] README: Rewrite Extra Attribute section Focus less on what was. This section should be moved into reference docs that aren't the README. It is not at all essential. --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 30dae8d..6664625 100644 --- a/README.md +++ b/README.md @@ -157,10 +157,9 @@ sbtix.buildSbtProgram { #### `nix` Extra Attribute -Source dependencies do not need locking. Originally this was implemented by completely ignoring dependencies with `nix` [extra attribute](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Extra+Attributes), but this is not necessary anymore. - -Additionally, `sbtix` will generate a `type = "built";` entry for JARs it finds in the `sbtix-build-inputs.nix` repository, and handle them appropriately during the build. +By adding the `nix` [extra attribute](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Extra+Attributes), `sbtix` will ignore the dependency for the purpose of locking. +This used to be the only mechanism for handling local dependencies, but is now a legacy solution and/or escape hatch. ### Authentication From d43d071b5facf5f57f16b6bb0dd4543d60dddf8c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 6 Oct 2023 17:34:45 +0200 Subject: [PATCH 5/7] NixBuiltArtifact: do not generate repo attribute It wasn't used by the Nix code. --- plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala index 41fd8a4..e09f7df 100644 --- a/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala +++ b/plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala @@ -90,7 +90,6 @@ case class NixBuiltArtifact(repoName: String, path: String) extends NixArtifact def toNixValue = s"""{ | type = "built"; - | repo = ${quote(repoName)}; | path = ${quote(path)}; |}""".stripMargin } From b8bb800a94970005aa6ddd6017140fcc9fa7b872 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 6 Oct 2023 17:38:45 +0200 Subject: [PATCH 6/7] README: Add more syntax highlighting --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6664625..8dcf5ec 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,13 @@ Additionally, this means that Nix can do a better job of enforcing purity where To install sbtix either: -``` +```bash nix shell github:natural-transformation/sbtix ``` Or clone the sbtix git repo and: -``` +```bash cd sbtix nix-env -f . -i sbtix ``` @@ -123,7 +123,7 @@ the local dependency must be available both when running 'sbtix-gen' and when building. You provide it in a `sbtix-build-inputs.nix` file, which could hold a single dependency: -``` +```nix { pkgs ? import {} }: pkgs.callPackage ./path/to/dependency/derivation {} @@ -131,7 +131,7 @@ pkgs.callPackage ./path/to/dependency/derivation {} .. or multiple: -``` +```nix { pkgs ? import {} }: pkgs.symlinkJoin { @@ -147,7 +147,7 @@ pkgs.symlinkJoin { This file is picked up (by name) by `sbtix-gen`, and also should be passed in as a parameter in the `buildSbtProgram` invocation in your `default.nix`: -``` +```nix sbtix.buildSbtProgram { ... sbtixBuildInputs = (pkgs.callPackage ./sbtix-build-inputs.nix {}); @@ -179,7 +179,7 @@ Q: Why I am getting errors trying to generate `repo.nix` files when using the Pl A: You probably need to add the following resolver to your project for Sbtix to find. -``` +```scala // if using PlayFramework resolvers += Resolver.url("sbt-plugins-releases", url("https://dl.bintray.com/playframework/sbt-plugin-releases"))(Resolver.ivyStylePatterns) ``` @@ -189,12 +189,12 @@ Q: When I `nix-build` it sbt complains `java.io.IOException: Cannot run program A: You are likely depending on a project via git. This isn't recommended usage for sbtix since it leads to non-deterministic builds. However you can enable this by making two small changes to sbtix.nix, in order to make git a buildInput. top of sbtix.nix with git as buildinput -``` +```nix { runCommand, fetchurl, lib, stdenv, jdk, sbt, writeText, git }: ``` bottom of sbtix.nix with git as buildinput -``` +```nix buildInputs = [ jdk sbt git ] ++ buildInputs; ``` From 117a8cabca0bd2b4ab66eb5ca8898c77b9773b84 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 6 Oct 2023 17:45:51 +0200 Subject: [PATCH 7/7] src/sbtix.sh: Support ivy-style built repo Co-authored-by: Arnout Engelen --- src/sbtix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbtix.sh b/src/sbtix.sh index 75c10d8..2677821 100644 --- a/src/sbtix.sh +++ b/src/sbtix.sh @@ -26,7 +26,7 @@ new MavenCache("sbtix-local-dependencies", file("$(nix-build sbtix-build-inputs. Resolver.file("sbtix-local-dependencies-ivy", file("$(nix-build sbtix-build-inputs.nix)"))(Resolver.ivyStylePatterns), ) EOF - export SBTIX_NIX_BUILT_REPOS="nix-sbtix-local-dependencies" + export SBTIX_NIX_BUILT_REPOS="nix-sbtix-local-dependencies,nix-sbtix-local-dependencies-ivy" fi # if sbtix_plugin.sbt is a link or does not exist then update the link. If it is a regular file do not replace it.