From-source nix-built artifacts without the "nix" Extra Attribute

CONTRIBUTING.md

@@ -10,6 +10,13 @@ The `tests` directory contains further integration tests. Move
 to the `tests` directory and run the appropriate `run.sh` from
 there.
 
+```console
+nix develop    # load the NIX_PATH and other dependencies
+nix shell .    # build sbtix and add it to PATH
+./tests/multi-build/run.sh
+./tests/template-generation/run.sh
+```
+
 #### CI preview
 
 The integration tests aren't Nix builds because they require network access and build access to the Nix store.

README.md

@@ -24,13 +24,13 @@ Additionally, this means that Nix can do a better job of enforcing purity where
 
 To install sbtix either:
 
-```
+```bash
 nix shell github:natural-transformation/sbtix
 ```
 
 Or clone the sbtix git repo and:
 
-```
+```bash
 cd sbtix
 nix-env -f . -i sbtix
 ```
@@ -123,15 +123,15 @@ the local dependency must be available both when running 'sbtix-gen'
 and when building. You provide it in a `sbtix-build-inputs.nix` file,
 which could hold a single dependency:
 
-```
+```nix
 { pkgs ? import <nixpkgs> {} }:
 
 pkgs.callPackage ./path/to/dependency/derivation {}
 ```
 
 .. or multiple:
 
-```
+```nix
 { pkgs ? import <nixpkgs> {} }:
 
 pkgs.symlinkJoin {
@@ -147,14 +147,20 @@ pkgs.symlinkJoin {
 This file is picked up (by name) by `sbtix-gen`, and also should be passed
 in as a parameter in the `buildSbtProgram` invocation in your `default.nix`:
 
-```
+```nix
 sbtix.buildSbtProgram {
     ...
     sbtixBuildInputs = (pkgs.callPackage ./sbtix-build-inputs.nix {});
     ...
 }
 ```
 
+#### `nix` Extra Attribute
+
+By adding the `nix` [extra attribute](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Extra+Attributes), `sbtix` will ignore the dependency for the purpose of locking.
+
+This used to be the only mechanism for handling local dependencies, but is now a legacy solution and/or escape hatch.
+
 ### Authentication
 
 In order to use a private repository, add your credentials to `coursierCredentials`. Note that the key should be the name of the repository, see
@@ -173,7 +179,7 @@ Q: Why I am getting errors trying to generate `repo.nix` files when using the Pl
 
 A: You probably need to add the following resolver to your project for Sbtix to find.
 
-```
+```scala
 // if using PlayFramework
 resolvers += Resolver.url("sbt-plugins-releases", url("https://dl.bintray.com/playframework/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
 ```
@@ -183,12 +189,12 @@ Q: When I `nix-build` it sbt complains `java.io.IOException: Cannot run program
 A: You are likely depending on a project via git.  This isn't recommended usage for sbtix since it leads to non-deterministic builds. However you can enable this by making two small changes to sbtix.nix, in order to make git a buildInput.
 
 top of sbtix.nix with git as buildinput
-```
+```nix
 { runCommand, fetchurl, lib, stdenv, jdk, sbt, writeText, git }:
 ```
 
 bottom of sbtix.nix with git as buildinput
-```
+```nix
 buildInputs = [ jdk sbt git ] ++ buildInputs;
 ```
 

flake.nix

@@ -23,6 +23,8 @@
             # See CONTRIBUTING.md
             pkgs.hci
           ];
+          # TODO: Don't rely on NIX_PATH in tests.
+          NIX_PATH = "nixpkgs=${inputs.nixpkgs}";
         };
       };
       flake = {

plugin/nix-exprs/sbtix.nix

@@ -102,6 +102,11 @@ let
             concatStringsSep "\n" (["mkdir -p $out"] ++ concatLists (map copyTemplate templates))
         );
 
+  copyFile = name: input:
+    runCommand name { inherit input; } ''
+      cp -a $input $out
+    '';
+
 in rec {
     mkRepo = name: artifacts: localBuildsRepo: runCommand name {}
         (let
@@ -110,23 +115,28 @@ in rec {
             linkArtifact = outputPath: urlAttrs:
                 let
                   artifact =
-                    (if builtins.substring 0 5 urlAttrs.url == "file:" then
+                    if urlAttrs.type or null == "built" then
                       if localBuildsRepo == "" then
                         abort "'sbtixBuildInputs' parameter missing from 'buildSbtProgram'/'buildSbtLibrary', but local dependencies found."
                       else
-                        # replace the nix store path prefix to make sure
-                        # a repo.nix generated with a different version of
-                        # nixpkgs but the same dependencies will still work:
-                        localBuildsRepo + "/" + (lib.strings.concatStringsSep "/" (lib.lists.drop 4 (lib.strings.splitString "/" urlAttrs.url)))
+                        # Unlike fetched JARs which are content addressed derivations by virtue of being fixed-output,
+                        # these copies are not content addressed, because ca-derivations is still experimental.
+                        # This is unfortunate, because it will create duplication in the store during builds and development,
+                        # but at least these can be GC-ed and allow the end result to only reference the single JARs that
+                        # result from this copying operation.
+                        copyFile (baseNameOf urlAttrs.path) (localBuildsRepo + "/" + urlAttrs.path)
                     else
-                      fetchurl urlAttrs
-                    );
+                      fetchurl urlAttrs;
+                  hashBash =
+                    if urlAttrs.type or null == "built"
+                    then ''$(sha256sum "${artifact}" | cut -c -64)''
+                    else ''$(echo ${toLower urlAttrs.sha256} | tr / _)'';
                 in
                 [ ''mkdir -p "$out/${parentDirs outputPath}"''
                   ''ln -fsn "${artifact}" "$out/${outputPath}"''
                   # TODO: include the executable bit as a suffix of the hash.
                   #       Shouldn't matter in our use case though.
-                  ''ln -fsn "${artifact}" "$out/cas/$(echo ${toLower urlAttrs.sha256} | tr / _)"''
+                  ''ln -fsn "${artifact}" "$out/cas/${hashBash}"''
                 ];
         in
             ''
@@ -232,6 +242,17 @@ in rec {
               runHook postDist
             '';
 
+            # These inputs are only meant for the build process. If they stick
+            # around in the outputs, they'd just bloat the user package for no
+            # good reason.
+            disallowedReferences = [
+              combinedCas
+              sbtixRepos
+              nixrepo
+            ]
+            ++ lib.optional (localBuildsRepo != "") [
+              localBuildsRepo
+            ];
         } // args // {
             repo = null;
             buildInputs = [ makeWrapper jdk sbt ] ++ buildInputs;

plugin/src/main/scala/se/nullable/sbtix/FindArtifacts.scala

@@ -44,7 +44,15 @@ object FindArtifactsOfRepo {
     }
 }
 
+object NixBuiltReposSetting {
+  val builtRepos = sys.env.get("SBTIX_NIX_BUILT_REPOS").map(_.split(",").toSet).getOrElse(Set.empty)
+}
+
 class FindArtifactsOfRepo(repoName: String, root: String) {
+  /**
+   * Whether this repo is a nix-built repo.
+   */
+  val isNixBuiltRepo: Boolean = NixBuiltReposSetting.builtRepos.contains(repoName)
 
   def findArtifacts(logger: Logger, modules: Set[GenericModule]): Set[NixArtifact] =
     modules.flatMap { ga =>
@@ -63,14 +71,14 @@ class FindArtifactsOfRepo(repoName: String, root: String) {
       targetArtifacts.map { artifactLocalFile =>
         val calcUrl = ga.calculateURI(artifactLocalFile).toURL
 
-        NixArtifact(
+        improveArtifact(NixFetchedArtifact(
           repoName,
           calcUrl.toString.replace(authedRootURI.toString, "").stripPrefix("/"),
           calcUrl.toString,
           FindArtifactsOfRepo
             .fetchChecksum(calcUrl.toString, "Artifact", artifactLocalFile.toURI.toURL)
             .get
-        )
+        ))
       }
     }
 
@@ -80,12 +88,26 @@ class FindArtifactsOfRepo(repoName: String, root: String) {
       metaArtifacts.filter(f => """.*(\.jar|\.pom|ivy.xml)$""".r.findFirstIn(f.artifactUrl).isDefined)
 
     targetMetaArtifacts.map { meta =>
-      NixArtifact(
+      improveArtifact(NixFetchedArtifact(
         repoName = repoName,
         relative = meta.artifactUrl.replace(root, "").stripPrefix("/"),
         url = meta.artifactUrl,
         sha256 = meta.checkSum
-      )
+      ))
+    }
+  }
+
+  /**
+   * Artifacts start out as fetched artifacts, because that's all SBT knows.
+   * This function picks out the ones that are built artifacts, and returns
+   * either a NixBuiltArtifact or a NixFetchedArtifact.
+   */
+  def improveArtifact(artifact: NixFetchedArtifact): NixArtifact = {
+    if (isNixBuiltRepo) {
+      NixBuiltArtifact(repoName, artifact.relative)
+    }
+    else {
+      artifact
     }
   }
 

plugin/src/main/scala/se/nullable/sbtix/NixWriter.scala

@@ -71,7 +71,10 @@ case class NixArtifactCollection(artifacts: Seq[NixArtifact]) extends NixBuilder
 
 }
 
-case class NixArtifact(repoName: String, relative: String, url: String, sha256: String) extends NixBuilder {
+trait NixArtifact extends NixBuilder {
+}
+
+case class NixFetchedArtifact(repoName: String, relative: String, url: String, sha256: String) extends NixArtifact {
   val toNixRef = s"${quote(repoName + "/" + relative)}"
 
   def toNixValue =
@@ -80,3 +83,13 @@ case class NixArtifact(repoName: String, relative: String, url: String, sha256:
        |  sha256 = ${quote(sha256)};
        |}""".stripMargin
 }
+
+case class NixBuiltArtifact(repoName: String, path: String) extends NixArtifact {
+  val toNixRef = s"${quote(repoName + "/" + path)}"
+
+  def toNixValue =
+    s"""{
+       |  type = "built";
+       |  path = ${quote(path)};
+       |}""".stripMargin
+}

src/sbtix.sh

@@ -26,6 +26,7 @@ new MavenCache("sbtix-local-dependencies", file("$(nix-build sbtix-build-inputs.
 Resolver.file("sbtix-local-dependencies-ivy", file("$(nix-build sbtix-build-inputs.nix)"))(Resolver.ivyStylePatterns),
 )
 EOF
+  export SBTIX_NIX_BUILT_REPOS="nix-sbtix-local-dependencies,nix-sbtix-local-dependencies-ivy"
 fi
 
 # if sbtix_plugin.sbt is a link or does not exist then update the link. If it is a regular file do not replace it.

tests/multi-build/three/build.sbt

@@ -8,7 +8,7 @@ name := "mb-three"
 
 version := ver
 
-libraryDependencies += org %% "mb-two" % ver extra ("nix" -> "")
+libraryDependencies += org %% "mb-two" % ver
 
 addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.4")
 enablePlugins(JavaAppPackaging)

tests/multi-build/two/build.sbt

@@ -10,4 +10,5 @@ version := ver
 
 libraryDependencies += org %% "mb-one" % ver extra ("nix" -> "")
 
-projectID := projectID.value.extra("nix" -> "")
+// For this one, we don't set the "nix" extra attribute, to test that we can detect nix-built dependencies without it.
+// projectID := projectID.value.extra("nix" -> "")