Improve visualize user experience (#3438)

lihaoyi · web-flow · commit 0c6f0b9b2e95 · 2024-08-31T09:40:43.000+08:00
* Swap over from graphviz-java's default J2V8 engine to https://github.com/caoccao/Javet since J2V8 is unmaintained and does not support mac-arm64. This allows `visualize` to be used on M1 Macbooks without needing to install `dot`/`graphviz`. The implementation was adapted from https://github.com/nidi3/graphviz-java/blob/master/graphviz-java/src/main/java/guru/nidi/graphviz/engine/V8JavascriptEngine.java but with the J2V8 code adjusted to fit Janet * Print out the `visualize` output by default without needing `show` * Bundle `logback` to make the annoying log4j warnings go away Tested OS-X locally on my machine after uninstalling graphviz, Windows and Linux are exercised in CI (GH actions doesn't have graphviz installed)
diff --git a/build.sc b/build.sc
@@ -128,10 +128,19 @@ object Deps {
   val castor = ivy"com.lihaoyi::castor:0.3.0"
   val fastparse = ivy"com.lihaoyi::fastparse:3.1.1"
   val flywayCore = ivy"org.flywaydb:flyway-core:8.5.13"
-  val graphvizJava = ivy"guru.nidi:graphviz-java-all-j2v8:0.18.1"
+  val graphvizJava = Seq(
+    ivy"guru.nidi:graphviz-java-min-deps:0.18.1",
+    ivy"org.webjars.npm:viz.js-graphviz-java:2.1.3",
+    ivy"org.apache.xmlgraphics:batik-rasterizer:1.17"
+  )
   val junixsocket = ivy"com.kohlschutter.junixsocket:junixsocket-core:2.10.0"
 
   val jgraphtCore = ivy"org.jgrapht:jgrapht-core:1.4.0" // 1.5.0+ dont support JDK8
+  val javet = Seq(
+    ivy"com.caoccao.javet:javet:3.1.5",
+    ivy"com.caoccao.javet:javet-linux-arm64:3.1.5",
+    ivy"com.caoccao.javet:javet-macos:3.1.5",
+  )
 
   val jline = ivy"org.jline:jline:3.26.3"
   val jnaVersion = "5.14.0"
@@ -180,6 +189,7 @@ object Deps {
   val fansi = ivy"com.lihaoyi::fansi:0.5.0"
   val jarjarabrams = ivy"com.eed3si9n.jarjarabrams::jarjar-abrams-core:1.14.0"
   val requests = ivy"com.lihaoyi::requests:0.9.0"
+  val logback = ivy"ch.qos.logback:logback-classic:1.2.11"
   val sonatypeCentralClient = ivy"com.lumidion::sonatype-central-client-requests:0.3.0"
 
   object RuntimeDeps {
@@ -579,7 +589,8 @@ object main extends MillStableScalaModule with BuildInfo {
     Deps.windowsAnsi,
     Deps.mainargs,
     Deps.coursierInterface,
-    Deps.requests
+    Deps.requests,
+    Deps.logback
   )
 
   def compileIvyDeps = Agg(Deps.scalaReflect(scalaVersion()))
@@ -748,7 +759,7 @@ object main extends MillStableScalaModule with BuildInfo {
   }
   object graphviz extends MillPublishScalaModule {
     def moduleDeps = Seq(main, scalalib)
-    def ivyDeps = Agg(Deps.graphvizJava, Deps.jgraphtCore)
+    def ivyDeps = Agg(Deps.jgraphtCore) ++ Deps.graphvizJava ++ Deps.javet
   }
 
 
diff --git a/docs/modules/ROOT/pages/Case_Study_Mill_vs_Gradle.adoc b/docs/modules/ROOT/pages/Case_Study_Mill_vs_Gradle.adoc
@@ -186,7 +186,7 @@ remains faster than Gradle by about 2.0x.
 Another area that Mill does better than Gradle is providing builtin tools for you to understand
 what your build is doing. For example, the Mockito project build discussed has 22 submodules
 and associated test suites, but how do these different modules depend on each other? With
-Mill, you can run `./mill show visualize __.compile`, and it will show you how the
+Mill, you can run `./mill visualize __.compile`, and it will show you how the
 `compile` task of each module depends on the others:
 
 image::MockitoCompileGraph.svg[]
diff --git a/docs/modules/ROOT/pages/Case_Study_Mill_vs_Maven.adoc b/docs/modules/ROOT/pages/Case_Study_Mill_vs_Maven.adoc
@@ -596,7 +596,7 @@ rather than configuring a bunch of third-party plugins to try and achieve what y
 Another area that Mill does better than Maven is providing builtin tools for you to understand
 what your build is doing. For example, the Netty project build discussed has 47 submodules
 and associated test suites, but how do these different modules depend on each other? With
-Mill, you can run `./mill show visualize __.compile`, and it will show you how the
+Mill, you can run `./mill visualize __.compile`, and it will show you how the
 `compile` task of each module depends on the others:
 
 image::NettyCompileGraph.svg[]
diff --git a/docs/modules/ROOT/pages/Case_Study_Mill_vs_SBT.adoc b/docs/modules/ROOT/pages/Case_Study_Mill_vs_SBT.adoc
@@ -233,7 +233,7 @@ wondering what your build tool is doing.
 Another area that Mill does better than SBT is providing builtin tools for you to understand
 what your build is doing. For example, the Gatling project build discussed has 21 submodules
 and associated test suites, but how do these different modules depend on each other? With
-Mill, you can run `./mill show visualize __.compile`, and it will show you how the
+Mill, you can run `./mill visualize __.compile`, and it will show you how the
 `compile` task of each module depends on the others:
 
 image::GatlingCompileGraph.svg[]
diff --git a/example/scalalib/basic/4-builtin-commands/build.sc b/example/scalalib/basic/4-builtin-commands/build.sc
@@ -291,7 +291,7 @@ foo.compileClasspath
 // == visualize
 //
 /** Usage
-> mill show visualize foo._
+> mill visualize foo._
 [
   ".../out/visualize.dest/out.txt",
   ".../out/visualize.dest/out.dot",
diff --git a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala
@@ -1,10 +1,16 @@
 package mill.main.graphviz
 
+import com.caoccao.javet.annotations.V8Function
+import com.caoccao.javet.interception.logging.JavetStandardConsoleInterceptor
+import com.caoccao.javet.interop.{V8Host, V8Runtime}
 import guru.nidi.graphviz.attribute.Rank.RankDir
 import guru.nidi.graphviz.attribute.{Rank, Shape, Style}
+import guru.nidi.graphviz.engine.{AbstractJavascriptEngine, AbstractJsGraphvizEngine, ResultHandler}
 import mill.api.PathRef
 import mill.define.NamedTask
 import org.jgrapht.graph.{DefaultEdge, SimpleDirectedGraph}
+import org.slf4j.LoggerFactory
+import org.slf4j.Logger
 
 object GraphvizTools {
 
@@ -61,17 +67,58 @@ object GraphvizTools {
 
     g = g.graphAttr().`with`(Rank.dir(RankDir.LEFT_TO_RIGHT))
 
-    val gv = Graphviz.fromGraph(g).totalMemory(100 * 1000 * 1000)
+    Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new V8JavascriptEngine()) {})
+    val gv = Graphviz.fromGraph(g).totalMemory(128 * 1024 * 1024)
     val outputs = Seq(
       Format.PLAIN -> "out.txt",
       Format.XDOT -> "out.dot",
       Format.JSON -> "out.json",
       Format.PNG -> "out.png",
       Format.SVG -> "out.svg"
     )
+
     for ((fmt, name) <- outputs) {
       gv.render(fmt).toFile((dest / name).toIO)
     }
     outputs.map(x => mill.PathRef(dest / x._2))
   }
 }
+
+class V8JavascriptEngine() extends AbstractJavascriptEngine {
+  val LOG: Logger = LoggerFactory.getLogger(classOf[V8JavascriptEngine])
+  val v8Runtime: V8Runtime = V8Host.getV8Instance().createV8Runtime()
+  LOG.info("Starting V8 runtime...")
+  LOG.info("Started V8 runtime. Initializing javascript...")
+  val resultHandler = new ResultHandler
+  val javetStandardConsoleInterceptor = new JavetStandardConsoleInterceptor(v8Runtime)
+  javetStandardConsoleInterceptor.register(v8Runtime.getGlobalObject)
+
+  class ResultHandlerInterceptor(resultHandler: ResultHandler) {
+    @V8Function
+    def result(s: String): Unit = resultHandler.setResult(s)
+
+    @V8Function
+    def error(s: String): Unit = resultHandler.setError(s)
+
+    @V8Function
+    def log(s: String): Unit = resultHandler.log(s)
+  }
+  val v8ValueObject = v8Runtime.createV8ValueObject
+  v8Runtime.getGlobalObject.set("resultHandlerInterceptor", v8ValueObject)
+  v8ValueObject.bind(new ResultHandlerInterceptor(resultHandler))
+
+  v8Runtime.getExecutor(
+    "var result = resultHandlerInterceptor.result; " +
+      "var error = resultHandlerInterceptor.error; " +
+      "var log = resultHandlerInterceptor.log; "
+  ).execute()
+
+  LOG.info("Initialized javascript.")
+
+  override protected def execute(js: String): String = {
+    v8Runtime.getExecutor(js).execute()
+    resultHandler.waitFor
+  }
+
+  override def close(): Unit = v8Runtime.close()
+}
diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala
@@ -435,7 +435,11 @@ trait MainModule extends BaseModule0 {
     ): Result[Seq[PathRef]] = {
       val (in, out) = vizWorker
       in.put((rs, allRs, ctx.dest))
-      out.take()
+      val res = out.take()
+      res.map { v =>
+        println(upickle.default.write(v.map(_.path.toString()), indent = 2))
+        v
+      }
     }
 
     Resolve.Tasks.resolve(
diff --git a/readme.adoc b/readme.adoc
@@ -362,6 +362,10 @@ as excluded {link-pr}/3329[#3329]
 ** Optimizations to Mill evaluation logic to reduce fixed overhead of running Mill
 on large projects {link-pr}/3388[#3388]
 
+** Improvements to `visualize` and `visualizePlan` such that they no longer need to be
+prefixed with `show` and no longer need a separate `graphviz`/`dot` install on Mac-OSX
+{link-pr}/3438[#3438]
+
 
 [#0-11-12]
 === 0.11.12 - 2024-08-20

Original file line number	Diff line number	Diff line change
`@@ -291,7 +291,7 @@ foo.compileClasspath`
`291`	`291`	`// == visualize`
`292`	`292`	`//`
`293`	`293`	`/** Usage`
`294`		`-> mill show visualize foo._`
	`294`	`+> mill visualize foo._`
`295`	`295`	`[`
`296`	`296`	`".../out/visualize.dest/out.txt",`
`297`	`297`	`".../out/visualize.dest/out.dot",`