Add Scala Native support

Project structure was changed in compliance with sbt-crossproject. googlecode's diffutils had to be removed, as Scala Native does not support java libraries. In it's place, part of diffutils was reimplemented in scala as part of scalafmt-cli. For similar reasons, the java interfaces were reimplemented in scala for Scala Native. Since Scala Native does not yet support glob expressions, a translation function to regex was added, heavily based on a linked stack-overflow answer. SN's regex implementation is currently based on the re2 instead of the standard java regexes. This means there are a number of differences between them, most notably the lack of backtracking in Scala Native. To accomodate that, some regex function reimplementations were done to accomodate the use of the existing scalafmt regexes in Scala Native. Some operations concerning tests were slightly modified to work on Native. For example, renaming a file to an existing directory path could throw errors. This is fine, as it does not concern the core of the test, only things like cleanup. Assumptions about dynamic tests not working on native were added. Scala Native does not support URLs and only has basic reflection support comparable to the one of GraalVM, so the used and expected version of scalafmt must match here as well. Term Display was made single-threaded for the Scala Native implementation, as it does not support concurrency. Since the new implementation caused display artifacts on JVM, the old multithreaded one was kept there. Scala Native setup and testing was added to the CI. The windows was temporarily disabled as it does not pass all of the tests. scala-native sbt command was added for easier scala-native cli building. Co-authored-by: Tomasz Godzik <[email protected]>
jchyb · Nov 4, 2021 · c2598fe · c2598fe
1 parent f7b1fd6
commit c2598fe
Show file tree

Hide file tree

Showing 64 changed files with 2,492 additions and 708 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -29,9 +29,38 @@ jobs:
       - run:
           # for GitOps tests
           git config --global user.email "[email protected]" && git config --global user.name "scalafmt"
-      - run: TEST="2.12" sbt ci-test
+      - run: TEST="2.12" sbt ci-test-jvm
         shell: bash
-      - run: TEST="2.13" sbt ci-test
+      - run: TEST="2.13" sbt ci-test-jvm
+        shell: bash
+  test-scala-native:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macOS-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: olafurpg/setup-scala@v13
+        with:
+          java-version: [email protected]
+      - if: matrix.os == 'windows-latest'
+        name: setup windows environment
+        run: ./bin/scala-native-setup/windows-setup.sh
+      - if: matrix.os == 'macOS-latest'
+        name: setup macOS environment
+        run: ./bin/scala-native-setup/macos-setup.sh
+      - if: matrix.os == 'ubuntu-latest'
+        name: setup ubuntu environment
+        run: ./bin/scala-native-setup/ubuntu-setup.sh
+      - name: run tests
+        run: |
+          git fetch --tags -f
+          # for GitOps tests	
+          git config --global user.email "[email protected]" && git config --global user.name "scalafmt"
+      - run: TEST="2.12" sbt ci-test-native
+        shell: bash
+      - run: TEST="2.13" sbt ci-test-native
         shell: bash
   formatting:
     runs-on: ubuntu-latest
@@ -81,46 +110,46 @@ jobs:
       - run: bin/build-native-image.sh
         env:
           CI: true
-      - uses: actions/upload-artifact@master
-        with:
-          name: ${{ matrix.artifact }}
-          path: scalafmt
-      - name: Upload release
-        if: github.event_name == 'release'
-        uses: actions/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }}
-          asset_path: scalafmt
-          asset_name: ${{ matrix.artifact }}
-          asset_content_type: application/zip
-  dockerize:
-    needs: [native-image,test]
-    runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'release'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Docker Meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: scalameta/scalafmt
-          tags: type=semver,pattern={{raw}}
-      - name: Downloading scalafmt-linux-musl for Docker Build
-        uses: actions/[email protected]
-        with:
-          name: scalafmt-linux-musl
-          path: tmp/scalafmt-linux-musl
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
+      # - uses: actions/upload-artifact@master
+      #   with:
+      #     name: ${{ matrix.artifact }}
+      #     path: scalafmt
+      # - name: Upload release
+      #   if: github.event_name == 'release'
+      #   uses: actions/[email protected]
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #   with:
+      #     upload_url: ${{ github.event.release.upload_url }}
+      #     asset_path: scalafmt
+      #     asset_name: ${{ matrix.artifact }}
+      #     asset_content_type: application/zip
+  # dockerize:
+  #   needs: [native-image,test]
+  #   runs-on: ubuntu-latest
+  #   if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'release'
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+  #     - name: Docker Meta
+  #       id: meta
+  #       uses: docker/metadata-action@v3
+  #       with:
+  #         images: scalameta/scalafmt
+  #         tags: type=semver,pattern={{raw}}
+  #     - name: Downloading scalafmt-linux-musl for Docker Build
+  #       uses: actions/[email protected]
+  #       with:
+  #         name: scalafmt-linux-musl
+  #         path: tmp/scalafmt-linux-musl
+  #     - name: Login to DockerHub
+  #       uses: docker/login-action@v1
+  #       with:
+  #         username: ${{ secrets.DOCKER_USERNAME }}
+  #         password: ${{ secrets.DOCKER_PASSWORD }}
+  #     - name: Build and push
+  #       uses: docker/build-push-action@v2
+  #       with:
+  #         context: .
+  #         push: true
+  #         tags: ${{ steps.meta.outputs.tags }}
diff --git a/bin/scala-native-setup/macos-setup.sh b/bin/scala-native-setup/macos-setup.sh
@@ -0,0 +1,3 @@
+apt-get install -y clang-12.0
+PATH=/usr/lib/llvm-12.0/bin:${PATH}
+clang --version
diff --git a/bin/scala-native-setup/ubuntu-setup.sh b/bin/scala-native-setup/ubuntu-setup.sh
@@ -0,0 +1,3 @@
+apt-get install -y clang-12.0
+PATH=/usr/lib/llvm-12.0/bin:${PATH}
+clang --version
diff --git a/bin/scala-native-setup/windows-setup.sh b/bin/scala-native-setup/windows-setup.sh
@@ -0,0 +1,3 @@
+choco install llvm
+echo "${env:ProgramFiles}\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+clang --version
diff --git a/build.sbt b/build.sbt
@@ -1,5 +1,6 @@
 import Dependencies._
 import sbtcrossproject.CrossPlugin.autoImport.crossProject
+import scala.scalanative.build._
 
 def parseTagVersion: String = {
   import scala.sys.process._
@@ -37,11 +38,6 @@ inThisBuild(
       Resolver.sonatypeRepo("releases"),
       Resolver.sonatypeRepo("snapshots")
     ),
-    libraryDependencies ++= List(
-      munit.value % Test,
-      scalacheck % Test,
-      scalametaTestkit % Test
-    ),
     testFrameworks += new TestFramework("munit.Framework")
   )
 )
@@ -50,8 +46,9 @@ name := "scalafmtRoot"
 publish / skip := true
 
 addCommandAlias("native-image", "cli/nativeImage")
+addCommandAlias("scala-native", "cliNative/compile;cliNative/nativeLink")
 
-commands += Command.command("ci-test") { s =>
+commands += Command.command("ci-test-jvm") { s =>
   val scalaVersion = sys.env.get("TEST") match {
     case Some("2.12") => scala212
     case _ => scala213
@@ -64,7 +61,18 @@ commands += Command.command("ci-test") { s =>
     s
 }
 
-lazy val dynamic = project
+commands += Command.command("ci-test-native") { s =>
+  val scalaVersion = sys.env.get("TEST") match {
+    case Some("2.12") => scala212
+    case _ => scala213
+  }
+  s"++$scalaVersion" ::
+    "testsNative/test" ::
+    s
+}
+
+lazy val dynamic = crossProject(JVMPlatform, NativePlatform)
+  .withoutSuffixFor(JVMPlatform)
   .in(file("scalafmt-dynamic"))
   .settings(
     moduleName := "scalafmt-dynamic",
@@ -76,14 +84,23 @@ lazy val dynamic = project
       "io.get-coursier" % "interface" % "0.0.17",
       "com.typesafe" % "config" % "1.4.1",
       munit.value % Test,
-      scalametaTestkit % Test
+      scalametaTestkit.value % Test
     ),
     scalacOptions ++= scalacJvmOptions.value
   )
+  .nativeSettings(
+    libraryDependencies ++= List(
+      "org.ekrich" %%% "sjavatime" % "1.1.5"
+    ),
+    scalaNativeNativeConfig
+  )
   .dependsOn(interfaces)
   .enablePlugins(BuildInfoPlugin)
 
-lazy val interfaces = project
+lazy val dynamicNative = dynamic.native
+
+lazy val interfaces = crossProject(JVMPlatform, NativePlatform)
+  .withoutSuffixFor(JVMPlatform)
   .in(file("scalafmt-interfaces"))
   .settings(
     moduleName := "scalafmt-interfaces",
@@ -100,14 +117,16 @@ lazy val interfaces = project
     }
   )
 
-lazy val core = crossProject(JVMPlatform)
+lazy val interfacesNative = interfaces.native
+
+lazy val core = crossProject(JVMPlatform, NativePlatform)
+  .withoutSuffixFor(JVMPlatform)
   .in(file("scalafmt-core"))
   .settings(
     moduleName := "scalafmt-core",
     buildInfoSettings,
     scalacOptions ++= scalacJvmOptions.value,
     libraryDependencies ++= Seq(
-      metaconfig.value,
       scalameta.value,
       // scala-reflect is an undeclared dependency of fansi, see #1252.
       // Scalafmt itself does not require scala-reflect.
@@ -128,21 +147,22 @@ lazy val core = crossProject(JVMPlatform)
       }
     }
   )
-  // .jsSettings(
-  //   libraryDependencies ++= List(
-  //     metaconfigHocon.value,
-  //     scalatest.value % Test // must be here for coreJS/test to run anything
-  //   )
-  // )
+  .nativeSettings(
+    libraryDependencies ++= List(
+      metaconfigSconfig.value
+    ),
+    scalaNativeNativeConfig
+  )
   .jvmSettings(
     Test / run / fork := true,
     libraryDependencies ++= List(
-      metaconfigTypesafe.value
+      metaconfigTypesafe.value,
+      metaconfig.value
     )
   )
   .enablePlugins(BuildInfoPlugin)
-lazy val coreJVM = core.jvm
-// lazy val coreJS = core.js
+
+lazy val coreNative = core.native
 
 import sbtassembly.AssemblyPlugin.defaultUniversalScript
 
@@ -160,24 +180,19 @@ val scalacJvmOptions = Def.setting {
   }
 }
 
-lazy val cli = project
+lazy val scalaNativeNativeConfig =
+  nativeConfig ~= {
+    _.withMode(Mode.debug)
+      .withLinkStubs(true)
+  }
+
+lazy val cli = crossProject(JVMPlatform, NativePlatform)
+  .withoutSuffixFor(JVMPlatform)
   .in(file("scalafmt-cli"))
   .settings(
     moduleName := "scalafmt-cli",
-    assembly / mainClass := Some("org.scalafmt.cli.Cli"),
-    assembly / assemblyOption := (assembly / assemblyOption).value
-      .withPrependShellScript(Some(defaultUniversalScript(shebang = false))),
-    assembly / assemblyJarName := "scalafmt.jar",
-    assembly / assemblyMergeStrategy := {
-      case "reflect.properties" => MergeStrategy.first
-      case x =>
-        val oldStrategy = (assembly / assemblyMergeStrategy).value
-        oldStrategy(x)
-    },
     libraryDependencies ++= Seq(
-      "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0",
-      "com.martiansoftware" % "nailgun-server" % "0.9.1",
-      "com.github.scopt" %% "scopt" % "4.0.1",
+      "com.github.scopt" %%% "scopt" % "4.0.1",
       // undeclared transitive dependency of coursier-small
       "org.scala-lang.modules" %% "scala-xml" % "1.3.0"
     ),
@@ -197,36 +212,62 @@ lazy val cli = project
           .toSeq
     }
   )
-  .dependsOn(coreJVM, dynamic)
+  .jvmSettings(
+    assembly / mainClass := Some("org.scalafmt.cli.Cli"),
+    assembly / assemblyOption := (assembly / assemblyOption).value
+      .withPrependShellScript(Some(defaultUniversalScript(shebang = false))),
+    assembly / assemblyJarName := "scalafmt.jar",
+    assembly / assemblyMergeStrategy := {
+      case "reflect.properties" => MergeStrategy.first
+      case x =>
+        val oldStrategy = (assembly / assemblyMergeStrategy).value
+        oldStrategy(x)
+    },
+    libraryDependencies += "com.martiansoftware" % "nailgun-server" % "0.9.1"
+  )
+  .nativeSettings(
+    scalaNativeNativeConfig
+  )
+  .dependsOn(core, dynamic)
   .enablePlugins(NativeImagePlugin)
 
-lazy val tests = project
+lazy val cliNative = cli.native
+
+lazy val tests = crossProject(JVMPlatform, NativePlatform)
+  .withoutSuffixFor(JVMPlatform)
   .in(file("scalafmt-tests"))
   .settings(
     publish / skip := true,
     libraryDependencies ++= Seq(
       // Test dependencies
-      "com.lihaoyi" %% "scalatags" % "0.10.0",
-      scalametaTestkit,
+      "com.lihaoyi" %%% "scalatags" % "0.10.0",
+      scalametaTestkit.value,
       munit.value
     ),
     scalacOptions ++= scalacJvmOptions.value,
-    javaOptions += "-Dfile.encoding=UTF8",
     buildInfoPackage := "org.scalafmt.tests",
     buildInfoKeys := Seq[BuildInfoKey](
-      "resourceDirectory" -> (Test / resourceDirectory).value
+      "resourceDirectory" -> (baseDirectory.value / ".." / "shared" / "src" / "test" / "resources")
     )
   )
   .enablePlugins(BuildInfoPlugin)
-  .dependsOn(coreJVM, dynamic, cli)
+  .dependsOn(core, dynamic, cli)
+  .nativeSettings(
+    scalaNativeNativeConfig
+  )
+  .jvmSettings(
+    javaOptions += "-Dfile.encoding=UTF8"
+  )
+
+lazy val testsNative = tests.native
 
 lazy val benchmarks = project
   .in(file("scalafmt-benchmarks"))
   .settings(
     publish / skip := true,
     moduleName := "scalafmt-benchmarks",
     libraryDependencies ++= Seq(
-      scalametaTestkit
+      scalametaTestkit.value
     ),
     run / javaOptions ++= Seq(
       "-Djava.net.preferIPv4Stack=true",
@@ -245,7 +286,7 @@ lazy val benchmarks = project
       "-server"
     )
   )
-  .dependsOn(coreJVM)
+  .dependsOn(core.jvm)
   .enablePlugins(JmhPlugin)
 
 lazy val docs = project
@@ -255,7 +296,7 @@ lazy val docs = project
     publish / skip := true,
     mdoc := (Compile / run).evaluated
   )
-  .dependsOn(cli, dynamic)
+  .dependsOn(cli.jvm, dynamic.jvm)
   .enablePlugins(DocusaurusPlugin)
 
 val V = "\\d+\\.\\d+\\.\\d+"