diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bb28d299400d..50c540e89b29 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -30,7 +30,7 @@ jobs: distribution: 'temurin' java-version: '19' - name: Check formatting - run: sbt ++2.13.8 scalafmtCheck test:scalafmtCheck + run: sbt scalafmtCheck Test/scalafmtCheck - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt Test/scalafmt'" if: ${{ failure() }} test-scripts: diff --git a/.scalafmt.conf b/.scalafmt.conf index 74a5d34cb57a..45635038b9eb 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,5 +1,5 @@ version = 3.7.3 -runner.dialect = scala213 +runner.dialect = scala3 preset = IntelliJ maxColumn = 120 align.preset = true diff --git a/README.md b/README.md index 806988d87f3d..7c640d8c3a6a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ Documentation: https://docs.joern.io/ Specification: https://cpg.joern.io + +## Announcement: upgrading from Joern 1.x to 2.x: see notes [below](#12x-upgrade-to-scala-3) + ## Requirements - JDK 19 (other versions _might_ work, but have not been properly tested) @@ -38,20 +41,16 @@ chmod +x ./joern-install.sh sudo ./joern-install.sh joern -Compiling (synthetic)/ammonite/predef/interpBridge.sc -Compiling (synthetic)/ammonite/predef/replBridge.sc -Compiling (synthetic)/ammonite/predef/DefaultPredef.sc -Compiling /home/tmp/shiftleft/joern/(console) - ██╗ ██████╗ ███████╗██████╗ ███╗ ██╗ ██║██╔═══██╗██╔════╝██╔══██╗████╗ ██║ ██║██║ ██║█████╗ ██████╔╝██╔██╗ ██║ ██ ██║██║ ██║██╔══╝ ██╔══██╗██║╚██╗██║ ╚█████╔╝╚██████╔╝███████╗██║ ██║██║ ╚████║ ╚════╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ +Version: 2.0.1 +Type `help` to begin joern> - ``` If the installation script fails for any reason, try @@ -122,3 +121,70 @@ are: For more instructions on how to run benchmarks individually head over to the `benchmarks` subproject. If you would like the benchmark results to be written to a file instead of printed to STDOUT, set the path to the environment variable `JOERN_BENCHMARK_RESULT_FILE`. + +## Upgrade notes + +### 2.0.x: Upgrade to Scala 3 +Joern is based on Scala. As of Joern 2.0.x we upgraded from Scala 2 to Scala 3. +This is a major version upgrade and Scala 3 is essentially a new language with a new REPL implementation, so this may sound scary. + +That being said, both the Scala as well as Joern maintainers have made an effort to minimize changes to the API, in order to ease the transition for users. Most importantly, the Joern workspace DSL (`importCode(...)` etc.) and the CPG Traversal DSL (e.g. `cpg.method.name("foo").l`) are unchanged. The latter is based on Scala collections API, which is actually identical (a shared library) between Scala 2 and Scala 3. + +Depending on your use case you may not even notice a difference: we've tried to keep as much as possible just like it was - most importantly the query DSL. +There are however a few changes - some that we believe are the better, and some were just unavoidable. Here's the most important ones as far as I can tell: + +1) The 'import a script' magic `$file.foo` from Ammonite was replaced by the `//> using file foo.sc` directive. This works in an active joern REPL as well as in scripts. + +2) Adding dependencies: the magic `$ivy.my-dependency` was replaced: +* `--dep` parameter, e.g. `./joern --dep com.michaelpollmeier:versionsort:1.0.7`. Can be specified multiple times. +* For scripts there's a slightly nicer alternative that let's you specify your dependencies within the script itself: `//> using dep com.michaelpollmeier:versionsort:1.0.7` +* all dependencies need to be known when joern starts, i.e. you can not dynamically add more dependencies to an active joern REPL session + +3) Script parameters: pass multiple `--param` parameters rather than one comma-separated list. Example: +``` +// old +./joern --script foo.sc --params paramA=valueA,paramB=valueB +// new +./joern --script foo.sc --param paramA=valueA --param paramB=valueB +``` +While that's slightly longer it is also less complex, easier to read, and you can pass values that contain commas :) + +Apart from that, Scala 3 is a bit stricter when it comes to adding or leaving out `()` for function application. The compiler messages are (on average) much better than before, so hopefully it'll guide you as good as possible. + +Depending on your level of integration with Joern you might not even notice anything. If you do, please check the lists above and below, and if that doesn't help: open a [github issue](https://github.com/joernio/joern/issues/new) or hit us up on [discord](https://discord.gg/vv4MH284Hc). + +Some more generic Scala-issues when upgrading Scala 2 to Scala 3: + +1. anonymous functions need an extra parenthesis around their parameter list: +```scala +Seq(1,2,3).map { i: Int => i + 1 } +// error: parentheses are required around the parameter of a lambda + +// option 1: add parentheses, as suggested by compiler: +Seq(1,2,3).map { (i: Int) => i + 1 } + +// option 2: drop type annotation (if possible): +Seq(1,2,3).map { i => i + 1 } +``` + +2. `main` entrypoint: `def main` instead of `extends App` +See https://docs.scala-lang.org/scala3/book/methods-main-methods.html +```scala +object Main extends App { + println("hello world") +} + +// depending on usage, may lead to NullPointerExceptions +// context: Scala3 doesn't support the 'magic' DelayedInit trait + +// rewrite to: +object Main { + def main(args: Array[String]) = { + println("hello world") + } +} +``` + + + + diff --git a/benchmarks/build.sbt b/benchmarks/build.sbt index 0d1f65b95bf3..38d4c933a8af 100644 --- a/benchmarks/build.sbt +++ b/benchmarks/build.sbt @@ -1,7 +1,5 @@ name := "benchmarks" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.dataflowengineoss) dependsOn(Projects.semanticcpg) dependsOn(Projects.console) diff --git a/build.sbt b/build.sbt index c6fbd119e4f3..822e86d73885 100644 --- a/build.sbt +++ b/build.sbt @@ -1,8 +1,8 @@ name := "joern" ThisBuild / organization := "io.joern" -ThisBuild / scalaVersion := "2.13.8" +ThisBuild / scalaVersion := "3.3.0" -val cpgVersion = "1.3.600" +val cpgVersion = "1.3.600+2-e07f2d9a" lazy val joerncli = Projects.joerncli lazy val querydb = Projects.querydb diff --git a/c2cpg.sh b/c2cpg.sh new file mode 120000 index 000000000000..1c7b5c42356d --- /dev/null +++ b/c2cpg.sh @@ -0,0 +1 @@ +joern-cli/frontends/c2cpg/c2cpg.sh \ No newline at end of file diff --git a/console/build.sbt b/console/build.sbt index fa3031601451..8c648dc057ca 100644 --- a/console/build.sbt +++ b/console/build.sbt @@ -5,7 +5,6 @@ enablePlugins(JavaAppPackaging) val ScoptVersion = "4.0.1" val CaskVersion = "0.8.3" val CirceVersion = "0.14.5" -val AmmoniteVersion = "2.5.8" val ZeroturnaroundVersion = "1.15" dependsOn( @@ -18,58 +17,17 @@ dependsOn( ) libraryDependencies ++= Seq( - "io.shiftleft" %% "codepropertygraph" % Versions.cpg, - "com.github.scopt" %% "scopt" % ScoptVersion, - "org.typelevel" %% "cats-effect" % Versions.cats, - "io.circe" %% "circe-generic" % CirceVersion, - "io.circe" %% "circe-parser" % CirceVersion, - "org.zeroturnaround" % "zt-zip" % ZeroturnaroundVersion, - "com.lihaoyi" %% "ammonite" % AmmoniteVersion cross CrossVersion.full, - "com.lihaoyi" %% "os-lib" % "0.8.1", - "com.lihaoyi" %% "cask" % CaskVersion, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "com.michaelpollmeier" %% "scala-repl-pp-all" % "0.1.26+1-5262c2c7", + "io.shiftleft" %% "codepropertygraph" % Versions.cpg, + "com.github.scopt" %% "scopt" % ScoptVersion, + "org.typelevel" %% "cats-effect" % Versions.cats, + "io.circe" %% "circe-generic" % CirceVersion, + "io.circe" %% "circe-parser" % CirceVersion, + "org.zeroturnaround" % "zt-zip" % ZeroturnaroundVersion, + "com.lihaoyi" %% "os-lib" % "0.8.1", + "com.lihaoyi" %% "pprint" % "0.7.3", + "com.lihaoyi" %% "cask" % CaskVersion, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) -Test / compile := (Test / compile).dependsOn((Projects.c2cpg / stage)).value - -scalacOptions ++= Seq( - "-deprecation", // Emit warning and location for usages of deprecated APIs. - "-encoding", - "utf-8", // Specify character encoding used by source files. - "-explaintypes", // Explain type errors in more detail. - "-feature", // Emit warning and location for usages of features that should be imported explicitly. - "-language:existentials", // Existential types (besides wildcard types) can be written and inferred - "-language:experimental.macros", // Allow macro definition (besides implementation and application) - "-language:higherKinds", // Allow higher-kinded types - "-language:implicitConversions", // Allow definition of implicit functions called views - "-unchecked", // Enable additional warnings where generated code depends on assumptions. - "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access. - "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver. - "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error. - "-Xlint:delayedinit-select", // Selecting member of DelayedInit. - "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element. - "-Xlint:inaccessible", // Warn about inaccessible types in method signatures. - "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`. - "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id. - "-Xlint:option-implicit", // Option.apply used implicit view. - "-Xlint:package-object-classes", // Class or object defined in package object. - "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds. - "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field. - "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component. - "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope. - "-Ywarn-dead-code", // Warn when dead code is identified. - "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. - "-Ywarn-numeric-widen", // Warn when numerics are widened. - "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. - "-Ywarn-unused:imports", // Warn if an import selector is not referenced. - "-Ywarn-unused:locals", // Warn if a local definition is unused. - "-Ywarn-unused:params", // Warn if a value parameter is unused. - "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. - "-Ywarn-unused:privates", // Warn if a private member is unused. - "-Yrangepos" -) - -// would love to reenable, but somehow StorageBackend.scala triggers a strange `[warn] method with a single empty parameter list overrides method without any parameter list` that doesn't make sense to me... -scalacOptions -= "-Xfatal-warnings" - -Test / fork := false +Test/compile := (Test/compile).dependsOn((Projects.c2cpg/stage)).value diff --git a/console/src/main/java/io/github/retronym/java9rtexport/Export.java b/console/src/main/java/io/github/retronym/java9rtexport/Export.java deleted file mode 100644 index ed33a8fb2be3..000000000000 --- a/console/src/main/java/io/github/retronym/java9rtexport/Export.java +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright (C) 2012-2014 EPFL -Copyright (C) 2012-2014 Typesafe, Inc. - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the EPFL nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package io.github.retronym.java9rtexport; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.nio.file.*; -import java.nio.file.StandardCopyOption; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - - -/* TODO MP delete this file once it's fixed upstream - we're shadowing io.github.retronym.java9rtexport.Export - * which contains a bug for an edge case: the following line from `rtTo` - * java.nio.file.Files.copy(rt().toPath(), dest.toPath()); - * throws an exception if run concurrently: - * - * java.nio.file.FileAlreadyExistsException: /home/runner/.ammonite/rt-17.0.5.jar - * at java.base/sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:573) - * at java.base/sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:257) - * at java.base/java.nio.file.Files.copy(Files.java:1305) - * at io.github.retronym.java9rtexport.Export.rtTo(Export.java:88) - * at io.github.retronym.java9rtexport.Export.rtAt(Export.java:100) - * at io.github.retronym.java9rtexport.Export.rtAt(Export.java:105) - * at ammonite.util.Classpath$.classpath(Classpath.scala:76) - */ - -public class Export { - private final static Object lock = new Object(); - private static File tempFile = null; - - public static String rtJarName = "rt-" + System.getProperty("java.version") + ".jar"; - - public static File rt() { - try { - synchronized (lock) { - if (tempFile == null) { - Path tempPath = Files.createTempFile("rt", ".jar"); - tempFile = tempPath.toFile(); - tempFile.deleteOnExit(); - tempFile.delete(); - FileSystem fileSystem = FileSystems.getFileSystem(URI.create("jrt:/")); - Path path = fileSystem.getPath("/modules"); - URI uri = URI.create("jar:" + tempPath.toUri()); - Map env = new HashMap<>(); - env.put("create", "true"); - try (FileSystem zipfs = FileSystems.newFileSystem(uri, env)) { - Iterator iterator = Files.list(path).iterator(); - while (iterator.hasNext()) { - Path next = iterator.next(); - Copy.copyDirectory(next, zipfs.getPath("/")); - } - } - } - } - } catch (IOException e) { - e.printStackTrace(); - System.exit(-1); - } - return tempFile; - } - - /** - * Needs to be `synchronized` because java.nio.file.Files.copy isn't thread safe: - * https://stackoverflow.com/questions/69796396/is-files-copy-a-thread-safe-function-in-java - * and our own handling of "if !exists, then copy" isn't either... - */ - public static synchronized boolean rtTo(File dest, boolean verbose) { - try { - if (!dest.exists()) { - if (verbose) { - System.out.println("Copying Java " + - System.getProperty("java.version") + - " runtime jar to " + - dest.getParentFile() + - " ..."); - System.out.flush(); - } - dest.getParentFile().mkdirs(); - java.nio.file.Files.copy(rt().toPath(), dest.toPath()); - return true; - } - } catch (IOException e) { - e.printStackTrace(); - System.exit(-1); - } - return false; - } - - public static File rtAt(File dir, boolean verbose) { - File f = new File(dir, rtJarName); - rtTo(f, verbose); - return f; - } - - public static File rtAt(File dir) { - return rtAt(dir, false); - } -} diff --git a/console/src/main/scala/io/joern/console/BridgeBase.scala b/console/src/main/scala/io/joern/console/BridgeBase.scala index e2fd4d4698c6..916eec378eac 100644 --- a/console/src/main/scala/io/joern/console/BridgeBase.scala +++ b/console/src/main/scala/io/joern/console/BridgeBase.scala @@ -1,18 +1,21 @@ package io.joern.console -import ammonite.interp.Watchable -import ammonite.util.{Colors, Res} -import better.files._ -import io.joern.console.cpgqlserver.CPGQLServer -import io.joern.console.embammonite.EmbeddedAmmonite +import better.files.* +import replpp.scripting.ScriptRunner + +import scala.jdk.CollectionConverters.* import io.shiftleft.codepropertygraph.generated.Languages -import os.{pwd, Path} +import java.io.{InputStream, PrintStream, File as JFile} +import java.net.URLClassLoader +import java.nio.file.{Files, Path, Paths} +import java.util.stream.Collectors +import scala.util.{Failure, Success, Try} case class Config( scriptFile: Option[Path] = None, command: Option[String] = None, params: Map[String, String] = Map.empty, - additionalImports: List[Path] = Nil, + additionalImports: Seq[Path] = Nil, addPlugin: Option[String] = None, rmPlugin: Option[String] = None, pluginToRun: Option[String] = None, @@ -24,24 +27,25 @@ case class Config( server: Boolean = false, serverHost: String = "localhost", serverPort: Int = 8080, - serverAuthUsername: String = "", - serverAuthPassword: String = "", + serverAuthUsername: Option[String] = None, + serverAuthPassword: Option[String] = None, nocolors: Boolean = false, cpgToLoad: Option[File] = None, forInputPath: Option[String] = None, - frontendArgs: Array[String] = Array.empty + frontendArgs: Array[String] = Array.empty, + verbose: Boolean = false, + dependencies: Seq[String] = Seq.empty, + resolvers: Seq[String] = Seq.empty ) -/** Base class for Ammonite Bridge, split by topic into multiple self types. +/** Base class for ReplBridge, split by topic into multiple self types. */ -trait BridgeBase extends ScriptExecution with PluginHandling with ServerHandling { +trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandling with ServerHandling { - protected def parseConfig(args: Array[String]): Config = { - implicit def pathRead: scopt.Read[Path] = - scopt.Read.stringRead - .map(Path(_, pwd)) // support both relative and absolute paths + def slProduct: SLProduct - val parser = new scopt.OptionParser[Config]("(joern|ocular)") { + protected def parseConfig(args: Array[String]): Config = { + val parser = new scopt.OptionParser[Config](slProduct.name) { override def errorOnUnknownArgument = false note("Script execution") @@ -50,15 +54,40 @@ trait BridgeBase extends ScriptExecution with PluginHandling with ServerHandling .action((x, c) => c.copy(scriptFile = Some(x))) .text("path to script file: will execute and exit") - opt[Map[String, String]]('p', "params") - .valueName("k1=v1,k2=v2") - .action((x, c) => c.copy(params = x)) - .text("key values for script") + opt[String]("param") + .valueName("param1=value1") + .unbounded() + .optional() + .action { (x, c) => + x.split("=", 2) match { + case Array(key, value) => c.copy(params = c.params + (key -> value)) + case _ => throw new IllegalArgumentException(s"unable to parse param input $x") + } + } + .text("key/value pair for main function in script - may be passed multiple times") + + opt[Path]("import") + .valueName("script1.sc") + .unbounded() + .optional() + .action((x, c) => c.copy(additionalImports = c.additionalImports :+ x)) + .text("import (and run) additional script(s) on startup - may be passed multiple times") - opt[Seq[Path]]("import") - .valueName("script1.sc,script2.sc,...") - .action((x, c) => c.copy(additionalImports = x.toList)) - .text("import additional additional script(s): will execute and keep console open") + opt[String]('d', "dep") + .valueName("com.michaelpollmeier:versionsort:1.0.7") + .unbounded() + .optional() + .action((x, c) => c.copy(dependencies = c.dependencies :+ x)) + .text( + "add artifacts (including transitive dependencies) for given maven coordinate to classpath - may be passed multiple times" + ) + + opt[String]('r', "repo") + .valueName("https://repository.apache.org/content/groups/public/") + .unbounded() + .optional() + .action((x, c) => c.copy(resolvers = c.resolvers :+ x)) + .text("additional repositories to resolve dependencies - may be passed multiple times") opt[String]("command") .action((x, c) => c.copy(command = Some(x))) @@ -113,11 +142,11 @@ trait BridgeBase extends ScriptExecution with PluginHandling with ServerHandling .text("Port on which to expose the CPGQL server") opt[String]("server-auth-username") - .action((x, c) => c.copy(serverAuthUsername = x)) + .action((x, c) => c.copy(serverAuthUsername = Option(x))) .text("Basic auth username for the CPGQL server") opt[String]("server-auth-password") - .action((x, c) => c.copy(serverAuthPassword = x)) + .action((x, c) => c.copy(serverAuthPassword = Option(x))) .text("Basic auth password for the CPGQL server") note("Misc") @@ -135,6 +164,10 @@ trait BridgeBase extends ScriptExecution with PluginHandling with ServerHandling .action((_, c) => c.copy(nocolors = true)) .text("turn off colors") + opt[Unit]("verbose") + .action((_, c) => c.copy(verbose = true)) + .text("enable verbose output (predef, resolved dependency jars, ...)") + help("help") .text("Print this help text") } @@ -143,105 +176,97 @@ trait BridgeBase extends ScriptExecution with PluginHandling with ServerHandling parser.parse(args, Config()).get } - /** Entry point for Joern's integrated ammonite shell - */ - protected def runAmmonite(config: Config, slProduct: SLProduct = OcularProduct): Unit = { + /** Entry point for Joern's integrated REPL and plugin manager */ + protected def run(config: Config): Unit = { if (config.listPlugins) { - printPluginsAndLayerCreators(config, slProduct) + printPluginsAndLayerCreators(config) } else if (config.addPlugin.isDefined) { new PluginManager(InstallConfig().rootPath).add(config.addPlugin.get) } else if (config.rmPlugin.isDefined) { new PluginManager(InstallConfig().rootPath).rm(config.rmPlugin.get) - } else { - config.scriptFile match { - case None => - if (config.server) { - GlobalReporting.enable() - startHttpServer(config) - } else if (config.pluginToRun.isDefined) { - runPlugin(config, slProduct.name) - } else { - startInteractiveShell(config, slProduct) - } - case Some(scriptFile) => - runScript(scriptFile, config) + } else if (config.scriptFile.isDefined) { + val scriptReturn = runScript(config) + if (scriptReturn.isFailure) { + println(scriptReturn.failed.get.getMessage) + System.exit(1) } + } else if (config.server) { + GlobalReporting.enable() + startHttpServer(config) + } else if (config.pluginToRun.isDefined) { + runPlugin(config, slProduct.name) + } else { + startInteractiveShell(config) } } - protected def additionalImportCode(config: Config): List[String] = - config.additionalImports.flatMap { importScript => - val file = importScript.toIO - assert(file.canRead, s"unable to read $file") - readScript(file.toScala) - } - - private def readScript(scriptFile: File): List[String] = { - val code = scriptFile.lines.toList - println(s"importing $scriptFile (${code.size} lines)") - code + protected def createPredefFile(additionalLines: Seq[String] = Nil): Path = { + val tmpFile = Files.createTempFile("joern-predef", "sc") + Files.write(tmpFile, (predefLines ++ additionalLines).asJava) + tmpFile.toAbsolutePath } - protected def predefPlus(lines: List[String]): String + /** code that is executed on startup */ + protected def predefLines: Seq[String] - protected def shutdownHooks: List[String] + protected def greeting: String - protected def promptStr(): String + protected def promptStr: String + protected def onExitCode: String } -trait ScriptExecution { - this: BridgeBase => - - protected def startInteractiveShell(config: Config, slProduct: SLProduct): (Res[Any], Seq[(Watchable, Long)]) = { - val configurePPrinterMaybe = - if (config.nocolors) "" - else "repl.pprinter.update(io.joern.console.pprinter.create())" - - val replConfig = List( - "repl.prompt() = \"" + promptStr() + "\"", - configurePPrinterMaybe, - "implicit val implicitPPrinter = repl.pprinter()", - "banner()" - ) ++ config.cpgToLoad.map { cpgFile => +trait InteractiveShell { this: BridgeBase => + protected def startInteractiveShell(config: Config) = { + val replConfig = config.cpgToLoad.map { cpgFile => "importCpg(\"" + cpgFile + "\")" } ++ config.forInputPath.map { name => s""" |openForInputPath(\"$name\") |""".stripMargin } - ammonite - .Main( - predefCode = predefPlus(additionalImportCode(config) ++ replConfig ++ shutdownHooks), - welcomeBanner = None, - storageBackend = new StorageBackend(slProduct), - remoteLogging = false, - colors = ammoniteColors(config) + + val predefFile = createPredefFile(replConfig.toSeq) + + replpp.InteractiveShell.run( + replpp.Config( + predefFiles = predefFile +: config.additionalImports, + nocolors = config.nocolors, + dependencies = config.dependencies, + resolvers = config.resolvers, + verbose = config.verbose, + greeting = greeting, + prompt = Option(promptStr), + onExitCode = Option(onExitCode) ) - .run() + ) } - protected def runScript(scriptFile: Path, config: Config): AnyVal = { - System.err.println(s"executing $scriptFile with params=${config.params}") - val scriptArgs: Seq[String] = { - val commandArgs = config.command.toList - val parameterArgs = config.params.flatMap { case (key, value) => Seq(s"--$key", value) } - commandArgs ++ parameterArgs - } - val predefCode = predefPlus(additionalImportCode(config) ++ importCpgCode(config) ++ shutdownHooks) - - ammonite - .Main(predefCode = predefCode, remoteLogging = false, colors = ammoniteColors(config)) - .runScript(scriptFile, scriptArgs) - ._1 match { - case Res.Success(r) => - System.err.println(s"script finished successfully") - System.err.println(r) - case Res.Failure(msg) => - throw new AssertionError(s"script failed: $msg") - case Res.Exception(e, msg) => - throw new AssertionError(s"script errored: $msg", e) - case _ => ??? +} + +trait ScriptExecution { this: BridgeBase => + + def runScript(config: Config): Try[Unit] = { + val scriptFile = config.scriptFile.getOrElse(throw new AssertionError("no script file configured")) + if (!Files.exists(scriptFile)) { + Try(throw new AssertionError(s"given script file `$scriptFile` does not exist")) + } else { + val predefFile = createPredefFile(importCpgCode(config)) + val scriptReturn = ScriptRunner.exec( + replpp.Config( + predefFiles = predefFile +: config.additionalImports, + scriptFile = Option(scriptFile), + command = config.command, + params = config.params, + dependencies = config.dependencies, + resolvers = config.resolvers, + verbose = config.verbose + ) + ) + if (config.verbose && scriptReturn.isFailure) { + println(scriptReturn.failed.get.getMessage) + } + scriptReturn } } @@ -256,26 +281,20 @@ trait ScriptExecution { |""".stripMargin } } - - private def ammoniteColors(config: Config) = - if (config.nocolors) Colors.BlackWhite - else Colors.Default - } -trait PluginHandling { - this: BridgeBase => +trait PluginHandling { this: BridgeBase => /** Print a summary of the available plugins and layer creators to the terminal. */ - protected def printPluginsAndLayerCreators(config: Config, slProduct: SLProduct): Unit = { + protected def printPluginsAndLayerCreators(config: Config): Unit = { println("Installed plugins:") println("==================") new PluginManager(InstallConfig().rootPath).listPlugins().foreach(println) println("Available layer creators") println() withTemporaryScript(codeToListPlugins(), slProduct.name) { file => - runScript(os.Path(file.path.toString), config) + runScript(config.copy(scriptFile = Some(file.path))).get } } @@ -286,8 +305,7 @@ trait PluginHandling { |""".stripMargin } - /** Run plugin by generating a temporary script based on the given config and executing the script via ammonite. - */ + /** Run plugin by generating a temporary script based on the given config and execute the script */ protected def runPlugin(config: Config, productName: String): Unit = { if (config.src.isEmpty) { println("You must supply a source directory with the --src flag") @@ -295,7 +313,7 @@ trait PluginHandling { } val code = loadOrCreateCpg(config, productName) withTemporaryScript(code, productName) { file => - runScript(os.Path(file.path.toString), config) + runScript(config.copy(scriptFile = Some(file.path))).get } } @@ -370,35 +388,20 @@ trait PluginHandling { trait ServerHandling { this: BridgeBase => protected def startHttpServer(config: Config): Unit = { - val predef = predefPlus(additionalImportCode(config)) - val ammonite = new EmbeddedAmmonite(predef) - ammonite.start() - Runtime.getRuntime.addShutdownHook(new Thread(() => { - ammonite.shutdown() - })) - val server = new CPGQLServer( - ammonite, - config.serverHost, - config.serverPort, - config.serverAuthUsername, - config.serverAuthPassword + val predefFile = createPredefFile(Nil) + + replpp.server.ReplServer.startHttpServer( + replpp.Config( + predefFiles = predefFile +: config.additionalImports, + dependencies = config.dependencies, + resolvers = config.resolvers, + verbose = true, // always print what's happening - helps debugging + serverHost = config.serverHost, + serverPort = config.serverPort, + serverAuthUsername = config.serverAuthUsername, + serverAuthPassword = config.serverAuthPassword + ) ) - println("Starting CPGQL server ...") - try { - server.main(Array.empty) - } catch { - case _: java.net.BindException => - println("Could not bind socket for CPGQL server, exiting.") - ammonite.shutdown() - System.exit(1) - case e: Throwable => - println("Unhandled exception thrown while attempting to start CPGQL server: ") - println(e.getMessage) - println("Exiting.") - - ammonite.shutdown() - System.exit(1) - } } } diff --git a/console/src/main/scala/io/joern/console/Console.scala b/console/src/main/scala/io/joern/console/Console.scala index 32b8da8a279d..e360f1e1998a 100644 --- a/console/src/main/scala/io/joern/console/Console.scala +++ b/console/src/main/scala/io/joern/console/Console.scala @@ -1,9 +1,10 @@ package io.joern.console -import better.files.Dsl._ import better.files.File +import dotty.tools.repl.State +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.joern.console.cpgcreation.ImportCode -import io.joern.console.scripting.{AmmoniteExecutor, ScriptManager} import io.joern.console.workspacehandling.{Project, WorkspaceLoader, WorkspaceManager} import io.joern.x2cpg.X2Cpg.defaultOverlayCreators import io.shiftleft.codepropertygraph.Cpg @@ -17,12 +18,8 @@ import scala.sys.process.Process import scala.util.control.NoStackTrace import scala.util.{Failure, Success, Try} -class Console[T <: Project]( - executor: AmmoniteExecutor, - loader: WorkspaceLoader[T], - baseDir: File = File.currentWorkingDirectory -) extends ScriptManager(executor) - with Reporting { +class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.currentWorkingDirectory) + extends Reporting { import Console._ @@ -349,7 +346,7 @@ class Console[T <: Project]( return None } } else { - cp(cpgFile, cpgDestinationPath) + cpgFile.copyTo(cpgDestinationPath, overwrite = true) } val cpgOpt = open(name).flatMap(_.cpg) @@ -419,7 +416,7 @@ class Console[T <: Project]( if (projectOpt.get.appliedOverlays.contains(creator.overlayName)) { report(s"Overlay ${creator.overlayName} already exists - skipping") } else { - mkdirs(File(overlayDirName)) + File(overlayDirName).createDirectories() runCreator(creator, Some(overlayDirName)) } } diff --git a/console/src/main/scala/io/joern/console/DefaultAmmoniteExecutor.scala b/console/src/main/scala/io/joern/console/DefaultAmmoniteExecutor.scala deleted file mode 100644 index ccfde33b07c7..000000000000 --- a/console/src/main/scala/io/joern/console/DefaultAmmoniteExecutor.scala +++ /dev/null @@ -1,7 +0,0 @@ -package io.joern.console - -import io.joern.console.scripting.AmmoniteExecutor - -object DefaultAmmoniteExecutor extends AmmoniteExecutor { - override lazy val predef: String = "" -} diff --git a/console/src/main/scala/io/joern/console/Help.scala b/console/src/main/scala/io/joern/console/Help.scala index 654768020977..97f4cba4bdc2 100644 --- a/console/src/main/scala/io/joern/console/Help.scala +++ b/console/src/main/scala/io/joern/console/Help.scala @@ -59,19 +59,17 @@ object Help { val membersCode = DocFinder .findDocumentedMethodsOf(clazz) .map { case StepDoc(_, funcName, doc) => - s"val $funcName : String = ${Help.format(doc.longInfo)}" + s" val $funcName: String = ${Help.format(doc.longInfo)}" } .mkString("\n") val overview = Help.overview(clazz) s""" | class Helper() { + | def run: String = Help.runLongHelp + | override def toString: String = \"\"\"$overview\"\"\" | - | $membersCode - | - | def run : String = Help.runLongHelp - | - | override def toString : String = \"\"\"$overview\"\"\" + | $membersCode | } | | val help = new Helper diff --git a/console/src/main/scala/io/joern/console/PPrinter.scala b/console/src/main/scala/io/joern/console/PPrinter.scala deleted file mode 100644 index 6b5d36b6fb91..000000000000 --- a/console/src/main/scala/io/joern/console/PPrinter.scala +++ /dev/null @@ -1,51 +0,0 @@ -package io.joern.console - -import pprint.{PPrinter, Renderer, Result, Tree, Truncated} -import scala.util.matching.Regex - -object pprinter { - - val AnsiEncodedRegexp: Regex = "\u001b\\[[\\d;]+m".r - def isAnsiEncoded(s: String): Boolean = - AnsiEncodedRegexp.findFirstIn(s).isDefined - - /** We use source-highlight to encode source as ansi strings, e.g. the .dump step Ammonite uses fansi for it's - * colour-coding, and while both pledge to follow the ansi codec, they aren't compatible TODO: PR for fansi to - * support these standard encodings out of the box - */ - def fixForFansi(ansiEncoded: String): String = - ansiEncoded - .replaceAll("\u001b\\[m", "\u001b[39m") // encoding ends with [39m for fansi instead of [m - .replaceAll("\u001b\\[0(\\d)m", "\u001b[$1m") // `[01m` is encoded as `[1m` in fansi for all single digit numbers - .replaceAll("\u001b\\[0?(\\d+);0?(\\d+)m", "\u001b[$1m\u001b[$2m") // `[01;34m` is encoded as `[1m[34m` in fansi - .replaceAll( - "\u001b\\[[00]+;0?(\\d+);0?(\\d+);0?(\\d+)m", - "\u001b[$1;$2;$3m" - ) // `[00;38;05;70m` is encoded as `[38;5;70m` in fansi - 8bit color encoding - - def create(): PPrinter = - new PPrinter(defaultHeight = 99999) { - override def tokenize( - x: Any, - width: Int = defaultWidth, - height: Int = defaultHeight, - indent: Int = defaultIndent, - initialOffset: Int = 0, - escapeUnicode: Boolean, - showFieldNames: Boolean - ): Iterator[fansi.Str] = { - val tree = this.treeify(x, escapeUnicode = escapeUnicode, showFieldNames = showFieldNames) - val renderer = new Renderer(width, colorApplyPrefix, colorLiteral, indent) { - override def rec(x: Tree, leftOffset: Int, indentCount: Int): Result = x match { - case Tree.Literal(body) if isAnsiEncoded(body) => - // this is the part we're overriding, everything else is just boilerplate - Result.fromString(fixForFansi(body)) - case _ => super.rec(x, leftOffset, indentCount) - } - } - val rendered = renderer.rec(tree, initialOffset, 0).iter - new Truncated(rendered, width, height) - } - } - -} diff --git a/console/src/main/scala/io/joern/console/Run.scala b/console/src/main/scala/io/joern/console/Run.scala index b770b0e3da63..e66c2fab71a2 100644 --- a/console/src/main/scala/io/joern/console/Run.scala +++ b/console/src/main/scala/io/joern/console/Run.scala @@ -47,7 +47,7 @@ object Run { .filterNot(t => t.isAnonymousClass || t.isLocalClass || t.isMemberClass || t.isSynthetic) .filterNot(t => t.getName.startsWith("io.joern.console.Run")) .toList - .map(t => (t.getSimpleName.toLowerCase, t.getName)) + .map(t => (t.getSimpleName.toLowerCase, s"_root_.${t.getName}")) .filter(t => !exclude.contains(t._2)) } @@ -59,46 +59,45 @@ object Run { val optsCode = s""" |class OptsDynamic { - | $optsMembersCode + |$optsMembersCode |} | |val opts = new OptsDynamic() | - | import overflowdb.BatchedUpdate.DiffGraphBuilder + | import _root_.overflowdb.BatchedUpdate.DiffGraphBuilder | implicit def _diffGraph: DiffGraphBuilder = opts.commit.diffGraphBuilder | def diffGraph = _diffGraph |""".stripMargin val membersCode = layerCreatorTypeNames - .map { case (varName, typeName) => s"def $varName: Cpg = _runAnalyzer(new $typeName(opts.$varName))" } + .map { case (varName, typeName) => s" def $varName: Cpg = _runAnalyzer(new $typeName(opts.$varName))" } .mkString("\n") val toStringCode = s""" - | import overflowdb.traversal.help.Table - | override def toString() : String = { - | val columnNames = List("name", "description") - | val rows = - | ${layerCreatorTypeNames.map { case (varName, typeName) => + | import overflowdb.traversal.help.Table + | override def toString() : String = { + | val columnNames = List("name", "description") + | val rows = + | ${layerCreatorTypeNames.map { case (varName, typeName) => s"""List("$varName",$typeName.description.trim)""" }} - | "\\n" + Table(columnNames, rows).render - | } + | "\\n" + Table(columnNames, rows).render + | } |""".stripMargin optsCode + s""" - | class OverlaysDynamic { + |class OverlaysDynamic { | - | def apply(query : io.shiftleft.semanticcpg.language.HasStoreMethod) { - | io.joern.console.Run.runCustomQuery(console, query) - | } + | def apply(query: _root_.io.shiftleft.semanticcpg.language.HasStoreMethod) = + | _root_.io.joern.console.Run.runCustomQuery(console, query) | - | $membersCode + |$membersCode | - | $toStringCode - | } - | val run = new OverlaysDynamic() + |$toStringCode + |} + |val run = new OverlaysDynamic() |""".stripMargin } diff --git a/console/src/main/scala/io/joern/console/StorageBackend.scala b/console/src/main/scala/io/joern/console/StorageBackend.scala deleted file mode 100644 index 037494bab76d..000000000000 --- a/console/src/main/scala/io/joern/console/StorageBackend.scala +++ /dev/null @@ -1,18 +0,0 @@ -package io.joern.console - -import ammonite.runtime.Storage -import ammonite.util.Tag -import os.Path - -/** like the default ammonite folder storage (which gives us e.g. command history), but without the CodePredef error - * when using multiple ocular installations (see https://github.com/ShiftLeftSecurity/product/issues/2082) - */ -class StorageBackend(slProduct: SLProduct) extends Storage.Folder(StorageBackend.consoleHome(slProduct)) { - override def compileCacheSave(path: String, tag: Tag, data: Storage.CompileCache): Unit = () - override def compileCacheLoad(path: String, tag: Tag): Option[Storage.CompileCache] = None -} - -object StorageBackend { - def consoleHome(slProduct: SLProduct): Path = - os.Path(System.getProperty("user.home")) / ".shiftleft" / slProduct.name -} diff --git a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala index 4ccc571c259b..955f880a62c6 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala @@ -93,7 +93,7 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) extends Rep def fromString(str: String, args: List[String] = List()): Cpg = { withCodeInTmpFile(str, "tmp." + extension) { dir => - apply(dir.path.toString, args = args) + super.apply(dir.path.toString, args = args) } match { case Failure(exception) => throw new ConsoleException(s"unable to generate cpg from given String", exception) case Success(value) => value diff --git a/console/src/main/scala/io/joern/console/cpgqlserver/CPGQLServer.scala b/console/src/main/scala/io/joern/console/cpgqlserver/CPGQLServer.scala deleted file mode 100644 index e0ee3b04127f..000000000000 --- a/console/src/main/scala/io/joern/console/cpgqlserver/CPGQLServer.scala +++ /dev/null @@ -1,162 +0,0 @@ -package io.joern.console.cpgqlserver - -import cask.model.{Request, Response} -import io.joern.console.embammonite.{EmbeddedAmmonite, HasUUID, QueryResult} - -import java.util.concurrent.ConcurrentHashMap -import java.util.{Base64, UUID} -import ammonite.compiler.{Parsers => AmmoniteParser} -import cask.model.Response.Raw -import cask.router.Result -import ujson.Obj - -object CPGLSError extends Enumeration { - val parseError: CPGLSError.Value = Value("cpgqls_query_parse_error") -} - -class CPGQLServer( - ammonite: EmbeddedAmmonite, - serverHost: String, - serverPort: Int, - serverAuthUsername: String = "", - serverAuthPassword: String = "" -) extends WebServiceWithWebSocket[QueryResult](serverHost, serverPort, serverAuthUsername, serverAuthPassword) { - - @cask.websocket("/connect") - override def handler(): cask.WebsocketResult = super.handler() - - @basicAuth() - @cask.get("/result/:uuidParam") - override def getResult(uuidParam: String)(isAuthorized: Boolean): Response[Obj] = - super.getResult(uuidParam)(isAuthorized) - - @basicAuth() - @cask.postJson("/query") - def postQuery(query: String)(isAuthorized: Boolean): Response[Obj] = { - val res = if (!isAuthorized) { - unauthorizedResponse - } else { - val hasErrorOnParseQuery = - // With ignoreIncomplete = false the result is always Some. Thus .get is ok. - AmmoniteParser.split(query, ignoreIncomplete = false, "N/A").get.isLeft - if (hasErrorOnParseQuery) { - val result = new QueryResult("", CPGLSError.parseError.toString, UUID.randomUUID()) - returnResult(result) - Response(ujson.Obj("success" -> false, "uuid" -> result.uuid.toString), 200) - } else { - val uuid = ammonite.queryAsync(query) { result => - returnResult(result) - } - Response(ujson.Obj("success" -> true, "uuid" -> uuid.toString), 200) - } - } - res - } - - override def resultToJson(result: QueryResult, success: Boolean): Obj = { - ujson.Obj("success" -> success, "uuid" -> result.uuid.toString, "stdout" -> result.out, "stderr" -> result.err) - } - - initialize() -} - -abstract class WebServiceWithWebSocket[T <: HasUUID]( - serverHost: String, - serverPort: Int, - serverAuthUsername: String = "", - serverAuthPassword: String = "" -) extends cask.MainRoutes { - - class basicAuth extends cask.RawDecorator { - - def wrapFunction(ctx: Request, delegate: Delegate): Result[Raw] = { - val authString = requestToAuthString(ctx) - val Array(user, password): Array[String] = authStringToUserAndPwd(authString) - val isAuthorized = - if (serverAuthUsername == "" && serverAuthPassword == "") - true - else - user == serverAuthUsername && password == serverAuthPassword - delegate(Map("isAuthorized" -> isAuthorized)) - } - - private def requestToAuthString(ctx: Request): String = { - try { - val authHeader = ctx.exchange.getRequestHeaders.get("authorization").getFirst - val strippedHeader = authHeader.replaceFirst("Basic ", "") - new String(Base64.getDecoder.decode(strippedHeader)) - } catch { - case _: Exception => "" - } - } - - private def authStringToUserAndPwd(authString: String): Array[String] = { - val split = authString.split(":") - if (split.length == 2) { - Array(split(0), split(1)) - } else { - Array("", "") - } - } - } - - override def port: Int = serverPort - - override def host: String = serverHost - - var openConnections = Set.empty[cask.WsChannelActor] - val resultMap = new ConcurrentHashMap[UUID, (T, Boolean)]() - val unauthorizedResponse: Response[Obj] = Response(ujson.Obj(), 401, headers = Seq("WWW-Authenticate" -> "Basic")) - - def handler(): cask.WebsocketResult = { - cask.WsHandler { connection => - connection.send(cask.Ws.Text("connected")) - openConnections += connection - cask.WsActor { - case cask.Ws.Error(e) => - println("Connection error: " + e.getMessage) - openConnections -= connection - case cask.Ws.Close(_, _) | cask.Ws.ChannelClosed() => - println("Connection closed.") - openConnections -= connection - } - } - } - - def getResult(uuidParam: String)(isAuthorized: Boolean): Response[Obj] = { - val res = if (!isAuthorized) { - unauthorizedResponse - } else { - val uuid = - try { - UUID.fromString(uuidParam) - } catch { - case _: IllegalArgumentException => null - } - val finalRes = if (uuid == null) { - ujson.Obj("success" -> false, "err" -> "UUID parameter is incorrectly formatted") - } else { - val resFromMap = resultMap.remove(uuid) - if (resFromMap == null) { - ujson.Obj("success" -> false, "err" -> "No result found for specified UUID") - } else { - resultToJson(resFromMap._1, resFromMap._2) - } - } - Response(finalRes, 200) - } - res - } - - def returnResult(result: T): Unit = { - resultMap.put(result.uuid, (result, true)) - openConnections.foreach { connection => - connection.send(cask.Ws.Text(result.uuid.toString)) - } - Response(ujson.Obj("success" -> true, "uuid" -> result.uuid.toString), 200) - } - - def resultToJson(result: T, b: Boolean): Obj - - initialize() -} diff --git a/console/src/main/scala/io/joern/console/embammonite/EmbeddedAmmonite.scala b/console/src/main/scala/io/joern/console/embammonite/EmbeddedAmmonite.scala deleted file mode 100644 index b9724735f290..000000000000 --- a/console/src/main/scala/io/joern/console/embammonite/EmbeddedAmmonite.scala +++ /dev/null @@ -1,150 +0,0 @@ -package io.joern.console.embammonite - -import ammonite.util.Colors -import org.slf4j.{Logger, LoggerFactory} - -import java.io.{BufferedReader, InputStreamReader, PipedInputStream, PipedOutputStream, PrintWriter} -import java.util.UUID -import java.util.concurrent.{BlockingQueue, LinkedBlockingQueue, Semaphore} - -/** Result of executing a query, containing in particular output received on standard out and on standard error. - */ -class QueryResult(val out: String, val err: String, val uuid: UUID) extends HasUUID - -trait HasUUID { - def uuid: UUID -} - -private[embammonite] case class Job(uuid: UUID, query: String, observer: QueryResult => Unit) - -class EmbeddedAmmonite(predef: String = "") { - - import EmbeddedAmmonite.logger - - val jobQueue: BlockingQueue[Job] = new LinkedBlockingQueue[Job]() - - val (inStream, toStdin) = pipePair() - val (fromStdout, outStream) = pipePair() - val (fromStderr, errStream) = pipePair() - - val writer = new PrintWriter(toStdin) - val reader = new BufferedReader(new InputStreamReader(fromStdout)) - val errReader = new BufferedReader(new InputStreamReader(fromStderr)) - - val userThread = new Thread(new UserRunnable(jobQueue, writer, reader, errReader)) - - val shellThread = new Thread(() => { - val ammoniteShell = - ammonite - .Main( - predefCode = EmbeddedAmmonite.predef + predef, - welcomeBanner = None, - remoteLogging = false, - colors = Colors.BlackWhite, - inputStream = inStream, - outputStream = outStream, - errorStream = errStream - ) - ammoniteShell.run() - }) - - private def pipePair(): (PipedInputStream, PipedOutputStream) = { - val out = new PipedOutputStream() - val in = new PipedInputStream() - in.connect(out) - (in, out) - } - - /** Start the embedded ammonite shell - */ - def start(): Unit = { - shellThread.start() - userThread.start() - } - - /** Submit query `q` to shell and call `observer` when the result is ready. - */ - def queryAsync(q: String)(observer: QueryResult => Unit): UUID = { - val uuid = UUID.randomUUID() - jobQueue.add(Job(uuid, q, observer)) - uuid - } - - /** Submit query `q` to the shell and return result. - */ - def query(q: String): QueryResult = { - val mutex = new Semaphore(0) - var result: QueryResult = null - queryAsync(q) { r => - result = r - mutex.release() - } - mutex.acquire() - result - } - - /** Shutdown the embedded ammonite shell and associated threads. - */ - def shutdown(): Unit = { - shutdownShellThread() - logger.info("Shell terminated gracefully") - shutdownWriterThread() - - def shutdownWriterThread(): Unit = { - jobQueue.add(Job(null, null, null)) - userThread.join() - } - def shutdownShellThread(): Unit = { - writer.println("exit") - writer.close() - shellThread.join() - } - } - -} - -object EmbeddedAmmonite { - - /* The standard frontend attempts to query /dev/tty - in multiple places, e.g., to query terminal dimensions. - This does not work in intellij tests - (see https://github.com/lihaoyi/Ammonite/issues/276) - The below hack overrides the default frontend with - a custom frontend that does not require /dev/tty. - This also enables us to disable terminal echo - by passing a `displayTransform` that returns - an empty string on all input. - */ - - val predef: String = - """class CustomFrontend extends ammonite.repl.AmmoniteFrontEnd(ammonite.compiler.Parsers) { - | override def width = 65536 - | override def height = 65536 - | - | override def readLine(reader: java.io.Reader, - | output: java.io.OutputStream, - | prompt: String, - | colors: ammonite.util.Colors, - | compilerComplete: (Int, String) => (Int, Seq[String], Seq[String]), - | history: IndexedSeq[String]) = { - | - | val writer = new java.io.OutputStreamWriter(output) - | - | val multilineFilter = ammonite.terminal.Filter.action( - | ammonite.terminal.SpecialKeys.NewLine, - | ti => ammonite.compiler.Parsers.split(ti.ts.buffer.mkString).isEmpty) { - | case ammonite.terminal.TermState(rest, b, c, _) => ammonite.terminal.filters.BasicFilters.injectNewLine(b, c, rest) - | } - | - | val allFilters = ammonite.terminal.Filter.merge(extraFilters, multilineFilter, ammonite.terminal.filters.BasicFilters.all) - | - | new ammonite.terminal.LineReader(width, prompt, reader, writer, allFilters, displayTransform = { (_: Vector[Char], i: Int) => (fansi.Str(""), i) } ) - | .readChar(ammonite.terminal.TermState(ammonite.terminal.LazyList.continually(reader.read()), Vector.empty, 0, ""), 0) - | } - |} - | - |repl.frontEnd() = new CustomFrontend() - |""".stripMargin - - private val logger: Logger = LoggerFactory.getLogger(classOf[EmbeddedAmmonite]) -} diff --git a/console/src/main/scala/io/joern/console/embammonite/UserRunnable.scala b/console/src/main/scala/io/joern/console/embammonite/UserRunnable.scala deleted file mode 100644 index 6ee4cb0b28ca..000000000000 --- a/console/src/main/scala/io/joern/console/embammonite/UserRunnable.scala +++ /dev/null @@ -1,92 +0,0 @@ -package io.joern.console.embammonite - -import io.joern.console.GlobalReporting -import org.slf4j.{Logger, LoggerFactory} - -import java.io.{BufferedReader, PrintWriter} -import java.util.UUID -import java.util.concurrent.BlockingQueue -import scala.util.Try - -class UserRunnable(queue: BlockingQueue[Job], writer: PrintWriter, reader: BufferedReader, errReader: BufferedReader) - extends Runnable { - - private val logger: Logger = LoggerFactory.getLogger(classOf[UserRunnable]) - - private val magicEchoSeq: Seq[Char] = List(27, 91, 57, 57, 57, 57, 68, 27, 91, 48, 74, 64, 32).map(_.toChar) - private val endMarker = """.*END: ([0-9a-f\-]+)""".r - - override def run(): Unit = { - try { - var terminate = false - while (!(terminate && queue.isEmpty)) { - val job = queue.take() - if (isTerminationMarker(job)) { - terminate = true - } else { - sendQueryToAmmonite(job) - val stdoutPair = stdOutUpToMarker() - val stdOutput = GlobalReporting.getAndClearGlobalStdOut() + stdoutPair.get - - val errOutput = exhaustStderr() - val result = new QueryResult(stdOutput, errOutput, job.uuid) - job.observer(result) - } - } - } catch { - case _: InterruptedException => - logger.info("Interrupted WriterThread") - } - logger.debug("WriterThread terminated gracefully") - } - - private def isTerminationMarker(job: Job): Boolean = { - job.uuid == null && job.query == null - } - - private def sendQueryToAmmonite(job: Job): Unit = { - writer.println(job.query.trim) - writer.println(s""""END: ${job.uuid}"""") - writer.println(s"""throw new RuntimeException("END: ${job.uuid}")""") - writer.flush() - } - - private def stdOutUpToMarker(): Option[String] = { - var currentOutput: String = "" - var line = reader.readLine() - while (line != null) { - if (!line.startsWith(magicEchoSeq) && line.nonEmpty) { - val uuid = uuidFromLine(line) - if (uuid.isEmpty) { - currentOutput += line + "\n" - } else { - return Some(currentOutput) - } - } - line = reader.readLine() - } - None - } - - private def uuidFromLine(line: String): Iterator[UUID] = { - endMarker.findAllIn(line).matchData.flatMap { m => - Try { UUID.fromString(m.group(1)) }.toOption - } - } - - private def exhaustStderr(): String = { - var currentOutput = "" - var line = errReader.readLine() - while (line != null) { - val uuid = uuidFromLine(line) - if (uuid.isEmpty) { - currentOutput += line - } else { - return currentOutput - } - line = errReader.readLine() - } - currentOutput - } - -} diff --git a/console/src/main/scala/io/joern/console/qdbwserver/QDBWServer.scala b/console/src/main/scala/io/joern/console/qdbwserver/QDBWServer.scala deleted file mode 100644 index 545c29cee1b3..000000000000 --- a/console/src/main/scala/io/joern/console/qdbwserver/QDBWServer.scala +++ /dev/null @@ -1,19 +0,0 @@ -package io.joern.console.qdbwserver - -import cask.MainRoutes - -class QDBWServer(serverHost: String, serverPort: Int, contentDir: String) extends MainRoutes { - - override def port: Int = { - serverPort - } - - override def host: String = { - serverHost - } - - @cask.staticFiles("/") - def staticFileRoutes(): String = contentDir - - initialize() -} diff --git a/console/src/main/scala/io/joern/console/scripting/AmmoniteExecutor.scala b/console/src/main/scala/io/joern/console/scripting/AmmoniteExecutor.scala deleted file mode 100644 index 93e424e43d3f..000000000000 --- a/console/src/main/scala/io/joern/console/scripting/AmmoniteExecutor.scala +++ /dev/null @@ -1,141 +0,0 @@ -package io.joern.console.scripting - -import ammonite.Main -import ammonite.runtime.Storage -import ammonite.util.{Bind, Res} -import cats.effect.IO -import cats.instances.list._ -import cats.syntax.traverse._ -import io.shiftleft.codepropertygraph.Cpg - -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} - -/** Provides an interface for the execution of scripts using the Ammonite interpreter. - * - * All scripts are compiled in-memory and no caching is performed. - */ -trait AmmoniteExecutor { - - protected def predef: String - - protected lazy val ammoniteMain: Main = ammonite.Main( - predefCode = predef, - remoteLogging = false, - verboseOutput = false, - welcomeBanner = None, - storageBackend = Storage.InMemory() - ) - - /** Runs the given script, passing any defined parameters in addition to bringing the provided variable bindings into - * scope. - * - * @param scriptPath - * A path pointing to the Ammonite script to be executed. - * @param parameters - * A map of parameters to be passed to the script, useful if you have a @main method in the script. - * @param bindings - * A list of variable bindings made implicitly available to scripts. - * @return - * The result of running the script. - */ - def runScript(scriptPath: Path, parameters: Map[String, String], bindings: IndexedSeq[Bind[_]]): IO[Any] = { - val args: Seq[String] = parameters.flatMap { case (key, value) => Seq(s"--$key", value) }.toSeq - for { - replInstance <- IO(ammoniteMain.instantiateRepl(bindings)) - repl <- IO.fromEither(replInstance.left.map { case (err, _) => new RuntimeException(err.msg) }) - ammoniteResult <- IO { - repl.initializePredef() - val wd = if (ammoniteMain.wd.wrapped.getRoot != scriptPath.getRoot) { - os.Path(scriptPath.getParent) - } else { - ammoniteMain.wd - } - ammonite.main.Scripts.runScript(wd, os.Path(scriptPath), repl.interp, args) - } - result <- ammoniteResult match { - case Res.Success(res) => IO.pure(res) - case Res.Exception(ex, _) => IO.raiseError(ex) - case Res.Failure(msg) => IO.raiseError(new RuntimeException(msg)) - case _ => IO.unit - } - } yield result - } - - /** Runs the given script, passing any defined parameters in addition to bringing a cpg into scope. - * - * @param scriptPath - * A path pointing to the Ammonite script to be executed. - * @param parameters - * A map of parameters to be passed to the script, useful if you have a @main method in the script. - * @param cpg - * A CPG that is made implicitly available in the script. - * @return - * The result of running the script. - */ - def runScript(scriptPath: Path, parameters: Map[String, String], cpg: Cpg): IO[Any] = { - runScript(scriptPath, parameters, bindings = IndexedSeq("cpg" -> cpg)) - } - - /** Runs multiple scripts in the order they are specified in `scriptPaths`. - * - * @param scriptPaths - * A list of paths pointing to Ammonite scripts to be executed. - * @param parameters - * A map from script path to a set of parameter key/values. If no entry is found for a script, an empty set of - * params will be passed to the interpreter. - * @param bindings - * A list of variable bindings made implicitly available to scripts. - * @return - * A list containing the results of running each script, in order. - */ - def runScripts( - scriptPaths: List[Path], - parameters: Map[Path, Map[String, String]], - bindings: IndexedSeq[Bind[_]] - ): IO[List[Any]] = { - scriptPaths.map { scriptPath => - val scriptParams = parameters.getOrElse(scriptPath, Map.empty) - runScript(scriptPath, scriptParams, bindings) - }.sequence - } - - /** Runs multiple scripts in the order they are specified in `scriptPaths`. - * - * @param scriptPaths - * A list of paths pointing to Ammonite scripts to be executed. - * @param parameters - * A map from script path to a set of parameter key/values. If no entry is found for a script, an empty set of - * params will be passed to the interpreter. - * @param cpg - * A CPG that is made implicitly available in the scripts. - * @return - * A list containing the results of running each script, in order. - */ - def runScripts(scriptPaths: List[Path], parameters: Map[Path, Map[String, String]], cpg: Cpg): IO[Any] = { - runScripts(scriptPaths, parameters, bindings = IndexedSeq("cpg" -> cpg)) - } - - /** Runs a query against the provided CPG. - * - * @param query - * The query to run against the CPG. - * @param cpg - * The CPG made implicitly available in the query - * @return - * The result of running the query. - */ - def runQuery(query: String, cpg: Cpg): IO[Any] = { - val queryContent = - s"""|@main def main() = { - |$query - |} - |""".stripMargin - - for { - tempFile <- IO(Files.createTempFile("sl_query", ".sc")) - _ <- IO(Files.write(tempFile, queryContent.getBytes(StandardCharsets.UTF_8))) - result <- runScript(tempFile, Map.empty, cpg) - } yield result - } -} diff --git a/console/src/main/scala/io/joern/console/scripting/ScriptManager.scala b/console/src/main/scala/io/joern/console/scripting/ScriptManager.scala deleted file mode 100644 index 5aac28f4e857..000000000000 --- a/console/src/main/scala/io/joern/console/scripting/ScriptManager.scala +++ /dev/null @@ -1,138 +0,0 @@ -package io.joern.console.scripting - -import better.files._ -import cats.effect.IO -import io.circe.generic.auto._ -import io.circe.parser._ -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.cpgloading.CpgLoader -import org.zeroturnaround.zip.{NameMapper, ZipUtil} - -import java.nio.file.{Files, NoSuchFileException} -import java.util.regex.Pattern -import scala.util.Try - -object ScriptManager { - final case class ScriptCollections(collection: String, scripts: ScriptDescriptions) - final case class ScriptDescriptions(description: String, scripts: List[ScriptDescription]) - final case class ScriptDescription(name: String, description: String) - - private val SCRIPT_DESCS: String = "scripts.json" -} - -/** This class manages a hierarchy of scripts, and provides an interface that allows users to easily discover and run - * scripts on their CPGs. - * - * Scripts should be grouped inside folders placed within the application's `resources/scripts` directory, for example: - * - * resources - * -- scripts - * ---- java - * ------ my-java-script.sc - * ---- go - * ---- csharp - * - * To run `my-java-script.sc` you would run: `runScript("java/my-java-script.sc", cpg)` - * - * @param executor - * An executor that is used to run the managed scripts. - */ -abstract class ScriptManager(executor: AmmoniteExecutor) { - - import ScriptManager._ - - implicit class CpgScriptRunner(cpg: Cpg) { - - /** Run an arbitrary script over this CPG. - * - * @param name - * The name of the script to run. - * @return - * The result of running the script against this CPG. - */ - def runScript(name: String): Any = - runScript(name, Map.empty) - - /** Run an arbitrary script over this CPG with parameters. - * - * @param name - * The name of the script to run. - * @param parameters - * The parameters to pass to the script. - * @return - * The result of running the script against this CPG. - */ - def runScript(name: String, parameters: Map[String, String]): Any = - ScriptManager.this.runScript(name, parameters, cpg) - } - - private val absoluteJarPathRegex = ("jar:file:(.*)!" + Pattern.quote(java.io.File.separator) + "scripts").r - private val scriptFileRegex = ("(scripts" + Pattern.quote(java.io.File.separator) + ".*)").r - private val scriptDir = "scripts" - - // This is to work around Ammonite failing to read resource files on the classpath. - // We simply copy the files into a temporary directory and read from there. - private lazy val scriptsTempDir: File = { - val newScriptsDir = File(Files.createTempDirectory("sl_scripts")) - - val scriptsPath = this.getClass.getClassLoader.getResource(scriptDir).toURI - if (scriptsPath.getScheme.contains("jar")) { - // get absolute jar path from classpath URI - scriptsPath.toString match { - case absoluteJarPathRegex(jarPath) => - ZipUtil.unpack( - new java.io.File(jarPath), - newScriptsDir.toJava, - new NameMapper { - override def map(name: String): String = name match { - case scriptFileRegex(scriptFile) => scriptFile - case _ => null - } - } - ) - } - } else { - File(scriptsPath).copyToDirectory(newScriptsDir) - } - - newScriptsDir / scriptDir - } - - def scripts(): List[ScriptCollections] = { - scriptsTempDir - .collectChildren(f => f.isDirectory && f != scriptsTempDir) - .map { dir => - val relativeDir = scriptsTempDir.relativize(dir) - - val scriptDescs = - Try((dir / SCRIPT_DESCS).lines.mkString(System.lineSeparator())).toEither - .flatMap(v => decode[ScriptDescriptions](v)) - .toOption - .getOrElse(ScriptDescriptions("", List.empty)) - - ScriptCollections(relativeDir.toString, scriptDescs) - } - .toList - } - - protected def withScriptFile[T](scriptName: String)(f: File => IO[T]): IO[T] = { - val scriptPath = scriptsTempDir / scriptName - if (scriptPath.exists) { - f(scriptPath) - } else { - IO.raiseError(new NoSuchFileException(s"Script [$scriptPath] was not found.")) - } - } - - def runScript(scriptName: String, parameters: Map[String, String], cpgFileName: String): Any = { - runScript(scriptName, parameters, CpgLoader.load(cpgFileName)) - } - - def runScript(scriptName: String, parameters: Map[String, String], cpg: Cpg): Any = { - // needed in the latest cast-effect version to run .unsafeRunSync() - import cats.effect.unsafe.implicits.global - withScriptFile(scriptName) { script => - executor.runScript(script.path, parameters, cpg) - }.unsafeRunSync() - } -} diff --git a/console/src/main/scala/io/joern/console/workspacehandling/Project.scala b/console/src/main/scala/io/joern/console/workspacehandling/Project.scala index 01fc9d23e61f..1bee29a97999 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/Project.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/Project.scala @@ -27,6 +27,8 @@ case class Project(projectFile: ProjectFile, var path: Path, var cpg: Option[Cpg def inputPath: String = projectFile.inputPath + def isOpen: Boolean = cpg.isDefined + def appliedOverlays: Seq[String] = { cpg.map(Overlays.appliedOverlays).getOrElse(Nil) } @@ -55,7 +57,7 @@ case class Project(projectFile: ProjectFile, var path: Path, var cpg: Option[Cpg def close: Project = { cpg.foreach { c => c.close() - System.err.println("Turning working copy into new persistent CPG") + System.err.println(s"closing/saving project `$name`") val workingCopy = path.resolve(workCpgFileName) val persistent = path.resolve(persistentCpgFileName) cp(workingCopy, persistent) diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala index a7e33fe1e843..9019e10bdc8b 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceLoader.scala @@ -21,10 +21,7 @@ abstract class WorkspaceLoader[ProjectType <: Project] { val dirFile = File(path) val dirPath = dirFile.path.toAbsolutePath - if (!dirFile.exists) { - println(s"creating workspace directory: ${dirFile.path.toString}") - mkdirs(dirFile) - } + mkdirs(dirFile) new Workspace(ListBuffer.from(loadProjectsFromFs(dirPath))) } @@ -43,7 +40,7 @@ abstract class WorkspaceLoader[ProjectType <: Project] { case Success(v) => Some(v) case Failure(e) => System.err.println(s"Error loading project at $path - skipping: ") - System.err.println(e) + e.printStackTrace None } } diff --git a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala index d6150b5e4a33..e27e12414ec1 100644 --- a/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala +++ b/console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala @@ -98,7 +98,7 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo /** Delete the workspace from disk, then initialize it again. */ - def reset(): Unit = { + def reset: Unit = { Try(cpg.close()) deleteWorkspace() workspace = loader.load(path) @@ -213,7 +213,10 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo case Some(p) => p.cpg match { case Some(value) => value - case None => throw new RuntimeException(s"No CPG loaded for project ${p.name}") + case None => + throw new RuntimeException( + s"No CPG loaded for project ${p.name} - try e.g. `help|importCode|importCpg|open`" + ) } case None => throw new RuntimeException("No projects loaded") } diff --git a/console/src/test/scala/io/joern/console/ConsoleTests.scala b/console/src/test/scala/io/joern/console/ConsoleTests.scala index 49421db03b24..0356441dff53 100644 --- a/console/src/test/scala/io/joern/console/ConsoleTests.scala +++ b/console/src/test/scala/io/joern/console/ConsoleTests.scala @@ -27,18 +27,19 @@ class ConsoleTests extends AnyWordSpec with Matchers { "importCode" should { "warn about non-existent dir" in ConsoleFixture() { (console, _) => val nonExistentDir = "/does/not/exist/" - the[ConsoleException] thrownBy console.importCode( - nonExistentDir - ) should have message s"Input path does not exist: '$nonExistentDir'" - the[ConsoleException] thrownBy console.importCode.c( - nonExistentDir - ) should have message s"Input path does not exist: '$nonExistentDir'" - the[ConsoleException] thrownBy console.importCode.jssrc( - nonExistentDir - ) should have message s"Input path does not exist: '$nonExistentDir'" - the[ConsoleException] thrownBy console.importCode.java( - nonExistentDir - ) should have message s"Input path does not exist: '$nonExistentDir'" + + intercept[ConsoleException] { + console.importCode(nonExistentDir) + }.getMessage shouldBe s"Input path does not exist: '$nonExistentDir'" + intercept[ConsoleException] { + console.importCode.c(nonExistentDir) + }.getMessage shouldBe s"Input path does not exist: '$nonExistentDir'" + intercept[ConsoleException] { + console.importCode.jssrc(nonExistentDir) + }.getMessage shouldBe s"Input path does not exist: '$nonExistentDir'" + intercept[ConsoleException] { + console.importCode.java(nonExistentDir) + }.getMessage shouldBe s"Input path does not exist: '$nonExistentDir'" } "provide overview of available language modules" in ConsoleFixture() { (console, _) => diff --git a/console/src/test/scala/io/joern/console/PPrinterTest.scala b/console/src/test/scala/io/joern/console/PPrinterTest.scala deleted file mode 100644 index efe0b09161ea..000000000000 --- a/console/src/test/scala/io/joern/console/PPrinterTest.scala +++ /dev/null @@ -1,114 +0,0 @@ -package io.joern.console - -import fansi.Color -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -/** We use source-highlight to encode source as ansi strings, e.g. the .dump step Ammonite uses fansi for it's - * colour-coding, and while both pledge to follow the ansi codec, they aren't compatible TODO: PR for fansi to support - * these standard encodings out of the box - */ -class PPrinterTest extends AnyWordSpec with Matchers { - - "a regular List" in { - val pp = pprinter.create() - pp(List(1, 2, 3)).plainText shouldBe "List(1, 2, 3)" - pp(List(1, 2, 3)) shouldBe fansi.Str.join( - Seq( - Color.Yellow("List"), - Color.Reset("("), - Color.Green("1"), - Color.Reset(", "), - Color.Green("2"), - Color.Reset(", "), - Color.Green("3"), - Color.Reset(")") - ) - ) - } - - "a case class" in { - val pp = pprinter.create() - case class Foo(i: Int, s: String) - pp(Foo(42, "bar")).plainText shouldBe """Foo(i = 42, s = "bar")""" - pp(Foo(42, "bar")) shouldBe fansi.Str.join( - Seq( - Color.Yellow("Foo"), - Color.Reset("(i = "), - Color.Green("42"), - Color.Reset(", s = "), - Color.Green("\"bar\""), - Color.Reset(")") - ) - ) - } - - "a Product with productElementNames" in { - val pp = pprinter.create() - - val product = new Product { - def canEqual(that: Any): Boolean = false - def productArity: Int = 2 - def productElement(n: Int): Any = { - n match { - case 0 => 42 - case 1 => "println(foo)" - } - } - override def productElementName(n: Int): String = { - n match { - case 0 => "id" - case 1 => "code" - } - } - } - - pp(product).plainText shouldBe """(id = 42, code = "println(foo)")""" - pp(product) shouldBe fansi.Str.join( - Seq( - Color.Reset("(id = "), - Color.Green("42"), - Color.Reset(", code = "), - Color.Green("\"println(foo)\""), - Color.Reset(")") - ) - ) - } - - // ansi colour-encoded strings as source-highlight produces them - val IntGreenForeground = "\u001b[32mint\u001b[m" - val IfBlueBold = "\u001b[01;34mif\u001b[m" - val FBold = "\u001b[01mF\u001b[m" - val X8bit = "\u001b[00;38;05;70mX\u001b[m" - - "fansi encoding fix" must { - "handle different ansi encoding termination" in { - // encoding ends with [39m for fansi instead of [m - val fixedForFansi = pprinter.fixForFansi(IntGreenForeground) - fixedForFansi shouldBe "\u001b[32mint\u001b[39m" - fansi.Str(fixedForFansi) shouldBe fansi.Color.Green("int") - } - - "handle different single-digit encodings" in { - // `[01m` is encoded as `[1m` in fansi for all single digit numbers - val fixedForFansi = pprinter.fixForFansi(FBold) - fixedForFansi shouldBe "\u001b[1mF\u001b[39m" - fansi.Str(fixedForFansi) shouldBe fansi.Str("F").overlay(fansi.Bold.On) - } - - "handle multi-encoded parts" in { - // `[01;34m` is encoded as `[1m[34m` in fansi - val fixedForFansi = pprinter.fixForFansi(IfBlueBold) - fixedForFansi shouldBe "\u001b[1m\u001b[34mif\u001b[39m" - fansi.Str(fixedForFansi) shouldBe fansi.Color.Blue("if").overlay(fansi.Bold.On) - } - - "handle 8bit (256) 'full' colors" in { - // `[00;38;05;70m` is encoded as `[38;5;70m` in fansi - val fixedForFansi = pprinter.fixForFansi(X8bit) - fixedForFansi shouldBe "\u001b[38;5;70mX\u001b[39m" - fansi.Str(fixedForFansi) shouldBe fansi.Color.Full(70)("X") - } - } - -} diff --git a/console/src/test/scala/io/joern/console/cpgqlserver/CPGQLServerTests.scala b/console/src/test/scala/io/joern/console/cpgqlserver/CPGQLServerTests.scala deleted file mode 100644 index dec653fdc390..000000000000 --- a/console/src/test/scala/io/joern/console/cpgqlserver/CPGQLServerTests.scala +++ /dev/null @@ -1,353 +0,0 @@ -package io.joern.console.cpgqlserver - -import cask.util.Logger.Console._ -import castor.Context.Simple.global -import io.joern.console.embammonite.EmbeddedAmmonite -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec -import requests.RequestFailedException -import ujson.Value.Value - -import java.net.URLEncoder -import java.util.UUID -import java.util.concurrent.locks.{Lock, ReentrantLock} -import scala.collection.mutable.ListBuffer -import scala.concurrent._ -import scala.concurrent.duration._ - -class CPGQLServerTests extends AnyWordSpec with Matchers { - private val validBasicAuthHeaderVal: String = "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" - - private val DefaultPromiseAwaitTimeout: FiniteDuration = Duration(10, SECONDS) - - private def postQuery(host: String, query: String, authHeaderVal: String = validBasicAuthHeaderVal): Value = { - val postResponse = requests.post( - s"$host/query", - data = ujson.Obj("query" -> query).toString, - headers = Seq("authorization" -> authHeaderVal) - ) - val res = - if (postResponse.bytes.length > 0) - ujson.read(postResponse.bytes) - else - ujson.Obj() - res - } - - private def getResponse(host: String, uuidParam: String, authHeaderVal: String = validBasicAuthHeaderVal): Value = { - val uri = s"$host/result/${URLEncoder.encode(uuidParam, "utf-8")}" - val getResponse = requests.get(uri, headers = Seq("authorization" -> authHeaderVal)) - ujson.read(getResponse.bytes) - } - - /** These tests happen to fail on github actions for the windows runner with the following output: WARNING: Unable to - * create a system terminal, creating a dumb terminal (enable debug logging for more information) Apr 21, 2022 - * 3:08:54 PM org.jboss.threads.Version INFO: JBoss Threads version 3.1.0.Final Apr 21, 2022 3:08:55 PM - * io.undertow.server.HttpServerExchange endExchange ERROR: UT005090: Unexpected failure - * java.lang.NoClassDefFoundError: Could not initialize class org.xnio.channels.Channels at - * io.undertow.io.UndertowOutputStream.close(UndertowOutputStream.java:348) - * - * This happens for both windows 2019 and 2022, and isn't reproducable elsewhere. Explicitly adding a dependency on - * `org.jboss.xnio/xnio-api` didn't help, as well as other debug attempts. So we gave up and disabled this - * specifically for github actions' windows runner. - */ - val isGithubActions = scala.util.Properties.envOrElse("GITHUB_ACTIONS", "false").toLowerCase == "true" - val isWindows = scala.util.Properties.isWin - - if (isGithubActions && isWindows) { - info("tests were cancelled because github actions windows doesn't support them for some unknown reason...") - } else { - "CPGQLServer" should { - - "allow websocket connections to the `/connect` endpoint" in Fixture() { host => - val wsMsgPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - val wsMsg = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - wsMsg shouldBe "connected" - } - - "allow posting a simple query without any websocket connections established" in Fixture() { host => - val postQueryResponse = postQuery(host, "1") - postQueryResponse.obj.keySet should contain("success") - val UUIDResponse = postQueryResponse("uuid").str - UUIDResponse should not be empty - postQueryResponse("success").bool shouldBe true - } - - "disallow posting a query when request headers do not include a valid authentication value" in Fixture() { host => - assertThrows[RequestFailedException] { - postQuery(host, "1", authHeaderVal = "Basic b4df00d") - } - } - - "return a valid JSON response when trying to retrieve the result of a query without a connection" in Fixture() { - host => - val postQueryResponse = postQuery(host, "1") - postQueryResponse.obj.keySet should contain("uuid") - val UUIDResponse = postQueryResponse("uuid").str - val getResultResponse = getResponse(host, UUIDResponse) - getResultResponse.obj.keySet should contain("success") - getResultResponse.obj.keySet should contain("err") - getResultResponse("success").bool shouldBe false - getResultResponse("err").str.length should not be 0 - } - - "allow fetching the result of a completed query using its UUID" in Fixture() { host => - val wsMsgPromise = scala.concurrent.Promise[String]() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { - case cask.Ws.Text(msg) if msg == "connected" => - connectedPromise.success(msg) - case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - val postQueryResponse = postQuery(host, "1") - val queryUUID = postQueryResponse("uuid").str - queryUUID.length should not be 0 - - val queryResultWSMessage = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - queryResultWSMessage.length should not be 0 - - val getResultResponse = getResponse(host, queryUUID) - getResultResponse.obj.keySet should contain("success") - getResultResponse("uuid").str shouldBe queryResultWSMessage - getResultResponse("stdout").str shouldBe "res0: Int = 1\n" - } - - "disallow fetching the result of a completed query with an invalid auth header" in Fixture() { host => - val wsMsgPromise = scala.concurrent.Promise[String]() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { - case cask.Ws.Text(msg) if msg == "connected" => - connectedPromise.success(msg) - case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - val postQueryResponse = postQuery(host, "1") - val queryUUID = postQueryResponse("uuid").str - queryUUID.length should not be 0 - - val queryResultWSMessage = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - queryResultWSMessage.length should not be 0 - - assertThrows[RequestFailedException] { - getResponse(host, queryUUID, "Basic b4df00d") - } - } - - "write a well-formatted message to a websocket connection when a query has finished evaluation" in Fixture() { - host => - val wsMsgPromise = scala.concurrent.Promise[String]() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { - case cask.Ws.Text(msg) if msg == "connected" => - connectedPromise.success(msg) - case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - - val postQueryResponse = postQuery(host, "1") - val queryUUID = postQueryResponse("uuid").str - queryUUID.length should not be 0 - - val queryResultWSMessage = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - queryResultWSMessage.length should not be 0 - - val getResultResponse = getResponse(host, queryUUID) - getResultResponse.obj.keySet should contain("success") - getResultResponse.obj.keySet should contain("stdout") - getResultResponse.obj.keySet should contain("stderr") - getResultResponse.obj.keySet should not contain "err" - getResultResponse("uuid").str shouldBe queryResultWSMessage - getResultResponse("stdout").str shouldBe "res0: Int = 1\n" - } - - "write a well-formatted message to a websocket connection when a query failed evaluation" in Fixture() { host => - val wsMsgPromise = scala.concurrent.Promise[String]() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { - case cask.Ws.Text(msg) if msg == "connected" => - connectedPromise.success(msg) - case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - - val postQueryResponse = postQuery(host, "if else for loop soup // i.e., an invalid Ammonite query") - val queryUUID = postQueryResponse("uuid").str - queryUUID.length should not be 0 - - val wsMsg = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - wsMsg.length should not be 0 - - val resp = getResponse(host, queryUUID) - resp.obj.keySet should contain("success") - resp.obj.keySet should contain("stdout") - resp.obj.keySet should contain("stderr") - resp.obj.keySet should not contain "err" - - resp("success").bool shouldBe true - resp("uuid").str shouldBe wsMsg - resp("stdout").str shouldBe "" - resp("stderr").str.length should not be 0 - } - - "write a well-formatted message to a websocket connection when a query containing an invalid char is submitted" in Fixture() { - host => - val wsMsgPromise = scala.concurrent.Promise[String]() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { - case cask.Ws.Text(msg) if msg == "connected" => - connectedPromise.success(msg) - case cask.Ws.Text(msg) => - wsMsgPromise.success(msg) - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - - val postQueryResponse = postQuery(host, "@1") - val queryUUID = postQueryResponse("uuid").str - queryUUID.length should not be 0 - - val wsMsg = Await.result(wsMsgPromise.future, DefaultPromiseAwaitTimeout) - wsMsg.length should not be 0 - - val resp = getResponse(host, queryUUID) - resp.obj.keySet should contain("success") - resp.obj.keySet should contain("stdout") - resp.obj.keySet should contain("stderr") - resp.obj.keySet should not contain "err" - - resp("success").bool shouldBe true - resp("uuid").str shouldBe wsMsg - resp("stdout").str shouldBe "" - resp("stderr").str.length should not be 0 - } - } - - "receive error when attempting to retrieve result with invalid uuid" in Fixture() { host => - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { case cask.Ws.Text(msg) => - connectedPromise.success(msg) - } - Await.result(connectedPromise.future, Duration(1, SECONDS)) - val getResultResponse = getResponse(host, UUID.randomUUID().toString) - getResultResponse.obj.keySet should contain("success") - getResultResponse.obj.keySet should contain("err") - getResultResponse("success").bool shouldBe false - } - - "return a valid JSON response when calling /result with incorrectly-formatted UUID parameter" in Fixture() { host => - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { case cask.Ws.Text(msg) => - connectedPromise.success(msg) - } - Await.result(connectedPromise.future, Duration(1, SECONDS)) - val getResultResponse = getResponse(host, "INCORRECTLY_FORMATTED_UUID_PARAM") - getResultResponse.obj.keySet should contain("success") - getResultResponse.obj.keySet should contain("err") - getResultResponse("success").bool shouldBe false - getResultResponse("err").str.length should not equal 0 - } - - "return websocket responses for all queries when posted quickly in a large number" in Fixture() { host => - val numQueries = 10 - val correctNumberOfUUIDsReceived = scala.concurrent.Promise[String]() - val wsUUIDs = ListBuffer[String]() - - val rtl: Lock = new ReentrantLock() - val connectedPromise = scala.concurrent.Promise[String]() - cask.util.WsClient.connect(s"$host/connect") { case cask.Ws.Text(msg) => - if (msg == "connected") { - connectedPromise.success(msg) - } else { - rtl.lock() - try { - wsUUIDs += msg - } finally { - rtl.unlock() - if (wsUUIDs.size == numQueries) { - correctNumberOfUUIDsReceived.success("") - } - } - } - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - - val postQueriesResponseUUIDs = - for (_ <- 1 to numQueries) yield { - val postQueryResponse = postQuery(host, "1") - postQueryResponse("uuid").str - } - - Await.result(correctNumberOfUUIDsReceived.future, DefaultPromiseAwaitTimeout * numQueries.toLong) - wsUUIDs.toSet should be(postQueriesResponseUUIDs.toSet) - } - - "return websocket responses for all queries when some are invalid" in Fixture() { host => - val queries = List("1", "1 + 1", "open(", "open)", "open{", "open}") - val correctNumberOfUUIDsReceived = scala.concurrent.Promise[String]() - val wsUUIDs = ListBuffer[String]() - val connectedPromise = scala.concurrent.Promise[String]() - - val rtl: Lock = new ReentrantLock() - cask.util.WsClient.connect(s"$host/connect") { case cask.Ws.Text(msg) => - if (msg == "connected") { - connectedPromise.success(msg) - } else { - rtl.lock() - try { - wsUUIDs += msg - } finally { - rtl.unlock() - if (wsUUIDs.size == queries.size) { - correctNumberOfUUIDsReceived.success("") - } - } - } - } - Await.result(connectedPromise.future, DefaultPromiseAwaitTimeout) - - val postQueriesResponseUUIDs = { - queries - .map(q => { - val res = postQuery(host, q) - res("uuid").str - }) - } - Await.result(correctNumberOfUUIDsReceived.future, DefaultPromiseAwaitTimeout * queries.size.toLong) - wsUUIDs.toSet should be(postQueriesResponseUUIDs.toSet) - } - } -} - -object Fixture { - - def apply[T]()(f: String => T): T = { - val ammonite = new EmbeddedAmmonite() - ammonite.start() - - val host = "localhost" - val port = 8081 - val authUsername = "username" - val authPassword = "password" - val httpEndpoint = "http://" + host + ":" + port.toString - val ammServer = new CPGQLServer(ammonite, host, port, authUsername, authPassword) - val server = io.undertow.Undertow.builder - .addHttpListener(ammServer.port, ammServer.host) - .setHandler(ammServer.defaultHandler) - .build - server.start() - val res = - try { f(httpEndpoint) } - finally { - server.stop() - ammonite.shutdown() - } - res - } -} diff --git a/console/src/test/scala/io/joern/console/embammonite/EmbeddedAmmoniteTests.scala b/console/src/test/scala/io/joern/console/embammonite/EmbeddedAmmoniteTests.scala deleted file mode 100644 index a505302bd8f2..000000000000 --- a/console/src/test/scala/io/joern/console/embammonite/EmbeddedAmmoniteTests.scala +++ /dev/null @@ -1,42 +0,0 @@ -package io.joern.console.embammonite - -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import java.util.concurrent.Semaphore - -class EmbeddedAmmoniteTests extends AnyWordSpec with Matchers { - - "EmbeddedAmmoniteShell" should { - "start and shutdown without hanging" in { - val shell = new EmbeddedAmmonite() - shell.start() - shell.shutdown() - } - - "execute a command synchronously" in { - val shell = new EmbeddedAmmonite() - shell.start() - val result = shell.query("""def foo() = { - | 1 - |} - |foo()""".stripMargin) - result.out shouldBe "defined function foo\nres1: Int = 1\n" - shell.shutdown() - } - - "execute a command asynchronously" in { - val shell = new EmbeddedAmmonite() - val mutex = new Semaphore(0) - shell.start() - shell.queryAsync("val x = 0") { result => - result.out shouldBe "x: Int = 0\n" - mutex.release() - } - mutex.acquire() - shell.shutdown() - } - - } - -} diff --git a/console/src/test/scala/io/joern/console/scripting/AmmoniteExecutorTest.scala b/console/src/test/scala/io/joern/console/scripting/AmmoniteExecutorTest.scala deleted file mode 100644 index 325780734f94..000000000000 --- a/console/src/test/scala/io/joern/console/scripting/AmmoniteExecutorTest.scala +++ /dev/null @@ -1,85 +0,0 @@ -package io.joern.console.scripting - -import better.files.File -import io.shiftleft.codepropertygraph.Cpg -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -// needed in the latest cast-effect version to run .unsafeRunSync() -import cats.effect.unsafe.implicits.global - -import java.nio.file.{Path, Paths} - -class AmmoniteExecutorTest extends AnyWordSpec with Matchers { - private object TestAmmoniteExecutor extends AmmoniteExecutor { - override protected def predef: String = - """ - |import io.shiftleft.semanticcpg.language._ - |""".stripMargin - } - - private def getScriptPath(script: String): Path = { - val scriptURI = getClass.getClassLoader.getResource(script).toURI - Paths.get(scriptURI) - } - - private def withExecutor[T](f: AmmoniteExecutor => T): T = { - f(TestAmmoniteExecutor) - } - - "An AmmoniteExecutor" should { - "execute a single script with an implicit cpg in scope" in withExecutor { executor => - val script = getScriptPath("scripts/general/list-funcs.sc") - - executor.runScript(script, Map.empty, Cpg.emptyCpg).unsafeRunSync() shouldBe List() - } - - "pass arguments to a script" in withExecutor { executor => - val script = getScriptPath("scripts/general/arguments-concatenate.sc") - - executor - .runScript(script, Map("one" -> "hello", "two" -> "world"), Cpg.emptyCpg) - .unsafeRunSync() shouldBe "hello world" - } - - "execute multiple scripts" in withExecutor { executor => - val script = getScriptPath("scripts/general/list-funcs.sc") - val secondScript = getScriptPath("scripts/java/list-sl-ns.sc") - - executor.runScripts(List(script, secondScript), Map.empty, Cpg.emptyCpg).unsafeRunSync() shouldBe - List(List(), List()) - } - - "return a failure if the script can not be found" in withExecutor { executor => - val script = (File(os.pwd.toNIO) / "cake.sc").path - - val ex = intercept[RuntimeException] { - executor.runScript(script, Map.empty, Cpg.emptyCpg).unsafeRunSync() - } - - ex.getMessage shouldBe s"Script file not found: ${script.toString}" - } - - "propagate any exceptions thrown by a script" in withExecutor { executor => - val script = getScriptPath("scripts/general/divide_by_zero.sc") - - intercept[ArithmeticException] { - executor.runScript(script, Map.empty, Cpg.emptyCpg).unsafeRunSync() - } - } - - "run a string query" in withExecutor { executor => - val query = "cpg.method.l" - - executor.runQuery(query, Cpg.emptyCpg).unsafeRunSync() shouldBe List() - } - - "propagate errors if the string query fails" in withExecutor { executor => - val query = "cake" - - intercept[RuntimeException] { - executor.runQuery(query, Cpg.emptyCpg).unsafeRunSync() - } - } - } -} diff --git a/console/src/test/scala/io/joern/console/scripting/ScriptManagerTest.scala b/console/src/test/scala/io/joern/console/scripting/ScriptManagerTest.scala deleted file mode 100644 index b691d3c9557e..000000000000 --- a/console/src/test/scala/io/joern/console/scripting/ScriptManagerTest.scala +++ /dev/null @@ -1,102 +0,0 @@ -package io.joern.console.scripting - -import cats.effect.IO -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.cpgloading.TestProtoCpg -import io.joern.console.scripting.ScriptManager.{ScriptCollections, ScriptDescription, ScriptDescriptions} -import io.shiftleft.utils.IOUtils -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec -import org.scalatest.{BeforeAndAfterAll, Inside} - -import java.nio.file.{NoSuchFileException, Path} -import scala.util.Try - -class ScriptManagerTest extends AnyWordSpec with Matchers with Inside with BeforeAndAfterAll { - - var zipFile: better.files.File = _ - protected var DEFAULT_CPG_NAME: String = _ - - override def beforeAll(): Unit = { - zipFile = TestProtoCpg.createTestProtoCpg - DEFAULT_CPG_NAME = zipFile.pathAsString - } - - override def afterAll(): Unit = { - zipFile.delete() - } - - private object TestScriptExecutor extends AmmoniteExecutor { - override protected def predef: String = "" - - override def runScript(scriptPath: Path, parameters: Map[String, String], cpg: Cpg): IO[Any] = - IO.fromTry(Try(IOUtils.readLinesInFile(scriptPath).mkString("\n"))) - } - - private object TestScriptManager extends ScriptManager(TestScriptExecutor) - - def withScriptManager(f: ScriptManager => Unit): Unit = { - f(TestScriptManager) - } - - "listing scripts" should { - "be correct" in withScriptManager { scriptManager => - val scripts = scriptManager.scripts() - val expected = List( - ScriptCollections( - "general", - ScriptDescriptions( - "A collection of general purpose scripts.", - List(ScriptDescription("list-funcs.sc", "Lists all functions.")) - ) - ), - ScriptCollections( - "java", - ScriptDescriptions( - "A collection of java-specific scripts.", - List(ScriptDescription("list-sl-ns.sc", "Lists all shiftleft namespaces.")) - ) - ), - ScriptCollections( - s"general${java.io.File.separator}general_plus", - ScriptDescriptions("Even more general purpose scripts.", List.empty) - ) - ) - - scripts should contain theSameElementsAs expected - } - } - - "running scripts" should { - "be correct when explicitly specifying a CPG" in withScriptManager { scriptManager => - val expected = - """|@main def main() = { - | cpg.method.name.l - |}""".stripMargin - - scriptManager.runScript("general/list-funcs.sc", Map.empty, Cpg.emptyCpg) shouldBe expected - } - - "be correct when specifying a CPG filename" in withScriptManager { scriptManager => - val expected = - """|@main def main() = { - | cpg.method.name.l - |}""".stripMargin - - scriptManager.runScript("general/list-funcs.sc", Map.empty, DEFAULT_CPG_NAME) shouldBe expected - } - - "throw an exception if the specified CPG can not be found" in withScriptManager { scriptManager => - intercept[Exception] { - scriptManager.runScript("general/list-funcs.sc", Map.empty, "cake.bin.zip") - } - } - - "throw an exception if the specified script can not be found" in withScriptManager { scriptManager => - intercept[NoSuchFileException] { - scriptManager.runScript("list-funcs.sc", Map.empty, Cpg.emptyCpg) - } - } - } - -} diff --git a/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala b/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala index 4c7627c034f1..9c13907952eb 100644 --- a/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala +++ b/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala @@ -4,7 +4,7 @@ import better.files.Dsl.mkdir import better.files.File import io.joern.console.cpgcreation.{CCpgGenerator, CpgGenerator, CpgGeneratorFactory, ImportCode} import io.joern.console.workspacehandling.{Project, ProjectFile, WorkspaceLoader} -import io.joern.console.{Console, ConsoleConfig, DefaultAmmoniteExecutor, FrontendConfig, InstallConfig} +import io.joern.console.{Console, ConsoleConfig, FrontendConfig, InstallConfig} import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.utils.ProjectRoot @@ -25,7 +25,7 @@ object ConsoleFixture { val console = constructor(workspaceDir.toString) fun(console, codeDir) Try(console.cpgs.foreach(cpg => cpg.close())) - Try(console.workspace.reset()) + Try(console.workspace.reset) } } } @@ -36,8 +36,7 @@ object TestWorkspaceLoader extends WorkspaceLoader[Project] { override def createProject(projectFile: ProjectFile, path: Path): Project = Project(projectFile, path) } -class TestConsole(workspaceDir: String) - extends Console[Project](DefaultAmmoniteExecutor, TestWorkspaceLoader, File(workspaceDir)) { +class TestConsole(workspaceDir: String) extends Console[Project](TestWorkspaceLoader, File(workspaceDir)) { override def config = new ConsoleConfig(install = new InstallConfig(Map("SHIFTLEFT_OCULAR_INSTALL_DIR" -> workspaceDir))) diff --git a/console/src/test/scala/io/joern/console/testing/package.scala b/console/src/test/scala/io/joern/console/testing/package.scala index 1cd94da66d64..b30149b4a961 100644 --- a/console/src/test/scala/io/joern/console/testing/package.scala +++ b/console/src/test/scala/io/joern/console/testing/package.scala @@ -15,7 +15,7 @@ package object testing { val cpgPath = project.get.path.resolve("cpg.bin") File.usingTemporaryFile("console", suffix = "cpg.bin") { tmpCpg => cp(cpgPath, tmpCpg) - Try(console.workspace.reset()) + Try(console.workspace.reset) fun(tmpCpg) } } diff --git a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala index f6a7a26cdbe3..009f850b1419 100644 --- a/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala +++ b/console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala @@ -54,7 +54,7 @@ class WorkspaceManagerTests extends AnyWordSpec with Matchers { WorkspaceTests.createFakeProject(tmpDir, "1") val manager = new WorkspaceManager(tmpDir.toString) manager.numberOfProjects shouldBe 1 - manager.reset() + manager.reset manager.numberOfProjects shouldBe 0 } } @@ -90,7 +90,7 @@ class WorkspaceManagerTests extends AnyWordSpec with Matchers { val manager = new WorkspaceManager[Project](workspaceFile.toString) manager.openProject( projectName, - { fileName: String => + (fileName: String) => { fileName.endsWith("cpg.bin.tmp") shouldBe true Some(Cpg.emptyCpg) } @@ -176,12 +176,7 @@ class WorkspaceManagerTests extends AnyWordSpec with Matchers { def createFakeProjectAndOpen(workspaceFile: File, projectName: String): WorkspaceManager[Project] = { WorkspaceTests.createFakeProject(workspaceFile, projectName) val manager = new WorkspaceManager[Project](workspaceFile.toString) - manager.openProject( - projectName, - { _: String => - Some(Cpg.emptyCpg) - } - ) + manager.openProject(projectName, (_: String) => Some(Cpg.emptyCpg)) manager } diff --git a/dataflowengineoss/build.sbt b/dataflowengineoss/build.sbt index da7d75b7f1c5..07167eecd7c1 100644 --- a/dataflowengineoss/build.sbt +++ b/dataflowengineoss/build.sbt @@ -1,7 +1,5 @@ name := "dataflowengineoss" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.semanticcpg, Projects.x2cpg) libraryDependencies ++= Seq( diff --git a/joern-cli/frontends/c2cpg/build.sbt b/joern-cli/frontends/c2cpg/build.sbt index 96b85d05a426..fcaa628c5607 100644 --- a/joern-cli/frontends/c2cpg/build.sbt +++ b/joern-cli/frontends/c2cpg/build.sbt @@ -1,6 +1,4 @@ name := "c2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % Test, Projects.x2cpg % "compile->compile;test->test") @@ -34,49 +32,6 @@ dependencyOverrides ++= Seq( Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) -scalacOptions ++= Seq() ++ ( - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => Seq() - case _ => - Seq( - "-deprecation", // Emit warning and location for usages of deprecated APIs. - "-encoding", - "utf-8", // Specify character encoding used by source files. - "-explaintypes", // Explain type errors in more detail. - "-feature", // Emit warning and location for usages of features that should be imported explicitly. - "-language:existentials", // Existential types (besides wildcard types) can be written and inferred - "-language:experimental.macros", // Allow macro definition (besides implementation and application) - "-language:higherKinds", // Allow higher-kinded types - "-language:implicitConversions", // Allow definition of implicit functions called views - "-unchecked", // Enable additional warnings where generated code depends on assumptions. - "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access. - "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver. - "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error. - "-Xlint:delayedinit-select", // Selecting member of DelayedInit. - "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element. - "-Xlint:inaccessible", // Warn about inaccessible types in method signatures. - "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`. - "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id. - "-Xlint:option-implicit", // Option.apply used implicit view. - "-Xlint:package-object-classes", // Class or object defined in package object. - "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds. - "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field. - "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component. - "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope. - "-Ywarn-dead-code", // Warn when dead code is identified. - "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. - "-Xlint:nullary-unit", // Warn when nullary methods return Unit. - "-Ywarn-numeric-widen", // Warn when numerics are widened. - "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. - "-Ywarn-unused:imports", // Warn if an import selector is not referenced. - "-Ywarn-unused:locals", // Warn if a local definition is unused. - "-Ywarn-unused:params", // Warn if a value parameter is unused. - "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. - "-Ywarn-unused:privates" // Warn if a private member is unused. - ) - } -) - compile / javacOptions ++= Seq("-Xlint:all", "-Xlint:-cast", "-g") Test / fork := true diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala index 3e32b3c46e13..0ae31c0fcf1d 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/passes/FunctionPass.scala @@ -4,7 +4,7 @@ import ghidra.app.util.template.TemplateSimplifier import ghidra.program.model.address.GenericAddress import ghidra.program.model.lang.Register import ghidra.program.model.listing.{CodeUnitFormat, CodeUnitFormatOptions, Function, Instruction, Program} -import ghidra.program.model.pcode.HighFunction +import ghidra.program.model.pcode.{HighFunction, HighSymbol} import ghidra.program.model.scalar.Scalar import io.joern.ghidra2cpg._ import io.joern.ghidra2cpg.processors._ @@ -183,7 +183,7 @@ abstract class FunctionPass( .map { highFunction => highFunction.getLocalSymbolMap.getSymbols.asScala.toSeq.filter(_.isParameter).toArray } - .getOrElse(Array.empty) + .getOrElse(Array.empty[HighSymbol]) checkedParameters = parameters.map { parameter => val checkedParameter = diff --git a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Decompiler.scala b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Decompiler.scala index 206231c19872..02a2a9a454f4 100644 --- a/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Decompiler.scala +++ b/joern-cli/frontends/ghidra2cpg/src/main/scala/io/joern/ghidra2cpg/utils/Decompiler.scala @@ -17,7 +17,7 @@ object Decompiler { opts.grabFromProgram(program) decompilerInterface.setOptions(opts) if (!decompilerInterface.openProgram(program)) { - println("Decompiler error: %s\n", decompilerInterface.getLastMessage) + println(s"Decompiler error: ${decompilerInterface.getLastMessage}") None } else { Some(new Decompiler(decompilerInterface)) diff --git a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala index 068d54a70247..917da1ffa137 100644 --- a/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala +++ b/joern-cli/frontends/ghidra2cpg/src/test/scala/io/joern/ghidra2cpg/fixtures/GhidraBinToCpgSuite.scala @@ -16,10 +16,14 @@ class GhidraFrontend extends LanguageFrontend { override def execute(inputFile: java.io.File): Cpg = { val dir = Files.createTempDirectory("ghidra2cpg-tests").toFile - Runtime.getRuntime.addShutdownHook(new Thread(() => FileUtils.deleteQuietly(dir))) + Runtime.getRuntime.addShutdownHook(new Thread(new Runnable { + override def run(): Unit = FileUtils.deleteQuietly(dir) + })) val tempDir = Files.createTempDirectory("ghidra2cpg").toFile - Runtime.getRuntime.addShutdownHook(new Thread(() => FileUtils.deleteQuietly(tempDir))) + Runtime.getRuntime.addShutdownHook(new Thread(new Runnable { + override def run(): Unit = FileUtils.deleteQuietly(tempDir) + })) val cpgBin = dir.getAbsolutePath implicit val defaultConfig: Config = Config() diff --git a/joern-cli/frontends/javasrc2cpg/build.sbt b/joern-cli/frontends/javasrc2cpg/build.sbt index 97226b0eef9c..1e927d653063 100644 --- a/joern-cli/frontends/javasrc2cpg/build.sbt +++ b/joern-cli/frontends/javasrc2cpg/build.sbt @@ -1,8 +1,5 @@ name := "javasrc2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( @@ -16,10 +13,6 @@ libraryDependencies ++= Seq( "net.lingala.zip4j" % "zip4j" % "2.11.5" ) -scalacOptions ++= Seq( - "-deprecation" // Emit warning and location for usages of deprecated APIs. -) - enablePlugins(JavaAppPackaging, LauncherJarPlugin) trapExit := false Global / onChangedBuildSource := ReloadOnSourceChanges @@ -33,23 +26,26 @@ packTestCode := { import java.nio.file.Paths val pkgRoot = "io" - val testClassOutputPath = Paths.get("joern-cli", "frontends", "javasrc2cpg", "target", "scala-2.13", "test-classes") + val testClassOutputPath = target.value / ("scala-" + scalaVersion.value) / "test-classes" val relativeTestCodePath = Paths.get(pkgRoot, "joern", "javasrc2cpg", "jartypereader", "testcode") - File(testClassOutputPath.resolve(relativeTestCodePath)).list.filter(_.exists).foreach { testDir => - val tmpDir = File.newTemporaryDirectory() + val jarFileRoot = target.value.toScala / "testjars" + if (jarFileRoot.exists()) jarFileRoot.delete() + jarFileRoot.createDirectories() + + File(testClassOutputPath.toPath.resolve(relativeTestCodePath)).list.filter(_.exists).foreach { testDir => + val tmpDir = File.newTemporaryDirectory() val tmpDirWithCorrectPkgStruct = File(tmpDir.path.resolve(relativeTestCodePath)).createDirectoryIfNotExists() testDir.copyToDirectory(tmpDirWithCorrectPkgStruct) val testRootPath = tmpDir.path.resolve(pkgRoot) - val jarFilePath = testClassOutputPath.resolve(testDir.name ++ ".jar") - val jarFile = new ZipFile(jarFilePath.toAbsolutePath.toString) + val jarFilePath = jarFileRoot / (testDir.name + ".jar") + if (jarFilePath.exists()) jarFilePath.delete() + val jarFile = new ZipFile(jarFilePath.canonicalPath) val zipParameters = new ZipParameters() zipParameters.setCompressionMethod(CompressionMethod.DEFLATE) zipParameters.setCompressionLevel(CompressionLevel.NORMAL) zipParameters.setRootFolderNameInZip(relativeTestCodePath.toString) - - File(jarFilePath).delete(swallowIOExceptions = true) jarFile.addFolder(File(testRootPath).toJava) } } diff --git a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/jartypereader/JarTypeReaderTests.scala b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/jartypereader/JarTypeReaderTests.scala index 59b72ab8ecb8..0948c40e9dfd 100644 --- a/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/jartypereader/JarTypeReaderTests.scala +++ b/joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/jartypereader/JarTypeReaderTests.scala @@ -230,7 +230,7 @@ class JarTypeReaderTests extends AnyFreeSpec with Matchers { } private def getTypes(name: String): List[ResolvedTypeDecl] = { - val path = Paths.get("joern-cli", "frontends", "javasrc2cpg", "target", "scala-2.13", "test-classes", s"$name.jar") + val path = Paths.get("joern-cli", "frontends", "javasrc2cpg", "target", "testjars", s"$name.jar") val inputPath = ProjectRoot.relativise(path.toString) JarTypeReader.getTypes(inputPath) } diff --git a/joern-cli/frontends/jimple2cpg/build.sbt b/joern-cli/frontends/jimple2cpg/build.sbt index 85bf04bba4a1..339822ec2f87 100644 --- a/joern-cli/frontends/jimple2cpg/build.sbt +++ b/joern-cli/frontends/jimple2cpg/build.sbt @@ -1,8 +1,5 @@ name := "jimple2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( @@ -11,10 +8,6 @@ libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) -scalacOptions ++= Seq( - "-deprecation" // Emit warning and location for usages of deprecated APIs. -) - enablePlugins(JavaAppPackaging, LauncherJarPlugin) trapExit := false Test / fork := true diff --git a/joern-cli/frontends/jssrc2cpg/build.sbt b/joern-cli/frontends/jssrc2cpg/build.sbt index 3ba6d8c60187..6d6ac5df646b 100644 --- a/joern-cli/frontends/jssrc2cpg/build.sbt +++ b/joern-cli/frontends/jssrc2cpg/build.sbt @@ -4,8 +4,6 @@ import versionsort.VersionHelper import com.typesafe.config.{Config, ConfigFactory} name := "jssrc2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") @@ -30,50 +28,6 @@ libraryDependencies ++= Seq( Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) -scalacOptions ++= Seq() ++ ( - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => Seq() - case _ => - Seq( - "-deprecation", // Emit warning and location for usages of deprecated APIs. - "-encoding", - "utf-8", // Specify character encoding used by source files. - "-explaintypes", // Explain type errors in more detail. - "-feature", // Emit warning and location for usages of features that should be imported explicitly. - "-language:existentials", // Existential types (besides wildcard types) can be written and inferred - "-language:experimental.macros", // Allow macro definition (besides implementation and application) - "-language:higherKinds", // Allow higher-kinded types - "-language:implicitConversions", // Allow definition of implicit functions called views - "-unchecked", // Enable additional warnings where generated code depends on assumptions. - "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access. - // "-Xfatal-warnings", // Fail the compilation if there are any warnings. - "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver. - "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error. - "-Xlint:delayedinit-select", // Selecting member of DelayedInit. - "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element. - "-Xlint:inaccessible", // Warn about inaccessible types in method signatures. - "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`. - "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id. - "-Xlint:option-implicit", // Option.apply used implicit view. - "-Xlint:package-object-classes", // Class or object defined in package object. - "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds. - "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field. - "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component. - "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope. - "-Ywarn-dead-code", // Warn when dead code is identified. - "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. - "-Xlint:nullary-unit", // Warn when nullary methods return Unit. - "-Ywarn-numeric-widen", // Warn when numerics are widened. - "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. - "-Ywarn-unused:imports", // Warn if an import selector is not referenced. - "-Ywarn-unused:locals", // Warn if a local definition is unused. - "-Ywarn-unused:params", // Warn if a value parameter is unused. - "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. - "-Ywarn-unused:privates" // Warn if a private member is unused. - ) - } -) - compile / javacOptions ++= Seq("-Xlint:all", "-Xlint:-cast", "-g") Test / fork := false diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala index f5733ba70fee..5a315ede6209 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/DefaultTypeInfoProvider.scala @@ -93,7 +93,7 @@ class DefaultTypeInfoProvider(environment: KotlinCoreEnvironment) extends TypeIn def usedAsExpression(expr: KtExpression): Option[Boolean] = { val mapForEntity = bindingsForEntity(bindingContext, expr) - Option(mapForEntity.get(BindingContext.USED_AS_EXPRESSION.getKey)) + Option(mapForEntity.get(BindingContext.USED_AS_EXPRESSION.getKey)).map(_.booleanValue()) } def fullName(expr: KtTypeAlias, defaultValue: String): String = { diff --git a/joern-cli/frontends/php2cpg/build.sbt b/joern-cli/frontends/php2cpg/build.sbt index 5b161f2071bc..81c20bb028de 100644 --- a/joern-cli/frontends/php2cpg/build.sbt +++ b/joern-cli/frontends/php2cpg/build.sbt @@ -3,9 +3,6 @@ import scala.util.Properties.isWin name := "php2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - val phpParserVersion = "4.15.6" val phpParserBinName = "php-parser.phar" val phpParserDlUrl = s"https://github.com/joernio/PHP-Parser/releases/download/v$phpParserVersion/$phpParserBinName" @@ -19,10 +16,6 @@ libraryDependencies ++= Seq( "io.circe" %% "circe-core" % "0.15.0-M1" ) -scalacOptions ++= Seq( - "-deprecation" // Emit warning and location for usages of deprecated APIs. -) - lazy val phpParseInstallTask = taskKey[Unit]("Install PHP-Parse using PHP Composer") phpParseInstallTask := { val phpBinDir = baseDirectory.value / "bin" diff --git a/joern-cli/frontends/pysrc2cpg/build.sbt b/joern-cli/frontends/pysrc2cpg/build.sbt index f7d16eae7414..ccdae4ec3233 100644 --- a/joern-cli/frontends/pysrc2cpg/build.sbt +++ b/joern-cli/frontends/pysrc2cpg/build.sbt @@ -1,8 +1,5 @@ name := "pysrc2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt index e70da907e36b..51bd91b2153c 100644 --- a/joern-cli/frontends/rubysrc2cpg/build.sbt +++ b/joern-cli/frontends/rubysrc2cpg/build.sbt @@ -1,8 +1,5 @@ name := "rubysrc2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test") libraryDependencies ++= Seq( @@ -11,10 +8,6 @@ libraryDependencies ++= Seq( "org.antlr" % "antlr4-runtime" % Versions.antlr ) -scalacOptions ++= Seq( - "-deprecation" // Emit warning and location for usages of deprecated APIs. -) - enablePlugins(JavaAppPackaging, LauncherJarPlugin, Antlr4Plugin) Antlr4 / antlr4PackageName := Some("io.joern.rubysrc2cpg.parser") diff --git a/joern-cli/frontends/x2cpg/build.sbt b/joern-cli/frontends/x2cpg/build.sbt index bedb6238b683..8ac2fd348fd6 100644 --- a/joern-cli/frontends/x2cpg/build.sbt +++ b/joern-cli/frontends/x2cpg/build.sbt @@ -1,6 +1,4 @@ name := "x2cpg" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") dependsOn(Projects.semanticcpg) @@ -11,52 +9,6 @@ libraryDependencies ++= Seq( Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) -scalacOptions ++= Seq() ++ ( - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => Seq() - case _ => - Seq( - "-deprecation", // Emit warning and location for usages of deprecated APIs. - "-encoding", - "utf-8", // Specify character encoding used by source files. - "-explaintypes", // Explain type errors in more detail. - "-feature", // Emit warning and location for usages of features that should be imported explicitly. - "-language:existentials", // Existential types (besides wildcard types) can be written and inferred - "-language:experimental.macros", // Allow macro definition (besides implementation and application) - "-language:higherKinds", // Allow higher-kinded types - "-language:implicitConversions", // Allow definition of implicit functions called views - "-unchecked", // Enable additional warnings where generated code depends on assumptions. - "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access. - // "-Xfatal-warnings", //-Werror is incompatible with the concept of @deprecate. - // TODO: Find the right incantation to ensure that deprecation warnings are - // not suppressed but are not treated as error either - "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver. - "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error. - "-Xlint:delayedinit-select", // Selecting member of DelayedInit. - "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element. - "-Xlint:inaccessible", // Warn about inaccessible types in method signatures. - "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`. - "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id. - "-Xlint:option-implicit", // Option.apply used implicit view. - "-Xlint:package-object-classes", // Class or object defined in package object. - "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds. - "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field. - "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component. - "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope. - "-Ywarn-dead-code", // Warn when dead code is identified. - "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. - "-Xlint:nullary-unit", // Warn when nullary methods return Unit. - "-Ywarn-numeric-widen", // Warn when numerics are widened. - "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. - "-Ywarn-unused:imports", // Warn if an import selector is not referenced. - "-Ywarn-unused:locals", // Warn if a local definition is unused. - "-Ywarn-unused:params", // Warn if a value parameter is unused. - "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. - "-Ywarn-unused:privates" // Warn if a private member is unused. - ) - } -) - compile / javacOptions ++= Seq("-Xlint:all", "-Xlint:-cast", "-g") Test / fork := true diff --git a/joern-cli/src/main/resources/scripts/assertions.sc b/joern-cli/src/main/resources/scripts/assertions.sc new file mode 100644 index 000000000000..07bf864a76ce --- /dev/null +++ b/joern-cli/src/main/resources/scripts/assertions.sc @@ -0,0 +1,12 @@ +def assertContains[A](name: String, actual: IterableOnce[A], expected: IterableOnce[A]) = { + val actualSet = actual.iterator.to(Set) + + val missing = expected.iterator.filterNot(actualSet.contains).toSeq + if (missing.nonEmpty) + throw new AssertionError( + s"""$name did not contain the following expected element(s): + |$missing + |Actual elements: + |${actual.iterator.mkString(System.lineSeparator)}""".mkString + ) with scala.util.control.NoStackTrace +} \ No newline at end of file diff --git a/joern-cli/src/main/resources/scripts/c/assertions.sc b/joern-cli/src/main/resources/scripts/c/assertions.sc new file mode 100644 index 000000000000..07bf864a76ce --- /dev/null +++ b/joern-cli/src/main/resources/scripts/c/assertions.sc @@ -0,0 +1,12 @@ +def assertContains[A](name: String, actual: IterableOnce[A], expected: IterableOnce[A]) = { + val actualSet = actual.iterator.to(Set) + + val missing = expected.iterator.filterNot(actualSet.contains).toSeq + if (missing.nonEmpty) + throw new AssertionError( + s"""$name did not contain the following expected element(s): + |$missing + |Actual elements: + |${actual.iterator.mkString(System.lineSeparator)}""".mkString + ) with scala.util.control.NoStackTrace +} \ No newline at end of file diff --git a/joern-cli/src/main/resources/scripts/c/const-ish.sc b/joern-cli/src/main/resources/scripts/c/const-ish.sc index 26d1daf1f227..df1a4d17dd02 100644 --- a/joern-cli/src/main/resources/scripts/c/const-ish.sc +++ b/joern-cli/src/main/resources/scripts/c/const-ish.sc @@ -1,13 +1,19 @@ -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Member, Method} -import io.joern.dataflowengineoss.language._ -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal._ +//> using file assertions.sc -@main def main() = { - cpg.method.internal.filter { method => +@main def main(inputPath: String) = { + importCode(inputPath) + val methods = cpg.method.internal.filter { method => method.start.assignment.target .reachableBy(method.parameter.filter(_.code.contains("const"))) .nonEmpty - }.toSetImmutable + }.name + + val expected = Set( + "modify_const_struct_member_cpp_cast", + "modify_const_struct_member_c_cast", + "modify_const_struct_cpp_cast", + "modify_const_struct_c_cast" + ) + assertContains("methods", methods, expected) + } diff --git a/joern-cli/src/main/resources/scripts/c/malloc-leak.sc b/joern-cli/src/main/resources/scripts/c/malloc-leak.sc index d6cc0d4e1381..ca0f223d612b 100644 --- a/joern-cli/src/main/resources/scripts/c/malloc-leak.sc +++ b/joern-cli/src/main/resources/scripts/c/malloc-leak.sc @@ -4,9 +4,16 @@ import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment import overflowdb.traversal._ -@main def main() = { +//> using file assertions.sc + +@main def main(inputPath: String) = { + importCode(inputPath) def allocated = cpg.call("malloc").inAssignment.target.dedup def freed = cpg.call("free").argument(1) def flowsFromAllocToFree = freed.reachableBy(allocated).toSetImmutable - allocated.map(_.code).toSetImmutable.diff(flowsFromAllocToFree.map(_.code)) + val leaks = allocated.map(_.code).toSetImmutable.diff(flowsFromAllocToFree.map(_.code)) + + val expected = Set("leak") + assertContains("leaks", leaks, expected) + } diff --git a/joern-cli/src/main/resources/scripts/c/malloc-overflow.sc b/joern-cli/src/main/resources/scripts/c/malloc-overflow.sc index 01a4f169d3fc..6a9a1c6da89b 100644 --- a/joern-cli/src/main/resources/scripts/c/malloc-overflow.sc +++ b/joern-cli/src/main/resources/scripts/c/malloc-overflow.sc @@ -1,11 +1,10 @@ -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.codepropertygraph.generated.nodes.Call -import io.shiftleft.semanticcpg.language._ -import overflowdb.traversal._ -@main def main(): List[Call] = { - cpg +//> using file assertions.sc + +@main def main(inputPath: String) = { + importCode(inputPath) + val calls = cpg .call("malloc") .filter { mallocCall => mallocCall.argument(1) match { @@ -13,6 +12,12 @@ import overflowdb.traversal._ subCall.name == Operators.addition || subCall.name == Operators.multiplication case _ => false } - } - .l + }.code + + val expected = Set( + "malloc(sizeof(int) * 42)", + "malloc(sizeof(int) * 3)", + "malloc(sizeof(int) + 55)" + ) + assertContains("calls", calls, expected) } diff --git a/joern-cli/src/main/resources/scripts/c/pointer-to-int.sc b/joern-cli/src/main/resources/scripts/c/pointer-to-int.sc index 42b1408a5b51..e350b9f0e218 100644 --- a/joern-cli/src/main/resources/scripts/c/pointer-to-int.sc +++ b/joern-cli/src/main/resources/scripts/c/pointer-to-int.sc @@ -5,6 +5,8 @@ import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.language.operatorextension._ import overflowdb.traversal._ +//> using file assertions.sc + private def expressionIsPointer(argument: Expression, isSubExpression: Boolean = false): Boolean = { argument match { case identifier: Identifier => @@ -20,8 +22,9 @@ private def expressionIsPointer(argument: Expression, isSubExpression: Boolean = } } -@main def main(): List[nodes.Call] = { - cpg.assignment +@main def main(inputPath: String) = { + importCode(inputPath) + val calls = cpg.assignment .filter(assign => assign.source.isInstanceOf[Call] && assign.target.isInstanceOf[Identifier]) .filter { assignment => val target = assignment.target.asInstanceOf[Identifier] @@ -31,5 +34,21 @@ private def expressionIsPointer(argument: Expression, isSubExpression: Boolean = target.typeFullName == "int" && expressionIsPointer(source) } - .l + .code + + val expected = Seq( + "simple_subtraction = p - q", + "nested_subtraction = p - q - r", + "literal_subtraction = p - i", + "addrOf_subtraction = p - &i", + "nested_addrOf_subtraction = 3 - &i - 4", + "literal_addrOf_subtraction = 3 - &i", + "array_subtraction = x - p", + "array_literal_subtraction = x - 3", + "array_addrOf_subtraction = x - &i", + // TODO: We don't have access to type info for indirect field member access. + // "unsafe_struct = foo_t->p - 1" + ) + + assertContains("calls", calls, expected) } diff --git a/joern-cli/src/main/resources/scripts/c/syscalls.sc b/joern-cli/src/main/resources/scripts/c/syscalls.sc index a4db0eab6391..3693b5ab6dc7 100644 --- a/joern-cli/src/main/resources/scripts/c/syscalls.sc +++ b/joern-cli/src/main/resources/scripts/c/syscalls.sc @@ -2,6 +2,8 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.semanticcpg.language._ +//> using file assertions.sc + // Up-to-date as of Kernel version 4.11 private val linuxSyscalls: Set[String] = Set( "_llseek", @@ -409,6 +411,10 @@ private val linuxSyscalls: Set[String] = Set( "writev" ) -@main def main(): List[Call] = { - cpg.call.filter(c => linuxSyscalls.contains(c.name)).l +@main def main(inputPath: String) = { + importCode(inputPath) + val calls = cpg.call.filter(c => linuxSyscalls.contains(c.name)).name + + val expected = Set("gettimeofday", "exit") + assertContains("calls", calls, expected) } diff --git a/joern-cli/src/main/resources/scripts/c/userspace-memory-access.sc b/joern-cli/src/main/resources/scripts/c/userspace-memory-access.sc index 92aa0e2642ea..633f6023600d 100644 --- a/joern-cli/src/main/resources/scripts/c/userspace-memory-access.sc +++ b/joern-cli/src/main/resources/scripts/c/userspace-memory-access.sc @@ -2,6 +2,8 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import io.shiftleft.semanticcpg.language._ +//> using file assertions.sc + // Find more information at http://www.makelinux.net/ldd3/ (Chapter 3) // and https://www.kernel.org/doc/html/latest/core-api/mm-api.html private val calls: Set[String] = Set( @@ -15,6 +17,10 @@ private val calls: Set[String] = Set( "vm_iomap_memory" ) -@main def main(): List[Call] = { - cpg.call.filter(call => calls.contains(call.name)).l +@main def main(inputPath: String) = { + importCode(inputPath) + val calls0 = cpg.call.filter(call => calls.contains(call.name)).name + + val expected = Set("get_user") + assertContains("calls", calls0, expected) } diff --git a/joern-cli/src/main/resources/scripts/general/help.sc b/joern-cli/src/main/resources/scripts/general/help.sc new file mode 100644 index 000000000000..b36ca1bcb12b --- /dev/null +++ b/joern-cli/src/main/resources/scripts/general/help.sc @@ -0,0 +1,6 @@ +@main def main(inputPath: String) = { + importCode(inputPath) + + if (help.cpg.toString.isEmpty) + throw new AssertionError("no help text available!") with scala.util.control.NoStackTrace +} diff --git a/joern-cli/src/main/resources/scripts/general/list-funcs.sc b/joern-cli/src/main/resources/scripts/general/list-funcs.sc deleted file mode 100644 index 2ff416a27fd9..000000000000 --- a/joern-cli/src/main/resources/scripts/general/list-funcs.sc +++ /dev/null @@ -1,21 +0,0 @@ -/* list-funcs.scala - - This script simply returns a list of the names as String of all methods contained in the currently loaded CPG. - - Input: A valid CPG - Output: scala.List[String] - - Running the Script - ------------------ - see: README.md - - Sample Output - ------------- - List(".indirectMemberAccess", ".assignment", "free_list", "free", ".notEquals") - */ - -import io.shiftleft.semanticcpg.language._ - -@main def main(): List[String] = { - cpg.method.name.l -} diff --git a/joern-cli/src/main/resources/scripts/general/run.sc b/joern-cli/src/main/resources/scripts/general/run.sc new file mode 100644 index 000000000000..e73ac1c96a60 --- /dev/null +++ b/joern-cli/src/main/resources/scripts/general/run.sc @@ -0,0 +1,10 @@ +@main def main(inputPath: String) = { + importCode(inputPath) + + if (!run.toString.contains("base")) + throw new AssertionError( + s"""base layer not applied...? + |output of `run`: + |$run""".mkString + ) with scala.util.control.NoStackTrace +} diff --git a/joern-cli/src/main/resources/scripts/trigger-error.sc b/joern-cli/src/main/resources/scripts/trigger-error.sc new file mode 100644 index 000000000000..bbc2174fb90d --- /dev/null +++ b/joern-cli/src/main/resources/scripts/trigger-error.sc @@ -0,0 +1 @@ +assert(true == false, "trigger an error for testing purposes") diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala index 6a8be0199323..aacf64e61f21 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernScan.scala @@ -1,19 +1,19 @@ package io.joern.joerncli -import better.files._ +import better.files.* import io.joern.console.scan.{ScanPass, outputFindings} import io.joern.console.{BridgeBase, DefaultArgumentProvider, JoernProduct, Query, QueryDatabase} import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext} import io.joern.dataflowengineoss.semanticsloader.Semantics import io.joern.joerncli.JoernScan.getQueriesFromQueryDb import io.joern.joerncli.Scan.{allTag, defaultTag} -import io.joern.joerncli.console.AmmoniteBridge +import io.joern.joerncli.console.ReplBridge import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.semanticcpg.language.{DefaultNodeExtensionFinder, NodeExtensionFinder} import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions} +import java.io.PrintStream import org.json4s.native.Serialization import org.json4s.{Formats, NoTypeHints} - import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -39,6 +39,7 @@ case class JoernScanConfig( ) object JoernScan extends BridgeBase { + override val slProduct = JoernProduct val implementationVersion = getClass.getPackage.getImplementationVersion @@ -173,7 +174,7 @@ object JoernScan extends BridgeBase { language = config.language, frontendArgs = frontendArgs.toArray ) - runAmmonite(shellConfig, JoernProduct) + run(shellConfig) println(s"Run `joern --for-input-path ${config.src}` to explore interactively") } @@ -217,7 +218,7 @@ object JoernScan extends BridgeBase { val rmPluginConfig = io.joern.console .Config() .copy(rmPlugin = Some("querydb")) - runAmmonite(rmPluginConfig, JoernProduct) + run(rmPluginConfig) } private def addQueryDatabase(absPath: String): Unit = { @@ -225,7 +226,7 @@ object JoernScan extends BridgeBase { val addPluginConfig = io.joern.console .Config() .copy(addPlugin = Some(absPath)) - runAmmonite(addPluginConfig, JoernProduct) + run(addPluginConfig) } private def urlForVersion(version: String): String = { @@ -236,9 +237,12 @@ object JoernScan extends BridgeBase { } } - override protected def predefPlus(lines: List[String]): String = AmmoniteBridge.predefPlus(lines) - override protected def shutdownHooks: List[String] = AmmoniteBridge.shutdownHooks - override protected def promptStr() = AmmoniteBridge.promptStr() + override protected def predefLines = ReplBridge.predefLines + override protected def promptStr = ReplBridge.promptStr + + override protected def greeting = ReplBridge.greeting + + override protected def onExitCode = ReplBridge.onExitCode } object Scan { diff --git a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala index d5afc9a91742..9d1d2adca9c3 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/JoernVectors.scala @@ -144,7 +144,7 @@ object JoernVectors { val embedding = generator.embed(cpg) println("{") println("\"objects\":") - traversalToJson(embedding.objects, { x: String => generator.defaultToString(x) }) + traversalToJson(embedding.objects, generator.defaultToString) if (config.dimToFeature) { println(",\"dimToFeature\": ") println(Serialization.write(embedding.dimToStructure)) @@ -156,7 +156,7 @@ object JoernVectors { cpg.graph.edges().map { x => Map("src" -> x.outNode().id(), "dst" -> x.inNode().id(), "label" -> x.label()) }, - { x: Map[String, Any] => generator.defaultToString(x) } + generator.defaultToString ) println("}") } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/AmmoniteBridge.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/AmmoniteBridge.scala deleted file mode 100644 index e77f8d4bd8cd..000000000000 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/AmmoniteBridge.scala +++ /dev/null @@ -1,31 +0,0 @@ -package io.joern.joerncli.console - -import io.joern.console.{BridgeBase, JoernProduct} - -object AmmoniteBridge extends App with BridgeBase { - - runAmmonite(parseConfig(args), JoernProduct) - - /** Code that is executed when starting the shell - */ - override def predefPlus(lines: List[String]): String = { - s"""${Predefined.forInteractiveShell} - |${lines.mkString("\n")} - |""".stripMargin - } - - override def promptStr(): String = "joern> " - - override def shutdownHooks: List[String] = List("""interp.beforeExitHooks.append{_ => - |println("Would you like to save changes? (y/N)") - |val answer = scala.Console.in.read.toChar - |if (answer == 'Y' || answer == 'y') { - | System.err.println("saving.") - | workspace.projects.foreach { p => - | p.close - | } - | } - |} - |""".stripMargin) - -} diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernAmmoniteExecutor.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernAmmoniteExecutor.scala deleted file mode 100644 index 67579693b93c..000000000000 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernAmmoniteExecutor.scala +++ /dev/null @@ -1,9 +0,0 @@ -package io.joern.joerncli.console - -import io.joern.console.scripting.AmmoniteExecutor - -object JoernAmmoniteExecutor extends AmmoniteExecutor { - - override lazy val predef: String = - Predefined.forScripts -} diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala index 7d7044de2156..3d6e801d1a12 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/JoernConsole.scala @@ -20,7 +20,7 @@ class JoernWorkspaceLoader extends WorkspaceLoader[JoernProject] { } } -class JoernConsole extends Console[JoernProject](JoernAmmoniteExecutor, new JoernWorkspaceLoader) { +class JoernConsole extends Console[JoernProject](new JoernWorkspaceLoader) { override val config: ConsoleConfig = JoernConsole.defaultConfig @@ -34,25 +34,6 @@ class JoernConsole extends Console[JoernProject](JoernAmmoniteExecutor, new Joer .map(x => x.asInstanceOf[JoernProject].context) .getOrElse(EngineContext()) - def banner(): Unit = { - println(s""" - | ██╗ ██████╗ ███████╗██████╗ ███╗ ██╗ - | ██║██╔═══██╗██╔════╝██╔══██╗████╗ ██║ - | ██║██║ ██║█████╗ ██████╔╝██╔██╗ ██║ - |██ ██║██║ ██║██╔══╝ ██╔══██╗██║╚██╗██║ - |╚█████╔╝╚██████╔╝███████╗██║ ██║██║ ╚████║ - | ╚════╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ - |Version: $version - |$helpMsg - """.stripMargin) - } - - private def helpMsg: String = - s"""Type `help` or `browse(help)` to begin""".stripMargin - - def version: String = - getClass.getPackage.getImplementationVersion - def loadCpg(inputPath: String): Option[Cpg] = { report("Deprecated. Please use `importCpg` instead") importCpg(inputPath) @@ -67,20 +48,24 @@ class JoernConsole extends Console[JoernProject](JoernAmmoniteExecutor, new Joer object JoernConsole { - def defaultConfig: ConsoleConfig = new ConsoleConfig() + def banner(): String = + s""" + | ██╗ ██████╗ ███████╗██████╗ ███╗ ██╗ + | ██║██╔═══██╗██╔════╝██╔══██╗████╗ ██║ + | ██║██║ ██║█████╗ ██████╔╝██╔██╗ ██║ + |██ ██║██║ ██║██╔══╝ ██╔══██╗██║╚██╗██║ + |╚█████╔╝╚██████╔╝███████╗██║ ██║██║ ╚████║ + | ╚════╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝╚═╝ ╚═══╝ + |Version: $version + |$helpMsg + """.stripMargin - def runScriptTest(scriptName: String, params: Map[String, String], cpg: Cpg): Any = { - class TempConsole(workspaceDir: String) extends JoernConsole { - override def context: EngineContext = EngineContext() - override val config = - new ConsoleConfig(install = new InstallConfig(Map("SHIFTLEFT_CONSOLE_INSTALL_DIR" -> workspaceDir))) - } - val workspaceDir = File.newTemporaryDirectory("console") - try { - new TempConsole(workspaceDir.toString).runScript(scriptName, params, cpg) - } finally { - workspaceDir.delete() - } - } + def version: String = + getClass.getPackage.getImplementationVersion + + private def helpMsg: String = + s"""Type `help` to begin""".stripMargin + + def defaultConfig: ConsoleConfig = new ConsoleConfig() } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala index 1fdee9988712..d44bb0eb7d8b 100644 --- a/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/Predefined.scala @@ -4,46 +4,31 @@ import io.joern.console.{Help, Run} object Predefined { - /* ammonite tab completion is partly broken for scala > 2.12.8 - * applying workaround for package wildcard imports from https://github.com/lihaoyi/Ammonite/issues/1009 */ - val shared: String = - """ - |import io.joern.console.{`package` => _, _} - |import io.joern.joerncli.console.JoernConsole._ - |import io.shiftleft.codepropertygraph.Cpg - |import io.shiftleft.codepropertygraph.Cpg.docSearchPackages - |import io.shiftleft.codepropertygraph.cpgloading._ - |import io.shiftleft.codepropertygraph.generated._ - |import io.shiftleft.codepropertygraph.generated.nodes._ - |import io.shiftleft.codepropertygraph.generated.edges._ - |import io.joern.dataflowengineoss.language.{`package` => _, _} - |import io.shiftleft.semanticcpg.language.{`package` => _, _} - |import overflowdb.{`package` => _, _} - |import overflowdb.traversal.{`package` => _, help => _, _} - |import scala.jdk.CollectionConverters._ - |implicit val resolver: ICallResolver = NoResolve - |implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder - """.stripMargin + val shared: Seq[String] = + Seq( + "import _root_.io.joern.console._", + "import _root_.io.joern.joerncli.console.JoernConsole._", + "import _root_.io.shiftleft.codepropertygraph.Cpg", + "import _root_.io.shiftleft.codepropertygraph.Cpg.docSearchPackages", + "import _root_.io.shiftleft.codepropertygraph.cpgloading._", + "import _root_.io.shiftleft.codepropertygraph.generated._", + "import _root_.io.shiftleft.codepropertygraph.generated.nodes._", + "import _root_.io.shiftleft.codepropertygraph.generated.edges._", + "import _root_.io.joern.dataflowengineoss.language._", + "import _root_.io.shiftleft.semanticcpg.language._", + "import overflowdb._", + "import overflowdb.traversal.{`package` => _, help => _, _}", + "import scala.jdk.CollectionConverters._", + "implicit val resolver: ICallResolver = NoResolve", + "implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder", + ) - val forInteractiveShell: String = - shared + - """ - |import io.joern.joerncli.console.Joern._ - |def script(x: String) : Any = console.runScript(x, Map(), cpg) - """.stripMargin + - dynamicPredef() - - val forScripts: String = - shared + - """ - |import io.joern.joerncli.console.Joern.{cpg =>_, _} - """.stripMargin + - dynamicPredef() - - def dynamicPredef(): String = { - Run.codeForRunCommand() + - Help.codeForHelpCommand(classOf[io.joern.joerncli.console.JoernConsole]) + - "ossDataFlowOptions = opts.ossdataflow" + val forInteractiveShell: Seq[String] = { + shared ++ + Seq("import _root_.io.joern.joerncli.console.Joern._") ++ + Run.codeForRunCommand().linesIterator ++ + Help.codeForHelpCommand(classOf[io.joern.joerncli.console.JoernConsole]).linesIterator ++ + Seq("ossDataFlowOptions = opts.ossdataflow") } } diff --git a/joern-cli/src/main/scala/io/joern/joerncli/console/ReplBridge.scala b/joern-cli/src/main/scala/io/joern/joerncli/console/ReplBridge.scala new file mode 100644 index 000000000000..fed6be35686a --- /dev/null +++ b/joern-cli/src/main/scala/io/joern/joerncli/console/ReplBridge.scala @@ -0,0 +1,26 @@ +package io.joern.joerncli.console + +import io.joern.console.{BridgeBase, JoernProduct} + +import java.io.PrintStream + +object ReplBridge extends BridgeBase { + + override val slProduct = JoernProduct + + def main(args: Array[String]): Unit = { + run(parseConfig(args)) + } + + /** Code that is executed when starting the shell + */ + override def predefLines = + Predefined.forInteractiveShell + + override def greeting = JoernConsole.banner() + + override def promptStr: String = "joern" + + override def onExitCode: String = "workspace.projects.foreach(_.close)" + +} diff --git a/joern-cli/src/test/scala/io/joern/joerncli/ConsoleTests.scala b/joern-cli/src/test/scala/io/joern/joerncli/ConsoleTests.scala index 89ff48e6de2f..e167b6bd6384 100644 --- a/joern-cli/src/test/scala/io/joern/joerncli/ConsoleTests.scala +++ b/joern-cli/src/test/scala/io/joern/joerncli/ConsoleTests.scala @@ -2,28 +2,24 @@ package io.joern.joerncli import better.files.Dsl.mkdir import better.files.File -import io.joern.console.cpgcreation.ImportCode -import io.joern.console.testing.TestCpgGeneratorFactory -import io.joern.console.workspacehandling.Project -import io.joern.console.{Console, ConsoleConfig, InstallConfig} -import io.joern.joerncli.console.JoernConsole import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -class MyImportCode[T <: Project](console: Console[T]) extends ImportCode(console) { - override val generatorFactory = new TestCpgGeneratorFactory(console.config) -} +class ConsoleTests extends AnyWordSpec with Matchers { -class TestJoernConsole(workspaceDir: String) extends JoernConsole { - override val config = - new ConsoleConfig(install = new InstallConfig(Map("SHIFTLEFT_CONSOLE_INSTALL_DIR" -> workspaceDir))) - override val importCode = new MyImportCode(this) -} + "run" should { + "provide a human readable overview of overlay creators" in withTestCode { codeDir => + RunScriptTests.exec("general/run.sc", codeDir.toString) + } + } -object ConsoleFixture { - def apply(constructor: String => JoernConsole = { x => - new TestJoernConsole(x) - })(fun: (JoernConsole, File) => Unit): Unit = { + "help" should { + "allow getting long description via help object" in withTestCode { codeDir => + RunScriptTests.exec("general/help.sc", codeDir.toString) + } + } + + def withTestCode(fun: File => Unit): Unit = { File.usingTemporaryDirectory("console") { workspaceDir => File.usingTemporaryDirectory("console") { codeDir => mkdir(codeDir / "dir1") @@ -31,45 +27,8 @@ object ConsoleFixture { (codeDir / "dir1" / "foo.c") .write("int main(int argc, char **argv) { char *ptr = 0x1 + argv; return argc; }") (codeDir / "dir2" / "bar.c").write("int bar(int x) { return x; }") - val console = constructor(workspaceDir.toString) - fun(console, codeDir) - } - } - } -} - -class ConsoleTests extends AnyWordSpec with Matchers { - - "run" should { - "provide a human readable overview of overlay creators" in ConsoleFixture({ dir => - new TestJoernConsole(dir) - }) { (console, codeDir) => - File.usingTemporaryFile("console") { myScript => - console.importCode(codeDir.toString) - val cpg = console.cpg - myScript.write(s""" - | if (!run.toString.contains("base")) - | throw new RuntimeException("base layer not applied...?") - |""".stripMargin) - console.CpgScriptRunner(cpg).runScript(myScript.toString) + fun(codeDir) } } } - - "help" should { - "allow getting long description via help object" in ConsoleFixture({ dir => - new TestJoernConsole(dir) - }) { (console, codeDir) => - File.usingTemporaryFile("console") { myScript => - console.importCode(codeDir.toString) - val cpg = console.cpg - myScript.write(s""" - | if (help.cpg.toString.isEmpty) - | throw new RuntimeException("no help text available...?") - |""".stripMargin) - console.CpgScriptRunner(cpg).runScript(myScript.toString) - } - } - } - } diff --git a/joern-cli/src/test/scala/io/joern/joerncli/RunScriptTests.scala b/joern-cli/src/test/scala/io/joern/joerncli/RunScriptTests.scala index e2b74461dcd0..9e5deda87c10 100644 --- a/joern-cli/src/test/scala/io/joern/joerncli/RunScriptTests.scala +++ b/joern-cli/src/test/scala/io/joern/joerncli/RunScriptTests.scala @@ -1,112 +1,50 @@ package io.joern.joerncli -import better.files.File -import io.joern.{console, joerncli} -import io.joern.joerncli.console.JoernConsole -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} -import io.shiftleft.semanticcpg.language._ +import io.joern.console.Config +import io.joern.joerncli.console.ReplBridge +import io.shiftleft.utils.ProjectRoot import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -class RunScriptTests extends AnyWordSpec with Matchers with AbstractJoernCliTest { - - "Executing scripts for example code 'testcode/unsafe-ptr" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/unsafe-ptr")) - ) { case (cpg: Cpg, _) => - "work correctly for 'pointer-to-int.sc'" in { - val calls = - joerncli.console.JoernConsole.runScriptTest("c/pointer-to-int.sc", Map.empty, cpg).asInstanceOf[List[Call]] - - calls.map(_.code) should contain theSameElementsAs List( - "simple_subtraction = p - q", - "nested_subtraction = p - q - r", - "literal_subtraction = p - i", - "addrOf_subtraction = p - &i", - "nested_addrOf_subtraction = 3 - &i - 4", - "literal_addrOf_subtraction = 3 - &i", - "array_subtraction = x - p", - "array_literal_subtraction = x - 3", - "array_addrOf_subtraction = x - &i" - // TODO: We don't have access to type info for indirect field member access. - // "unsafe_struct = foo_t->p - 1" - ) +class RunScriptTests extends AnyWordSpec with Matchers { + import RunScriptTests._ + + Seq( + ("c/pointer-to-int.sc", "unsafe-ptr"), + ("c/syscalls.sc", "syscalls"), + ("c/userspace-memory-access.sc", "syscalls"), + ("c/malloc-overflow.sc", "malloc-overflow"), + ("c/malloc-leak.sc", "leak"), + ("c/const-ish.sc", "const-ish") + ).foreach { case (scriptFileName, codePathRelative) => + s"Executing '$scriptFileName' on '$codePathRelative'" in { + exec(scriptFileName, s"$testCodeRoot/$codePathRelative") } } - "Executing scripts for example code 'testcode/syscalls" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/syscalls")) - ) { case (cpg: Cpg, _) => - "work correctly for 'syscalls.sc'" in { - val calls = - joerncli.console.JoernConsole.runScriptTest("c/syscalls.sc", Map.empty, cpg).asInstanceOf[List[Call]] - - calls.map(_.name) should contain theSameElementsAs List("gettimeofday", "exit") + "should return Failure if" when { + "script doesn't exist" in { + val result = ReplBridge.runScript(Config(scriptFile = Some(scriptsRoot.resolve("does-not-exist.sc")))) + result.failed.get.getMessage should include("does not exist") } - "work correctly for 'userspace-memory-access.sc'" in { - val calls = - joerncli.console.JoernConsole - .runScriptTest("c/userspace-memory-access.sc", Map.empty, cpg) - .asInstanceOf[List[Call]] - - calls.map(_.name) should contain theSameElementsAs List("get_user") + "script runs ins an exception" in { + val result = ReplBridge.runScript(Config(scriptFile = Some(scriptsRoot.resolve("trigger-error.sc")))) + result.failed.get.getMessage should include("exit code was 1") } } +} - "Executing scripts for example code 'testcode/malloc-overflow" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/malloc-overflow")) - ) { case (cpg: Cpg, _) => - "work correctly for 'malloc-overflow.sc'" in { - val calls = - joerncli.console.JoernConsole.runScriptTest("c/malloc-overflow.sc", Map.empty, cpg).asInstanceOf[List[Call]] +object RunScriptTests { + val projectRoot = ProjectRoot.find.path.toAbsolutePath + val scriptsRoot = projectRoot.resolve("scripts") + val testCodeRoot = s"${projectRoot}/joern-cli/src/test/resources/testcode" - calls.map(_.code) should contain theSameElementsAs List( - "malloc(sizeof(int) * 42)", - "malloc(sizeof(int) * 3)", - "malloc(sizeof(int) + 55)" + def exec(scriptFileName: String, codePathAbsolute: String): Unit = { + ReplBridge + .runScript( + Config(scriptFile = Some(scriptsRoot.resolve(scriptFileName)), params = Map("inputPath" -> codePathAbsolute)) ) - } - } - - "Executing scripts for example code 'testcode/leak" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/leak")) - ) { case (cpg: Cpg, _) => - "work correctly for 'malloc-leak.sc'" in { - val calls = - joerncli.console.JoernConsole.runScriptTest("c/malloc-leak.sc", Map.empty, cpg).asInstanceOf[Set[String]] - - calls should contain theSameElementsAs Set("leak") - } - } - - "Executing scripts for example code 'testcode/const-ish" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/const-ish")) - ) { case (cpg: Cpg, _) => - "work correctly for 'const-ish.sc'" in { - val methods = - JoernConsole.runScriptTest("c/const-ish.sc", Map.empty, cpg).asInstanceOf[Set[Method]] - - // "side_effect_number" is included here as we are simply trying to emulate a side effect. - methods.map(_.name) should contain theSameElementsAs - Set( - "modify_const_struct_member_cpp_cast", - "modify_const_struct_member_c_cast", - "modify_const_struct_cpp_cast", - "modify_const_struct_c_cast" - ) - } + .get } - - "Executing scripts for example code 'testcode/free'" should withTestCpg( - File(getClass.getClassLoader.getResource("testcode/free")) - ) { case (cpg: Cpg, _) => - "work correctly for 'list-funcs'" in { - val expected = cpg.method.name.l - val actual = joerncli.console.JoernConsole.runScriptTest("general/list-funcs.sc", Map.empty, cpg) - actual shouldBe expected - } - - } - } diff --git a/joern-cli/src/universal/joern b/joern-cli/src/universal/joern index dfff80431878..bde888d92884 100755 --- a/joern-cli/src/universal/joern +++ b/joern-cli/src/universal/joern @@ -6,7 +6,7 @@ else SCRIPT_ABS_PATH=$(readlink -f "$0") fi SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH") -SCRIPT="$SCRIPT_ABS_DIR"/bin/ammonite-bridge +SCRIPT="$SCRIPT_ABS_DIR"/bin/repl-bridge if [ ! -f "$SCRIPT" ]; then echo "Unable to find $SCRIPT, have you created the distribution?" diff --git a/joern-cli/src/universal/joern.bat b/joern-cli/src/universal/joern.bat index 6916ccc9726d..f2f3f45b81e4 100644 --- a/joern-cli/src/universal/joern.bat +++ b/joern-cli/src/universal/joern.bat @@ -1,6 +1,6 @@ @echo off set "SCRIPT_ABS_DIR=%~dp0" -set "SCRIPT=%SCRIPT_ABS_DIR%bin\ammonite-bridge.bat" +set "SCRIPT=%SCRIPT_ABS_DIR%bin\repl-bridge.bat" "%SCRIPT%" "-J-XX:+UseG1GC" "-J-XX:CompressedClassSpaceSize=128m" "-Dlog4j.configurationFile=%SCRIPT_ABS_DIR%conf\log4j2.xml" %* diff --git a/joern.bat b/joern.bat index c40f2618debf..12c562c0b215 100644 --- a/joern.bat +++ b/joern.bat @@ -1,3 +1,3 @@ @echo off -.\joern-cli\target\universal\stage\bin\ammonite-bridge.bat %* +.\joern-cli\target\universal\stage\bin\repl-bridge.bat %* diff --git a/macros/build.sbt b/macros/build.sbt index a3d5e103c44c..d54e400c5e88 100644 --- a/macros/build.sbt +++ b/macros/build.sbt @@ -1,25 +1,10 @@ name := "macros" -scalaVersion := "2.13.8" -crossScalaVersions := Seq("2.13.8", "3.2.2") - dependsOn(Projects.semanticcpg % Test) libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test -) ++ ( - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => Seq() - case _ => Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value) - } -) - -scalacOptions ++= Seq() ++ ( - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((3, _)) => Seq() - case _ => Seq("-Yrangepos") - } ) enablePlugins(JavaAppPackaging) diff --git a/macros/src/main/scala-2/io/joern/macros/QueryMacros.scala b/macros/src/main/scala-2/io/joern/macros/QueryMacros.scala deleted file mode 100644 index 74302a629f0d..000000000000 --- a/macros/src/main/scala-2/io/joern/macros/QueryMacros.scala +++ /dev/null @@ -1,26 +0,0 @@ -package io.joern.macros - -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.StoredNode -import io.joern.console.TraversalWithStrRep -import overflowdb.traversal.Traversal - -import scala.language.experimental.macros -import scala.reflect.macros.whitebox - -object QueryMacros { - - def withStrRep(traversal: Cpg => Traversal[_ <: StoredNode]): TraversalWithStrRep = macro withStrRepImpl - - def withStrRepImpl(c: whitebox.Context)(traversal: c.Tree): c.Expr[TraversalWithStrRep] = { - import c.universe._ - val fileContent = new String(traversal.pos.source.content) - val start = traversal.pos.start - val end = traversal.pos.end - val traversalAsString: String = fileContent.slice(start, end) - - c.Expr(q""" - TraversalWithStrRep($traversal, $traversalAsString) - """) - } -} diff --git a/macros/src/main/scala-3/io/joern/macros/QueryMacros.scala b/macros/src/main/scala/io/joern/macros/QueryMacros.scala similarity index 63% rename from macros/src/main/scala-3/io/joern/macros/QueryMacros.scala rename to macros/src/main/scala/io/joern/macros/QueryMacros.scala index 7dacfc05d9b0..df90b64600a0 100644 --- a/macros/src/main/scala-3/io/joern/macros/QueryMacros.scala +++ b/macros/src/main/scala/io/joern/macros/QueryMacros.scala @@ -10,13 +10,14 @@ import scala.quoted.{Expr, Quotes} object QueryMacros { inline def withStrRep(inline traversal: Cpg => Traversal[_ <: StoredNode]): TraversalWithStrRep = - ${withStrRepImpl('{traversal})} + ${ withStrRepImpl('{ traversal }) } - private def withStrRepImpl(travExpr: Expr[Cpg => Traversal[_ <: StoredNode]]) - (using quotes: Quotes): Expr[TraversalWithStrRep] = { + private def withStrRepImpl( + travExpr: Expr[Cpg => Traversal[_ <: StoredNode]] + )(using quotes: Quotes): Expr[TraversalWithStrRep] = { import quotes.reflect._ - val pos = travExpr.asTerm.pos + val pos = travExpr.asTerm.pos val code = Position(pos.sourceFile, pos.start, pos.end).sourceCode.getOrElse("N/A") - '{TraversalWithStrRep(${travExpr}, ${Expr(code)})} + '{ TraversalWithStrRep(${ travExpr }, ${ Expr(code) }) } } } diff --git a/querydb/README.md b/querydb/README.md index d6ca5e241fe6..4c8666c68b41 100644 --- a/querydb/README.md +++ b/querydb/README.md @@ -56,7 +56,7 @@ Please follow the rules below for a tear-free query writing experience: A query bundle is simply an `object` that derives from `QueryBundle` * Queries can have parameters,but you must provide a default value for each parameter * Please add unit tests for queries. These also serve as a spec for what your query does. -* Please format the code before sending a PR using `sbt scalafmt` and `sbt test:scalafmt` +* Please format the code before sending a PR using `sbt scalafmt Test/scalafmt` Take a look at the query bundle `Metrics` at `src/main/scala/io/joern/scanners/c/Metrics.scala` as an example: diff --git a/querydb/build.sbt b/querydb/build.sbt index 555015ab9ce3..134e24a8a41e 100644 --- a/querydb/build.sbt +++ b/querydb/build.sbt @@ -63,7 +63,7 @@ createDistribution := { dstArchive.toJava } -Compile / scalacOptions ++= Seq("-Xfatal-warnings", "-feature", "-deprecation", "-language:implicitConversions") +Compile / scalacOptions ++= Seq("-Xfatal-warnings", "-feature", "-language:implicitConversions") fork := true diff --git a/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala b/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala index 999026e723a3..fa2fb259396a 100644 --- a/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala +++ b/querydb/src/main/scala/io/joern/scanners/android/UnprotectedAppParts.scala @@ -21,7 +21,7 @@ object UnprotectedAppParts extends QueryBundle { title = "Intent redirected without validation", description = "-", score = 4, - withStrRep({ cpg => + withStrRep { cpg => cpg.method .nameExact("getParcelableExtra") .callIn @@ -31,8 +31,7 @@ object UnprotectedAppParts extends QueryBundle { def sink = startActivityCalls.whereNot(_.controlledBy.astParent.isControlStructure).argument sink.reachableByFlows(c).nonEmpty } - .l - }), + }, tags = List(QueryTags.android), codeExamples = CodeExamples( List(""" diff --git a/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala b/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala index ea57b54641aa..e7e58ca2933c 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/HeapBasedOverflow.scala @@ -28,9 +28,9 @@ object HeapBasedOverflow extends QueryBundle { score = 4, withStrRep({ cpg => val src = - cpg.method(".*malloc$").callIn.where(_.argument(1).arithmetic).l + cpg.method(".*malloc$").callIn.where(_.argument(1).arithmetic) - cpg.method("(?i)memcpy").callIn.l.filter { memcpyCall => + cpg.method("(?i)memcpy").callIn.filter { memcpyCall => memcpyCall .argument(1) .reachableBy(src) diff --git a/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala b/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala index b74242ae9121..99bbc63a5b3d 100644 --- a/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala +++ b/querydb/src/main/scala/io/joern/scanners/c/NullTermination.scala @@ -29,7 +29,7 @@ object NullTermination extends QueryBundle { |""".stripMargin, score = 4, withStrRep({ cpg => - val allocations = cpg.method(".*malloc$").callIn.argument(1).l + val allocations = cpg.method(".*malloc$").callIn.argument(1) cpg .method("(?i)strncpy") .callIn diff --git a/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala b/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala index f6dfb76bc256..21310c543181 100644 --- a/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala +++ b/querydb/src/main/scala/io/joern/scanners/ghidra/UserInputIntoDangerousFunctions.scala @@ -25,7 +25,7 @@ object UserInputIntoDangerousFunctions extends QueryBundle { def source = cpg.call.methodFullName("getenv").cfgNext.isCall.argument(2) def sink = cpg.method.fullName("strcpy").parameter.index(2) - sink.reachableBy(source).l + sink.reachableBy(source) }), tags = List(QueryTags.badfn) ) diff --git a/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala b/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala index fb04bf7f2ab1..1e43586657f0 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/CryptographyMisuse.scala @@ -32,7 +32,7 @@ object CryptographyMisuse extends QueryBundle { def sink = cpg.method.fullName("java.security.MessageDigest.getInstance.*").parameter - sink.reachableBy(source).l + sink.reachableBy(source) }), tags = List(QueryTags.cryptography, QueryTags.default), codeExamples = CodeExamples( @@ -70,7 +70,7 @@ object CryptographyMisuse extends QueryBundle { def sink = cpg.method.fullName("javax.crypto.spec.PBEKeySpec..*").parameter - sink.reachableBy(source).dedup.filter(f => Integer.parseInt(f.code) < 1000).l + sink.reachableBy(source).dedup.filter(f => Integer.parseInt(f.code) < 1000) }), tags = List(QueryTags.cryptography, QueryTags.default), codeExamples = CodeExamples( diff --git a/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala b/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala index a65cf3bc2ce5..67aebe735261 100644 --- a/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala +++ b/querydb/src/main/scala/io/joern/scanners/java/SQLInjection.scala @@ -32,7 +32,7 @@ object SQLInjection extends QueryBundle { def sink = cpg.method.name("query").parameter.order(1) - sink.reachableBy(source).l + sink.reachableBy(source) }), tags = List(QueryTags.sqlInjection, QueryTags.default) ) diff --git a/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkProtocols.scala b/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkProtocols.scala index da7b1c6a3daf..098f9f25a5ac 100644 --- a/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkProtocols.scala +++ b/querydb/src/main/scala/io/joern/scanners/kotlin/NetworkProtocols.scala @@ -26,7 +26,6 @@ object NetworkProtocols extends QueryBundle { .fullNameExact("java.net.URL.:void(java.lang.String)") .callIn .where(_.argument.isLiteral.code("^[^h]*http:.*")) - .l }), tags = List(QueryTags.insecureNetworkTraffic, QueryTags.android), codeExamples = CodeExamples( diff --git a/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala b/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala index f20c2909a22f..9fc370a5284f 100644 --- a/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala +++ b/querydb/src/test/scala/io/joern/suites/AllBundlesTestSuite.scala @@ -2,8 +2,6 @@ package io.joern.suites import io.joern.console.QueryDatabase import org.scalatest.wordspec.AnyWordSpec -import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper - import org.scalatest.matchers.should.Matchers._ class AllBundlesTestSuite extends AnyWordSpec { diff --git a/semanticcpg/build.sbt b/semanticcpg/build.sbt index 0ddbc46cf15d..df7a0161e0d9 100644 --- a/semanticcpg/build.sbt +++ b/semanticcpg/build.sbt @@ -1,7 +1,5 @@ name := "semanticcpg" -crossScalaVersions := Seq("2.13.8", "3.2.2") - libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "org.json4s" %% "json4s-native" % Versions.json4s, @@ -9,5 +7,3 @@ libraryDependencies ++= Seq( ) Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value) - -compile / javacOptions ++= Seq("-g") //debug symbols diff --git a/test-additionalfuncs.sc b/test-additionalfuncs.sc new file mode 100644 index 000000000000..54f5cb911e6c --- /dev/null +++ b/test-additionalfuncs.sc @@ -0,0 +1,8 @@ + +// ./joern --import test-additionalfuncs.sc +// joern> exportMethods("workspace/src/cpg.bin", "a.txt") + +def exportMethods(cpgFile: String, outFile: String) = { + loadCpg(cpgFile) + cpg.method.name.l |> outFile +} diff --git a/test-cpg-callotherscript.sc b/test-cpg-callotherscript.sc new file mode 100644 index 000000000000..97024fb8d03f --- /dev/null +++ b/test-cpg-callotherscript.sc @@ -0,0 +1,6 @@ +// to test, run e.g. +// ./joern --script test-cpg-callotherscript.sc --import test-additionalfuncs.sc --param cpgFile=workspace/src/cpg.bin --param outFile=a.txt + +@main def exec(cpgFile: String, outFile: String) = { + exportMethods(cpgFile, outFile) +} diff --git a/test-cpg.sc b/test-cpg.sc new file mode 100644 index 000000000000..9c10d1ff1822 --- /dev/null +++ b/test-cpg.sc @@ -0,0 +1,4 @@ +@main def exec(cpgFile: String, outFile: String) = { + loadCpg(cpgFile) + cpg.method.name.l |> outFile +} \ No newline at end of file diff --git a/test-dependencies.sc b/test-dependencies.sc new file mode 100644 index 000000000000..cdec67909f3f --- /dev/null +++ b/test-dependencies.sc @@ -0,0 +1,5 @@ +//> using com.michaelpollmeier:versionsort:1.0.7 + +val compareResult = versionsort.VersionHelper.compare("1.0", "0.9") +assert(compareResult == 1, + s"result of comparison should be `1`, but was `$compareResult`") \ No newline at end of file diff --git a/test-main-withargs.sc b/test-main-withargs.sc new file mode 100644 index 000000000000..364e0b20b8ce --- /dev/null +++ b/test-main-withargs.sc @@ -0,0 +1,3 @@ +@main def main(name: String) = { + println(s"Hello, $name!") +} diff --git a/test-main.sc b/test-main.sc new file mode 100644 index 000000000000..9fe313e18910 --- /dev/null +++ b/test-main.sc @@ -0,0 +1,3 @@ +@main def main() = { + println("Hello, world!") +} diff --git a/test-simple.sc b/test-simple.sc new file mode 100644 index 000000000000..c4267130d162 --- /dev/null +++ b/test-simple.sc @@ -0,0 +1 @@ +println("Hello!") diff --git a/tests/frontends-tests.sh b/tests/frontends-tests.sh index 042c179312fc..7fe012d131f8 100755 --- a/tests/frontends-tests.sh +++ b/tests/frontends-tests.sh @@ -33,7 +33,7 @@ declare -A expectedMethod=( for frontend in "${frontends[@]}"; do rm -rf workspace - $JOERN --script tests/frontends-testscript.sc --params inputPath=tests/code/$frontend,minMethodCount=${minMethodCount[$frontend]},expectedMethod=${expectedMethod[$frontend]},frontend=$frontend + $JOERN --script tests/frontends-testscript.sc --param inputPath=tests/code/$frontend --param minMethodCount=${minMethodCount[$frontend]} --param expectedMethod=${expectedMethod[$frontend]} --param frontend=$frontend JOERN_EXIT_CODE=$? if [ $JOERN_EXIT_CODE != 0 ]; then diff --git a/tests/loadcpg.sc b/tests/loadcpg.sc deleted file mode 100644 index 06b8bcbcbbe4..000000000000 --- a/tests/loadcpg.sc +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Utility script fed into ammonite before testing each individual script. - */ - -import io.joern.joerncli.console.JoernConsole._ - -importCpg("cpg.bin") diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index de8bdc9282ac..826888b5bdcb 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -6,22 +6,35 @@ SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH") # Setup JOERN="$SCRIPT_ABS_DIR"/../joern -JOERN_PARSER="$SCRIPT_ABS_DIR"/../joern-parse JOERN_SCRIPTS_DIR="$SCRIPT_ABS_DIR/../scripts" +TESTCODE_ROOT="$SCRIPT_ABS_DIR/../joern-cli/src/test/resources/testcode" -# Generate a CPG for use in the script tests. -$JOERN_PARSER "$SCRIPT_ABS_DIR"/code +# Run all scripts +scripts=( + c/pointer-to-int.sc + c/syscalls.sc + c/userspace-memory-access.sc + c/malloc-overflow.sc + c/malloc-leak.sc + c/const-ish.sc +) +declare -A code=( + [c/pointer-to-int.sc]=unsafe-ptr + [c/syscalls.sc]=syscalls + [c/userspace-memory-access.sc]=syscalls + [c/malloc-overflow.sc]=malloc-overflow + [c/malloc-leak.sc]=leak + [c/const-ish.sc]=const-ish +) -# Run each script. -for script in "$JOERN_SCRIPTS_DIR"/**/*.sc; do - echo "Testing script [$script]..." - $JOERN --import "$SCRIPT_ABS_DIR"/loadcpg.sc --script "$script" +for script in "${scripts[@]}"; do + $JOERN --script "$JOERN_SCRIPTS_DIR/$script" --param inputPath="$TESTCODE_ROOT/${code[$script]}" JOERN_EXIT_CODE=$? + if [ $JOERN_EXIT_CODE != 0 ]; then echo "Script [$script] failed to run successfully." exit 1 fi - echo "Script [$script] passed..." echo "" done