Redact passwords from conda output (apache#243)

Author: dansanduleac

core/src/main/scala/org/apache/spark/api/conda/CondaEnvironmentManager.scala

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.attribute.PosixFilePermission
+import java.util.regex.Pattern
 
 import scala.collection.JavaConverters._
 import scala.sys.process.BasicIO
@@ -160,16 +161,21 @@ final class CondaEnvironmentManager(condaBinaryPath: String,
    * @return the stdout of the process
    */
   private[this] def runOrFail(command: ProcessBuilder, description: String): String = {
+    import CondaEnvironmentManager.redactCredentials
     val out = new StringBuffer
     val err = new StringBuffer
     val collectErrOutToBuffer = new ProcessIO(
-    BasicIO.input(false),
-    BasicIO.processFully(out),
-    BasicIO.processFully(line => {
-      err append line
-      err append BasicIO.Newline
-      log.info(s"<conda> $line")
-    }))
+      BasicIO.input(false),
+      BasicIO.processFully((redactCredentials _).andThen(line => {
+        out.append(line)
+        out.append(BasicIO.Newline)
+        ()
+      })),
+      BasicIO.processFully((redactCredentials _).andThen(line => {
+        err.append(line)
+        err.append(BasicIO.Newline)
+        log.info(s"<conda> $line")
+      })))
     val exitCode = command.run(collectErrOutToBuffer).exitValue()
     if (exitCode != 0) {
       throw new SparkException(s"Attempt to $description exited with code: "
@@ -196,6 +202,13 @@ object CondaEnvironmentManager extends Logging {
     sparkConf.contains(CONDA_BINARY_PATH)
   }
 
+  private[this] val httpUrlToken =
+    Pattern.compile("(\\b\\w+://[^:/@]*:)([^/@]+)(?=@([\\w-.]+)(:\\d+)?\\b)")
+
+  private[conda] def redactCredentials(line: String): String = {
+    httpUrlToken.matcher(line).replaceAll("$1<password>")
+  }
+
   def fromConf(sparkConf: SparkConf): CondaEnvironmentManager = {
     val condaBinaryPath = sparkConf.get(CONDA_BINARY_PATH).getOrElse(
       sys.error(s"Expected config ${CONDA_BINARY_PATH.key} to be set"))

core/src/test/scala/org/apache/spark/api/conda/CondaEnvironmentManagerTest.scala

@@ -28,4 +28,19 @@ class CondaEnvironmentManagerTest extends org.apache.spark.SparkFunSuite with Te
     CondaEnvironmentManager.ensureExecutable(path.toString)
     assert(Files.isExecutable(path), "File should now be executable")
   }
+
+  test("CondaEnvironmentManager.redactCredentials") {
+    val original = "24u0f8 adfghjfouh https://:[email protected]" +
+      ".baz:12345/whatever/else"
+    val redacted = "24u0f8 adfghjfouh https://:<password>@my-host-name-5.foo.bar" +
+      ".baz:12345/whatever/else"
+    assert(CondaEnvironmentManager.redactCredentials(original) == redacted)
+  }
+
+  test("CondaEnvironmentManager.redactTwoCredentials") {
+    val original = "random:https://:[email protected]/whatever/else][http://us_r:[email protected]:222"
+    val redacted = "random:https://:<password>@x-5.bar/whatever/else]" +
+      "[http://us_r:<password>@yy.bar:222"
+    assert(CondaEnvironmentManager.redactCredentials(original) == redacted)
+  }
 }