Replace synchronization blocks in polyglot Java initialization

.github/workflows/engine-checks.yml

@@ -513,6 +513,7 @@ jobs:
           ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
           ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPORT_ALL_TESTS: true
       - if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         name: Standard Library Test Reporter
         uses: dorny/test-reporter@v1

engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/RuntimeManagementTest.scala

@@ -208,5 +208,45 @@ class RuntimeManagementTest extends InterpreterTest {
       def all                         = 0.to(4).map(mkAccessStr) ++ List(1, 3).map(mkFreeStr)
       totalOut should contain theSameElementsAs all
     }
+
+    "Allow for multithreaded polyglot class loading" in {
+      val langCtx = interpreterContext
+        .ctx()
+        .getBindings(LanguageInfo.ID)
+        .invokeMember(MethodNames.TopScope.LEAK_CONTEXT)
+        .asHostObject[EnsoContext]()
+
+      val code =
+        """import Standard.Base.Data.Numbers
+          |polyglot java import org.enso.example.TestClass
+          |main =
+          |    instance = TestClass.new (x -> x * 2)
+          |    instance.callFunctionAndIncrement 10
+          |""".stripMargin
+
+      val main = getMain(code)
+
+      def runMain()
+        : java.util.concurrent.CompletableFuture[org.graalvm.polyglot.Value] =
+        langCtx.getThreadManager.submit(() => {
+          main.execute()
+        })
+
+      def runTest(): Unit = {
+        val futures = 0.until(parallelism).map(_ => runMain())
+        val combinedFuture = java.util.concurrent.CompletableFuture
+          .allOf(futures: _*)
+          .thenApply(_ => {
+            futures
+              .map(_.get(10, java.util.concurrent.TimeUnit.SECONDS).asInt())
+          })
+        val result =
+          combinedFuture.get(20, java.util.concurrent.TimeUnit.SECONDS)
+        result should equal(List(21, 21, 21, 21, 21))
+        futures.forall(_.isDone) shouldBe true
+      }
+
+      runTest()
+    }
   }
 }

engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java

@@ -30,7 +30,7 @@ Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) {
     var file = ctx.getTruffleFile(new File(expectStringNode.execute(path)));
     if (getRootNode() instanceof ClosureRootNode crn) {
       var pkg = crn.getModuleScope().getModule().getPackage();
-      ctx.addToClassPath(pkg, file);
+      ctx.addToClassPath(pkg, file, true);
     } else {
       throw ctx.raiseAssertionPanic(this, "Cannot find package", null);
     }

engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java

@@ -513,16 +513,17 @@ public Optional<Module> findModuleByExpressionId(UUID expressionId) {
    *
    * @param who who requests the addition
    * @param file the file to register
+   * @param polyglotContextEntered true if a polyglot context has been entered, false otherwise
    */
   @TruffleBoundary
-  public void addToClassPath(Package<?> who, TruffleFile file) {
+  public void addToClassPath(Package<?> who, TruffleFile file, boolean polyglotContextEntered) {
     assert who != null;
     var path = new File(file.toUri()).getAbsoluteFile();
     if (!path.exists()) {
       throw new IllegalStateException("File not found " + path);
     }
     try {
-      EnsoPolyglotJava.addToClassPath(this, who, path);
+      EnsoPolyglotJava.addToClassPath(this, who, path, polyglotContextEntered);
     } catch (InteropException ex) {
       throw raiseAssertionPanic(null, "Cannot add " + file + " to classpath", ex);
     }
@@ -998,10 +999,12 @@ public State currentState() {
     return singleStateProfile.profile(language.currentState());
   }
 
-  private Object extraValues(int index, Function<EnsoContext, ?> init) {
+  private synchronized Object extraValues(int index, Function<EnsoContext, ?> init) {
     if (index >= extraValues.length || extraValues[index] == null) {
       CompilerDirectives.transferToInterpreterAndInvalidate();
-      extraValues = Arrays.copyOf(extraValues, Extra.COUNTER.get());
+      if (index >= extraValues.length) {
+        extraValues = Arrays.copyOf(extraValues, index + 1);
+      }
       extraValues[index] = init.apply(this);
       assert extraValues[index] != null;
     }

engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoPolyglotJava.java

@@ -1,6 +1,7 @@
 package org.enso.interpreter.runtime;
 
 import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.TruffleSafepoint;
 import com.oracle.truffle.api.interop.ArityException;
 import com.oracle.truffle.api.interop.InteropException;
 import com.oracle.truffle.api.interop.InteropLibrary;
@@ -17,6 +18,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Semaphore;
 import org.enso.common.HostEnsoUtils;
 import org.enso.common.RuntimeOptions;
 import org.enso.interpreter.runtime.util.TruffleFileSystem;
@@ -36,6 +38,7 @@ final class EnsoPolyglotJava {
   private final boolean isHostClassLoading;
   private final List<File> pendingPath = new ArrayList<>();
   private Object polyglotJava = this;
+  private Semaphore lock = new Semaphore(1, true);
 
   private EnsoPolyglotJava(EnsoContext ctx, boolean isHostClassLoading) {
     this.ctx = ctx;
@@ -152,58 +155,86 @@ static void close(EnsoContext ctx) {
   }
 
   @CompilerDirectives.TruffleBoundary
-  private synchronized Object findPolyglotJava() throws InteropException {
-    if (polyglotJava != this) {
-      return polyglotJava;
-    }
-    polyglotJava = createPolyglotJava(ctx);
-    while (!pendingPath.isEmpty()) {
-      addToClassPath(pendingPath.remove(0));
-    }
+  private Object findPolyglotJava() throws InteropException {
+    TruffleSafepoint.setBlockedThreadInterruptible(null, Semaphore::acquire, lock);
     try {
-      InteropLibrary.getUncached()
-          .invokeMember(polyglotJava, "findLibraries", new LibraryResolver());
-    } catch (InteropException ex) {
-      logger.log(Level.WARNING, "Cannot register findLibraries", ex);
+      if (polyglotJava != this) {
+        return polyglotJava;
+      }
+      polyglotJava = createPolyglotJava(ctx);
+      while (!pendingPath.isEmpty()) {
+        InteropLibrary.getUncached()
+            .invokeMember(polyglotJava, "addPath", pendingPath.remove(0).toString());
+      }
+      try {
+        InteropLibrary.getUncached()
+            .invokeMember(polyglotJava, "findLibraries", new LibraryResolver());
+      } catch (InteropException ex) {
+        logger.log(Level.WARNING, "Cannot register findLibraries", ex);
+      }
+      return polyglotJava;
+    } finally {
+      lock.release();
     }
-    return polyglotJava;
   }
 
   /**
    * This method ensure that hosted as well as guest classpath is the same. This is necessary until
    * real isolation between libraries is implemented.
    */
-  static void addToClassPath(EnsoContext ctx, Object whoIsIgnored, File path)
+  static void addToClassPath(
+      EnsoContext ctx, Object whoIsIgnored, File path, boolean polyglotContextEntered)
       throws InteropException {
     var data = KEY.get(ctx);
-    data.hosted.addToClassPath(path);
-    data.guest.addToClassPath(path);
+    data.hosted.addToClassPath(path, polyglotContextEntered);
+    data.guest.addToClassPath(path, polyglotContextEntered);
   }
 
   /**
    * Modifies the classpath to use to lookup {@code polyglot java} imports.
    *
    * @param file the file to register
+   * @param polyglotContextEntered if true, any lock acquisition will be interruptable for Truffle's
+   *     Safepoints purposes
    */
   @CompilerDirectives.TruffleBoundary
-  private final synchronized void addToClassPath(File file) throws InteropException {
-    if (polyglotJava == this) {
-      pendingPath.add(file);
+  private final void addToClassPath(File file, boolean polyglotContextEntered)
+      throws InteropException {
+    if (polyglotContextEntered) {
+      TruffleSafepoint.setBlockedThreadInterruptible(null, Semaphore::acquire, lock);
     } else {
-      InteropLibrary.getUncached().invokeMember(polyglotJava, "addPath", file.toString());
+      try {
+        lock.acquire();
+      } catch (InterruptedException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    try {
+      if (polyglotJava == this) {
+        pendingPath.add(file);
+      } else {
+        InteropLibrary.getUncached().invokeMember(polyglotJava, "addPath", file.toString());
+      }
+    } finally {
+      lock.release();
     }
   }
 
-  private final synchronized void close() {
-    if (polyglotJava instanceof TruffleObject closeJava) {
-      polyglotJava = null;
-      try {
-        InteropLibrary.getUncached().invokeMember(closeJava, "close");
-      } catch (InteropException ex) {
-        logger.log(Level.WARNING, "Cannot close " + closeJava, ex);
+  private final void close() {
+    TruffleSafepoint.setBlockedThreadInterruptible(null, Semaphore::acquire, lock);
+    try {
+      if (polyglotJava instanceof TruffleObject closeJava) {
+        polyglotJava = null;
+        try {
+          InteropLibrary.getUncached().invokeMember(closeJava, "close");
+        } catch (InteropException ex) {
+          logger.log(Level.WARNING, "Cannot close " + closeJava, ex);
+        }
+      } else {
+        polyglotJava = null;
       }
-    } else {
-      polyglotJava = null;
+    } finally {
+      lock.release();
     }
   }
 

engine/runtime/src/main/java/org/enso/interpreter/runtime/ThreadExecutors.java

@@ -40,7 +40,8 @@ ExecutorService newCachedThreadPool(
   }
 
   ScheduledExecutorService newScheduledThreadPool(int cnt, String name, boolean systemThread) {
-    var s = Executors.newScheduledThreadPool(cnt, new Factory(name, systemThread));
+    var s = new ScheduledThreadPoolExecutor(cnt, new Factory(name, systemThread));
+    s.allowCoreThreadTimeOut(true);
     pools.put(s, name);
     return s;
   }

engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala

@@ -212,7 +212,7 @@ private class DefaultPackageRepository(
     isLibrary: Boolean
   ): Unit = {
     val extensions = pkg.listPolyglotExtensions("java")
-    extensions.foreach(context.addToClassPath(pkg, _))
+    extensions.foreach(context.addToClassPath(pkg, _, false))
 
     val (regularModules, syntheticModulesMetadata) = pkg
       .listSources()