Add API to choose non-default preopens (#1317)

squarejesse · web-flow · commit c7e7ad357784 · 2023-07-30T10:33:33.000-04:00
I need to find a proper document explaining how this all works.
As is it works but is not following a proper spec.
diff --git a/okio-wasifilesystem/build.gradle.kts b/okio-wasifilesystem/build.gradle.kts
@@ -64,8 +64,12 @@ val injectWasiInit by tasks.creating {
   outputs.file(entryPointMjs)
 
   doLast {
-    val tmpdir = File(System.getProperty("java.io.tmpdir"), "okio-wasifilesystem-test")
-    tmpdir.mkdirs()
+    val base = File(System.getProperty("java.io.tmpdir"), "okio-wasifilesystem-test")
+    val baseA = File(base, "a")
+    val baseB = File(base, "b")
+    base.mkdirs()
+    baseA.mkdirs()
+    baseB.mkdirs()
 
     entryPointMjs.writeText(
       """
@@ -75,7 +79,9 @@ val injectWasiInit by tasks.creating {
       export const wasi = new WASI({
         version: 'preview1',
         preopens: {
-          '/tmp': '$tmpdir'
+          '/tmp': '$base',
+          '/a': '$baseA',
+          '/b': '$baseB'
         }
       });
 
diff --git a/okio-wasifilesystem/src/wasmMain/kotlin/okio/WasiFileSystem.kt b/okio-wasifilesystem/src/wasmMain/kotlin/okio/WasiFileSystem.kt
@@ -21,7 +21,6 @@ import okio.Path.Companion.toPath
 import okio.internal.ErrnoException
 import okio.internal.fdClose
 import okio.internal.preview1.Errno
-import okio.internal.preview1.FirstPreopenDirectoryTmp
 import okio.internal.preview1.dirnamelen
 import okio.internal.preview1.fd
 import okio.internal.preview1.fd_readdir
@@ -60,7 +59,18 @@ import okio.internal.write
  *
  * [WASI]: https://wasi.dev/
  */
-object WasiFileSystem : FileSystem() {
+class WasiFileSystem(
+  private val relativePathPreopen: Int = DEFAULT_FIRST_PREOPEN,
+  pathToPreopen: Map<Path, Int> = mapOf("/".toPath() to DEFAULT_FIRST_PREOPEN),
+) : FileSystem() {
+  private val pathSegmentsToPreopen = pathToPreopen.mapKeys { (key, _) -> key.segmentsBytes }
+
+  init {
+    require(pathSegmentsToPreopen.isNotEmpty()) {
+      "pathToPreopen must be non-empty"
+    }
+  }
+
   override fun canonicalize(path: Path): Path {
     // There's no APIs in preview1 to canonicalize a path. We give it a best effort by resolving
     // all symlinks, but this could result in a relative path.
@@ -108,7 +118,7 @@ object WasiFileSystem : FileSystem() {
       val (pathAddress, pathSize) = allocator.write(path.toString())
 
       val errno = path_filestat_get(
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd(path) ?: return null,
         flags = 0,
         path = pathAddress.address.toInt(),
         pathSize = pathSize,
@@ -144,7 +154,7 @@ object WasiFileSystem : FileSystem() {
           val bufPointer = allocator.allocate(bufLen)
           val readlinkReturnPointer = allocator.allocate(4) // `size` is u32, 4 bytes.
           val readlinkErrno = path_readlink(
-            fd = FirstPreopenDirectoryTmp,
+            fd = preopenFd(path) ?: return null,
             path = pathAddress.address.toInt(),
             pathSize = pathSize,
             buf = bufPointer.address.toInt(),
@@ -174,7 +184,7 @@ object WasiFileSystem : FileSystem() {
 
   override fun list(dir: Path): List<Path> {
     val fd = pathOpen(
-      path = dir.toString(),
+      path = dir,
       oflags = oflag_directory,
       rightsBase = right_fd_readdir,
     )
@@ -252,7 +262,7 @@ object WasiFileSystem : FileSystem() {
       right_fd_seek or
       right_fd_sync
     val fd = pathOpen(
-      path = file.toString(),
+      path = file,
       oflags = 0,
       rightsBase = rightsBase,
     )
@@ -275,7 +285,7 @@ object WasiFileSystem : FileSystem() {
       right_fd_sync or
       right_fd_write
     val fd = pathOpen(
-      path = file.toString(),
+      path = file,
       oflags = oflags,
       rightsBase = rightsBase,
     )
@@ -285,7 +295,7 @@ object WasiFileSystem : FileSystem() {
   override fun source(file: Path): Source {
     return FileSource(
       fd = pathOpen(
-        path = file.toString(),
+        path = file,
         oflags = 0,
         rightsBase = right_fd_read,
       ),
@@ -300,7 +310,7 @@ object WasiFileSystem : FileSystem() {
 
     return FileSink(
       fd = pathOpen(
-        path = file.toString(),
+        path = file,
         oflags = oflags,
         rightsBase = right_fd_write or right_fd_sync,
       ),
@@ -315,7 +325,7 @@ object WasiFileSystem : FileSystem() {
 
     return FileSink(
       fd = pathOpen(
-        path = file.toString(),
+        path = file,
         oflags = oflags,
         rightsBase = right_fd_write,
         fdflags = fdflags_append,
@@ -328,7 +338,7 @@ object WasiFileSystem : FileSystem() {
       val (pathAddress, pathSize) = allocator.write(dir.toString())
 
       val errno = path_create_directory(
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd(dir) ?: throw FileNotFoundException("no preopen: $dir"),
         path = pathAddress.address.toInt(),
         pathSize = pathSize,
       )
@@ -347,10 +357,10 @@ object WasiFileSystem : FileSystem() {
       val (targetPathAddress, targetPathSize) = allocator.write(target.toString())
 
       val errno = path_rename(
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd(source) ?: throw FileNotFoundException("no preopen: $source"),
         old_path = sourcePathAddress.address.toInt(),
         old_pathSize = sourcePathSize,
-        new_fd = FirstPreopenDirectoryTmp,
+        new_fd = preopenFd(target) ?: throw FileNotFoundException("no preopen: $target"),
         new_path = targetPathAddress.address.toInt(),
         new_pathSize = targetPathSize,
       )
@@ -365,9 +375,10 @@ object WasiFileSystem : FileSystem() {
   override fun delete(path: Path, mustExist: Boolean) {
     withScopedMemoryAllocator { allocator ->
       val (pathAddress, pathSize) = allocator.write(path.toString())
+      val preopenFd = preopenFd(path) ?: throw FileNotFoundException("no preopen: $path")
 
       var errno = path_unlink_file(
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd,
         path = pathAddress.address.toInt(),
         pathSize = pathSize,
       )
@@ -382,7 +393,7 @@ object WasiFileSystem : FileSystem() {
         Errno.isdir.ordinal,
         -> {
           errno = path_remove_directory(
-            fd = FirstPreopenDirectoryTmp,
+            fd = preopenFd,
             path = pathAddress.address.toInt(),
             pathSize = pathSize,
           )
@@ -400,7 +411,7 @@ object WasiFileSystem : FileSystem() {
       val errno = path_symlink(
         old_path = targetPathAddress.address.toInt(),
         old_pathSize = targetPathSize,
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd(source) ?: throw FileNotFoundException("no preopen: $source"),
         new_path = sourcePathAddress.address.toInt(),
         new_pathSize = sourcePathSize,
       )
@@ -409,17 +420,18 @@ object WasiFileSystem : FileSystem() {
   }
 
   private fun pathOpen(
-    path: String,
+    path: Path,
     oflags: oflags,
     rightsBase: rights,
     fdflags: fdflags = 0,
   ): fd {
     withScopedMemoryAllocator { allocator ->
-      val (pathAddress, pathSize) = allocator.write(path)
+      val preopenFd = preopenFd(path) ?: throw FileNotFoundException("no preopen: $path")
+      val (pathAddress, pathSize) = allocator.write(path.toString())
 
       val returnPointer: Pointer = allocator.allocate(4) // fd is u32.
       val errno = path_open(
-        fd = FirstPreopenDirectoryTmp,
+        fd = preopenFd,
         dirflags = 0,
         path = pathAddress.address.toInt(),
         pathSize = pathSize,
@@ -437,5 +449,31 @@ object WasiFileSystem : FileSystem() {
     }
   }
 
+  /**
+   * Returns the file descriptor of the preopened path that is an ancestor of [path]. Returns null
+   * if there is no such file descriptor.
+   */
+  private fun preopenFd(path: Path): fd? {
+    if (path.isRelative) return relativePathPreopen
+
+    val pathSegmentsBytes = path.segmentsBytes
+    for ((candidate, fd) in pathSegmentsToPreopen) {
+      if (pathSegmentsBytes.size < candidate.size) continue
+      if (pathSegmentsBytes.subList(0, candidate.size) != candidate) continue
+      return fd
+    }
+    return null
+  }
+
   override fun toString() = "okio.WasiFileSystem"
+
+  companion object {
+    /**
+     * File descriptor of the first preopen in the `WASI` instance's configured `preopens` property.
+     * This is 3 by default, assuming `stdin` is 0, `stdout` is 1, and `stderr` is 2.
+     *
+     * Other preopens are assigned sequentially starting at this value.
+     */
+    val DEFAULT_FIRST_PREOPEN = 3
+  }
 }
diff --git a/okio-wasifilesystem/src/wasmMain/kotlin/okio/internal/preview1/Preview1.kt b/okio-wasifilesystem/src/wasmMain/kotlin/okio/internal/preview1/Preview1.kt
@@ -52,17 +52,6 @@ typealias dirnamelen = Int
  */
 typealias PointerU8 = Int
 
-val Stdin: fd = 0
-val Stdout: fd = 1
-val Stderr: fd = 2
-
-/**
- * Assume the /tmp directory is fd 3.
- *
- * TODO: look this up at runtime from whatever parent directory is requested.
- */
-val FirstPreopenDirectoryTmp: fd = 3
-
 /**
  * path_create_directory(fd: fd, path: string) -> Result<(), errno>
  *
diff --git a/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiFileSystemPreopensTest.kt b/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiFileSystemPreopensTest.kt
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2023 Square, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package okio
+
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertNull
+import okio.Path.Companion.toPath
+import okio.WasiFileSystem.Companion.DEFAULT_FIRST_PREOPEN
+
+/**
+ * Confirm the [WasiFileSystem] can operate on different preopened directories independently.
+ *
+ * This tracks the `preopens` attribute in `.mjs` script in `okio-wasifilesystem/build.gradle.kts`.
+ */
+class WasiFileSystemPreopensTest {
+  private val fileSystem = WasiFileSystem(
+    relativePathPreopen = DEFAULT_FIRST_PREOPEN,
+    pathToPreopen = mapOf(
+      "/tmp".toPath() to DEFAULT_FIRST_PREOPEN,
+      "/a".toPath() to DEFAULT_FIRST_PREOPEN + 1,
+      "/b".toPath() to DEFAULT_FIRST_PREOPEN + 2,
+    ),
+  )
+
+  private val testId = "${this::class.simpleName}-${randomToken(16)}"
+  private val baseA: Path = "/a".toPath() / testId
+  private val baseB: Path = "/b".toPath() / testId
+
+  @BeforeTest
+  fun setUp() {
+    fileSystem.createDirectory(baseA)
+    fileSystem.createDirectory(baseB)
+  }
+
+  @Test
+  fun operateOnPreopens() {
+    fileSystem.write(baseA / "a.txt") {
+      writeUtf8("hello world a")
+    }
+    fileSystem.write(baseB / "b.txt") {
+      writeUtf8("bello burld")
+    }
+    assertEquals(
+      "hello world a".length.toLong(),
+      fileSystem.metadata(baseA / "a.txt").size,
+    )
+    assertEquals(
+      "bello burld".length.toLong(),
+      fileSystem.metadata(baseB / "b.txt").size,
+    )
+  }
+
+  @Test
+  fun operateAcrossPreopens() {
+    fileSystem.write(baseA / "a.txt") {
+      writeUtf8("hello world")
+    }
+
+    fileSystem.atomicMove(baseA / "a.txt", baseB / "b.txt")
+
+    assertEquals(
+      "hello world",
+      fileSystem.read(baseB / "b.txt") {
+        readUtf8()
+      },
+    )
+  }
+
+  @Test
+  fun cannotOperateOutsideOfPreopens() {
+    val noPreopen = "/c".toPath() / testId
+    assertFailsWith<FileNotFoundException> {
+      fileSystem.createDirectory(noPreopen)
+    }
+    assertFailsWith<FileNotFoundException> {
+      fileSystem.sink(noPreopen)
+    }
+    assertNull(fileSystem.metadataOrNull(noPreopen))
+    assertFailsWith<FileNotFoundException> {
+      fileSystem.metadata(noPreopen)
+    }
+    assertNull(fileSystem.listOrNull(noPreopen))
+    assertFailsWith<FileNotFoundException> {
+      fileSystem.list(noPreopen)
+    }
+  }
+}
diff --git a/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiFileSystemTest.kt b/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiFileSystemTest.kt
@@ -19,7 +19,7 @@ import okio.Path.Companion.toPath
 
 class WasiFileSystemTest : AbstractFileSystemTest(
   clock = WasiClock,
-  fileSystem = WasiFileSystem,
+  fileSystem = WasiFileSystem(),
   windowsLimitations = Path.DIRECTORY_SEPARATOR == "\\",
   allowClobberingEmptyDirectories = Path.DIRECTORY_SEPARATOR == "\\",
   allowAtomicMoveFromFileToDirectory = false,
diff --git a/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiTest.kt b/okio-wasifilesystem/src/wasmTest/kotlin/okio/WasiTest.kt
@@ -24,7 +24,7 @@ import okio.ByteString.Companion.encodeUtf8
 import okio.Path.Companion.toPath
 
 class WasiTest {
-  private val fileSystem = WasiFileSystem
+  private val fileSystem = WasiFileSystem()
   private val base: Path = "/tmp".toPath() / "${this::class.simpleName}-${randomToken(16)}"
 
   @BeforeTest
