Problem: error Too many open files

src/aleph/vm/hypervisors/firecracker/microvm.py

@@ -6,6 +6,7 @@
 import shutil
 import string
 import traceback
+import typing
 from asyncio import Task
 from asyncio.base_events import Server
 from dataclasses import dataclass
@@ -93,6 +94,8 @@ class MicroVM:
     mounted_rootfs: Path | None = None
     _unix_socket: Server | None = None
     enable_log: bool
+    journal_stdout: typing.IO | None
+    journal_stderr: typing.IO | None
 
     def __repr__(self):
         return f"<MicroVM {self.vm_id}>"
@@ -219,19 +222,19 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce
             str(config_path),
         )
         if self.enable_log:
-            journal_stdout = journal.stream(self._journal_stdout_name)
-            journal_stderr = journal.stream(self._journal_stderr_name)
+            self.journal_stdout = journal.stream(self._journal_stdout_name)
+            self.journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = asyncio.subprocess.DEVNULL
-            journal_stderr = asyncio.subprocess.DEVNULL
+            self.journal_stdout = asyncio.subprocess.DEVNULL
+            self.journal_stderr = asyncio.subprocess.DEVNULL
 
         logger.debug(" ".join(options))
 
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=journal_stdout,
-            stderr=journal_stderr,
+            stdout=self.journal_stdout,
+            stderr=self.journal_stderr,
         )
         return self.proc
 
@@ -252,11 +255,11 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
 
         self.config_file_path = config_path
         if self.enable_log:
-            journal_stdout = journal.stream(self._journal_stdout_name)
-            journal_stderr = journal.stream(self._journal_stderr_name)
+            self.journal_stdout = journal.stream(self._journal_stdout_name)
+            self.journal_stderr = journal.stream(self._journal_stderr_name)
         else:
-            journal_stdout = asyncio.subprocess.DEVNULL
-            journal_stderr = asyncio.subprocess.DEVNULL
+            self.journal_stdout = asyncio.subprocess.DEVNULL
+            self.journal_stderr = asyncio.subprocess.DEVNULL
 
         options = (
             str(self.jailer_bin_path),
@@ -280,8 +283,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces
         self.proc = await asyncio.create_subprocess_exec(
             *options,
             stdin=asyncio.subprocess.PIPE,
-            stdout=journal_stdout,
-            stderr=journal_stderr,
+            stdout=self.journal_stdout,
+            stderr=self.journal_stderr,
         )
         return self.proc
 
@@ -480,6 +483,11 @@ async def teardown(self):
         if self.stderr_task:
             self.stderr_task.cancel()
 
+        if self.journal_stdout != asyncio.subprocess.DEVNULL:
+            self.journal_stdout.close()
+        if self.journal_stderr != asyncio.subprocess.DEVNULL:
+            self.journal_stderr.close()
+
         # Clean mounted block devices
         if self.mounted_rootfs:
             logger.debug("Waiting for one second for the VM to shutdown")

src/aleph/vm/hypervisors/qemu/qemuvm.py

@@ -1,4 +1,5 @@
 import asyncio
+import typing
 from asyncio.subprocess import Process
 from dataclasses import dataclass
 from pathlib import Path
@@ -28,6 +29,8 @@ class QemuVM:
     interface_name: str
     qemu_process: Process | None = None
     host_volumes: list[HostVolume]
+    journal_stdout: typing.IO | None
+    journal_stderr: typing.IO | None
 
     def __repr__(self) -> str:
         if self.qemu_process:
@@ -149,3 +152,8 @@ def send_shutdown_message(self):
     async def stop(self):
         """Stop the VM."""
         self.send_shutdown_message()
+
+        if self.journal_stdout != asyncio.subprocess.DEVNULL:
+            self.journal_stdout.close()
+        if self.journal_stderr != asyncio.subprocess.DEVNULL:
+            self.journal_stderr.close()