aero.py: speed up prepare_iso!

cold cached build:

real    0m59.722s
user    1m36.639s
sys     0m12.023s

Most of the time is spent in the `prepare_iso` function copying and creating
the initramfs. This can be improved by copying the userland binaries into sysroot
instead of copying the sysroot and userland binaries into one `build/initramfs_root` since
that copy is expensive.

After the improvement, cold cached build:

real    0m28.471s
user    1m28.544s
sys     0m6.706s

Signed-off-by: Andy-Python-Programmer <[email protected]>

Author: Andy-Python-Programmer

aero.py

@@ -422,19 +422,22 @@ def prepare_iso(args, kernel_bin, user_bins):
     shutil.copy(os.path.join(limine_path, 'limine-cd.bin'), iso_root)
     shutil.copy(os.path.join(limine_path, 'limine-cd-efi.bin'), iso_root)
 
-    initramfs_root = os.path.join(BUILD_DIR, 'initramfs_root')
-    initramfs_bin = os.path.join(initramfs_root, 'usr', 'bin')
+    sysroot_dir = os.path.join(SYSROOT_DIR, 'system-root')
+    shutil.copytree(BASE_FILES_DIR, sysroot_dir, dirs_exist_ok=True)
 
-    if os.path.exists(initramfs_root):
-        shutil.rmtree(initramfs_root)
+    # dynamic linker (ld.so)
+    mlibc = os.path.join(get_userland_package(), "mlibc")
+    # gcc libraries required for rust programs
+    gcc = os.path.join(get_userland_package(), "gcc")
 
-    sysroot_dir = os.path.join(SYSROOT_DIR, 'system-root')
+    # FIXME
+    if "host-rust-prebuilt" in str(mlibc):
+        shutil.copytree(mlibc, sysroot_dir, dirs_exist_ok=True)
+        shutil.copytree(gcc, sysroot_dir, dirs_exist_ok=True)
 
-    if os.path.exists(sysroot_dir):
-        # copying the sysroot will auto-magically create the bin directory.
-        shutil.copytree(sysroot_dir, initramfs_root)
-    else:
-        os.makedirs(initramfs_bin)
+    for file in user_bins:
+        bin_name = os.path.basename(file)
+        shutil.copy(file, os.path.join(sysroot_dir, "usr", "bin", bin_name))
 
     def find(path) -> List[str]:
         _, find_output, _ = run_command(['find', '.', '-type', 'f'],
@@ -447,40 +450,15 @@ def find(path) -> List[str]:
             lambda x: remove_prefix(x, './'), files_without_dot)
         files = list(files_without_prefix)
 
-        return files
-
-    def cp(src, dest):
-        files = find(src)
-
-        for line in files:
-            file = os.path.join(src, line)
-            dest_file = os.path.join(dest, line)
-
-            os.makedirs(os.path.dirname(dest_file), exist_ok=True)
-            shutil.copy(file, dest_file)
-
-    # dynamic linker (ld.so)
-    mlibc = os.path.join(get_userland_package(), "mlibc")
-    # gcc libraries required for rust programs
-    gcc = os.path.join(get_userland_package(), "gcc")
-
-    if "host-rust-prebuilt" in str(mlibc):
-        cp(mlibc, initramfs_root)
-        cp(gcc, initramfs_root)
-
-    cp(BASE_FILES_DIR, initramfs_root)
-
-    for file in user_bins:
-        bin_name = os.path.basename(file)
-
-        shutil.copy(file, os.path.join(initramfs_bin, bin_name))
+        files.append("usr/lib/libiconv.so.2")
+        return files 
 
-    files = find(initramfs_root)
+    files = find(sysroot_dir)
 
     with open(os.path.join(iso_root, 'initramfs.cpio'), 'wb') as initramfs:
         cpio_input = '\n'.join(files)
         code, _, _ = run_command(['cpio', '-o', '-v'],
-                                 cwd=initramfs_root,
+                                 cwd=sysroot_dir,
                                  stdout=initramfs,
                                  stderr=subprocess.PIPE,
                                  input=cpio_input.encode('utf-8'))
@@ -529,7 +507,7 @@ def run_in_emulator(args, iso_path):
 
     qemu_args = ['-cdrom', iso_path,
                  '-M', 'q35',
-                 '-m', '9G',
+                 '-m', '9800M',
                  '-smp', '5',
                  '-serial', 'stdio']
 

src/Cargo.lock

@@ -1,5 +1,2 @@
 [workspace]
 members = ["aero_kernel", "aero_syscall", "aero_proc"]
-
-[profile.release]
-debug = true

src/Cargo.toml

@@ -36,6 +36,7 @@ intrusive-collections = "0.9.2"
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 lai = { git = "https://github.com/aero-os/lai-rs" }
 uapi = { path = "../uapi" }
+cpio_reader = { git = "https://github.com/Andy-Python-Programmer/cpio_reader" }
 
 [dependencies.vte]
 git = "https://github.com/Andy-Python-Programmer/vte"
@@ -51,8 +52,5 @@ path = "../aero_proc"
 [dependencies.aero_syscall]
 path = "../aero_syscall"
 
-[dependencies.cpio_reader]
-git = "https://github.com/czapek1337/cpio_reader"
-
 [build-dependencies]
 nasm-rs = { version = "0.2", features = ["parallel"] }

src/aero_kernel/Cargo.toml

@@ -18,14 +18,15 @@
  */
 
 use alloc::sync::Arc;
+use cpio_reader::Mode;
 
 use crate::fs::{FileSystemError, Path};
 use crate::mem::paging::PhysAddr;
 
 use super::cache::DirCacheItem;
 use super::ramfs::RamFs;
 
-use super::{root_dir, FileSystem, Result, MOUNT_MANAGER};
+use super::{root_dir, FileSystem, LookupMode, Result, MOUNT_MANAGER};
 
 lazy_static::lazy_static! {
     static ref INIT_FILESYSTEM: Arc<InitRamFs> = InitRamFs::new();
@@ -56,8 +57,24 @@ pub(super) fn init() -> Result<()> {
         core::slice::from_raw_parts(base.as_ptr(), length as usize)
     };
 
+    let mut symlinks = alloc::vec![];
+
     for entry in cpio_reader::iter_files(initrd) {
         let path = Path::new(entry.name());
+
+        if entry.mode().contains(Mode::SYMBOLIK_LINK) {
+            // CPIO symbolically linked file's contain the target path as their contents.
+            let target =
+                core::str::from_utf8(entry.file()).map_err(|_| FileSystemError::InvalidPath)?;
+
+            let (parent, _) = path.parent_and_basename();
+
+            // We need to create symbolically linked files at the end, after all the
+            // other files.
+            symlinks.push((alloc::format!("{}/{}", parent.as_str(), target), path));
+            continue;
+        }
+
         let component_count = path.components().count();
 
         let mut cwd = root_dir().clone();
@@ -78,7 +95,16 @@ pub(super) fn init() -> Result<()> {
         }
     }
 
-    MOUNT_MANAGER.mount(root_dir().clone(), INIT_FILESYSTEM.clone())?;
+    for (src, target) in symlinks {
+        let src = super::lookup_path_with(root_dir().clone(), Path::new(&src), LookupMode::None)
+            .expect(&alloc::format!("your mom {:?}", src));
+        let (target_dir, target_name) = target.parent_and_basename();
 
+        let target = super::lookup_path_with(root_dir().clone(), target_dir, LookupMode::None)
+            .expect(&alloc::format!("your dad {:?}", target));
+        target.inode().link(target_name, src.inode()).unwrap();
+    }
+
+    MOUNT_MANAGER.mount(root_dir().clone(), INIT_FILESYSTEM.clone())?;
     Ok(())
 }