Works on real hardware again!

This commit adds various changes to ClashOS to get the stack traces
and bootloader to work again in qemu and on real hardware.

The linker scripts are changed to allow the binary to be linked at the
address 0x80000. This is the default address at which the raspberry pi
will load kernel8.img when no 'config.txt' file is present on the sd card.

To allow stack traces to work, the link register had to be zero'ed, because
it contains random stuff on start up on real hardware. It work on qemu,
because it sets the link register to zero.

The code for the framebuffer is currently comment out, because for some reason
the loading over serial fails to work when that code is compiled. But
loading over serial now works!

Author: FireFox317

README.md

@@ -54,7 +54,7 @@ the `-Dgdb` to both `zig build` commands.
 1. Mount an sdcard with a single FAT32 partition.
 2. Copy `boot/*` to `/path/to/sdcard/*`.
 3. `zig build`
-4. Copy `clashos.bin` to `/path/to/sdcard/kernel7.img`.
+4. Copy `clashos.bin` to `/path/to/sdcard/kernel8.img`.
 
 For further changes repeat steps 3 and 4.
 

boot/config.txt

@@ -7,6 +7,7 @@ pub fn build(b: *Builder) !void {
     const want_gdb = b.option(bool, "gdb", "Build for using gdb with qemu") orelse false;
     const want_pty = b.option(bool, "pty", "Create a separate TTY path") orelse false;
     const want_nodisplay = b.option(bool, "nodisplay", "No display for qemu") orelse false;
+    const want_stack_traces = b.option(bool, "stacktraces", "Adds debug info to the executable for runtime stack traces") orelse false;
 
     const arch = builtin.Arch{ .aarch64 = builtin.Arch.Arm64.v8 };
     const environ = builtin.Abi.eabihf;
@@ -20,24 +21,32 @@ pub fn build(b: *Builder) !void {
     bootloader.strip = true;
     bootloader.setOutputDir("zig-cache");
 
+    const run_objcopy_bootloader = b.addSystemCommand(&[_][]const u8{
+        "llvm-objcopy",             bootloader.getOutputPath(),
+        "-O",                       "binary",
+        "zig-cache/bootloader.bin",
+    });
+    run_objcopy_bootloader.step.dependOn(&bootloader.step);
+
     const exec_name = if (want_gdb) "clashos-dbg" else "clashos";
     const exe = b.addExecutable(exec_name, "src/main.zig");
     exe.setOutputDir("zig-cache");
     exe.setBuildMode(mode);
     exe.setTarget(arch, builtin.Os.freestanding, environ);
-    const linker_script = if (want_gdb) "src/qemu-gdb.ld" else "src/linker.ld";
+    const linker_script = if (want_gdb or !want_stack_traces) "src/qemu-gdb.ld" else "src/linker.ld";
     exe.setLinkerScriptPath(linker_script);
-    exe.addBuildOption([]const u8, "bootloader_exe_path", b.fmt("\"{}\"", bootloader.getOutputPath()));
-    exe.step.dependOn(&bootloader.step);
+    exe.addBuildOption([]const u8, "bootloader_exe_path", b.fmt("\"{}\"", .{"zig-cache/bootloader.bin"}));
+    exe.addBuildOption(bool, "want_stack_traces", want_stack_traces);
+    exe.step.dependOn(&run_objcopy_bootloader.step);
 
-    const run_objcopy = b.addSystemCommand(&[_][]const u8{
+    const run_objcopy_clashos = b.addSystemCommand(&[_][]const u8{
         "llvm-objcopy", exe.getOutputPath(),
         "-O",           "binary",
         "clashos.bin",
     });
-    run_objcopy.step.dependOn(&exe.step);
+    run_objcopy_clashos.step.dependOn(&exe.step);
 
-    b.default_step.dependOn(&run_objcopy.step);
+    b.default_step.dependOn(&run_objcopy_clashos.step);
 
     const qemu = b.step("qemu", "Run the OS in qemu");
     var qemu_args = std.ArrayList([]const u8).init(b.allocator);
@@ -71,6 +80,6 @@ pub fn build(b: *Builder) !void {
     }
 
     const upload = b.step("upload", "Send a new kernel image to a running instance. (See -Dtty option)");
-    upload.dependOn(&run_objcopy.step);
+    upload.dependOn(&run_objcopy_clashos.step);
     upload.dependOn(&run_send_image_tool.step);
 }

build.zig

@@ -1,7 +1,7 @@
 ENTRY(boot)
 
 SECTIONS {
-    kernelMainAt0x1100 = 0x1100; /* Must match value from kernel linker script */
+    kernel_main = 0x80100; /* Must match value from kernel linker script */
 
     . = 0x8800000; /* Must match debug.bootloader_address */
 

src/bootloader.ld

@@ -10,19 +10,24 @@ const builtin = @import("builtin");
 const debug = @import("debug.zig");
 const serial = @import("serial.zig");
 
-export fn bootloader_main(start_addr: [*]u8, len: usize) linksection(".text.first") noreturn {
+export fn bootloaderMain(start_addr: [*]u8, len: usize) linksection(".text.first") noreturn {
     var i: usize = 0;
     while (i < len) : (i += 1) {
         start_addr[i] = serial.readByte();
     }
+
+    serial.log("Jumping to new code...\n", .{});
+    while (!serial.isDoneWriting()) {}
     asm volatile (
-        \\mov sp,#0x08000000
-        \\bl kernelMainAt0x1100
+        \\ mov sp,#0x08000000
+        \\ eor x29, x29, x29
+        \\ eor x30, x30, x30
+        \\ b kernel_main
     );
     unreachable;
 }
 
 pub fn panic(message: []const u8, stack_trace: ?*builtin.StackTrace) noreturn {
-    serial.log("BOOTLOADER PANIC: {}", message);
-    debug.wfe_hang();
+    serial.log("BOOTLOADER PANIC: {}", .{message});
+    debug.wfeHang();
 }

src/bootloader.zig

@@ -1,5 +1,6 @@
 const builtin = @import("builtin");
 const std = @import("std");
+const build_options = @import("build_options");
 const assert = std.debug.assert;
 const serial = @import("serial.zig");
 
@@ -15,40 +16,53 @@ extern var __debug_ranges_start: u8;
 extern var __debug_ranges_end: u8;
 
 const source_files = [_][]const u8{
-    "src/debug.zig",
-    "src/video_core_mailboxes.zig",
+    "src/main.zig",
     "src/bootloader.zig",
+    "src/debug.zig",
     "src/serial.zig",
+    "src/time.zig",
+    "src/mmio.zig",
+    "src/exceptions.zig",
     "src/video_core_metrics.zig",
     "src/video_core_properties.zig",
     "src/video_core_frame_buffer.zig",
-    "src/time.zig",
-    "src/main.zig",
-    "src/mmio.zig",
+    "src/video_core_mailboxes.zig",
     "src/slice_iterator.zig",
 };
 
 var already_panicking: bool = false;
 
-pub fn panic(stack_trace: ?*builtin.StackTrace, comptime fmt: []const u8, args: ...) noreturn {
+pub fn panic(stack_trace: ?*builtin.StackTrace, comptime fmt: []const u8, args: var) noreturn {
     @setCold(true);
     if (already_panicking) {
-        serial.log("\npanicked during kernel panic");
-        wfe_hang();
+        serial.log("panicked during kernel panic!!\n", .{});
+        wfeHang();
     }
     already_panicking = true;
 
-    serial.log("panic: " ++ fmt, args);
+    serial.log("panic: " ++ fmt ++ "\n", args);
 
     const first_trace_addr = @returnAddress();
-    if (stack_trace) |t| {
-        dumpStackTrace(t);
+    if (build_options.want_stack_traces) {
+        if (stack_trace) |t| {
+            dumpStackTrace(t);
+        }
+        dumpCurrentStackTrace(first_trace_addr);
+    } else {
+        serial.log("Stack trace:\n", .{});
+        var it = std.debug.StackIterator{ .first_addr = null, .fp = @frameAddress() };
+        while (it.next()) |return_address| {
+            if (return_address == 0) break;
+
+            serial.log("  0x{x}\n", .{return_address - 4});
+        }
+        serial.log("The stack trace can be decoded using `addr2line -e zig-cache/clashos <address>`\n\n", .{});
+        serial.log("To get stack traces during runtime, enable them using `zig build -Dstacktraces`\n", .{});
     }
-    dumpCurrentStackTrace(first_trace_addr);
-    wfe_hang();
+    wfeHang();
 }
 
-pub fn wfe_hang() noreturn {
+pub fn wfeHang() noreturn {
     while (true) {
         asm volatile ("wfe");
     }
@@ -116,7 +130,7 @@ fn getSelfDebugInfo() !*std.debug.DwarfInfo {
         .debug_info = dwarfSectionFromSymbol(&__debug_info_start, &__debug_info_end),
         .debug_abbrev = dwarfSectionFromSymbolAbs(&__debug_abbrev_start, &__debug_abbrev_end),
         .debug_str = dwarfSectionFromSymbolAbs(&__debug_str_start, &__debug_str_end),
-        .debug_line = dwarfSectionFromSymbol(&__debug_line_start, &__debug_line_end),
+        .debug_line = dwarfSectionFromSymbolAbs(&__debug_line_start, &__debug_line_end),
         .debug_ranges = dwarfSectionFromSymbolAbs(&__debug_ranges_start, &__debug_ranges_end),
         .abbrev_table_list = undefined,
         .compile_unit_list = undefined,
@@ -140,22 +154,22 @@ const kernel_panic_allocator = &kernel_panic_allocator_state.allocator;
 
 pub fn dumpStackTrace(stack_trace: *const builtin.StackTrace) void {
     const dwarf_info = getSelfDebugInfo() catch |err| {
-        serial.log("Unable to dump stack trace: Unable to open debug info: {}", @errorName(err));
+        serial.log("Unable to dump stack trace: Unable to open debug info: {}", .{@errorName(err)});
         return;
     };
     writeStackTrace(stack_trace, dwarf_info) catch |err| {
-        serial.log("Unable to dump stack trace: {}", @errorName(err));
+        serial.log("Unable to dump stack trace: {}", .{@errorName(err)});
         return;
     };
 }
 
 pub fn dumpCurrentStackTrace(start_addr: ?usize) void {
     const dwarf_info = getSelfDebugInfo() catch |err| {
-        serial.log("Unable to dump stack trace: Unable to open debug info: {}", @errorName(err));
+        serial.log("Unable to dump stack trace: Unable to open debug info: {}", .{@errorName(err)});
         return;
     };
     writeCurrentStackTrace(dwarf_info, start_addr) catch |err| {
-        serial.log("Unable to dump stack trace: {}", @errorName(err));
+        serial.log("Unable to dump stack trace: {}", .{@errorName(err)});
         return;
     };
 }
@@ -189,15 +203,17 @@ fn printLineFromFile(out_stream: var, line_info: std.debug.LineInfo) anyerror!vo
             return;
         }
     }
-    try out_stream.print("(source file {} not added in std/debug.zig)\n", line_info.file_name);
+    try out_stream.print("(source file {} not added in std.debug.zig)\n", .{line_info.file_name});
 }
 
 fn writeCurrentStackTrace(dwarf_info: *std.debug.DwarfInfo, start_addr: ?usize) !void {
     var it = std.debug.StackIterator.init(start_addr);
     while (it.next()) |return_address| {
+        if (return_address == 0) break;
+
         try dwarf_info.printSourceAtAddress(
             serial_out_stream,
-            return_address,
+            return_address - 4,
             true,
             printLineFromFile,
         );

src/debug.zig

@@ -0,0 +1,60 @@
+const serial = @import("serial.zig");
+const debug = @import("debug.zig");
+
+comptime {
+    asm (
+        \\.section .text
+        \\.balign 0x800
+        \\exception_vector_table:
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+        \\.balign 0x80
+        \\ bl exceptionHandler
+    );
+}
+
+pub fn init() void {
+    asm volatile (
+        \\ adr x0, exception_vector_table
+        \\ msr vbar_el2,x0
+    );
+}
+
+export fn exceptionHandler() void {
+    serial.log("arm exception taken\n", .{});
+    var current_el = asm ("mrs %[current_el], CurrentEL\n"
+        : [current_el] "=r" (-> usize)
+    );
+    serial.log("CurrentEL {x} exception level {}\n", .{ current_el, current_el >> 2 & 0x3 });
+    var esr_el2 = asm ("mrs %[esr_el2], esr_el2"
+        : [esr_el2] "=r" (-> usize)
+    );
+    serial.log("esr_el2 {x} code 0x{x}\n", .{ esr_el2, esr_el2 >> 26 & 0x3f });
+    var elr_el2 = asm ("mrs %[elr_el2], elr_el2"
+        : [elr_el2] "=r" (-> usize)
+    );
+    serial.log("elr_el2 {x}\n", .{elr_el2});
+    var spsr_el2 = asm ("mrs %[spsr_el2], spsr_el2"
+        : [spsr_el2] "=r" (-> usize)
+    );
+    serial.log("spsr_el2 {x}\n", .{spsr_el2});
+    var far_el2 = asm ("mrs %[far_el2], far_el2"
+        : [far_el2] "=r" (-> usize)
+    );
+    serial.log("far_el2 {x}\n", .{far_el2});
+    serial.log("execution is now stopped in arm exception handler\n", .{});
+
+    debug.panic(null, "exception!!!", .{});
+}

src/exceptions.zig

@@ -1,14 +1,12 @@
 ENTRY(_start)
 
 SECTIONS {
-    . = 0x0000;
+    . = 0x80000;
 
     .text : ALIGN(4K) {
         KEEP(*(.text.boot))
         __end_init = .;
-        . = 0x1000;
-        KEEP(*(.text.exception))
-        . = 0x1100; /* Must match address from bootloader.ld */
+        . = 0x80100; /* Must match address from bootloader.ld */
         KEEP(*(.text.main))
         *(.text)
     }
@@ -43,5 +41,5 @@ SECTIONS {
         __bss_end = .;
     }
 
-    bootloader_main = 0x8800000; /* Must match bootloader linker script */
+    bootloaderMain = 0x8800000; /* Must match bootloader linker script */
 }