Merge remote-tracking branch 'upstream/master' into pagecache

(cherry picked from commit dfd283f49b42164608a4f38d9e01e9e4e28b4247)
Signed-off-by: greatbridf <greatbridf@icloud.com>

crates/eonix_hal/src/arch/riscv64/bootstrap.rs

@@ -8,19 +8,23 @@ use super::{
 use crate::{
     arch::{
         cpu::CPU,
-        fdt::{init_dtb_and_fdt, FdtExt},
+        fdt::{init_dtb_and_fdt, FdtExt, FDT},
         mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE},
     },
     bootstrap::BootStrapData,
     mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator},
 };
-use core::arch::naked_asm;
 use core::{
     alloc::Allocator,
     arch::asm,
     cell::RefCell,
     sync::atomic::{AtomicBool, AtomicUsize},
 };
+use core::{
+    arch::{global_asm, naked_asm},
+    hint::spin_loop,
+    sync::atomic::{AtomicPtr, Ordering},
+};
 use eonix_hal_traits::mm::Memory;
 use eonix_mm::{
     address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange},
@@ -30,13 +34,18 @@ use eonix_mm::{
 use eonix_percpu::PercpuArea;
 use fdt::Fdt;
 use riscv::{asm::sfence_vma_all, register::satp};
-use sbi::legacy::console_putchar;
+use sbi::{hsm::hart_start, legacy::console_putchar, PhysicalAddress};
 
 #[unsafe(link_section = ".bootstrap.stack")]
 static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16];
 
 static BOOT_STACK_START: &'static [u8; 4096 * 16] = &BOOT_STACK;
 
+#[unsafe(link_section = ".bootstrap.stack")]
+static TEMP_AP_STACK: [u8; 256] = [0; 256];
+
+static TEMP_AP_STACK_START: &'static [u8; 256] = &TEMP_AP_STACK;
+
 #[repr(C, align(4096))]
 struct PageTable([u64; PTES_PER_PAGE]);
 
@@ -60,6 +69,12 @@ static PT1: PageTable = {
     PageTable(arr)
 };
 
+static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = AtomicPtr::new(core::ptr::null_mut());
+
+static AP_COUNT: AtomicUsize = AtomicUsize::new(0);
+static AP_STACK: AtomicUsize = AtomicUsize::new(0);
+static AP_SEM: AtomicBool = AtomicBool::new(false);
+
 /// bootstrap in rust
 #[unsafe(naked)]
 #[unsafe(no_mangle)]
@@ -94,8 +109,6 @@ unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) -> ! {
     )
 }
 
-/// TODO:
-/// 启动所有的cpu
 pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! {
     let fdt = Fdt::from_ptr(ArchPhysAccess::as_ptr(dtb_addr).as_ptr())
         .expect("Failed to parse DTB from static memory.");
@@ -131,6 +144,9 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! {
         allocator: Some(real_allocator),
     };
 
+    // set current hart's mtimecmp register
+    set_next_timer();
+
     unsafe {
         _eonix_hal_main(bootstrap_data);
     }
@@ -138,7 +154,6 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! {
 
 unsafe extern "C" {
     fn BSS_LENGTH();
-    fn KIMAGE_PAGES();
 }
 
 /// TODO:
@@ -225,13 +240,152 @@ fn setup_cpu(alloc: impl PageAlloc, hart_id: usize) {
             .as_mut()
             .set_kernel_tp(PercpuArea::get_for(cpu.cpuid()).unwrap().cast());
     }
+}
+
+fn get_ap_start_addr() -> usize {
+    unsafe extern "C" {
+        fn _ap_start();
+    }
+    static AP_START_VALUE: &'static unsafe extern "C" fn() =
+        &(_ap_start as unsafe extern "C" fn());
+    unsafe { (AP_START_VALUE as *const _ as *const usize).read_volatile() }
+}
+
+fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) {
+    let local_hart_id = CPU::local().cpuid();
+    let mut ap_count = 0;
+
+    for hart_id in FDT.harts().filter(|&id| id != local_hart_id) {
+        let stack_range = {
+            let page_alloc = BasicPageAllocRef::new(&page_alloc);
+            let ap_stack = Page::alloc_order_in(4, page_alloc);
+            let stack_range = ap_stack.range();
+            ap_stack.into_raw();
+            stack_range
+        };
+
+        let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release);
+        assert!(old.is_null());
+
+        while AP_STACK
+            .compare_exchange_weak(
+                0,
+                stack_range.end().addr(),
+                Ordering::Release,
+                Ordering::Relaxed,
+            )
+            .is_err()
+        {
+            spin_loop();
+        }
+
+        unsafe {
+            hart_start(hart_id, PhysicalAddress::new(get_ap_start_addr()), 0);
+        }
+
+        while AP_COUNT.load(Ordering::Acquire) == ap_count {
+            spin_loop();
+        }
+
+        let old = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::Acquire);
+        assert_eq!(old as *const _, &raw const *page_alloc);
+        ap_count += 1;
+    }
+}
+
+#[unsafe(naked)]
+#[unsafe(no_mangle)]
+#[unsafe(link_section = ".bootstrap.apentry")]
+unsafe extern "C" fn _ap_start(hart_id: usize) -> ! {
+    naked_asm!(
+        "
+            la    sp, 1f        // set temp stack
+            mv    s0, a0        // save hart id
+
+            ld    t0, 2f
+            srli  t0, t0, 12
+            li    t1, 9 << 60
+            or    t0, t0, t1
+            csrw  satp, t0
+            sfence.vma
+
+            ld    t0, 3f
+            jalr  t0
+            mv    sp, a0
+
+            mv    a0, s0
+            ld    t0, 4f
+            jalr  t0
+
+            .pushsection .bootstrap.data, \"aw\", @progbits
+            1: .8byte {temp_stack}
+            2: .8byte {page_table}
+            3: .8byte {get_ap_stack}
+            4: .8byte {ap_entry}
+            .popsection
+        ",
+        temp_stack = sym TEMP_AP_STACK_START,
+        page_table = sym BOOT_PAGE_TABLE,
+        get_ap_stack = sym get_ap_stack,
+        ap_entry = sym ap_entry,
+    )
+}
+
+fn get_ap_stack() -> usize {
+    while AP_SEM
+        .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
+        .is_err()
+    {
+        core::hint::spin_loop();
+    }
+
+    let stack_addr = loop {
+        let addr = AP_STACK.swap(0, Ordering::AcqRel);
+        if addr != 0 {
+            break addr;
+        }
+        core::hint::spin_loop();
+    };
+
+    AP_SEM.store(false, Ordering::Release);
+
+    stack_addr
+}
+
+fn ap_entry(hart_id: usize, stack_bottom: PAddr) -> ! {
+    let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom);
+
+    {
+        // SAFETY: Acquire all the work done by the BSP and other APs.
+        let alloc = loop {
+            let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel);
+
+            if !alloc.is_null() {
+                break alloc;
+            }
+        };
+
+        let ref_alloc = unsafe { &*alloc };
+        setup_cpu(BasicPageAllocRef::new(&ref_alloc), hart_id);
+
+        // SAFETY: Release our allocation work.
+        BSP_PAGE_ALLOC.store(alloc, Ordering::Release);
+    }
+
+    // SAFETY: Make sure the allocator is set before we increment the AP count.
+    AP_COUNT.fetch_add(1, Ordering::Release);
+
+    unsafe extern "Rust" {
+        fn _eonix_hal_ap_main(stack_range: PRange) -> !;
+    }
 
     // set current hart's mtimecmp register
     set_next_timer();
-}
 
-/// TODO
-fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) {}
+    unsafe {
+        _eonix_hal_ap_main(stack_range);
+    }
+}
 
 pub fn early_console_write(s: &str) {
     write_str(s);

crates/eonix_hal/src/arch/riscv64/cpu.rs

@@ -57,15 +57,6 @@ impl CPU {
         sscratch::write(TRAP_SCRATCH.as_ptr() as usize);
     }
 
-    /// Boot all other hart.
-    pub unsafe fn bootstrap_cpus(&self) {
-        let total_harts = FDT.hart_count();
-        for i in (0..total_harts).filter(|&i| i != self.cpuid()) {
-            sbi::hsm::hart_start(i, todo!("AP entry"), 0)
-                .expect("Failed to start secondary hart via SBI");
-        }
-    }
-
     pub unsafe fn load_interrupt_stack(self: Pin<&mut Self>, sp: u64) {
         TRAP_SCRATCH
             .as_mut()

crates/eonix_hal/src/arch/riscv64/link.x

@@ -1,7 +1,8 @@
 SECTIONS {
     .bootstrap ORIGIN(RAM) :
     {
-        KEEP(*(.bootstrap.entry .bootstrap.data));
+        KEEP(*(.bootstrap.entry));
+        KEEP(*(.bootstrap.apentry .bootstrap.data));
 
         . = ORIGIN(RAM) + 0x1000;
         KEEP(*(.bootstrap.page_table.1));

src/driver/sbi_console.rs

@@ -1,4 +1,4 @@
-use crate::kernel::{block::make_device, CharDevice, CharDeviceType, Terminal, TerminalDevice};
+use crate::kernel::{Terminal, TerminalDevice};
 use alloc::sync::Arc;
 use eonix_log::ConsoleWrite;
 

src/fs/ext4.rs

@@ -13,7 +13,7 @@ use crate::{
             mount::{register_filesystem, Mount, MountCreator},
             s_isdir, s_isreg,
             vfs::Vfs,
-            DevId, FsContext, TimeSpec,
+            DevId, FsContext,
         },
     },
     path::Path,

src/kernel/console.rs

@@ -67,6 +67,7 @@ macro_rules! println_fatal {
     };
 }
 
+#[allow(unused_macros)]
 macro_rules! println_trace {
     ($feat:literal) => {
         #[deny(unexpected_cfgs)]

src/kernel/syscall/procops.rs

@@ -151,22 +151,20 @@ fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult<Vec<CString>>
 #[eonix_macros::define_syscall(SYS_EXECVE)]
 fn execve(exec: *const u8, argv: *const PtrT, envp: *const PtrT) -> KResult<SyscallNoReturn> {
     let exec = UserString::new(exec)?;
+    let exec = exec.as_cstr().to_owned();
+
     let argv = get_strings(UserPointer::new(argv)?)?;
     let envp = get_strings(UserPointer::new(envp)?)?;
 
-    let dentry = Dentry::open(
-        &thread.fs_context,
-        Path::new(exec.as_cstr().to_bytes())?,
-        true,
-    )?;
-
+    let dentry = Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true)?;
     if !dentry.is_valid() {
         Err(ENOENT)?;
     }
 
     // TODO: When `execve` is called by one of the threads in a process, the other threads
     //       should be terminated and `execve` is performed in the thread group leader.
-    let load_info = ProgramLoader::parse(dentry.clone())?.load(argv, envp)?;
+    let load_info =
+        ProgramLoader::parse(&thread.fs_context, exec, dentry.clone(), argv, envp)?.load()?;
 
     if let Some(robust_list) = thread.get_robust_list() {
         let _ = Task::block_on(robust_list.wake_all());

src/kernel/task/loader/mod.rs

@@ -1,11 +1,12 @@
 use crate::io::ByteBuffer;
 use crate::kernel::constants::ENOEXEC;
 use crate::kernel::task::loader::elf::ELF;
+use crate::kernel::vfs::FsContext;
+use crate::path::Path;
 use crate::{
     kernel::{mem::MMList, vfs::dentry::Dentry},
     prelude::*,
 };
-
 use alloc::{ffi::CString, sync::Arc};
 use eonix_mm::address::VAddr;
 
@@ -26,27 +27,88 @@ enum Object {
 }
 
 pub struct ProgramLoader {
+    args: Vec<CString>,
+    envs: Vec<CString>,
     object: Object,
 }
 
 impl ProgramLoader {
-    pub fn parse(file: Arc<Dentry>) -> KResult<Self> {
-        let mut magic = [0u8; 4];
-        file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0)?;
+    pub fn parse(
+        fs_context: &FsContext,
+        mut exec_path: CString,
+        mut file: Arc<Dentry>,
+        mut args: Vec<CString>,
+        envs: Vec<CString>,
+    ) -> KResult<Self> {
+        const RECURSION_LIMIT: usize = 4;
+        let mut nrecur = 0;
+
+        let object = loop {
+            if nrecur >= RECURSION_LIMIT {
+                return Err(ENOEXEC);
+            }
+
+            let mut magic = [0; 4];
+            file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0)?;
+
+            match magic {
+                [b'#', b'!', ..] => {
+                    let mut interpreter_line = [0; 256];
+                    let nread = file.read(&mut ByteBuffer::new(&mut interpreter_line), 0)?;
+
+                    // There is a tiny time gap between reading the magic number and
+                    // reading the interpreter line, so we need to check if the line
+                    // still starts with a shebang.
+                    if !interpreter_line.starts_with(b"#!") {
+                        return Err(ENOEXEC);
+                    }
+
+                    let line = &interpreter_line[2..nread];
+                    let line = line.split(|&b| b == b'\n' || b == b'\0').next().unwrap();
+
+                    let interpreter_name;
+                    let mut interpreter_arg = None;
+                    match line.iter().position(|&b| b == b' ') {
+                        Some(blank_pos) => {
+                            interpreter_name = CString::new(&line[..blank_pos]).unwrap();
+                            interpreter_arg = Some(CString::new(&line[blank_pos + 1..]).unwrap());
+                        }
+                        None => interpreter_name = CString::new(line).unwrap(),
+                    }
+
+                    let path = Path::new(interpreter_name.as_bytes())?;
+                    file = Dentry::open(fs_context, path, true)?;
+
+                    args.insert(0, interpreter_name.clone());
+                    if let Some(arg) = interpreter_arg {
+                        args.insert(1, arg);
+                    }
+
+                    if args.len() > 2 {
+                        args[2] = exec_path;
+                    } else {
+                        args.push(exec_path);
+                    }
+
+                    exec_path = interpreter_name;
+                }
+                ELF_MAGIC => break ELF::parse(file)?,
+                _ => return Err(ENOEXEC),
+            }
 
-        let object = match magic {
-            ELF_MAGIC => ELF::parse(file),
-            _ => Err(ENOEXEC),
-        }?;
+            nrecur += 1;
+        };
 
         Ok(ProgramLoader {
+            args,
+            envs,
             object: Object::ELF(object),
         })
     }
 
-    pub fn load(&self, args: Vec<CString>, envs: Vec<CString>) -> KResult<LoadInfo> {
-        match &self.object {
-            Object::ELF(elf) => elf.load(args, envs),
+    pub fn load(self) -> KResult<LoadInfo> {
+        match self.object {
+            Object::ELF(elf) => elf.load(self.args, self.envs),
         }
     }
 }

src/kernel/vfs/mod.rs

@@ -34,13 +34,6 @@ pub fn s_islnk(mode: Mode) -> bool {
     (mode & S_IFMT) == S_IFLNK
 }
 
-#[derive(Clone, Copy, Default)]
-#[repr(C)]
-pub struct TimeSpec {
-    pub sec: u64,
-    pub nsec: u64,
-}
-
 pub struct FsContext {
     pub fsroot: Arc<Dentry>,
     pub cwd: Spin<Arc<Dentry>>,
@@ -55,12 +48,6 @@ static GLOBAL_FS_CONTEXT: LazyLock<Arc<FsContext>> = LazyLock::new(|| {
     })
 });
 
-impl TimeSpec {
-    pub const fn default() -> Self {
-        Self { sec: 0, nsec: 0 }
-    }
-}
-
 impl FsContext {
     pub fn global() -> &'static Arc<Self> {
         &GLOBAL_FS_CONTEXT

src/lib.rs

@@ -69,8 +69,6 @@ static BSP_OK: AtomicBool = AtomicBool::new(false);
 fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! {
     setup_memory(&mut data);
 
-    BSP_OK.store(true, Ordering::Release);
-
     #[cfg(target_arch = "riscv64")]
     {
         driver::sbi_console::init_console();
@@ -87,6 +85,8 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! {
 
     Scheduler::get().spawn::<KernelStack, _>(FutureRun::new(init_process(data.get_early_stack())));
 
+    BSP_OK.store(true, Ordering::Release);
+
     drop(data);
     unsafe {
         // SAFETY: `preempt::count()` == 1.
@@ -163,19 +163,14 @@ async fn init_process(early_kstack: PRange) {
         )
         .unwrap();
 
-        let init_names = [
-            &b"/sbin/init"[..],
-            &b"/init"[..],
-            &b"/bin/busybox"[..],
-            &b"/mnt/busybox"[..],
-        ];
+        let init_names = [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]];
 
         let mut init_name = None;
         let mut init = None;
-        for name in &init_names {
+        for name in init_names {
             if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true) {
                 if dentry.is_valid() {
-                    init_name = Some(*name);
+                    init_name = Some(CString::new(name).unwrap());
                     init = Some(dentry);
                     break;
                 }
@@ -185,11 +180,7 @@ async fn init_process(early_kstack: PRange) {
         let init = init.expect("No init binary found in the system.");
         let init_name = init_name.unwrap();
 
-        let argv = vec![
-            CString::new(init_name).unwrap(),
-            CString::new("sh").unwrap(),
-            CString::new("/mnt/initsh").unwrap(),
-        ];
+        let argv = vec![init_name.clone()];
 
         let envp = vec![
             CString::new("LANG=C").unwrap(),
@@ -198,10 +189,10 @@ async fn init_process(early_kstack: PRange) {
             CString::new("PWD=/").unwrap(),
         ];
 
-        ProgramLoader::parse(init.clone())
-            .unwrap()
-            .load(argv, envp)
-            .unwrap()
+        ProgramLoader::parse(fs_context, init_name, init.clone(), argv, envp)
+            .expect("Failed to parse init program")
+            .load()
+            .expect("Failed to load init program")
     };
 
     let thread_builder = ThreadBuilder::new()