feat(page fault): refactor mmap page fault handle to deal with page cache and sharing mapping

src/kernel/mem/mm_area.rs

@@ -1,18 +1,23 @@
 use super::mm_list::EMPTY_PAGE;
 use super::paging::AllocZeroed as _;
 use super::{AsMemoryBlock, Mapping, Page, Permission};
-use crate::io::ByteBuffer;
+use crate::kernel::constants::EINVAL;
+use crate::kernel::mem::page_cache::PageCacheRawPage;
 use crate::KResult;
-use core::{borrow::Borrow, cell::UnsafeCell, cmp::Ordering};
+use core::sync::atomic;
+use core::{borrow::Borrow, cell::UnsafeCell, cmp};
+use eonix_hal::traits::fault::PageFaultErrorCode;
 use eonix_mm::address::{AddrOps as _, VAddr, VRange};
 use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE};
-use eonix_mm::paging::PFN;
+use eonix_mm::paging::{PAGE_SIZE, PFN};
+use eonix_runtime::task::Task;
 
 #[derive(Debug)]
 pub struct MMArea {
     range: UnsafeCell<VRange>,
     pub(super) mapping: Mapping,
     pub(super) permission: Permission,
+    pub is_shared: bool,
 }
 
 impl Clone for MMArea {
@@ -21,16 +26,18 @@ impl Clone for MMArea {
             range: UnsafeCell::new(self.range()),
             mapping: self.mapping.clone(),
             permission: self.permission,
+            is_shared: self.is_shared,
         }
     }
 }
 
 impl MMArea {
-    pub fn new(range: VRange, mapping: Mapping, permission: Permission) -> Self {
+    pub fn new(range: VRange, mapping: Mapping, permission: Permission, is_shared: bool) -> Self {
         Self {
             range: range.into(),
             mapping,
             permission,
+            is_shared,
         }
     }
 
@@ -56,9 +63,9 @@ impl MMArea {
         assert!(at.is_page_aligned());
 
         match self.range_borrow().cmp(&VRange::from(at)) {
-            Ordering::Less => (Some(self), None),
-            Ordering::Greater => (None, Some(self)),
-            Ordering::Equal => {
+            cmp::Ordering::Less => (Some(self), None),
+            cmp::Ordering::Greater => (None, Some(self)),
+            cmp::Ordering::Equal => {
                 let diff = at - self.range_borrow().start();
                 if diff == 0 {
                     return (None, Some(self));
@@ -71,6 +78,7 @@ impl MMArea {
                         Mapping::Anonymous => Mapping::Anonymous,
                         Mapping::File(mapping) => Mapping::File(mapping.offset(diff)),
                     },
+                    is_shared: self.is_shared,
                 };
 
                 let new_range = self.range_borrow().shrink(self.range_borrow().end() - at);
@@ -119,35 +127,75 @@ impl MMArea {
 
     /// # Arguments
     /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary.
-    pub fn handle_mmap(
+    pub async fn handle_mmap(
         &self,
         pfn: &mut PFN,
         attr: &mut PageAttribute,
         offset: usize,
+        error: PageFaultErrorCode,
     ) -> KResult<()> {
-        // TODO: Implement shared mapping
-        let Mapping::File(mapping) = &self.mapping else {
+        let Mapping::File(file_mapping) = &self.mapping else {
             panic!("Anonymous mapping should not be PA_MMAP");
         };
 
-        assert!(offset < mapping.length, "Offset out of range");
-        unsafe {
-            Page::with_raw(*pfn, |page| {
-                // SAFETY: `page` is marked as mapped, so others trying to read or write to
-                //         it will be blocked and enter the page fault handler, where they will
-                //         be blocked by the mutex held by us.
-                let page_data = page.as_memblk().as_bytes_mut();
+        assert!(offset < file_mapping.length, "Offset out of range");
 
-                let cnt_to_read = (mapping.length - offset).min(0x1000);
-                let cnt_read = mapping.file.read(
-                    &mut ByteBuffer::new(&mut page_data[..cnt_to_read]),
-                    mapping.offset + offset,
-                )?;
+        let Some(page_cache) = file_mapping.file.page_cache() else {
+            panic!("Mapping file should have pagecache");
+        };
+
+        let file_offset = file_mapping.offset + offset;
+        let cnt_to_read = (file_mapping.length - offset).min(0x1000);
+        let raw_page = page_cache.get_page(file_offset).await?.ok_or(EINVAL)?;
+
+        // Read or ifetch fault, we find page in pagecache and do mapping
+        // Write falut, we need to care about shared or private mapping.
+        if error.contains(PageFaultErrorCode::Read)
+            || error.contains(PageFaultErrorCode::InstructionFetch)
+        {
+            // Bss is embarrassing in pagecache!
+            // We have to assume cnt_to_read < PAGE_SIZE all bss
+            if cnt_to_read < PAGE_SIZE {
+                let new_page = Page::zeroed();
+                unsafe {
+                    let page_data = new_page.as_memblk().as_bytes_mut();
+                    page_data[..cnt_to_read]
+                        .copy_from_slice(&raw_page.as_memblk().as_bytes()[..cnt_to_read]);
+                }
+                *pfn = new_page.into_raw();
+            } else {
+                raw_page.refcount().fetch_add(1, atomic::Ordering::Relaxed);
+                *pfn = Into::<PFN>::into(raw_page);
+            }
 
-                page_data[cnt_read..].fill(0);
+            if self.permission.write {
+                if self.is_shared {
+                    // The page may will not be written,
+                    // But we simply assume page will be dirty
+                    raw_page.set_dirty();
+                    attr.insert(PageAttribute::WRITE);
+                } else {
+                    attr.insert(PageAttribute::COPY_ON_WRITE);
+                }
+            }
+        } else if error.contains(PageFaultErrorCode::Write) {
+            if self.is_shared {
+                raw_page.refcount().fetch_add(1, atomic::Ordering::Relaxed);
+                raw_page.set_dirty();
+                *pfn = Into::<PFN>::into(raw_page);
+            } else {
+                let new_page = Page::zeroed();
+                unsafe {
+                    let page_data = new_page.as_memblk().as_bytes_mut();
+                    page_data[..cnt_to_read]
+                        .copy_from_slice(&raw_page.as_memblk().as_bytes()[..cnt_to_read]);
+                }
+                *pfn = new_page.into_raw();
+            }
 
-                KResult::Ok(())
-            })?;
+            attr.insert(PageAttribute::WRITE);
+        } else {
+            unreachable!("Unexpected page fault error code: {:?}", error);
         }
 
         attr.insert(PageAttribute::PRESENT);
@@ -155,7 +203,12 @@ impl MMArea {
         Ok(())
     }
 
-    pub fn handle(&self, pte: &mut impl PTE, offset: usize) -> KResult<()> {
+    pub fn handle(
+        &self,
+        pte: &mut impl PTE,
+        offset: usize,
+        error: Option<PageFaultErrorCode>,
+    ) -> KResult<()> {
         let mut attr = pte.get_attr().as_page_attr().expect("Not a page attribute");
         let mut pfn = pte.get_pfn();
 
@@ -164,7 +217,9 @@ impl MMArea {
         }
 
         if attr.contains(PageAttribute::MAPPED) {
-            self.handle_mmap(&mut pfn, &mut attr, offset)?;
+            let error =
+                error.expect("Mapped area should not be accessed without a page fault error code");
+            Task::block_on(self.handle_mmap(&mut pfn, &mut attr, offset, error))?;
         }
 
         attr.set(PageAttribute::ACCESSED, true);

src/kernel/mem/mm_list.rs

@@ -256,6 +256,7 @@ impl MMListInner<'_> {
         len: usize,
         mapping: Mapping,
         permission: Permission,
+        is_shared: bool,
     ) -> KResult<()> {
         assert_eq!(at.floor(), at);
         assert_eq!(len & (PAGE_SIZE - 1), 0);
@@ -271,7 +272,8 @@ impl MMListInner<'_> {
             Mapping::File(_) => self.page_table.set_mmapped(range, permission),
         }
 
-        self.areas.insert(MMArea::new(range, mapping, permission));
+        self.areas
+            .insert(MMArea::new(range, mapping, permission, is_shared));
         Ok(())
     }
 }
@@ -343,9 +345,11 @@ impl MMList {
             let list_inner = list_inner.lock().await;
 
             for area in list_inner.areas.iter() {
-                list_inner
-                    .page_table
-                    .set_copy_on_write(&mut inner.page_table, area.range());
+                if !area.is_shared {
+                    list_inner
+                        .page_table
+                        .set_copy_on_write(&mut inner.page_table, area.range());
+                }
             }
         }
 
@@ -507,21 +511,22 @@ impl MMList {
         len: usize,
         mapping: Mapping,
         permission: Permission,
+        is_shared: bool,
     ) -> KResult<VAddr> {
         let inner = self.inner.borrow();
         let mut inner = Task::block_on(inner.lock());
 
         if hint == VAddr::NULL {
             let at = inner.find_available(hint, len).ok_or(ENOMEM)?;
-            inner.mmap(at, len, mapping, permission)?;
+            inner.mmap(at, len, mapping, permission, is_shared)?;
             return Ok(at);
         }
 
-        match inner.mmap(hint, len, mapping.clone(), permission) {
+        match inner.mmap(hint, len, mapping.clone(), permission, is_shared) {
             Ok(()) => Ok(hint),
             Err(EEXIST) => {
                 let at = inner.find_available(hint, len).ok_or(ENOMEM)?;
-                inner.mmap(at, len, mapping, permission)?;
+                inner.mmap(at, len, mapping, permission, is_shared)?;
                 Ok(at)
             }
             Err(err) => Err(err),
@@ -534,9 +539,10 @@ impl MMList {
         len: usize,
         mapping: Mapping,
         permission: Permission,
+        is_shared: bool,
     ) -> KResult<VAddr> {
         Task::block_on(self.inner.borrow().lock())
-            .mmap(at, len, mapping.clone(), permission)
+            .mmap(at, len, mapping.clone(), permission, is_shared)
             .map(|_| at)
     }
 
@@ -571,6 +577,7 @@ impl MMList {
                     write: true,
                     execute: false,
                 },
+                false,
             ));
         }
 
@@ -644,7 +651,7 @@ impl MMList {
                 let page_start = current.floor() + idx * 0x1000;
                 let page_end = page_start + 0x1000;
 
-                area.handle(pte, page_start - area_start)?;
+                area.handle(pte, page_start - area_start, None)?;
 
                 let start_offset;
                 if page_start < current {
@@ -718,7 +725,9 @@ impl PageTableExt for KernelPageTable<'_> {
 }
 
 trait PTEExt {
+    // private anonymous
     fn set_anonymous(&mut self, execute: bool);
+    // file mapped or shared anonymous
     fn set_mapped(&mut self, execute: bool);
     fn set_copy_on_write(&mut self, from: &mut Self);
 }
@@ -742,10 +751,7 @@ where
     fn set_mapped(&mut self, execute: bool) {
         // Writable flag is set during page fault handling while executable flag is
         // preserved across page faults, so we set executable flag now.
-        let mut attr = PageAttribute::READ
-            | PageAttribute::USER
-            | PageAttribute::MAPPED
-            | PageAttribute::COPY_ON_WRITE;
+        let mut attr = PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED;
         attr.set(PageAttribute::EXECUTE, execute);
 
         self.set(EMPTY_PAGE.clone().into_raw(), T::Attr::from(attr));

src/kernel/mem/mm_list/mapping.rs

@@ -1,23 +1,34 @@
-use crate::kernel::vfs::dentry::Dentry;
+use core::fmt::Debug;
+
+use crate::kernel::vfs::inode::Inode;
 use alloc::sync::Arc;
 use eonix_mm::paging::PAGE_SIZE;
 
 #[derive(Debug, Clone)]
 pub struct FileMapping {
-    pub file: Arc<Dentry>,
+    pub file: Arc<dyn Inode>,
     /// Offset in the file, aligned to 4KB boundary.
     pub offset: usize,
     /// Length of the mapping. Exceeding part will be zeroed.
     pub length: usize,
 }
+
+impl Debug for dyn Inode {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "Inode()")
+    }
+}
+
 #[derive(Debug, Clone)]
 pub enum Mapping {
+    // private anonymous memory
     Anonymous,
+    // file-backed memory or shared anonymous memory(tmp file)
     File(FileMapping),
 }
 
 impl FileMapping {
-    pub fn new(file: Arc<Dentry>, offset: usize, length: usize) -> Self {
+    pub fn new(file: Arc<dyn Inode>, offset: usize, length: usize) -> Self {
         assert_eq!(offset & (PAGE_SIZE - 1), 0);
         Self {
             file,

src/kernel/mem/mm_list/page_fault.rs

@@ -90,7 +90,7 @@ impl MMList {
             .next()
             .expect("If we can find the mapped area, we should be able to find the PTE");
 
-        area.handle(pte, addr.floor() - area.range().start())
+        area.handle(pte, addr.floor() - area.range().start(), Some(error))
             .map_err(|_| Signal::SIGBUS)?;
 
         flush_tlb(addr.floor().addr());
@@ -160,9 +160,11 @@ pub fn handle_kernel_page_fault(
         .next()
         .expect("If we can find the mapped area, we should be able to find the PTE");
 
-    if let Err(_) = area.handle(pte, addr.floor() - area.range().start()) {
+    if let Err(_) = area.handle(pte, addr.floor() - area.range().start(), Some(error)) {
         return Some(try_page_fault_fix(fault_pc, addr));
     }
 
+    flush_tlb(addr.addr());
+
     None
 }