Merge branch 'mem_fix' into exec

CMakeLists.txt

@@ -40,8 +40,8 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2")
 endif()
 
 if (NOT DEFINED FDISK_BIN)
@@ -100,7 +100,7 @@ set(KERNEL_MAIN_SOURCES src/fs/fat.cpp
                         include/types/allocator.hpp
                         include/types/cplusplus.hpp
                         include/types/list.hpp
-                        include/types/lock.h
+                        include/types/lock.hpp
                         include/types/string.hpp
                         include/types/vector.hpp
                         include/kernel_main.h

include/asm/sys.h

@@ -7,17 +7,17 @@
 extern "C" {
 #endif
 
-void asm_switch_pd(page_directory_entry* pd_addr);
-void asm_enable_paging(page_directory_entry* pd_addr);
+void asm_switch_pd(pd_t pd_addr);
+void asm_enable_paging(pd_t pd_addr);
 
-phys_ptr_t current_pd(void);
+pptr_t current_pd(void);
 
 // the limit should be set on the higher 16bit
 // e.g. (n * sizeof(segment_descriptor) - 1) << 16
 // the lower bit off the limit is either 0 or 1
 // indicating whether or not to enable interrupt
 // after loading gdt
-void asm_load_gdt(uint32_t limit, phys_ptr_t addr);
+void asm_load_gdt(uint32_t limit, pptr_t addr);
 
 void asm_load_tr(uint16_t index);
 

include/kernel/hw/port.hpp

@@ -22,11 +22,13 @@ public:
             "this type is not implemented yet.");
         port_size_t ret;
         if constexpr (types::is_same<port_size_t, uint8_t>::value)
-            asm("inb %1, %0"
+            asm volatile(
+                "inb %1, %0"
                 : "=a"(ret)
                 : "d"(mp));
         if constexpr (types::is_same<port_size_t, uint16_t>::value)
-            asm("inw %1, %0"
+            asm volatile(
+                "inw %1, %0"
                 : "=a"(ret)
                 : "d"(mp));
         return ret;
@@ -38,11 +40,13 @@ public:
             types::is_same<port_size_t, uint8_t>::value || types::is_same<port_size_t, uint16_t>::value,
             "this type is not implemented yet.");
         if constexpr (types::is_same<port_size_t, uint8_t>::value)
-            asm("outb %1, %0"
+            asm volatile(
+                "outb %1, %0"
                 :
                 : "d"(mp), "a"(n));
         if constexpr (types::is_same<port_size_t, uint16_t>::value)
-            asm("outw %1, %0"
+            asm volatile(
+                "outw %1, %0"
                 :
                 : "d"(mp), "a"(n));
         return n;

include/kernel/mem.h

@@ -40,23 +40,22 @@ struct e820_mem_map_entry_24 {
  * ps  : use 4MiB pages (ignored)
  * addr: page table address
  */
-struct page_directory_entry_in {
-    uint32_t p : 1;
-    uint32_t rw : 1;
-    uint32_t us : 1;
-    uint32_t pwt : 1;
-    uint32_t pcd : 1;
-    uint32_t a : 1;
-    uint32_t d : 1;
-    uint32_t ps : 1;
-    uint32_t ignored : 4;
-    page_t pt_page : 20;
-};
-
-typedef union page_directory_entry {
+typedef union pde_t {
     uint32_t v;
-    struct page_directory_entry_in in;
-} page_directory_entry;
+    struct {
+        uint32_t p : 1;
+        uint32_t rw : 1;
+        uint32_t us : 1;
+        uint32_t pwt : 1;
+        uint32_t pcd : 1;
+        uint32_t a : 1;
+        uint32_t d : 1;
+        uint32_t ps : 1;
+        uint32_t ignored : 4;
+        page_t pt_page : 20;
+    } in;
+} pde_t;
+typedef pde_t (*pd_t)[1024];
 
 /*
  * page table entry
@@ -72,24 +71,23 @@ typedef union page_directory_entry {
  * g   : used in cr4 mode (ignored)
  * addr: physical memory address
  */
-struct page_table_entry_in {
-    uint32_t p : 1;
-    uint32_t rw : 1;
-    uint32_t us : 1;
-    uint32_t pwt : 1;
-    uint32_t pcd : 1;
-    uint32_t a : 1;
-    uint32_t d : 1;
-    uint32_t pat : 1;
-    uint32_t g : 1;
-    uint32_t ignored : 3;
-    page_t page : 20;
-};
-
-typedef union page_table_entry {
+typedef union pte_t {
     uint32_t v;
-    struct page_table_entry_in in;
-} page_table_entry;
+    struct {
+        uint32_t p : 1;
+        uint32_t rw : 1;
+        uint32_t us : 1;
+        uint32_t pwt : 1;
+        uint32_t pcd : 1;
+        uint32_t a : 1;
+        uint32_t d : 1;
+        uint32_t pat : 1;
+        uint32_t g : 1;
+        uint32_t ignored : 3;
+        page_t page : 20;
+    } in;
+} pte_t;
+typedef pte_t (*pt_t)[1024];
 
 // in kernel_main.c
 extern uint8_t e820_mem_map[1024];
@@ -109,7 +107,7 @@ void* ki_malloc(size_t size);
 
 void ki_free(void* ptr);
 
-#define KERNEL_PAGE_DIRECTORY_ADDR ((page_directory_entry*)0x00000000)
+#define KERNEL_PAGE_DIRECTORY_ADDR ((pd_t)0x00001000)
 
 void init_mem(void);
 

include/kernel/mm.hpp

@@ -3,131 +3,212 @@
 #include <kernel/mem.h>
 #include <kernel/vfs.hpp>
 #include <types/allocator.hpp>
+#include <types/cplusplus.hpp>
 #include <types/list.hpp>
+#include <types/size.h>
+#include <types/status.h>
 #include <types/types.h>
 #include <types/vector.hpp>
 
-constexpr size_t THREAD_KERNEL_STACK_SIZE = 2 * PAGE_SIZE;
+#define invalidate_tlb(addr) asm("invlpg (%0)" \
+                                 :             \
+                                 : "r"(addr)   \
+                                 : "memory")
 
-struct page_attr {
-    uint32_t cow : 1;
-};
+constexpr size_t THREAD_KERNEL_STACK_SIZE = 2 * PAGE_SIZE;
 
 struct page {
     page_t phys_page_id;
-    page_table_entry* pte;
+    pte_t* pte;
     size_t* ref_count;
-    struct page_attr attr;
+    union {
+        uint32_t v;
+        struct {
+            uint32_t cow : 1;
+        } in;
+    } attr;
 };
 
+struct mm;
+
+// map the page to the end of the mm_area in pd
+int k_map(
+    mm* mm_area,
+    page* page,
+    int read,
+    int write,
+    int priv,
+    int cow);
+
+// unmap a whole mem area, making sure that we will never use it again
+int k_unmap(mm* mm_area);
+
+// private memory mapping
+// changes won't be neither written back to file nor shared between processes
+// TODO: shared mapping
+// @param len is aligned to 4kb boundary automatically, exceeding part will
+// be filled with '0's and not written back to the file
+int mmap(
+    void* hint,
+    size_t len,
+    fs::inode* file,
+    size_t offset,
+    int write,
+    int priv);
+
 using page_arr = types::vector<page, types::kernel_ident_allocator>;
 
-struct mm_attr {
-    uint32_t read : 1;
-    uint32_t write : 1;
-    uint32_t system : 1;
-};
+using mm_list = types::list<mm, types::kernel_ident_allocator>;
 
-class mm {
+struct mm {
 public:
-    linr_ptr_t start;
-    struct mm_attr attr;
-    page_directory_entry* pd;
-    page_arr* pgs;
-    fs::inode* mapped_file;
-    size_t file_offset;
+    void* start;
+    union {
+        uint32_t v;
+        struct {
+            uint32_t read : 1;
+            uint32_t write : 1;
+            uint32_t system : 1;
+        } in;
+    } attr;
+    pd_t pd;
+    page_arr* pgs = types::kernel_ident_allocator_new<page_arr>();
+    fs::inode* mapped_file = nullptr;
+    size_t file_offset = 0;
 
 public:
-    mm(const mm& val);
-    mm(linr_ptr_t start, page_directory_entry* pd, bool write, bool system);
+    static constexpr int mirror_mm_area(mm_list* dst, const mm* src, pd_t pd)
+    {
+        mm new_nn {
+            .start = src->start,
+            .attr { src->attr.v },
+            .pd = pd,
+            .mapped_file = src->mapped_file,
+            .file_offset = src->file_offset,
+        };
+
+        for (auto iter = src->pgs->begin(); iter != src->pgs->end(); ++iter) {
+            // TODO: preserve dirty flag, clear accessed flag
+            if (k_map(&new_nn, &*iter,
+                    src->attr.in.read, src->attr.in.write, src->attr.in.system, 1)
+                != GB_OK) {
+                return GB_FAILED;
+            }
+        }
+
+        dst->emplace_back(types::move(new_nn));
+        return GB_OK;
+    }
 };
 
-using mm_list = types::list<mm, types::kernel_ident_allocator>;
-
 // in mem.cpp
 extern mm_list* kernel_mms;
 extern page empty_page;
 
 // translate physical address to virtual(mapped) address
-void* p_ptr_to_v_ptr(phys_ptr_t p_ptr);
-
-// translate linear address to physical address
-phys_ptr_t l_ptr_to_p_ptr(const mm_list* mms, linr_ptr_t v_ptr);
-
-// translate virtual(mapped) address to physical address
-phys_ptr_t v_ptr_to_p_ptr(void* v_ptr);
+void* ptovp(pptr_t p_ptr);
 
 // @return the pointer to the mm_area containing l_ptr
 //         nullptr if not
-mm* find_mm_area(mm_list* mms, linr_ptr_t l_ptr);
-
-// find the corresponding page the l_ptr pointing to
-// @return the pointer to the struct if found, NULL if not found
-struct page* find_page_by_l_ptr(const mm_list* mms, linr_ptr_t l_ptr);
+mm* find_mm_area(mm_list* mms, void* l_ptr);
 
-static inline page_t phys_addr_to_page(phys_ptr_t ptr)
+inline constexpr size_t vptrdiff(void* p1, void* p2)
+{
+    return (uint8_t*)p1 - (uint8_t*)p2;
+}
+inline constexpr page* lto_page(const mm* mm_area, void* l_ptr)
+{
+    size_t offset = vptrdiff(l_ptr, mm_area->start);
+    return &mm_area->pgs->at(offset / PAGE_SIZE);
+}
+inline constexpr page_t to_page(pptr_t ptr)
 {
     return ptr >> 12;
 }
-
-static inline pd_i_t page_to_pd_i(page_t p)
+inline constexpr size_t to_pdi(page_t pg)
 {
-    return p >> 10;
+    return pg >> 10;
 }
-
-static inline pt_i_t page_to_pt_i(page_t p)
+inline constexpr size_t to_pti(page_t pg)
 {
-    return p & (1024 - 1);
+    return pg & (1024 - 1);
 }
-
-static inline phys_ptr_t page_to_phys_addr(page_t p)
+inline constexpr pptr_t to_pp(page_t p)
 {
     return p << 12;
 }
-
-static inline pd_i_t linr_addr_to_pd_i(linr_ptr_t ptr)
+inline constexpr size_t lto_pdi(pptr_t ptr)
 {
-    return page_to_pd_i(phys_addr_to_page(ptr));
+    return to_pdi(to_page(ptr));
 }
-
-static inline pd_i_t linr_addr_to_pt_i(linr_ptr_t ptr)
+inline constexpr size_t lto_pti(pptr_t ptr)
 {
-    return page_to_pt_i(phys_addr_to_page(ptr));
+    return to_pti(to_page(ptr));
 }
-
-static inline page_directory_entry* mms_get_pd(const mm_list* mms)
+inline constexpr pte_t* to_pte(pt_t pt, page_t pg)
+{
+    return *pt + to_pti(pg);
+}
+inline pd_t mms_get_pd(const mm_list* mms)
 {
     return mms->begin()->pd;
 }
-
-// map the page to the end of the mm_area in pd
-int k_map(
-    mm* mm_area,
-    page* page,
-    int read,
-    int write,
-    int priv,
-    int cow);
-
-// @param len is aligned to 4kb boundary automatically, exceeding part will
-// be filled with '0's and not written back to the file
-int mmap(
-    void* hint,
-    size_t len,
-    fs::inode* file,
-    size_t offset,
-    int write,
-    int priv);
-
-// allocate a raw page
-page_t alloc_raw_page(void);
+inline void* to_vp(page_t pg)
+{
+    return ptovp(to_pp(pg));
+}
+inline pd_t to_pd(page_t pg)
+{
+    return reinterpret_cast<pd_t>(to_vp(pg));
+}
+inline pt_t to_pt(page_t pg)
+{
+    return reinterpret_cast<pt_t>(to_vp(pg));
+}
+inline pt_t to_pt(pde_t* pde)
+{
+    return to_pt(pde->in.pt_page);
+}
+inline pde_t* to_pde(pd_t pd, void* addr)
+{
+    return *pd + lto_pdi((pptr_t)addr);
+}
+inline pte_t* to_pte(pt_t pt, void* addr)
+{
+    return *pt + lto_pti((pptr_t)addr);
+}
+inline pte_t* to_pte(pde_t* pde, void* addr)
+{
+    return to_pte(to_pt(pde), addr);
+}
+inline pte_t* to_pte(pd_t pd, void* addr)
+{
+    return to_pte(to_pde(pd, addr), addr);
+}
+inline pte_t* to_pte(pde_t* pde, page_t pg)
+{
+    return to_pte(to_pt(pde), pg);
+}
+inline constexpr void* mmend(const mm* mm_area)
+{
+    return (char*)mm_area->start + mm_area->pgs->size() * PAGE_SIZE;
+}
 
 // allocate n raw page(s)
 // @return the id of the first page allocated
 page_t alloc_n_raw_pages(size_t n);
 
+// allocate a raw page
+inline page_t alloc_raw_page(void)
+{
+    return alloc_n_raw_pages(1);
+}
+
 // allocate a struct page together with the raw page
 struct page allocate_page(void);
 
-page_directory_entry* alloc_pd(void);
-page_table_entry* alloc_pt(void);
+pd_t alloc_pd(void);
+pt_t alloc_pt(void);
+
+void dealloc_pd(pd_t pd);
+void dealloc_pt(pt_t pt);

include/kernel/syscall.hpp

@@ -1,11 +1,23 @@
 #pragma once
 
-#include <types/types.h>
 #include <kernel/interrupt.h>
+#include <types/types.h>
 
 // return value is stored in %eax and %edx
 typedef void (*syscall_handler)(interrupt_stack* data);
 
-#define syscall(eax) asm volatile("movl %0, %%eax\n\tint $0x80"::"r"(eax):"eax","edx")
+inline uint32_t syscall(uint32_t num, uint32_t arg1 = 0, uint32_t arg2 = 0)
+{
+    asm volatile(
+        "movl %1, %%edi\n"
+        "movl %2, %%esi\n"
+        "movl %3, %%eax\n"
+        "int $0x80\n"
+        "movl %%eax, %0"
+        : "=g"(num)
+        : "g"(arg1), "g"(arg2), "g"(num)
+        : "eax", "edx", "edi", "esi");
+    return num;
+}
 
 void init_syscall(void);

include/types/allocator.hpp

@@ -59,17 +59,45 @@ public:
 };
 
 template <typename T, typename... Args>
-T* kernel_allocator_new(Args&&... args)
+constexpr T* kernel_allocator_new(Args&&... args)
 {
     return allocator_traits<kernel_allocator<T>>::allocate_and_construct(forward<Args>(args)...);
 }
 
+template <PointerType T, typename... Args>
+constexpr auto kernel_allocator_pnew(T, Args&&... args)
+{
+    using value_type = typename traits::remove_pointer<T>::type;
+    return kernel_allocator_new<value_type>(forward<Args>(args)...);
+}
+
 template <typename T, typename... Args>
-T* kernel_ident_allocator_new(Args&&... args)
+constexpr T* kernel_ident_allocator_new(Args&&... args)
 {
     return allocator_traits<kernel_ident_allocator<T>>::allocate_and_construct(forward<Args>(args)...);
 }
 
+template <PointerType T, typename... Args>
+constexpr auto kernel_ident_allocator_pnew(T, Args&&... args)
+{
+    using value_type = typename traits::remove_pointer<T>::type;
+    return kernel_ident_allocator_new<value_type>(forward<Args>(args)...);
+}
+
+template <PointerType T>
+constexpr void kernel_allocator_delete(T ptr)
+{
+    using value_type = typename traits::remove_pointer<T>::type;
+    return allocator_traits<kernel_allocator<value_type>>::deconstruct_and_deallocate(ptr);
+}
+
+template <PointerType T>
+constexpr void kernel_ident_allocator_delete(T ptr)
+{
+    using value_type = typename traits::remove_pointer<T>::type;
+    return allocator_traits<kernel_ident_allocator<value_type>>::deconstruct_and_deallocate(ptr);
+}
+
 template <Allocator _allocator>
 class allocator_traits {
 public:

include/types/cplusplus.hpp

@@ -141,6 +141,9 @@ concept convertible_to = (traits::is_pointer<Src>::value && is_same<Dst, uint32_
     { static_cast<Dst>(_src) };
 };
 
+template <typename T>
+concept PointerType = traits::is_pointer<T>::value;
+
 template <typename A, typename B>
 concept same_as = is_same<A, B>::value;
 

include/types/hash_map.hpp

@@ -111,13 +111,13 @@ public:
         const key_type key;
         value_type value;
 
-        pair(void) = delete;
-        pair(const key_type _key, value_type _val)
+        constexpr pair(void) = delete;
+        constexpr pair(const key_type _key, value_type _val)
             : key(_key)
             , value(_val)
         {
         }
-        bool operator==(const pair& p)
+        constexpr bool operator==(const pair& p)
         {
             return key == p.key;
         }
@@ -129,49 +129,51 @@ public:
         using _Value = typename traits::remove_pointer<Pointer>::type;
         using Reference = typename traits::add_reference<_Value>::type;
 
+        friend class hash_map;
+
     public:
-        iterator(const iterator& iter) noexcept
+        constexpr iterator(const iterator& iter) noexcept
             : p(iter.p)
         {
         }
 
-        iterator(iterator&& iter) noexcept
+        constexpr iterator(iterator&& iter) noexcept
             : p(iter.p)
         {
             iter.p = nullptr;
         }
 
-        iterator& operator=(const iterator& iter)
+        constexpr iterator& operator=(const iterator& iter)
         {
             p = iter.p;
             return *this;
         }
 
-        explicit iterator(Pointer p) noexcept
+        explicit constexpr iterator(Pointer p) noexcept
             : p(p)
         {
         }
 
-        bool operator==(const iterator& iter) const noexcept
+        constexpr bool operator==(const iterator& iter) const noexcept
         {
             return this->p == iter.p;
         }
 
-        bool operator!=(const iterator& iter) const noexcept
+        constexpr bool operator!=(const iterator& iter) const noexcept
         {
             return !(*this == iter);
         }
 
-        bool operator!()
+        constexpr operator bool()
         {
-            return !p;
+            return p != nullptr;
         }
 
-        Reference operator*() const noexcept
+        constexpr Reference operator*() const noexcept
         {
             return *p;
         }
-        Pointer operator->() const noexcept
+        constexpr Pointer operator->() const noexcept
         {
             return p;
         }
@@ -202,19 +204,19 @@ protected:
     }
 
 public:
-    explicit hash_map(void)
+    explicit constexpr hash_map(void)
         : buckets(INITIAL_BUCKETS_ALLOCATED)
     {
         for (size_type i = 0; i < INITIAL_BUCKETS_ALLOCATED; ++i)
             buckets.emplace_back();
     }
 
-    hash_map(const hash_map& v)
+    constexpr hash_map(const hash_map& v)
         : buckets(v.buckets)
     {
     }
 
-    hash_map(hash_map&& v)
+    constexpr hash_map(hash_map&& v)
         : buckets(move(v.buckets))
     {
     }
@@ -224,22 +226,22 @@ public:
         buckets.clear();
     }
 
-    void insert(const pair& p)
+    constexpr void insert(const pair& p)
     {
         auto hash_value = _Hasher::hash(p.key, hash_length());
         buckets.at(hash_value).push_back(p);
     }
-    void insert(pair&& p)
+    constexpr void insert(pair&& p)
     {
         auto hash_value = _Hasher::hash(p.key, hash_length());
         buckets.at(hash_value).push_back(move(p));
     }
-    void insert(const key_type& key, const value_type& val)
+    constexpr void insert(const key_type& key, const value_type& val)
     {
         insert(pair { key, val });
     }
 
-    void remove(const key_type& key)
+    constexpr void remove(const key_type& key)
     {
         auto hash_value = _Hasher::hash(key, hash_length());
         auto& bucket = buckets.at(hash_value);
@@ -251,7 +253,18 @@ public:
         }
     }
 
-    iterator_type find(const key_type& key)
+    constexpr void remove(iterator_type iter)
+    {
+        remove(iter->key);
+        iter.p = nullptr;
+    }
+    constexpr void remove(const_iterator_type iter)
+    {
+        remove(iter->key);
+        iter.p = nullptr;
+    }
+
+    constexpr iterator_type find(const key_type& key)
     {
         auto hash_value = _Hasher::hash(key, hash_length());
         auto& bucket = buckets.at(hash_value);
@@ -262,7 +275,7 @@ public:
         return iterator_type(nullptr);
     }
 
-    const_iterator_type find(const key_type& key) const
+    constexpr const_iterator_type find(const key_type& key) const
     {
         auto hash_value = _Hasher::hash(key, hash_length());
         const auto& bucket = buckets.at(hash_value);
@@ -273,7 +286,7 @@ public:
         return const_iterator_type(nullptr);
     }
 
-    void clear(void)
+    constexpr void clear(void)
     {
         for (size_t i = 0; i < buckets.size(); ++i)
             buckets.at(i).clear();

include/types/list.hpp

@@ -36,7 +36,7 @@ private:
         node_type* prev = 0;
         node_type* next = 0;
 
-        void connect(node_type* _next) noexcept
+        constexpr void connect(node_type* _next) noexcept
         {
             this->next = _next;
             _next->prev = static_cast<node_type*>(this);
@@ -46,12 +46,12 @@ private:
     template <typename NodeValue>
     class node : public node_base {
     public:
-        explicit node(const NodeValue& v) noexcept
+        explicit constexpr node(const NodeValue& v) noexcept
             : value(v)
         {
         }
 
-        explicit node(NodeValue&& v) noexcept
+        explicit constexpr node(NodeValue&& v) noexcept
             : value(move(v))
         {
         }
@@ -67,80 +67,80 @@ public:
         using Reference = typename types::traits::add_reference<Value>::type;
 
     public:
-        iterator(const iterator& iter) noexcept
+        constexpr iterator(const iterator& iter) noexcept
             : n(iter.n)
         {
         }
 
-        iterator(iterator&& iter) noexcept
+        constexpr iterator(iterator&& iter) noexcept
             : n(iter.n)
         {
             iter.n = nullptr;
         }
 
-        iterator& operator=(const iterator& iter)
+        constexpr iterator& operator=(const iterator& iter)
         {
             n = iter.n;
             return *this;
         }
 
-        explicit iterator(node_type* _n) noexcept
+        explicit constexpr iterator(node_type* _n) noexcept
             : n(_n)
         {
         }
 
-        bool operator==(const iterator& iter) const noexcept
+        constexpr bool operator==(const iterator& iter) const noexcept
         {
             return this->_node() == iter._node();
         }
 
-        bool operator!=(const iterator& iter) const noexcept
+        constexpr bool operator!=(const iterator& iter) const noexcept
         {
             return !(*this == iter);
         }
 
-        iterator& operator++() noexcept
+        constexpr iterator& operator++() noexcept
         {
             n = n->next;
             return *this;
         }
 
-        iterator operator++(int) noexcept
+        constexpr iterator operator++(int) noexcept
         {
             iterator iter(*this);
             n = n->next;
             return iter;
         }
 
-        iterator& operator--() noexcept
+        constexpr iterator& operator--() noexcept
         {
             n = n->prev;
             return *this;
         }
 
-        iterator operator--(int) noexcept
+        constexpr iterator operator--(int) noexcept
         {
             iterator iter(*this);
             n = n->prev;
             return iter;
         }
 
-        Reference operator*() const noexcept
+        constexpr Reference operator*() const noexcept
         {
             return n->value;
         }
 
-        Pointer operator->() const noexcept
+        constexpr Pointer operator->() const noexcept
         {
             return &n->value;
         }
 
-        Pointer ptr(void) const noexcept
+        constexpr Pointer ptr(void) const noexcept
         {
             return &n->value;
         }
 
-        node_base_type* _node(void) const noexcept
+        constexpr node_base_type* _node(void) const noexcept
         {
             return n;
         }
@@ -153,17 +153,17 @@ private:
     node_base_type* head;
     node_base_type* tail;
 
-    const size_t& _size(void) const noexcept
+    constexpr const size_t& _size(void) const noexcept
     {
         return (static_cast<sentry_node_type*>(head))->value;
     }
 
-    size_t& _size(void) noexcept
+    constexpr size_t& _size(void) noexcept
     {
         return (static_cast<sentry_node_type*>(head))->value;
     }
 
-    void destroy(void)
+    constexpr void destroy(void)
     {
         if (!head || !tail)
             return;
@@ -173,7 +173,7 @@ private:
     }
 
 public:
-    list() noexcept
+    constexpr list() noexcept
         // size is stored in the 'head' node
         : head(allocator_traits<sentry_allocator_type>::allocate_and_construct(0))
         , tail(allocator_traits<sentry_allocator_type>::allocate_and_construct(0))
@@ -182,14 +182,14 @@ public:
         tail->connect(static_cast<node_type*>(head));
     }
 
-    list(const list& v)
+    constexpr list(const list& v)
         : list()
     {
         for (const auto& item : v)
             push_back(item);
     }
 
-    list(list&& v)
+    constexpr list(list&& v)
         : head(v.head)
         , tail(v.tail)
     {
@@ -197,7 +197,7 @@ public:
         v.tail = nullptr;
     }
 
-    list& operator=(const list& v)
+    constexpr list& operator=(const list& v)
     {
         clear();
         for (const auto& item : v)
@@ -205,7 +205,7 @@ public:
         return *this;
     }
 
-    list& operator=(list&& v)
+    constexpr list& operator=(list&& v)
     {
         destroy();
 
@@ -222,7 +222,7 @@ public:
         destroy();
     }
 
-    iterator_type find(const value_type& v) noexcept
+    constexpr iterator_type find(const value_type& v) noexcept
     {
         for (iterator_type iter = begin(); iter != end(); ++iter)
             if (*iter == v)
@@ -231,7 +231,7 @@ public:
     }
 
     // erase the node which iter points to
-    iterator_type erase(const iterator_type& iter) noexcept
+    constexpr iterator_type erase(const iterator_type& iter) noexcept
     {
         node_base_type* current_node = iter._node();
         iterator_type ret(current_node->next);
@@ -241,14 +241,14 @@ public:
         return ret;
     }
 
-    void clear(void)
+    constexpr void clear(void)
     {
         for (auto iter = begin(); iter != end();)
             iter = erase(iter);
     }
 
     // insert the value v in front of the given iterator
-    iterator_type insert(const iterator_type& iter, const value_type& v) noexcept
+    constexpr iterator_type insert(const iterator_type& iter, const value_type& v) noexcept
     {
         node_type* new_node = allocator_traits<allocator_type>::allocate_and_construct(v);
         iterator_type ret(new_node);
@@ -260,7 +260,7 @@ public:
     }
 
     // insert the value v in front of the given iterator
-    iterator_type insert(const iterator_type& iter, value_type&& v) noexcept
+    constexpr iterator_type insert(const iterator_type& iter, value_type&& v) noexcept
     {
         node_type* new_node = allocator_traits<allocator_type>::allocate_and_construct(move(v));
         iterator_type ret(new_node);
@@ -271,74 +271,74 @@ public:
         return ret;
     }
 
-    void push_back(const value_type& v) noexcept
+    constexpr void push_back(const value_type& v) noexcept
     {
         insert(end(), v);
     }
 
-    void push_back(value_type&& v) noexcept
+    constexpr void push_back(value_type&& v) noexcept
     {
         insert(end(), move(v));
     }
 
     template <typename... Args>
-    iterator_type emplace_back(Args&&... args)
+    constexpr iterator_type emplace_back(Args&&... args)
     {
         return insert(end(), value_type(forward<Args>(args)...));
     }
 
-    void push_front(const value_type& v) noexcept
+    constexpr void push_front(const value_type& v) noexcept
     {
         insert(begin(), v);
     }
 
-    void push_front(value_type&& v) noexcept
+    constexpr void push_front(value_type&& v) noexcept
     {
         insert(begin(), move(v));
     }
 
     template <typename... Args>
-    iterator_type emplace_front(Args&&... args)
+    constexpr iterator_type emplace_front(Args&&... args)
     {
         return insert(begin(), value_type(forward<Args>(args)...));
     }
 
-    size_t size(void) const noexcept
+    constexpr size_t size(void) const noexcept
     {
         return _size();
     }
 
-    iterator_type begin() noexcept
+    constexpr iterator_type begin() noexcept
     {
         return iterator_type(head->next);
     }
 
-    iterator_type end() noexcept
+    constexpr iterator_type end() noexcept
     {
         return iterator_type(static_cast<node_type*>(tail));
     }
 
-    const_iterator_type begin() const noexcept
+    constexpr const_iterator_type begin() const noexcept
     {
         return const_iterator_type(head->next);
     }
 
-    const_iterator_type end() const noexcept
+    constexpr const_iterator_type end() const noexcept
     {
         return const_iterator_type(static_cast<node_type*>(tail));
     }
 
-    const_iterator_type cbegin() const noexcept
+    constexpr const_iterator_type cbegin() const noexcept
     {
         return begin();
     }
 
-    const_iterator_type cend() const noexcept
+    constexpr const_iterator_type cend() const noexcept
     {
         return end();
     }
 
-    bool empty(void) const noexcept
+    constexpr bool empty(void) const noexcept
     {
         return size() == 0;
     }

include/types/lock.h

@@ -1,35 +0,0 @@
-#pragma once
-
-#include <types/stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-static inline void spin_lock(uint32_t volatile* lock_addr)
-{
-    asm volatile(
-            "_spin:\n\t\
-             movl $1, %%eax\n\t\
-             xchgl %%eax, (%0)\n\t\
-             test $0, %%eax\n\t\
-             jne _spin\n\t\
-            "
-            : "=r" (lock_addr)
-            : "0"  (lock_addr)
-            : "eax", "memory"
-            );
-}
-
-static inline void spin_unlock(uint32_t volatile* lock_addr)
-{
-    asm volatile("movl $0, %%eax\nxchgl %%eax, (%0)"
-                 :
-                 : "r"  (lock_addr)
-                 : "eax", "memory"
-                 );
-}
-
-#ifdef __cplusplus
-}
-#endif

include/types/lock.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <types/stdint.h>
+
+inline void spin_lock(uint32_t volatile* lock_addr)
+{
+    asm volatile(
+        "0:\n\t\
+         movl $1, %%eax\n\t\
+         xchgl %%eax, (%0)\n\t\
+         test $0, %%eax\n\t\
+         jne 0\n\t\
+        "
+        :
+        : "r"(lock_addr)
+        : "eax", "memory");
+}
+
+inline void spin_unlock(uint32_t volatile* lock_addr)
+{
+    asm volatile(
+        "movl $0, %%eax\n\
+         xchgl %%eax, (%0)"
+        :
+        : "r"(lock_addr)
+        : "eax", "memory");
+}

include/types/size.h

@@ -18,8 +18,5 @@ typedef uint64_t ptr_t;
 typedef int64_t diff_t;
 #endif
 
-typedef ptr_t phys_ptr_t;
-typedef ptr_t linr_ptr_t;
+typedef ptr_t pptr_t;
 typedef size_t page_t;
-typedef size_t pd_i_t;
-typedef size_t pt_i_t;

include/types/types.h

@@ -18,6 +18,14 @@
 #error "no definition for ((SECTION))"
 #endif
 
+#ifdef __GNUC__
+#define likely(expr) (__builtin_expect(!!(expr), 1))
+#define unlikely(expr) (__builtin_expect(!!(expr), 0))
+#else
+#define likely(expr) (!!(expr))
+#define unlikely(expr) (!!(expr))
+#endif
+
 #ifdef __cplusplus
 #include <types/allocator.hpp>
 #include <types/cplusplus.hpp>

include/types/vector.hpp

@@ -30,82 +30,82 @@ public:
         using Reference = typename types::traits::add_reference<Value>::type;
 
     public:
-        iterator(const iterator& iter) noexcept
+        constexpr iterator(const iterator& iter) noexcept
             : p(iter.p)
         {
         }
 
-        iterator(iterator&& iter) noexcept
+        constexpr iterator(iterator&& iter) noexcept
             : p(iter.p)
         {
             iter.p = nullptr;
         }
 
-        iterator& operator=(const iterator& iter)
+        constexpr iterator& operator=(const iterator& iter)
         {
             p = iter.p;
             return *this;
         }
 
-        explicit iterator(Pointer p) noexcept
+        explicit constexpr iterator(Pointer p) noexcept
             : p(p)
         {
         }
 
-        bool operator==(const iterator& iter) const noexcept
+        constexpr bool operator==(const iterator& iter) const noexcept
         {
             return this->p == iter.p;
         }
 
-        bool operator!=(const iterator& iter) const noexcept
+        constexpr bool operator!=(const iterator& iter) const noexcept
         {
             return !(*this == iter);
         }
 
-        iterator& operator++() noexcept
+        constexpr iterator& operator++() noexcept
         {
             ++p;
             return *this;
         }
 
-        iterator operator++(int) noexcept
+        constexpr iterator operator++(int) noexcept
         {
             iterator iter(*this);
             ++p;
             return iter;
         }
 
-        iterator& operator--() noexcept
+        constexpr iterator& operator--() noexcept
         {
             --p;
             return *this;
         }
 
-        iterator operator--(int) noexcept
+        constexpr iterator operator--(int) noexcept
         {
             iterator iter(*this);
             --p;
             return iter;
         }
 
-        iterator operator+(size_type n) noexcept
+        constexpr iterator operator+(size_type n) noexcept
         {
             iterator iter(p + n);
             return iter;
         }
 
-        iterator operator-(size_type n) noexcept
+        constexpr iterator operator-(size_type n) noexcept
         {
             iterator iter(p - n);
             return iter;
         }
 
-        Reference operator*() const noexcept
+        constexpr Reference operator*() const noexcept
         {
             return *p;
         }
 
-        Pointer operator->() const noexcept
+        constexpr Pointer operator->() const noexcept
         {
             return p;
         }
@@ -115,21 +115,21 @@ public:
     };
 
 public:
-    explicit vector(size_type capacity = 1) noexcept
+    explicit constexpr vector(size_type capacity = 1) noexcept
         : m_arr(nullptr)
         , m_size(0)
     {
         resize(capacity);
     }
 
-    vector(const vector& arr) noexcept
+    constexpr vector(const vector& arr) noexcept
         : vector(arr.capacity())
     {
         for (const auto& item : arr)
             push_back(item);
     }
 
-    vector(vector&& arr) noexcept
+    constexpr vector(vector&& arr) noexcept
     {
         m_arr = arr.m_arr;
         m_capacity = arr.m_capacity;
@@ -140,7 +140,7 @@ public:
         arr.m_size = 0;
     }
 
-    vector& operator=(vector&& arr)
+    constexpr vector& operator=(vector&& arr)
     {
         resize(0);
         m_arr = arr.m_arr;
@@ -154,7 +154,7 @@ public:
         return *this;
     }
 
-    vector& operator=(const vector& arr)
+    constexpr vector& operator=(const vector& arr)
     {
         return operator=(vector(arr));
     }
@@ -164,7 +164,7 @@ public:
         resize(0);
     }
 
-    void resize(size_type n)
+    constexpr void resize(size_type n)
     {
         value_type* new_ptr = allocator_traits<allocator_type>::allocate(n);
 
@@ -213,39 +213,39 @@ public:
     //     ++_size();
     // }
 
-    value_type* data(void) noexcept
+    constexpr value_type* data(void) noexcept
     {
         return m_arr;
     }
 
-    const value_type* data(void) const noexcept
+    constexpr const value_type* data(void) const noexcept
     {
         return m_arr;
     }
 
-    value_type& at(index_type i) noexcept
+    constexpr value_type& at(index_type i) noexcept
     {
         // TODO: boundary check
         return _at(i);
     }
 
-    const value_type& at(index_type i) const noexcept
+    constexpr const value_type& at(index_type i) const noexcept
     {
         // TODO: boundary check
         return _at(i);
     }
 
-    value_type& operator[](index_type i) noexcept
+    constexpr value_type& operator[](index_type i) noexcept
     {
         return at(i);
     }
 
-    const value_type& operator[](index_type i) const noexcept
+    constexpr const value_type& operator[](index_type i) const noexcept
     {
         return at(i);
     }
 
-    void push_back(const value_type& v) noexcept
+    constexpr void push_back(const value_type& v) noexcept
     {
         if (m_size == m_capacity)
             resize(m_capacity * 2);
@@ -253,7 +253,7 @@ public:
         ++m_size;
     }
 
-    void push_back(value_type&& v) noexcept
+    constexpr void push_back(value_type&& v) noexcept
     {
         if (m_size == m_capacity)
             resize(m_capacity * 2);
@@ -262,74 +262,74 @@ public:
     }
 
     template <typename... Args>
-    iterator_type emplace_back(Args&&... args)
+    constexpr iterator_type emplace_back(Args&&... args)
     {
         push_back(value_type(forward<Args>(args)...));
         return back();
     }
 
-    void pop_back(void) noexcept
+    constexpr void pop_back(void) noexcept
     {
         allocator_traits<allocator_type>::deconstruct(&*back());
         --m_size;
     }
 
-    size_type size(void) const noexcept
+    constexpr size_type size(void) const noexcept
     {
         return m_size;
     }
 
-    size_type capacity(void) const noexcept
+    constexpr size_type capacity(void) const noexcept
     {
         return m_capacity;
     }
 
-    const_iterator_type cbegin() const noexcept
+    constexpr const_iterator_type cbegin() const noexcept
     {
         return const_iterator_type(m_arr);
     }
 
-    const_iterator_type cend() const noexcept
+    constexpr const_iterator_type cend() const noexcept
     {
         return const_iterator_type(m_arr + m_size);
     }
 
-    iterator_type begin() noexcept
+    constexpr iterator_type begin() noexcept
     {
         return iterator_type(m_arr);
     }
 
-    const_iterator_type begin() const noexcept
+    constexpr const_iterator_type begin() const noexcept
     {
         return cbegin();
     }
 
-    iterator_type end() noexcept
+    constexpr iterator_type end() noexcept
     {
         return iterator_type(m_arr + m_size);
     }
 
-    const_iterator_type end() const noexcept
+    constexpr const_iterator_type end() const noexcept
     {
         return cend();
     }
 
-    iterator_type back() noexcept
+    constexpr iterator_type back() noexcept
     {
         return iterator_type(m_arr + m_size - 1);
     }
 
-    const_iterator_type back() const noexcept
+    constexpr const_iterator_type back() const noexcept
     {
         return const_iterator_type(m_arr + m_size - 1);
     }
 
-    bool empty(void) const noexcept
+    constexpr bool empty(void) const noexcept
     {
         return size() == 0;
     }
 
-    void clear(void)
+    constexpr void clear(void)
     {
         for (size_t i = 0; i < size(); ++i)
             allocator_traits<allocator_type>::deconstruct(m_arr + i);
@@ -345,11 +345,11 @@ public:
     // iterator_type cr_end() noexcept;
 
 protected:
-    inline const value_type& _at(index_type i) const
+    inline constexpr const value_type& _at(index_type i) const
     {
         return m_arr[i];
     }
-    inline value_type& _at(index_type i)
+    inline constexpr value_type& _at(index_type i)
     {
         return m_arr[i];
     }

src/boot.s

@@ -111,37 +111,37 @@ setup_early_kernel_page_table:
 # set up early kernel page table
 
 # the early kernel page directory is located at physical
-# address 0x00000000, size 4k, and the empty page is at
-# 0x5000-0x5fff, so we fill the first 6KiB
+# address 0x00001000, size 4k, and the empty page is at
+# 0x0000-0x0fff, so we fill the first 6KiB
     movl $0x00000000, %eax
     movl $0x6000, %ecx
     call _fill_zero
 
 # map the first 16MiB identically
-# 0x0000-0x0fff: early kernel pd
-# 0x1000-0x4fff: pde 0 - 4
-    movl $0x00000000, %eax
-    movl $0x00001003, %ebx
+# 0x1000-0x1fff: early kernel pd
+# 0x2000-0x5fff: pde 0 - 4
+    movl $0x00001000, %eax
+    movl $0x00002003, %ebx
 _fill_pde_loop:
     movl %ebx, (%eax)
     addl $4, %eax
     addl $0x1000, %ebx
-    cmpl $0x5003, %ebx
+    cmpl $0x6003, %ebx
     jne _fill_pde_loop
 
 # then, create page tables
     movl $0x00000003, %eax
-    movl $0x00001000, %ecx
+    movl $0x00002000, %ecx
 
 _create_page_table_loop1:
     movl %eax, (%ecx)
     addl $4, %ecx
     addl $0x1000, %eax
-    cmpl $0x4ffc, %ecx
+    cmpl $0x5ffc, %ecx
     jle _create_page_table_loop1
 
 load_early_kernel_page_table:
-    movl $0x00000000, %eax
+    movl $0x00001000, %eax
     movl %eax, %cr3
 
     movl %cr0, %eax

src/kernel/hw/ata.cpp

@@ -145,7 +145,7 @@ size_t _ata_read(fs::special_node* sn, char* buf, size_t buf_size, size_t offset
         end = sn->data1 + sn->data2;
     offset %= 512;
     for (size_t i = start; i < end; ++i) {
-        (*ata_bus)->read_sector(b, i, 0);
+        (void)(*ata_bus)->read_sector(b, i, 0);
         size_t to_copy = 0;
         if (offset)
             to_copy = 512 - offset;

src/kernel/interrupt.cpp

@@ -16,6 +16,7 @@
 #include <kernel_main.h>
 #include <types/size.h>
 #include <types/stdint.h>
+#include <types/types.h>
 
 static struct IDT_entry IDT[256];
 
@@ -141,7 +142,7 @@ extern "C" void int13_handler(
 }
 
 struct PACKED int14_data {
-    linr_ptr_t l_addr;
+    void* l_addr;
     struct regs_32 s_regs;
     struct page_fault_error_code error_code;
     void* v_eip;
@@ -149,7 +150,7 @@ struct PACKED int14_data {
     uint32_t eflags;
 };
 
-static inline void _int14_panic(void* eip, linr_ptr_t cr2, struct page_fault_error_code error_code)
+static inline void _int14_panic(void* eip, void* cr2, struct page_fault_error_code error_code)
 {
     char buf[256] {};
     snprintf(
@@ -171,24 +172,22 @@ extern "C" void int14_handler(int14_data* d)
         mms = kernel_mms;
 
     mm* mm_area = find_mm_area(mms, d->l_addr);
-    if (!mm_area)
+    if (unlikely(!mm_area))
         _int14_panic(d->v_eip, d->l_addr, d->error_code);
 
-    page_directory_entry* pde = mms_get_pd(mms) + linr_addr_to_pd_i(d->l_addr);
-    page_table_entry* pte = (page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pde->in.pt_page));
-    pte += linr_addr_to_pt_i(d->l_addr);
-    struct page* page = find_page_by_l_ptr(mms, d->l_addr);
+    pte_t* pte = to_pte(mms_get_pd(mms), d->l_addr);
+    page* page = lto_page(mm_area, d->l_addr);
 
-    if (d->error_code.present == 0 && !mm_area->mapped_file)
+    if (unlikely(d->error_code.present == 0 && !mm_area->mapped_file))
         _int14_panic(d->v_eip, d->l_addr, d->error_code);
 
     // copy on write
-    if (page->attr.cow == 1) {
+    if (page->attr.in.cow == 1) {
         // if it is a dying page
         if (*page->ref_count == 1) {
-            page->attr.cow = 0;
+            page->attr.in.cow = 0;
             pte->in.a = 0;
-            pte->in.rw = mm_area->attr.write;
+            pte->in.rw = mm_area->attr.in.write;
             return;
         }
         // duplicate the page
@@ -198,23 +197,23 @@ extern "C" void int14_handler(int14_data* d)
         if (d->error_code.present == 0)
             pte->in.p = 1;
 
-        char* new_page_data = (char*)p_ptr_to_v_ptr(page_to_phys_addr(new_page));
-        memcpy(new_page_data, p_ptr_to_v_ptr(page_to_phys_addr(page->phys_page_id)), PAGE_SIZE);
+        char* new_page_data = (char*)to_vp(new_page);
+        memcpy(new_page_data, to_vp(page->phys_page_id), PAGE_SIZE);
 
         pte->in.page = new_page;
-        pte->in.rw = mm_area->attr.write;
+        pte->in.rw = mm_area->attr.in.write;
         pte->in.a = 0;
 
         --*page->ref_count;
 
-        page->ref_count = (size_t*)k_malloc(sizeof(size_t));
+        page->ref_count = (size_t*)ki_malloc(sizeof(size_t));
         *page->ref_count = 1;
-        page->attr.cow = 0;
+        page->attr.in.cow = 0;
         page->phys_page_id = new_page;
 
         // memory mapped
         if (d->error_code.present == 0) {
-            size_t offset = (d->l_addr - mm_area->start) & 0xfffff000;
+            size_t offset = vptrdiff(d->l_addr, mm_area->start) & 0xfffff000;
             vfs_read(mm_area->mapped_file, new_page_data, PAGE_SIZE, mm_area->file_offset + offset, PAGE_SIZE);
         }
     }

src/kernel/mem.cpp

@@ -10,6 +10,7 @@
 #include <kernel/vga.h>
 #include <kernel_main.h>
 #include <types/bitmap.h>
+#include <types/size.h>
 #include <types/status.h>
 
 // global objects
@@ -20,8 +21,8 @@ mm_list* kernel_mms;
 
 // constant values
 
-#define EMPTY_PAGE_ADDR ((phys_ptr_t)0x5000)
-#define EMPTY_PAGE_END ((phys_ptr_t)0x6000)
+#define EMPTY_PAGE_ADDR ((pptr_t)0x0000)
+#define EMPTY_PAGE_END ((pptr_t)0x1000)
 
 #define IDENTICALLY_MAPPED_HEAP_SIZE ((size_t)0x400000)
 
@@ -59,9 +60,9 @@ private:
     brk_memory_allocator(const brk_memory_allocator&) = delete;
     brk_memory_allocator(brk_memory_allocator&&) = delete;
 
-    inline int brk(byte* addr)
+    inline constexpr int brk(byte* addr)
     {
-        if (addr >= p_limit)
+        if (unlikely(addr >= p_limit))
             return GB_FAILED;
         p_break = addr;
         return GB_OK;
@@ -70,7 +71,7 @@ private:
     // sets errno
     inline byte* sbrk(size_type increment)
     {
-        if (brk(p_break + increment) != GB_OK) {
+        if (unlikely(brk(p_break + increment) != GB_OK)) {
             errno = ENOMEM;
             return nullptr;
         } else {
@@ -99,7 +100,7 @@ private:
                 errno = 0;
                 return start_pos;
             } else {
-                if (!start_pos->flags.has_next) {
+                if (unlikely(!start_pos->flags.has_next)) {
                     errno = ENOTFOUND;
                     return start_pos;
                 }
@@ -113,7 +114,7 @@ private:
     {
         sbrk(sizeof(mem_blk) + size - 4 * sizeof(byte));
         // preserves errno
-        if (errno) {
+        if (unlikely(errno)) {
             return nullptr;
         }
 
@@ -200,7 +201,10 @@ static brk_memory_allocator
 
 void* k_malloc(size_t size)
 {
-    return kernel_heap_allocator->alloc(size);
+    void* ptr = kernel_heap_allocator->alloc(size);
+    if unlikely (!ptr)
+        MAKE_BREAK_POINT();
+    return ptr;
 }
 
 void k_free(void* ptr)
@@ -211,9 +215,8 @@ void k_free(void* ptr)
 void* ki_malloc(size_t size)
 {
     void* ptr = kernel_ident_mapped_allocator.alloc(size);
-    if (!ptr) {
+    if (unlikely(!ptr))
         MAKE_BREAK_POINT();
-    }
     return ptr;
 }
 
@@ -222,7 +225,7 @@ void ki_free(void* ptr)
     kernel_ident_mapped_allocator.free(ptr);
 }
 
-void* p_ptr_to_v_ptr(phys_ptr_t p_ptr)
+void* ptovp(pptr_t p_ptr)
 {
     if (p_ptr <= 0x30000000) {
         // memory below 768MiB is identically mapped
@@ -234,73 +237,46 @@ void* p_ptr_to_v_ptr(phys_ptr_t p_ptr)
     }
 }
 
-phys_ptr_t l_ptr_to_p_ptr(const mm_list* mms, linr_ptr_t v_ptr)
-{
-    for (const mm& item : *mms) {
-        if (v_ptr < item.start || v_ptr >= item.start + item.pgs->size() * PAGE_SIZE)
-            continue;
-        size_t offset = (size_t)(v_ptr - item.start);
-        const page& p = item.pgs->at(offset / PAGE_SIZE);
-        return page_to_phys_addr(p.phys_page_id) + (offset % PAGE_SIZE);
-    }
-
-    // TODO: handle error
-    return 0xffffffff;
-}
-
-phys_ptr_t v_ptr_to_p_ptr(const void* v_ptr)
-{
-    if (v_ptr < KERNEL_IDENTICALLY_MAPPED_AREA_LIMIT) {
-        return (phys_ptr_t)v_ptr;
-    }
-    return l_ptr_to_p_ptr(kernel_mms, (linr_ptr_t)v_ptr);
-}
-
-static inline void mark_page(page_t n)
+inline void mark_page(page_t n)
 {
     bm_set(mem_bitmap, n);
 }
 
-static inline void free_page(page_t n)
+inline void free_page(page_t n)
 {
     bm_clear(mem_bitmap, n);
 }
 
-static void mark_addr_len(phys_ptr_t start, size_t n)
+constexpr void mark_addr_len(pptr_t start, size_t n)
 {
-    if (n == 0)
+    if (unlikely(n == 0))
         return;
-    page_t start_page = phys_addr_to_page(start);
-    page_t end_page = phys_addr_to_page(start + n + 4095);
+    page_t start_page = to_page(start);
+    page_t end_page = to_page(start + n + 4095);
     for (page_t i = start_page; i < end_page; ++i)
         mark_page(i);
 }
 
-static void free_addr_len(phys_ptr_t start, size_t n)
+constexpr void free_addr_len(pptr_t start, size_t n)
 {
-    if (n == 0)
+    if (unlikely(n == 0))
         return;
-    page_t start_page = phys_addr_to_page(start);
-    page_t end_page = phys_addr_to_page(start + n + 4095);
+    page_t start_page = to_page(start);
+    page_t end_page = to_page(start + n + 4095);
     for (page_t i = start_page; i < end_page; ++i)
         free_page(i);
 }
 
-static inline void mark_addr_range(phys_ptr_t start, phys_ptr_t end)
+inline constexpr void mark_addr_range(pptr_t start, pptr_t end)
 {
     mark_addr_len(start, end - start);
 }
 
-static inline void free_addr_range(phys_ptr_t start, phys_ptr_t end)
+inline constexpr void free_addr_range(pptr_t start, pptr_t end)
 {
     free_addr_len(start, end - start);
 }
 
-page_t alloc_raw_page(void)
-{
-    return alloc_n_raw_pages(1);
-}
-
 // @return the max count (but less than n) of the pages continuously available
 static inline size_t _test_n_raw_pages(page_t start, size_t n)
 {
@@ -334,40 +310,55 @@ struct page allocate_page(void)
         .phys_page_id = alloc_raw_page(),
         .pte = nullptr,
         .ref_count = types::kernel_ident_allocator_new<size_t>(0),
-        .attr {},
+        .attr { 0 },
     };
 }
 
-static inline void make_page_table(page_table_entry* pt)
-{
-    memset(pt, 0x00, sizeof(page_table_entry) * 1024);
-}
-
-page_directory_entry* alloc_pd(void)
+pd_t alloc_pd(void)
 {
     // TODO: alloc page in low mem and gen struct page for it
     page_t pd_page = alloc_raw_page();
-    page_directory_entry* pd = (page_directory_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pd_page));
+    pd_t pd = to_pd(pd_page);
     memset(pd, 0x00, PAGE_SIZE);
     return pd;
 }
 
-page_table_entry* alloc_pt(void)
+pt_t alloc_pt(void)
 {
     // TODO: alloc page in low mem and gen struct page for it
     page_t pt_page = alloc_raw_page();
-    page_table_entry* pt = (page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pt_page));
-    make_page_table(pt);
+    pt_t pt = to_pt(pt_page);
+    memset(pt, 0x00, PAGE_SIZE);
     return pt;
 }
 
+void dealloc_pd(pd_t pd)
+{
+    for (pde_t* ent = (*pd) + 256; ent < (*pd) + 1024; ++ent) {
+        if (!ent->in.p)
+            continue;
+        dealloc_pt(to_pt(ent));
+    }
+    memset(pd, 0x00, sizeof(*pd));
+
+    page_t pg = to_page((pptr_t)pd);
+    free_page(pg);
+}
+void dealloc_pt(pt_t pt)
+{
+    memset(pt, 0x00, sizeof(*pt));
+
+    page_t pg = to_page((pptr_t)pt);
+    free_page(pg);
+}
+
 static inline void init_mem_layout(void)
 {
     mem_size = 1024 * mem_size_info.n_1k_blks;
     mem_size += 64 * 1024 * mem_size_info.n_64k_blks;
 
     // mark kernel page directory
-    mark_addr_range(0x00000000, 0x00005000);
+    mark_addr_range(0x00001000, 0x00006000);
     // mark empty page
     mark_addr_range(EMPTY_PAGE_ADDR, EMPTY_PAGE_END);
     // mark EBDA and upper memory as allocated
@@ -394,29 +385,16 @@ static inline void init_mem_layout(void)
     }
 }
 
-mm* find_mm_area(mm_list* mms, linr_ptr_t l_ptr)
+mm* find_mm_area(mm_list* mms, void* l_ptr)
 {
     for (auto iter = mms->begin(); iter != mms->end(); ++iter)
-        if (l_ptr >= iter->start && l_ptr < iter->start + iter->pgs->size() * PAGE_SIZE)
+        if (l_ptr >= iter->start && l_ptr < mmend(iter.ptr()))
             return iter.ptr();
     return nullptr;
 }
 
-struct page* find_page_by_l_ptr(const mm_list* mms, linr_ptr_t l_ptr)
-{
-    for (const mm& item : *mms) {
-        if (l_ptr >= item.start && l_ptr < item.start + item.pgs->size() * PAGE_SIZE) {
-            size_t offset = (size_t)(l_ptr - item.start);
-            return &item.pgs->at(offset / PAGE_SIZE);
-        }
-    }
-
-    // TODO: error handling
-    return nullptr;
-}
-
 static inline void map_raw_page_to_pte(
-    page_table_entry* pte,
+    pte_t* pte,
     page_t page,
     int present,
     int rw,
@@ -439,27 +417,27 @@ int k_map(
     int priv,
     int cow)
 {
-    linr_ptr_t addr = (linr_ptr_t)mm_area->start + mm_area->pgs->size() * PAGE_SIZE;
-    page_directory_entry* pde = mm_area->pd + linr_addr_to_pd_i(addr);
+    void* addr = mmend(mm_area);
+    pde_t* pde = to_pde(mm_area->pd, addr);
     // page table not exist
-    if (!pde->in.p) {
+    if (unlikely(!pde->in.p)) {
         // allocate a page for the page table
         pde->in.p = 1;
         pde->in.rw = 1;
         pde->in.us = (priv == 0);
         pde->in.pt_page = alloc_raw_page();
 
-        make_page_table((page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pde->in.pt_page)));
+        memset(to_pt(pde), 0x00, PAGE_SIZE);
     }
 
     // map the page in the page table
-    page_table_entry* pte = (page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pde->in.pt_page));
-    pte += linr_addr_to_pt_i(addr);
+    pte_t* pte = to_pte(pde, addr);
     map_raw_page_to_pte(pte, page->phys_page_id, read, (write && !cow), priv);
 
-    if (cow && !page->attr.cow) {
-        page->attr.cow = 1;
+    if (unlikely(cow && !page->attr.in.cow)) {
+        page->attr.in.cow = 1;
         page->pte->in.rw = 0;
+        invalidate_tlb(addr);
     }
     ++*page->ref_count;
 
@@ -468,10 +446,29 @@ int k_map(
     return GB_OK;
 }
 
+int k_unmap(mm* mm_area)
+{
+    for (auto iter = mm_area->pgs->begin(); iter != mm_area->pgs->end(); ++iter) {
+        if (*iter->ref_count == 1) {
+            ki_free(iter->ref_count);
+            free_page(iter->phys_page_id);
+        } else {
+            --*iter->ref_count;
+        }
+
+        iter->phys_page_id = 0;
+        iter->attr.v = 0;
+        iter->pte->v = 0;
+    }
+    mm_area->attr.v = 0;
+    mm_area->start = 0;
+    return GB_OK;
+}
+
 bool check_addr_range_avail(const mm* mm_area, void* start, void* end)
 {
-    void* m_start = (void*)mm_area->start;
-    void* m_end = (void*)(mm_area->start + PAGE_SIZE * mm_area->pgs->size());
+    void* m_start = mm_area->start;
+    void* m_end = mmend(mm_area);
 
     if (start >= m_end || end <= m_start)
         return true;
@@ -488,7 +485,7 @@ static inline int _mmap(
     int write,
     int priv)
 {
-    if (!file->flags.in.file && !file->flags.in.special_node) {
+    if (unlikely(!file->flags.in.file && !file->flags.in.special_node)) {
         errno = EINVAL;
         return GB_FAILED;
     }
@@ -502,7 +499,15 @@ static inline int _mmap(
             return GB_FAILED;
         }
 
-    auto iter_mm = mms->emplace_back((linr_ptr_t)hint, mms_get_pd(&current_process->mms), write, priv);
+    auto iter_mm = mms->emplace_back(mm {
+        .start = hint,
+        .attr { .in {
+            .read = 1,
+            .write = static_cast<uint32_t>(write),
+            .system = static_cast<uint32_t>(priv),
+        } },
+        .pd = mms_get_pd(&current_process->mms),
+    });
     iter_mm->mapped_file = file;
     iter_mm->file_offset = offset;
 
@@ -529,9 +534,9 @@ int mmap(
 // to avoid dead loops
 static inline void _init_map_page_identically(page_t page)
 {
-    page_directory_entry* pde = KERNEL_PAGE_DIRECTORY_ADDR + page_to_pd_i(page);
+    pde_t* pde = *KERNEL_PAGE_DIRECTORY_ADDR + to_pdi(page);
     // page table not exist
-    if (!pde->in.p) {
+    if (unlikely(!pde->in.p)) {
         // allocate a page for the page table
         // set the P bit of the pde in advance
         pde->in.p = 1;
@@ -539,24 +544,22 @@ static inline void _init_map_page_identically(page_t page)
         pde->in.us = 0;
         pde->in.pt_page = alloc_raw_page();
         _init_map_page_identically(pde->in.pt_page);
-
-        make_page_table((page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pde->in.pt_page)));
+        memset(to_pt(pde), 0x00, PAGE_SIZE);
     }
 
     // map the page in the page table
-    page_table_entry* pt = (page_table_entry*)p_ptr_to_v_ptr(page_to_phys_addr(pde->in.pt_page));
-    pt += page_to_pt_i(page);
+    pte_t* pt = to_pte(pde, page);
     pt->v = 0x00000003;
     pt->in.page = page;
 }
 
 static inline void init_paging_map_low_mem_identically(void)
 {
-    for (phys_ptr_t addr = 0x01000000; addr < 0x30000000; addr += 0x1000) {
+    for (pptr_t addr = 0x01000000; addr < 0x30000000; addr += 0x1000) {
         // check if the address is valid and not mapped
-        if (bm_test(mem_bitmap, phys_addr_to_page(addr)))
+        if (bm_test(mem_bitmap, to_page(addr)))
             continue;
-        _init_map_page_identically(phys_addr_to_page(addr));
+        _init_map_page_identically(to_page(addr));
     }
 }
 
@@ -570,29 +573,35 @@ void init_mem(void)
     init_paging_map_low_mem_identically();
 
     kernel_mms = types::kernel_ident_allocator_new<mm_list>();
-    auto heap_mm = kernel_mms->emplace_back((linr_ptr_t)KERNEL_HEAP_START, KERNEL_PAGE_DIRECTORY_ADDR, 1, 1);
+    auto heap_mm = kernel_mms->emplace_back(mm {
+        .start = KERNEL_HEAP_START,
+        .attr { .in {
+            .read = 1,
+            .write = 1,
+            .system = 1,
+        } },
+        .pd = KERNEL_PAGE_DIRECTORY_ADDR,
+    });
 
     page heap_first_page {
         .phys_page_id = alloc_raw_page(),
         .pte = nullptr,
         .ref_count = types::kernel_ident_allocator_new<size_t>(0),
-        .attr = {
-            .cow = 0,
-        },
+        .attr = { 0 },
     };
 
     k_map(heap_mm.ptr(), &heap_first_page, 1, 1, 1, 0);
     memset(KERNEL_HEAP_START, 0x00, PAGE_SIZE);
     kernel_heap_allocator = types::kernel_ident_allocator_new<brk_memory_allocator>(KERNEL_HEAP_START,
-        (uint32_t)KERNEL_HEAP_LIMIT - (uint32_t)KERNEL_HEAP_START);
+        vptrdiff(KERNEL_HEAP_LIMIT, KERNEL_HEAP_START));
 
     // create empty_page struct
-    empty_page.attr.cow = 0;
-    empty_page.phys_page_id = phys_addr_to_page(EMPTY_PAGE_ADDR);
+    empty_page.attr.in.cow = 0;
+    empty_page.phys_page_id = to_page(EMPTY_PAGE_ADDR);
     empty_page.ref_count = types::kernel_ident_allocator_new<size_t>(1);
-    empty_page.pte = ((page_table_entry*)p_ptr_to_v_ptr(
-                         page_to_phys_addr(KERNEL_PAGE_DIRECTORY_ADDR->in.pt_page)))
-        + page_to_pt_i(empty_page.phys_page_id);
+    empty_page.pte = to_pte(*KERNEL_PAGE_DIRECTORY_ADDR, empty_page.phys_page_id);
+    empty_page.pte->in.rw = 0;
+    invalidate_tlb(0x00000000);
 
     // TODO: improve the algorithm SO FREAKING SLOW
     // while (kernel_mm_head->len < 256 * 1024 * 1024 / PAGE_SIZE) {
@@ -618,31 +627,3 @@ void create_segment_descriptor(
     sd->access = access;
     sd->flags = flags;
 }
-
-mm::mm(linr_ptr_t start, page_directory_entry* pd, bool write, bool system)
-    : start(start)
-    , attr({
-          .read { 1 },
-          .write { write },
-          .system { system },
-      })
-    , pd(pd)
-    , pgs(types::kernel_ident_allocator_new<page_arr>())
-    , mapped_file(nullptr)
-    , file_offset(0)
-{
-}
-
-mm::mm(const mm& val)
-    : start(val.start)
-    , attr({
-          .read { val.attr.read },
-          .write { val.attr.write },
-          .system { val.attr.system },
-      })
-    , pd(val.pd)
-    , pgs(val.pgs)
-    , mapped_file(nullptr)
-    , file_offset(0)
-{
-}

src/kernel/process.cpp

@@ -57,34 +57,12 @@ process::process(const process& val, const thread& main_thd)
     auto iter_thd = thds.emplace_back(main_thd);
     iter_thd->owner = this;
 
-    if (!val.attr.system) {
-        // TODO: allocate low mem
-        k_esp = (void*)page_to_phys_addr(alloc_n_raw_pages(2));
-        memset((char*)k_esp, 0x00, THREAD_KERNEL_STACK_SIZE);
-        k_esp = (char*)k_esp + THREAD_KERNEL_STACK_SIZE;
-
-        page_directory_entry* pd = alloc_pd();
-        memcpy(pd, mms_get_pd(kernel_mms), PAGE_SIZE);
-
-        mms.begin()->pd = pd;
-        // skip kernel heap
-        for (auto iter_src = ++val.mms.cbegin(); iter_src != val.mms.cend(); ++iter_src) {
-            auto iter_dst = mms.emplace_back(iter_src->start, pd, iter_src->attr.write, iter_src->attr.system);
-            iter_dst->pd = pd;
-            for (auto pg = iter_src->pgs->begin(); pg != iter_src->pgs->end(); ++pg)
-                k_map(iter_dst.ptr(),
-                    &*pg,
-                    iter_src->attr.read,
-                    iter_src->attr.write,
-                    iter_src->attr.system,
-                    1);
-        }
-    } else {
-        // TODO: allocate low mem
-        k_esp = (void*)page_to_phys_addr(alloc_n_raw_pages(2));
-        memcpy(k_esp, main_thd.owner->k_esp, THREAD_KERNEL_STACK_SIZE);
-        k_esp = (char*)k_esp + THREAD_KERNEL_STACK_SIZE;
+    // TODO: allocate low mem
+    k_esp = (void*)to_pp(alloc_n_raw_pages(2));
+    memcpy(k_esp, (char*)main_thd.owner->k_esp - THREAD_KERNEL_STACK_SIZE, THREAD_KERNEL_STACK_SIZE);
+    k_esp = (char*)k_esp + THREAD_KERNEL_STACK_SIZE;
 
+    if (val.attr.system) {
         auto orig_k_esp = (uint32_t)main_thd.owner->k_esp;
 
         iter_thd->regs.ebp -= orig_k_esp;
@@ -92,6 +70,15 @@ process::process(const process& val, const thread& main_thd)
 
         iter_thd->regs.esp -= orig_k_esp;
         iter_thd->regs.esp += (uint32_t)k_esp;
+    } else {
+        pd_t pd = alloc_pd();
+        memcpy(pd, mms_get_pd(kernel_mms), PAGE_SIZE);
+
+        mms.begin()->pd = pd;
+
+        // skip kernel heap since it's already copied above
+        for (auto iter_src = ++val.mms.cbegin(); iter_src != val.mms.cend(); ++iter_src)
+            mm::mirror_mm_area(&mms, iter_src.ptr(), pd);
     }
 }
 
@@ -102,7 +89,7 @@ process::process(void* start_eip)
     , pid { max_pid++ }
 {
     // TODO: allocate low mem
-    k_esp = (void*)page_to_phys_addr(alloc_n_raw_pages(2));
+    k_esp = (void*)to_pp(alloc_n_raw_pages(2));
     memset((char*)k_esp, 0x00, THREAD_KERNEL_STACK_SIZE);
     k_esp = (char*)k_esp + THREAD_KERNEL_STACK_SIZE;
 
@@ -134,10 +121,10 @@ void NORETURN _kernel_init(void)
     // TODO: parse kernel parameters
     auto* _new_fs = fs::register_fs(types::kernel_allocator_new<fs::fat::fat32>(fs::vfs_open("/dev/hda1")->ind));
     int ret = fs::fs_root->ind->fs->mount(fs::vfs_open("/mnt"), _new_fs);
-    if (ret != GB_OK)
+    if (unlikely(ret != GB_OK))
         syscall(0x03);
 
-    page_directory_entry* new_pd = alloc_pd();
+    pd_t new_pd = alloc_pd();
     memcpy(new_pd, mms_get_pd(kernel_mms), PAGE_SIZE);
 
     asm_cli();
@@ -151,7 +138,7 @@ void NORETURN _kernel_init(void)
     types::elf::elf32_load("/mnt/INIT.ELF", &intrpt_stack, 0);
     // map stack area
     ret = mmap((void*)types::elf::ELF_STACK_TOP, types::elf::ELF_STACK_SIZE, fs::vfs_open("/dev/null")->ind, 0, 1, 0);
-    if (ret != GB_OK)
+    if (unlikely(ret != GB_OK))
         syscall(0x03);
 
     asm_cli();
@@ -177,10 +164,7 @@ void kernel_threadd_main(void)
             spin_unlock(&kthreadd_lock);
 
             // syscall_fork
-            asm volatile("movl $0x00, %%eax\nint $0x80\nmovl %%eax, %0"
-                         : "=a"(return_value)
-                         :
-                         :);
+            return_value = syscall(0x00);
 
             if (return_value != 0) {
                 // child
@@ -280,7 +264,7 @@ static inline void next_task(const types::list<thread*>::iterator_type& iter_to_
 
 void do_scheduling(interrupt_stack* intrpt_data)
 {
-    if (!is_scheduler_ready)
+    if (unlikely(!is_scheduler_ready))
         return;
 
     auto iter_thd = ready_thds->begin();

src/kernel_main.c

@@ -128,7 +128,7 @@ void load_new_gdt(void)
     create_segment_descriptor(new_gdt + 4, 0, ~0, 0b1100, SD_TYPE_DATA_USER);
     create_segment_descriptor(new_gdt + 5, (uint32_t)&tss, sizeof(tss), 0b0000, SD_TYPE_TSS);
 
-    asm_load_gdt((6 * 8 - 1) << 16, (phys_ptr_t)new_gdt);
+    asm_load_gdt((6 * 8 - 1) << 16, (pptr_t)new_gdt);
     asm_load_tr((6 - 1) * 8);
 
     asm_cli();

user-space-program/basic-lib.h

@@ -4,14 +4,14 @@ typedef __UINT8_TYPE__ uint8_t;
 
 static inline uint32_t syscall(uint32_t num, uint32_t arg1, uint32_t arg2)
 {
-    asm ("movl %3, %%eax\n\
-          movl %1, %%edi\n\
-          movl %2, %%esi\n\
-          int $0x80\n\
-          movl %%eax, %0"
-         :"=r"(num)
-         :"0"(arg1), "r"(arg2), "r"(num)
-         :"eax", "edi", "esi"
-         );
+    asm volatile(
+        "movl %1, %%edi\n"
+        "movl %2, %%esi\n"
+        "movl %3, %%eax\n"
+        "int $0x80\n"
+        "movl %%eax, %0"
+        : "=g"(num)
+        : "g"(arg1), "g"(arg2), "g"(num)
+        : "eax", "edx", "edi", "esi");
     return num;
 }