diff --git a/arch/x86-family/mm/address_space.c b/arch/x86-family/mm/address_space.c index a7e5bc1d..12148c84 100644 --- a/arch/x86-family/mm/address_space.c +++ b/arch/x86-family/mm/address_space.c @@ -40,7 +40,7 @@ -static x86_page_t __mm_copy_data(x86_page_t* __s, size_t* size, bool on_demand, int level) { +static x86_page_t __mm_copy_data(x86_page_t* __s, size_t* size, int level) { DEBUG_ASSERT(__s); DEBUG_ASSERT(size); @@ -71,22 +71,12 @@ static x86_page_t __mm_copy_data(x86_page_t* __s, size_t* size, bool on_demand, *size += pagesize; -#if defined(CONFIG_DEMAND_PAGING) - if (on_demand) { + uintptr_t page = __alloc_frame(pagesize, false); - return (*__s = (*__s & ~(X86_MMU_PG_RW | X86_MMU_PG_AP_TP_MASK)) | X86_MMU_PG_AP_TP_COW); + memcpy((void*)arch_vmm_p2v(page, ARCH_VMM_AREA_HEAP), (void*)arch_vmm_p2v(*__s & X86_MMU_ADDRESS_MASK, ARCH_VMM_AREA_HEAP), (size_t)pagesize); - } else -#endif - { - - uintptr_t page = __alloc_frame(pagesize, false); - - memcpy((void*)arch_vmm_p2v(page, ARCH_VMM_AREA_HEAP), (void*)arch_vmm_p2v(*__s & X86_MMU_ADDRESS_MASK, ARCH_VMM_AREA_HEAP), (size_t)pagesize); - - return page | X86_MMU_PG_AP_PFB | (*__s & ~X86_MMU_ADDRESS_MASK); - } + return page | X86_MMU_PG_AP_PFB | (*__s & ~X86_MMU_ADDRESS_MASK); } @@ -107,13 +97,13 @@ static void __mm_copy_page(x86_page_t* __s, x86_page_t* __d, size_t* size, int l if (flags & ARCH_VMM_CLONE_USERSPACE) { - if ((*__s & X86_MMU_PG_AP_TP_MASK) == X86_MMU_PG_AP_TP_COW) { + if (((*__s & X86_MMU_PG_AP_TP_MASK) == X86_MMU_PG_AP_TP_COW) && ((*__s & X86_MMU_ADDRESS_MASK) == 0)) { *__d = *__s; } else { - *__d = __mm_copy_data(__s, size, (flags & ARCH_VMM_CLONE_DEMAND), level); + *__d = __mm_copy_data(__s, size, level); } } } @@ -231,10 +221,9 @@ static void __mm_free_table(uintptr_t __s, int level) { __mm_free_table(((uintptr_t)s[i]) & X86_MMU_ADDRESS_MASK, level - 1); - // FIXME: not safe to free frame here, as it may be used by system page tables - // // if(s[i] & X86_MMU_PT_AP_PFB) { - // // __free_frame(((uintptr_t) s[i]) & X86_MMU_ADDRESS_MASK, X86_MMU_PAGESIZE); - // // } + if (s[i] & X86_MMU_PT_AP_PFB) { + __free_frame(((uintptr_t)s[i]) & X86_MMU_ADDRESS_MASK, X86_MMU_PAGESIZE); + } } } } @@ -253,6 +242,7 @@ __returns_nonnull vmm_address_space_t* arch_vmm_create_address_space(vmm_address dest->pm = __alloc_frame(X86_MMU_PAGESIZE, true); + dest->flags = ARCH_VMM_ADDRESS_SPACE_OWNED; @@ -305,7 +295,9 @@ void arch_vmm_free_address_space(vmm_address_space_t* space) { } - // TODO: free all mappings + if (!(space->flags & ARCH_VMM_ADDRESS_SPACE_OWNED)) { + return; + } #if defined(__x86_64__) scoped_lock(&space->lock) { @@ -320,11 +312,12 @@ void arch_vmm_free_address_space(vmm_address_space_t* space) { #endif - // FIXME: maybe unsafe - //__free_frame(space->pm, X86_MMU_PAGESIZE); + __free_frame(space->pm, X86_MMU_PAGESIZE); space->pm = 0UL; space->size = 0UL; space->mmap.heap_start = 0UL; space->mmap.heap_end = 0UL; + + kfree(space); } diff --git a/arch/x86-family/mm/getphysaddr.c b/arch/x86-family/mm/getphysaddr.c index dc44ca71..29c75e58 100644 --- a/arch/x86-family/mm/getphysaddr.c +++ b/arch/x86-family/mm/getphysaddr.c @@ -41,6 +41,8 @@ __nonnull(1) uintptr_t arch_vmm_getphysaddr(vmm_address_space_t* space, uintptr_t virtaddr) { +retry: + uintptr_t pagesize = X86_MMU_PAGESIZE; @@ -125,8 +127,11 @@ __nonnull(1) uintptr_t arch_vmm_getphysaddr(vmm_address_space_t* space, uintptr_ DEBUG_ASSERT((*d != X86_MMU_CLEAR) && "Page unmapped"); - if (unlikely((*d & X86_MMU_PG_AP_TP_MASK) != X86_MMU_PG_AP_TP_PAGE)) + if (unlikely((*d & X86_MMU_PG_AP_TP_MASK) != X86_MMU_PG_AP_TP_PAGE)) { + spinlock_unlock(&space->lock); PANIC_ASSERT(pagefault_handle(current_cpu->frame, virtaddr) == 0); + goto retry; + } DEBUG_ASSERT(((*d & X86_MMU_PG_AP_TP_MASK) == X86_MMU_PG_AP_TP_PAGE) && "Page bad type"); diff --git a/arch/x86-family/mm/pagefault.c b/arch/x86-family/mm/pagefault.c index f46686e1..6c5b3547 100644 --- a/arch/x86-family/mm/pagefault.c +++ b/arch/x86-family/mm/pagefault.c @@ -40,6 +40,8 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { + bool locked = false; + vmm_address_space_t* space = NULL; #if DEBUG_LEVEL_TRACE @@ -63,6 +65,12 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { if (unlikely(!pm)) PFE("no memory mapping", 0L); + if (current_task && current_task->address_space && current_task->address_space->pm == pm) { + space = current_task->address_space; + spinlock_lock(&space->lock); + locked = true; + } + uintptr_t pagesize = X86_MMU_PAGESIZE; uintptr_t s = cr2; @@ -120,7 +128,7 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { #elif defined(__i386__) /* CR3-L2 */ - { d = &((x86_page_t*)arch_vmm_p2v(space->pm, ARCH_VMM_AREA_HEAP))[(s >> 22) & 0x3FF]; } + { d = &((x86_page_t*)arch_vmm_p2v(pm, ARCH_VMM_AREA_HEAP))[(s >> 22) & 0x3FF]; } /* HUGE_4MB */ @@ -129,7 +137,7 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { /* PD-L1 */ { if (*d == X86_MMU_CLEAR) - PFE("PD-L1 doesn't not exist"); + PFE("PD-L1 doesn't not exist", *d); d = &((x86_page_t*)arch_vmm_p2v(*d & X86_MMU_ADDRESS_MASK, ARCH_VMM_AREA_HEAP))[(s >> 12) & 0x3FF]; } @@ -153,21 +161,30 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { //! Handle Copy on Write - uintptr_t page = __alloc_frame(pagesize, false); + uintptr_t old = *d; + uintptr_t page = __alloc_frame(pagesize, (old & X86_MMU_ADDRESS_MASK) == 0); - if ((*d & X86_MMU_ADDRESS_MASK) != 0) { + if ((old & X86_MMU_ADDRESS_MASK) != 0) { - memcpy((void*)arch_vmm_p2v(page, ARCH_VMM_AREA_HEAP), (void*)arch_vmm_p2v(*d & X86_MMU_ADDRESS_MASK, ARCH_VMM_AREA_HEAP), (size_t)pagesize); + memcpy((void*)arch_vmm_p2v(page, ARCH_VMM_AREA_HEAP), (void*)arch_vmm_p2v(old & X86_MMU_ADDRESS_MASK, ARCH_VMM_AREA_HEAP), (size_t)pagesize); page |= X86_MMU_PG_RW; } - *d = page | X86_MMU_PG_P | X86_MMU_PG_AP_PFB | X86_MMU_PG_AP_TP_PAGE | ((*d & ~X86_MMU_ADDRESS_MASK) & ~(X86_MMU_PG_AP_TP_MASK)); + *d = page | X86_MMU_PG_P | X86_MMU_PG_AP_PFB | X86_MMU_PG_AP_TP_PAGE | ((old & ~X86_MMU_ADDRESS_MASK) & ~(X86_MMU_PG_AP_TP_MASK)); } } + __asm__ __volatile__("invlpg (%0)" ::"r"(cr2) : "memory"); + + if (locked) { + spinlock_unlock(&space->lock); + } + - current_task->rusage.ru_majflt++; + if (current_task) { + current_task->rusage.ru_majflt++; + } #if DEBUG_LEVEL_TRACE @@ -180,6 +197,10 @@ __nonnull(1) int pagefault_handle(interrupt_frame_t* frame, uintptr_t cr2) { pfe: + if (locked) { + spinlock_unlock(&space->lock); + } + if (x86_intr_is_user_mode(frame)) return -1; diff --git a/arch/x86-family/mm/vm_access.c b/arch/x86-family/mm/vm_access.c index 89a58bbe..96bb37aa 100644 --- a/arch/x86-family/mm/vm_access.c +++ b/arch/x86-family/mm/vm_access.c @@ -130,6 +130,11 @@ __nonnull(1) int arch_vmm_access(vmm_address_space_t* space, uintptr_t virtaddr, /* Page Table */ { + check_or_fail(*d != X86_MMU_CLEAR); + + if (!(mode & S_OK) && current_cpu->frame && x86_intr_is_user_mode(current_cpu->frame)) { + check_or_fail(*d & X86_MMU_PG_U); + } if (mode & R_OK) { if (!(*d & X86_MMU_PG_P)) { diff --git a/arch/x86-family/mm/vm_lock.c b/arch/x86-family/mm/vm_lock.c index 4ef29053..16d5ae64 100644 --- a/arch/x86-family/mm/vm_lock.c +++ b/arch/x86-family/mm/vm_lock.c @@ -51,6 +51,6 @@ __nonnull(1) void arch_vmm_lock(vmm_address_space_t* space, uintptr_t virtaddr, #if defined(CONFIG_X86_ENABLE_SMAP) if (cpu_has(current_cpu->id, X86_FEATURE_SMAP)) - x86_set_cr4(x86_get_cr4() & ~(X86_CR4_SMAP_MASK)); + x86_stac(); #endif } diff --git a/arch/x86-family/mm/vm_map.c b/arch/x86-family/mm/vm_map.c index c588c155..7d1ce825 100644 --- a/arch/x86-family/mm/vm_map.c +++ b/arch/x86-family/mm/vm_map.c @@ -262,7 +262,7 @@ __nonnull(1) uintptr_t arch_vmm_map(vmm_address_space_t* space, uintptr_t virtad if (flags & ARCH_VMM_MAP_DEMAND) *d = X86_MMU_PG_AP_TP_COW | (b & ~X86_MMU_PG_P); else - *d = __alloc_frame(pagesize, false) | X86_MMU_PG_AP_PFB | b; + *d = __alloc_frame(pagesize, true) | X86_MMU_PG_AP_PFB | b; } } diff --git a/arch/x86-family/mm/vm_protect.c b/arch/x86-family/mm/vm_protect.c index d0c79ec5..d371d24c 100644 --- a/arch/x86-family/mm/vm_protect.c +++ b/arch/x86-family/mm/vm_protect.c @@ -81,23 +81,23 @@ __nonnull(1) uintptr_t arch_vmm_mprotect(vmm_address_space_t* space, uintptr_t v - uint64_t b = X86_MMU_PG_P; + uint64_t base = X86_MMU_PG_P; if (flags & ARCH_VMM_MAP_DISABLED) - b &= ~X86_MMU_PG_P; + base &= ~X86_MMU_PG_P; if (flags & ARCH_VMM_MAP_RDWR) - b |= X86_MMU_PG_RW; + base |= X86_MMU_PG_RW; if (flags & ARCH_VMM_MAP_USER) - b |= X86_MMU_PG_U; + base |= X86_MMU_PG_U; if (flags & ARCH_VMM_MAP_UNCACHED) - b |= X86_MMU_PG_CD; + base |= X86_MMU_PG_CD; if (flags & ARCH_VMM_MAP_SHARED) - b |= X86_MMU_PG_G; + base |= X86_MMU_PG_G; @@ -106,23 +106,23 @@ __nonnull(1) uintptr_t arch_vmm_mprotect(vmm_address_space_t* space, uintptr_t v //* Set No-Execute Bit if (flags & ARCH_VMM_MAP_NOEXEC) if (boot_cpu_has(X86_FEATURE_NX)) - b |= X86_MMU_PT_NX; /* NX */ + base |= X86_MMU_PT_NX; /* NX */ #endif if (flags & ARCH_VMM_MAP_HUGETLB) { - b |= X86_MMU_PG_PS; + base |= X86_MMU_PG_PS; if (flags & ARCH_VMM_MAP_VIDEO_MEMORY) if (boot_cpu_has(X86_FEATURE_PAT)) - b |= X86_MMU_PG_PAT; /* WC */ + base |= X86_MMU_PG_PAT; /* WC */ } else { if (flags & ARCH_VMM_MAP_VIDEO_MEMORY) if (boot_cpu_has(X86_FEATURE_PAT)) - b |= X86_MMU_PT_PAT; /* WC */ + base |= X86_MMU_PT_PAT; /* WC */ } @@ -203,10 +203,13 @@ __nonnull(1) uintptr_t arch_vmm_mprotect(vmm_address_space_t* space, uintptr_t v DEBUG_ASSERT((*d != X86_MMU_CLEAR) && "Page unmapped"); - if (!(*d & X86_MMU_PG_P)) - b &= ~X86_MMU_PG_P; + x86_page_t old = *d; + uint64_t page_flags = base; - *d = (*d & X86_MMU_ADDRESS_MASK) | (*d & X86_MMU_PG_AP_TP_MASK) | b; + if (!(flags & ARCH_VMM_MAP_DISABLED) && !(old & X86_MMU_PG_P) && ((old & X86_MMU_PG_AP_TP_MASK) == X86_MMU_PG_AP_TP_COW)) + page_flags &= ~X86_MMU_PG_P; + + *d = (old & X86_MMU_ADDRESS_MASK) | (old & (X86_MMU_PG_AP_TP_MASK | X86_MMU_PG_AP_PFB)) | page_flags; diff --git a/arch/x86-family/mm/vm_unlock.c b/arch/x86-family/mm/vm_unlock.c index 40dcea2a..de4419e2 100644 --- a/arch/x86-family/mm/vm_unlock.c +++ b/arch/x86-family/mm/vm_unlock.c @@ -50,6 +50,6 @@ __nonnull(1) void arch_vmm_unlock(vmm_address_space_t* space, uintptr_t virtaddr #if defined(CONFIG_X86_ENABLE_SMAP) if (cpu_has(current_cpu->id, X86_FEATURE_SMAP)) - x86_set_cr4(x86_get_cr4() | X86_CR4_SMAP_MASK); + x86_clac(); #endif } diff --git a/arch/x86-family/mm/vm_unmap.c b/arch/x86-family/mm/vm_unmap.c index 09e4d06e..add705d2 100644 --- a/arch/x86-family/mm/vm_unmap.c +++ b/arch/x86-family/mm/vm_unmap.c @@ -154,7 +154,7 @@ __nonnull(1) uintptr_t arch_vmm_unmap(vmm_address_space_t* space, uintptr_t virt } - __asm__ __volatile__("invlpg (%0)" ::"r"(virtaddr) : "memory"); + __asm__ __volatile__("invlpg (%0)" ::"r"(s) : "memory"); space->size -= pagesize >> 12; } diff --git a/include/aplus/memory.h b/include/aplus/memory.h index 047eaa85..562d55a7 100644 --- a/include/aplus/memory.h +++ b/include/aplus/memory.h @@ -90,6 +90,8 @@ #define ARCH_VMM_CLONE_NEW_SPACE (0) + #define ARCH_VMM_ADDRESS_SPACE_OWNED (1 << 0) + typedef struct { @@ -107,6 +109,7 @@ typedef struct vmm_address_space { uintptr_t pm; size_t size; size_t refcount; + uint32_t flags; struct { diff --git a/include/arch/x86/cpu.h b/include/arch/x86/cpu.h index 8baaf291..f1709253 100644 --- a/include/arch/x86/cpu.h +++ b/include/arch/x86/cpu.h @@ -650,6 +650,14 @@ static inline void x86_swapgs() { __asm__ __volatile__("swapgs"); } +static inline void x86_stac() { + __asm__ __volatile__("stac" ::: "memory"); +} + +static inline void x86_clac() { + __asm__ __volatile__("clac" ::: "memory"); +} + /*! * @brief Write to Extended Control Register. diff --git a/include/arch/x86/vmm.h b/include/arch/x86/vmm.h index 81614411..03d50dd7 100644 --- a/include/arch/x86/vmm.h +++ b/include/arch/x86/vmm.h @@ -85,7 +85,7 @@ #if defined(__x86_64__) typedef uint64_t x86_page_t; - #elif + #elif defined(__i386__) typedef uint32_t x86_page_t; #endif diff --git a/kernel/syscalls/009_mmap.c b/kernel/syscalls/009_mmap.c index cc46753c..6f62c7e3 100644 --- a/kernel/syscalls/009_mmap.c +++ b/kernel/syscalls/009_mmap.c @@ -78,6 +78,9 @@ SYSCALL( if (unlikely(flags == 0)) return -EINVAL; + if (unlikely(len == 0)) + return -EINVAL; + if (unlikely(!(flags & MAP_PRIVATE) && !(flags & MAP_SHARED) && !(flags & MAP_SHARED_VALIDATE))) return -EINVAL; @@ -106,6 +109,8 @@ SYSCALL( if (prot != PROT_NONE) arch_flags |= ARCH_VMM_MAP_USER; + else + arch_flags |= ARCH_VMM_MAP_DISABLED; // if(!(prot & PROT_READ)) // arch_flags |= ARCH_VMM_MAP_USER; @@ -136,7 +141,7 @@ SYSCALL( break; case MAP_HUGE_1GB: arch_flags |= ARCH_VMM_MAP_HUGE_1GB; - pagesize = arch_vmm_gethugepagesize(ARCH_VMM_MAP_HUGE_2MB); + pagesize = arch_vmm_gethugepagesize(ARCH_VMM_MAP_HUGE_1GB); break; default: @@ -159,31 +164,32 @@ SYSCALL( spinlock_lock(¤t_task->address_space->lock); - - int i; - for (i = 0; i < CONFIG_MMAP_MAX; i++) { + size_t i = 0; + for (; i < CONFIG_MMAP_MAX; i++) { if (current_task->address_space->mmap.mappings[i].start != 0UL) continue; - - current_task->address_space->mmap.mappings[i].start = current_task->address_space->mmap.heap_end - len; - current_task->address_space->mmap.mappings[i].end = current_task->address_space->mmap.heap_end; - current_task->address_space->mmap.mappings[i].fd = fd; - current_task->address_space->mmap.mappings[i].offset = offset; - - if (current_task->address_space->mmap.heap_end & (pagesize - 1)) { current_task->address_space->mmap.heap_end &= ~(pagesize - 1); current_task->address_space->mmap.heap_end += (pagesize); } + if (unlikely(current_task->address_space->mmap.heap_end > UINTPTR_MAX - len)) { + i = CONFIG_MMAP_MAX; + break; + } start = current_task->address_space->mmap.heap_end; current_task->address_space->mmap.heap_end += len; + current_task->address_space->mmap.mappings[i].start = start; + current_task->address_space->mmap.mappings[i].end = start + len; + current_task->address_space->mmap.mappings[i].fd = fd; + current_task->address_space->mmap.mappings[i].offset = offset; + break; } diff --git a/kernel/syscalls/010_mprotect.c b/kernel/syscalls/010_mprotect.c index 80eb7aba..89194be0 100644 --- a/kernel/syscalls/010_mprotect.c +++ b/kernel/syscalls/010_mprotect.c @@ -81,6 +81,8 @@ SYSCALL( if (prot != PROT_NONE) arch_flags |= ARCH_VMM_MAP_USER; + else + arch_flags |= ARCH_VMM_MAP_DISABLED; // if(!(prot & PROT_READ)) // arch_flags |= ARCH_VMM_MAP_USER; diff --git a/kernel/syscalls/011_munmap.c b/kernel/syscalls/011_munmap.c index c061c006..e0bf0911 100644 --- a/kernel/syscalls/011_munmap.c +++ b/kernel/syscalls/011_munmap.c @@ -51,4 +51,82 @@ * Auto-generated by scripts/gen-syscalls.js */ -SYSCALL(11, munmap, long sys_munmap(unsigned long addr, size_t len) { return -ENOSYS; }); +SYSCALL( + 11, munmap, long sys_munmap(unsigned long addr, size_t len) { + if (unlikely(!len)) + return -EINVAL; + + if (unlikely(addr & (arch_vmm_getpagesize() - 1))) + return -EINVAL; + + if (unlikely(len & (arch_vmm_getpagesize() - 1))) + len = (len & ~(arch_vmm_getpagesize() - 1)) + arch_vmm_getpagesize(); + + if (unlikely(addr > UINTPTR_MAX - len)) + return -EINVAL; + + uintptr_t end = addr + len; + int found = 0; + int error = 0; + + scoped_lock(¤t_task->address_space->lock) { + for (size_t i = 0; i < CONFIG_MMAP_MAX; i++) { + + mmap_mapping_t* mapping = ¤t_task->address_space->mmap.mappings[i]; + + if (!mapping->start) + continue; + + if (mapping->end <= addr || mapping->start >= end) + continue; + + found = 1; + + if (addr <= mapping->start && end >= mapping->end) { + memset(mapping, 0, sizeof(*mapping)); + continue; + } + + if (addr <= mapping->start) { + mapping->offset += end - mapping->start; + mapping->start = end; + continue; + } + + if (end >= mapping->end) { + mapping->end = addr; + continue; + } + + size_t slot; + for (slot = 0; slot < CONFIG_MMAP_MAX; slot++) { + if (current_task->address_space->mmap.mappings[slot].start == 0UL) + break; + } + + if (unlikely(slot == CONFIG_MMAP_MAX)) { + error = -ENOMEM; + break; + } + + current_task->address_space->mmap.mappings[slot] = (mmap_mapping_t){ + .start = end, + .end = mapping->end, + .fd = mapping->fd, + .offset = mapping->offset + (end - mapping->start), + }; + + mapping->end = addr; + } + } + + if (error) + return error; + + if (!found) + return -EINVAL; + + arch_vmm_unmap(current_task->address_space, addr, len); + + return 0; + });