rustubs/arch/x86_64/
paging.rs

1//! # paging structgure terminology:
2//! - PML4(E)   -> L4
3//! - PDPDE(E)  -> L3
4//! - PD(E)     -> L2
5//! - PT(E)     -> L1 -> (leaf) pagetable == 4K
6//!
7//! We use 4-level 4K paging. Huge pages for userspace is not yet supported.
8//! The kernel's ID-mapping uses 1G page (in the total of 512 G). See startup
9//! asm code.
10pub mod fault;
11pub mod pagetable;
12use crate::defs;
13use crate::defs::rounddown_4k;
14use crate::defs::P2V;
15use crate::defs::V2P;
16use crate::io::*;
17use crate::mm::allocate_frame_4k_zeroed;
18use crate::mm::vmm::VMArea;
19use crate::mm::vmm::VMType;
20use core::arch::asm;
21use core::ops::Range;
22use core::ptr;
23pub use pagetable::*;
24/// for x86_64, return the CR3 register. this is the **physical** address of the
25/// page table root.
26/// TODO: use page root in task struct instead of raw cr3
27#[inline]
28pub fn get_cr3() -> u64 {
29	let cr3: u64;
30	unsafe { asm!("mov {}, cr3", out(reg) cr3) };
31	cr3
32}
33
34#[inline]
35pub fn set_cr3(cr3: u64) {
36	unsafe {
37		asm!("mov cr3, {}", in(reg) cr3);
38	}
39}
40
41/// returns the identically mapped (+ kernel offset) virtual address of the page
42/// table
43#[inline]
44pub fn get_root() -> u64 { P2V(get_cr3()).unwrap() }
45
46// if we are out of memory in a paging handler we are screwed anyways..
47// TODO handle OOM properly
48fn get_free_page_zeroed() -> u64 {
49	P2V(allocate_frame_4k_zeroed().unwrap()).unwrap()
50}
51
52/// unsafe as it dereferences raw pointer pt_root. Must make sure it's a valid,
53/// 4k aligned _virtual_ address.
54// TODO use Result type instead of bool so that we can do early return with ?..
55pub unsafe fn map_vma(pt_root: u64, vma: &VMArea, do_copy: bool) -> bool {
56	// create mappings in pagetable
57	let flags = PTEFlags::PRESENT | PTEFlags::WRITABLE | PTEFlags::USER;
58	if !map_range(pt_root, &vma.vm_range, flags) {
59		panic!("failed to map range {:X?}, possibly OOM", vma.vm_range);
60	}
61	if !do_copy {
62		return true;
63	}
64	match vma.backing {
65		VMType::ANOM => {
66			return true;
67		}
68		VMType::FILE(f) => {
69			if !do_copy {
70				return true;
71			}
72			unsafe {
73				ptr::copy_nonoverlapping(
74					&f[0] as *const u8,
75					vma.vm_range.start as *mut u8,
76					f.len(),
77				)
78			}
79
80			if let Some(bss_range) = vma.bss_range() {
81				core::slice::from_raw_parts_mut(
82					bss_range.start as *mut u8,
83					(bss_range.end - bss_range.start) as usize,
84				)
85				.fill(0);
86			}
87
88			return true;
89		}
90		_ => {
91			panic!("unknown backing {:?}", vma);
92		}
93	}
94}
95
96pub fn map_range(pt_root: u64, r: &Range<u64>, flags: PTEFlags) -> bool {
97	let mut va_aligned = rounddown_4k(r.start);
98	while va_aligned < r.end {
99		if !map_page(pt_root, va_aligned, flags) {
100			panic!("failed to map page {:X?}, possibly OOM", va_aligned);
101		}
102		va_aligned += defs::Mem::PAGE_SIZE;
103	}
104	return true;
105}
106
107/// walk the page table, create missing tables, return mapped physical frame
108pub fn map_page(pt_root: u64, va: u64, _flags: PTEFlags) -> bool {
109	let pt = pt_root as *mut Pagetable;
110	if !defs::is_aligned_4k(va) {
111		println!("not aligned");
112		return false;
113	}
114	let flags: u64 = _flags.bits();
115	let l4idx = pagetable::p4idx(va) as usize;
116	let l3idx = pagetable::p3idx(va) as usize;
117	let l2idx = pagetable::p2idx(va) as usize;
118	let l1idx = pagetable::p1idx(va) as usize;
119	let mut require_new = false;
120	unsafe {
121		let l4_ent = &mut (*pt).entries[l4idx];
122		let l3_tbl: *mut Pagetable;
123		if l4_ent.is_unused() || require_new {
124			l3_tbl = get_free_page_zeroed() as *mut Pagetable;
125			l4_ent.entry = defs::V2P(l3_tbl as u64).unwrap() | flags;
126			require_new = true
127		} else {
128			l3_tbl = defs::P2V(l4_ent.addr()).unwrap() as *mut Pagetable;
129		}
130		let l3_ent = &mut (*l3_tbl).entries[l3idx];
131		let l2_tbl: *mut Pagetable;
132		if l3_ent.is_unused() || require_new {
133			l2_tbl = get_free_page_zeroed() as *mut Pagetable;
134			l3_ent.entry = defs::V2P(l2_tbl as u64).unwrap() | flags;
135			require_new = true
136		} else {
137			l2_tbl = defs::P2V(l3_ent.addr()).unwrap() as *mut Pagetable;
138		}
139		let l2_ent = &mut (*l2_tbl).entries[l2idx];
140		let l1_tbl: *mut Pagetable;
141		if l2_ent.is_unused() || require_new {
142			l1_tbl = get_free_page_zeroed() as *mut Pagetable;
143			l2_ent.entry = defs::V2P(l1_tbl as u64).unwrap() | flags;
144			require_new = true
145		} else {
146			l1_tbl = defs::P2V(l2_ent.addr()).unwrap() as *mut Pagetable;
147		}
148		let pte = &mut (*l1_tbl).entries[l1idx];
149		if pte.is_unused() || require_new {
150			let page = get_free_page_zeroed();
151			pte.entry = defs::V2P(page).unwrap() | flags;
152		} else {
153			// TODO we need to free this frame
154			panic!("PTE already taken: {:#X} for VA {:#X}", pte.entry, va);
155		}
156		// flush tlb
157		asm!("invlpg [{0}]", in(reg) va);
158	}
159	return true;
160}
161
162/// clone an paging hierarchy (fuck me why can't I spell this fucking word.)
163/// currently the kernel part is not deep copied i.e. all PML4 entries point to
164/// the original physical addresses of L3 tables. The user part is deep-copied
165/// without COW.
166///
167/// The actual virtual memory is also copied.
168/// unsafe: assume the pagetables are not ill-formed.
169///
170/// It both takes and returns VIRTUAL ADDRESSES
171pub unsafe fn address_space_clone(src_root_va: u64) -> Option<u64> {
172	let pt_src = src_root_va as *mut Pagetable;
173
174	let pt_new = get_free_page_zeroed() as *mut Pagetable;
175
176	// the kernel part: just use the same mapping.
177	for idx in 256..512 {
178		(*pt_new).entries[idx] = (*pt_src).entries[idx].clone();
179	}
180
181	// the user part:
182	// TODO: show not set GLOBAL bit for the user pages, TLB hazzard
183	for idx in 0..256 {
184		let pte_src = &(*pt_src).entries[idx];
185		if !pte_src.is_present() {
186			continue;
187		}
188
189		let next_level_va = get_free_page_zeroed();
190		(*pt_new).entries[idx].entry =
191			V2P(next_level_va).unwrap() | pte_src.flags().bits();
192
193		let next_src = P2V(pte_src.addr()).unwrap() as *mut Pagetable;
194		let next_tar = next_level_va as *mut Pagetable;
195		pt_clone_level(next_src, next_tar, 3);
196	}
197
198	Some(pt_new as u64)
199}
200
201/// THIS ONLY WORKS FOR 4K PAGES! Must not use this on kernel mappings, which
202/// use huge pages.
203/// on lv==1, it's a "page" not a pagetable, hence do memcpy.
204///
205/// TODO: just because there is page mapping doesn't mean you need to copy all
206/// the pages. Smarter: 1) COW, 2) copy by VMAs after rather than by raw pages.
207pub unsafe fn pt_clone_level(
208	pt_src: *mut Pagetable,
209	pt_new: *mut Pagetable,
210	lv: usize,
211) {
212	for idx in 0..512 {
213		let pte_src = &(*pt_src).entries[idx];
214		if !pte_src.is_present() {
215			continue;
216		}
217		let next_level_va = get_free_page_zeroed();
218		debug_assert!(0 != next_level_va);
219		(*pt_new).entries[idx].entry =
220			V2P(next_level_va).unwrap() | pte_src.flags().bits();
221
222		let next_src = P2V(pte_src.addr()).unwrap() as *mut Pagetable;
223		let next_tar = next_level_va as *mut Pagetable;
224		if lv == 1 {
225			ptr::copy_nonoverlapping(
226				P2V(pte_src.addr()).unwrap() as *const u8,
227				next_level_va as *mut u8,
228				defs::Mem::PAGE_SIZE as usize,
229			)
230		} else {
231			pt_clone_level(next_src, next_tar, lv - 1);
232		}
233	}
234}
rustubs/arch/x86_64/paging.rs

rustubs/arch/x86_64/
paging.rs
