From: Andi Kleen <ak@suse.de>
Converted to true 4levels. The address space per process is expanded to
47bits now, the supported physical address space is 46bits.
Lmbench fork/exit numbers are down a few percent because it has to walk much
more pagetables, but some planned future optimizations will hopefully recover
it.
See Documentation/x86_64/mm.txt for more details on the memory map.
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-x86_64-akpm/Documentation/x86_64/mm.txt | 168 +++---------------------
25-x86_64-akpm/arch/x86_64/ia32/syscall32.c | 31 ++--
25-x86_64-akpm/arch/x86_64/kernel/acpi/sleep.c | 8 -
25-x86_64-akpm/arch/x86_64/kernel/head.S | 1
25-x86_64-akpm/arch/x86_64/kernel/init_task.c | 2
25-x86_64-akpm/arch/x86_64/kernel/setup64.c | 11 -
25-x86_64-akpm/arch/x86_64/mm/fault.c | 87 +++++++-----
25-x86_64-akpm/arch/x86_64/mm/init.c | 43 ++----
25-x86_64-akpm/arch/x86_64/mm/ioremap.c | 39 ++++-
25-x86_64-akpm/arch/x86_64/mm/pageattr.c | 32 ++--
25-x86_64-akpm/include/asm-x86_64/e820.h | 3
25-x86_64-akpm/include/asm-x86_64/mmu_context.h | 5
25-x86_64-akpm/include/asm-x86_64/page.h | 6
25-x86_64-akpm/include/asm-x86_64/pda.h | 1
25-x86_64-akpm/include/asm-x86_64/pgalloc.h | 30 ++++
25-x86_64-akpm/include/asm-x86_64/pgtable.h | 89 ++++--------
25-x86_64-akpm/include/asm-x86_64/processor.h | 4
17 files changed, 238 insertions(+), 322 deletions(-)
diff -puN arch/x86_64/ia32/syscall32.c~4level-architecture-changes-for-x86_64 arch/x86_64/ia32/syscall32.c
--- 25-x86_64/arch/x86_64/ia32/syscall32.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/ia32/syscall32.c Fri Dec 10 16:40:04 2004
@@ -40,23 +40,30 @@ static int use_sysenter = -1;
*/
int __map_syscall32(struct mm_struct *mm, unsigned long address)
{
+ pml4_t *pml4;
+ pgd_t *pgd;
pte_t *pte;
pmd_t *pmd;
- int err = 0;
+ int err = -ENOMEM;
spin_lock(&mm->page_table_lock);
- pmd = pmd_alloc(mm, pgd_offset(mm, address), address);
- if (pmd && (pte = pte_alloc_map(mm, pmd, address)) != NULL) {
- if (pte_none(*pte)) {
- set_pte(pte,
- mk_pte(virt_to_page(syscall32_page),
- PAGE_KERNEL_VSYSCALL));
+ pml4 = pml4_offset(mm, address);
+ pgd = pgd_alloc(mm, pml4, address);
+ if (pgd) {
+ pmd = pmd_alloc(mm, pgd, address);
+ if (pmd && (pte = pte_alloc_map(mm, pmd, address)) != NULL) {
+ if (pte_none(*pte)) {
+ set_pte(pte,
+ mk_pte(virt_to_page(syscall32_page),
+ PAGE_KERNEL_VSYSCALL));
+ }
+ /* Flush only the local CPU. Other CPUs taking a fault
+ will just end up here again
+ This probably not needed and just paranoia. */
+ __flush_tlb_one(address);
+ err = 0;
}
- /* Flush only the local CPU. Other CPUs taking a fault
- will just end up here again */
- __flush_tlb_one(address);
- } else
- err = -ENOMEM;
+ }
spin_unlock(&mm->page_table_lock);
return err;
}
diff -puN arch/x86_64/kernel/acpi/sleep.c~4level-architecture-changes-for-x86_64 arch/x86_64/kernel/acpi/sleep.c
--- 25-x86_64/arch/x86_64/kernel/acpi/sleep.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/kernel/acpi/sleep.c Fri Dec 10 16:40:04 2004
@@ -61,9 +61,13 @@ extern char wakeup_start, wakeup_end;
extern unsigned long FASTCALL(acpi_copy_wakeup_routine(unsigned long));
+static pml4_t low_ptr;
+
static void init_low_mapping(void)
{
- cpu_pda[0].level4_pgt[0] = cpu_pda[0].level4_pgt[pml4_index(PAGE_OFFSET)];
+ pml4_t *slot0 = pml4_offset(current->mm, 0UL);
+ low_ptr = *slot0;
+ set_pml4(slot0, *pml4_offset(current->mm, PAGE_OFFSET));
flush_tlb_all();
}
@@ -97,7 +101,7 @@ int acpi_save_state_disk (void)
*/
void acpi_restore_state_mem (void)
{
- cpu_pda[0].level4_pgt[0] = 0;
+ set_pml4(pml4_offset(current->mm, 0UL), low_ptr);
flush_tlb_all();
}
diff -puN arch/x86_64/kernel/head.S~4level-architecture-changes-for-x86_64 arch/x86_64/kernel/head.S
--- 25-x86_64/arch/x86_64/kernel/head.S~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/kernel/head.S Fri Dec 10 16:40:04 2004
@@ -225,7 +225,6 @@ ENTRY(init_level4_pgt)
.quad 0x0000000000103007 /* -> level3_kernel_pgt */
.org 0x2000
-/* Kernel does not "know" about 4-th level of page tables. */
ENTRY(level3_ident_pgt)
.quad 0x0000000000104007
.fill 511,8,0
diff -puN arch/x86_64/kernel/init_task.c~4level-architecture-changes-for-x86_64 arch/x86_64/kernel/init_task.c
--- 25-x86_64/arch/x86_64/kernel/init_task.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/kernel/init_task.c Fri Dec 10 16:40:04 2004
@@ -47,5 +47,3 @@ EXPORT_SYMBOL(init_task);
DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_maxaligned_in_smp;
#define ALIGN_TO_4K __attribute__((section(".data.init_task")))
-
-pgd_t boot_vmalloc_pgt[512] ALIGN_TO_4K;
diff -puN arch/x86_64/kernel/setup64.c~4level-architecture-changes-for-x86_64 arch/x86_64/kernel/setup64.c
--- 25-x86_64/arch/x86_64/kernel/setup64.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/kernel/setup64.c Fri Dec 10 16:40:04 2004
@@ -100,7 +100,6 @@ void __init setup_per_cpu_areas(void)
void pda_init(int cpu)
{
- pml4_t *level4;
struct x8664_pda *pda = &cpu_pda[cpu];
/* Setup up data that may be needed in __get_free_pages early */
@@ -119,22 +118,14 @@ void pda_init(int cpu)
/* others are initialized in smpboot.c */
pda->pcurrent = &init_task;
pda->irqstackptr = boot_cpu_stack;
- level4 = init_level4_pgt;
} else {
- level4 = (pml4_t *)__get_free_pages(GFP_ATOMIC, 0);
- if (!level4)
- panic("Cannot allocate top level page for cpu %d", cpu);
pda->irqstackptr = (char *)
__get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
if (!pda->irqstackptr)
panic("cannot allocate irqstack for cpu %d", cpu);
}
- pda->level4_pgt = (unsigned long *)level4;
- if (level4 != init_level4_pgt)
- memcpy(level4, &init_level4_pgt, PAGE_SIZE);
- set_pml4(level4 + 510, mk_kernel_pml4(__pa_symbol(boot_vmalloc_pgt)));
- asm volatile("movq %0,%%cr3" :: "r" (__pa(level4)));
+ asm volatile("movq %0,%%cr3" :: "r" (__pa_symbol(&init_level4_pgt)));
pda->irqstackptr += IRQSTACKSIZE-64;
}
diff -puN arch/x86_64/mm/fault.c~4level-architecture-changes-for-x86_64 arch/x86_64/mm/fault.c
--- 25-x86_64/arch/x86_64/mm/fault.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/mm/fault.c Fri Dec 10 16:40:04 2004
@@ -232,7 +232,53 @@ static noinline void pgtable_bad(unsigne
do_exit(SIGKILL);
}
-int page_fault_trace;
+/*
+ * Handle a fault on the vmalloc or module mapping area
+ */
+static int vmalloc_fault(unsigned long address)
+{
+ pml4_t *pml4, *pml4_ref;
+ pgd_t *pgd, *pgd_ref;
+ pmd_t *pmd, *pmd_ref;
+ pte_t *pte, *pte_ref;
+
+ /* Copy kernel mappings over when needed. This can also
+ happen within a race in page table update. In the later
+ case just flush. */
+
+ pml4 = pml4_offset(current->mm ?: &init_mm, address);
+ pml4_ref = pml4_offset_k(address);
+ if (pml4_none(*pml4_ref))
+ return -1;
+ if (pml4_none(*pml4))
+ set_pml4(pml4, *pml4_ref);
+
+ /* Below here mismatches are bugs because these lower tables
+ are shared */
+
+ pgd = pml4_pgd_offset(pml4, address);
+ pgd_ref = pml4_pgd_offset(pml4_ref, address);
+ if (pgd_none(*pgd_ref))
+ return -1;
+ if (pgd_none(*pgd) || pgd_page(*pgd) != pgd_page(*pgd_ref))
+ BUG();
+ pmd = pmd_offset(pgd, address);
+ pmd_ref = pmd_offset(pgd_ref, address);
+ if (pmd_none(*pmd_ref))
+ return -1;
+ if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
+ BUG();
+ pte_ref = pte_offset_kernel(pmd_ref, address);
+ if (!pte_present(*pte_ref))
+ return -1;
+ pte = pte_offset_kernel(pmd, address);
+ if (!pte_present(*pte) || pte_page(*pte) != pte_page(*pte_ref))
+ BUG();
+ __flush_tlb_all();
+ return 0;
+}
+
+int page_fault_trace = 0;
int exception_trace = 1;
/*
@@ -300,8 +346,11 @@ asmlinkage void do_page_fault(struct pt_
* protection error (error_code & 1) == 0.
*/
if (unlikely(address >= TASK_SIZE)) {
- if (!(error_code & 5))
- goto vmalloc_fault;
+ if (!(error_code & 5)) {
+ if (vmalloc_fault(address) < 0)
+ goto bad_area_nosemaphore;
+ return;
+ }
/*
* Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock.
@@ -310,7 +359,7 @@ asmlinkage void do_page_fault(struct pt_
}
if (unlikely(error_code & (1 << 3)))
- goto page_table_corruption;
+ pgtable_bad(address, regs, error_code);
/*
* If we're in an interrupt or have no user
@@ -524,34 +573,4 @@ do_sigbus:
info.si_addr = (void __user *)address;
force_sig_info(SIGBUS, &info, tsk);
return;
-
-vmalloc_fault:
- {
- pgd_t *pgd;
- pmd_t *pmd;
- pte_t *pte;
-
- /*
- * x86-64 has the same kernel 3rd level pages for all CPUs.
- * But for vmalloc/modules the TLB synchronization works lazily,
- * so it can happen that we get a page fault for something
- * that is really already in the page table. Just check if it
- * is really there and when yes flush the local TLB.
- */
- pgd = pgd_offset_k(address);
- if (!pgd_present(*pgd))
- goto bad_area_nosemaphore;
- pmd = pmd_offset(pgd, address);
- if (!pmd_present(*pmd))
- goto bad_area_nosemaphore;
- pte = pte_offset_kernel(pmd, address);
- if (!pte_present(*pte))
- goto bad_area_nosemaphore;
-
- __flush_tlb_all();
- return;
- }
-
-page_table_corruption:
- pgtable_bad(address, regs, error_code);
}
diff -puN arch/x86_64/mm/init.c~4level-architecture-changes-for-x86_64 arch/x86_64/mm/init.c
--- 25-x86_64/arch/x86_64/mm/init.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/mm/init.c Fri Dec 10 16:40:04 2004
@@ -120,7 +120,7 @@ static void set_pte_phys(unsigned long v
printk("PML4 FIXMAP MISSING, it should be setup in head.S!\n");
return;
}
- pgd = level3_offset_k(level4, vaddr);
+ pgd = pml4_pgd_offset(level4, vaddr);
if (pgd_none(*pgd)) {
pmd = (pmd_t *) spp_getpage();
set_pgd(pgd, __pgd(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
@@ -306,25 +306,12 @@ void __init init_memory_mapping(void)
extern struct x8664_pda cpu_pda[NR_CPUS];
-static unsigned long low_pml4[NR_CPUS];
-
-void swap_low_mappings(void)
-{
- int i;
- for (i = 0; i < NR_CPUS; i++) {
- unsigned long t;
- if (!cpu_pda[i].level4_pgt)
- continue;
- t = cpu_pda[i].level4_pgt[0];
- cpu_pda[i].level4_pgt[0] = low_pml4[i];
- low_pml4[i] = t;
- }
- flush_tlb_all();
-}
-
+/* Assumes all CPUs still execute in init_mm */
void zap_low_mappings(void)
{
- swap_low_mappings();
+ pml4_t *pml4 = pml4_offset_k(0UL);
+ pml4_clear(pml4);
+ flush_tlb_all();
}
#ifndef CONFIG_DISCONTIGMEM
@@ -360,11 +347,15 @@ void __init clear_kernel_mapping(unsigne
BUG_ON(size & ~LARGE_PAGE_MASK);
for (; address < end; address += LARGE_PAGE_SIZE) {
- pgd_t *pgd = pgd_offset_k(address);
- pmd_t *pmd;
- if (!pgd || pgd_none(*pgd))
+ pml4_t *pml4 = pml4_offset_k(address);
+ pgd_t *pgd;
+ pmd_t *pmd;
+ if (pml4_none(*pml4))
+ continue;
+ pgd = pml4_pgd_offset(pml4, address);
+ if (pgd_none(*pgd))
continue;
- pmd = pmd_offset(pgd, address);
+ pmd = pmd_offset(pgd, address);
if (!pmd || pmd_none(*pmd))
continue;
if (0 == (pmd_val(*pmd) & _PAGE_PSE)) {
@@ -539,21 +530,21 @@ int kern_addr_valid(unsigned long addr)
if (above != 0 && above != -1UL)
return 0;
- pml4 = pml4_offset_k(addr);
+ pml4 = pml4_offset_k(addr);
if (pml4_none(*pml4))
return 0;
- pgd = pgd_offset_k(addr);
+ pgd = pml4_pgd_offset(pml4, addr);
if (pgd_none(*pgd))
return 0;
- pmd = pmd_offset(pgd, addr);
+ pmd = pmd_offset(pgd, addr);
if (pmd_none(*pmd))
return 0;
if (pmd_large(*pmd))
return pfn_valid(pmd_pfn(*pmd));
- pte = pte_offset_kernel(pmd, addr);
+ pte = pte_offset_kernel(pmd, addr);
if (pte_none(*pte))
return 0;
return pfn_valid(pte_pfn(*pte));
diff -puN arch/x86_64/mm/ioremap.c~4level-architecture-changes-for-x86_64 arch/x86_64/mm/ioremap.c
--- 25-x86_64/arch/x86_64/mm/ioremap.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/mm/ioremap.c Fri Dec 10 16:40:04 2004
@@ -67,31 +67,54 @@ static inline int remap_area_pmd(pmd_t *
return 0;
}
+static inline int remap_area_pgd(pgd_t * pgd, unsigned long address, unsigned long size,
+ unsigned long phys_addr, unsigned long flags)
+{
+ unsigned long end;
+
+ address &= ~PML4_MASK;
+ end = address + size;
+ if (end > PML4_SIZE)
+ end = PML4_SIZE;
+ phys_addr -= address;
+ if (address >= end)
+ BUG();
+ do {
+ pmd_t * pmd = pmd_alloc(&init_mm, pgd, address);
+ if (!pmd)
+ return -ENOMEM;
+ remap_area_pmd(pmd, address, end - address, address + phys_addr, flags);
+ address = (address + PGDIR_SIZE) & PGDIR_MASK;
+ pmd++;
+ } while (address && (address < end));
+ return 0;
+}
+
static int remap_area_pages(unsigned long address, unsigned long phys_addr,
unsigned long size, unsigned long flags)
{
int error;
- pgd_t * dir;
+ pml4_t *pml4;
unsigned long end = address + size;
phys_addr -= address;
- dir = pgd_offset_k(address);
+ pml4 = pml4_offset_k(address);
flush_cache_all();
if (address >= end)
BUG();
spin_lock(&init_mm.page_table_lock);
do {
- pmd_t *pmd;
- pmd = pmd_alloc(&init_mm, dir, address);
+ pgd_t *pgd;
+ pgd = pgd_alloc(&init_mm, pml4, address);
error = -ENOMEM;
- if (!pmd)
+ if (!pgd)
break;
- if (remap_area_pmd(pmd, address, end - address,
+ if (remap_area_pgd(pgd, address, end - address,
phys_addr + address, flags))
break;
error = 0;
- address = (address + PGDIR_SIZE) & PGDIR_MASK;
- dir++;
+ address = (address + PML4_SIZE) & PML4_MASK;
+ pml4++;
} while (address && (address < end));
spin_unlock(&init_mm.page_table_lock);
flush_tlb_all();
diff -puN arch/x86_64/mm/pageattr.c~4level-architecture-changes-for-x86_64 arch/x86_64/mm/pageattr.c
--- 25-x86_64/arch/x86_64/mm/pageattr.c~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/arch/x86_64/mm/pageattr.c Fri Dec 10 16:40:04 2004
@@ -16,10 +16,14 @@
static inline pte_t *lookup_address(unsigned long address)
{
- pgd_t *pgd = pgd_offset_k(address);
+ pml4_t *pml4 = pml4_offset_k(address);
+ pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
- if (!pgd || !pgd_present(*pgd))
+ if (pml4_none(*pml4))
+ return NULL;
+ pgd = pml4_pgd_offset(pml4, address);
+ if (!pgd_present(*pgd))
return NULL;
pmd = pmd_offset(pgd, address);
if (!pmd_present(*pmd))
@@ -101,16 +105,20 @@ static inline void save_page(unsigned lo
*/
static void revert_page(unsigned long address, pgprot_t ref_prot)
{
- pgd_t *pgd;
- pmd_t *pmd;
- pte_t large_pte;
-
- pgd = pgd_offset_k(address);
- pmd = pmd_offset(pgd, address);
- BUG_ON(pmd_val(*pmd) & _PAGE_PSE);
- pgprot_val(ref_prot) |= _PAGE_PSE;
- large_pte = mk_pte_phys(__pa(address) & LARGE_PAGE_MASK, ref_prot);
- set_pte((pte_t *)pmd, large_pte);
+ pml4_t *pml4;
+ pgd_t *pgd;
+ pmd_t *pmd;
+ pte_t large_pte;
+
+ pml4 = pml4_offset_k(address);
+ BUG_ON(pml4_none(*pml4));
+ pgd = pml4_pgd_offset(pml4,address);
+ BUG_ON(pgd_none(*pgd));
+ pmd = pmd_offset(pgd, address);
+ BUG_ON(pmd_val(*pmd) & _PAGE_PSE);
+ pgprot_val(ref_prot) |= _PAGE_PSE;
+ large_pte = mk_pte_phys(__pa(address) & LARGE_PAGE_MASK, ref_prot);
+ set_pte((pte_t *)pmd, large_pte);
}
static int
diff -puN Documentation/x86_64/mm.txt~4level-architecture-changes-for-x86_64 Documentation/x86_64/mm.txt
--- 25-x86_64/Documentation/x86_64/mm.txt~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/Documentation/x86_64/mm.txt Fri Dec 10 16:40:04 2004
@@ -1,148 +1,24 @@
-The paging design used on the x86-64 linux kernel port in 2.4.x provides:
-o per process virtual address space limit of 512 Gigabytes
-o top of userspace stack located at address 0x0000007fffffffff
-o PAGE_OFFSET = 0xffff800000000000
-o start of the kernel = 0xffffffff800000000
-o global RAM per system 2^64-PAGE_OFFSET-sizeof(kernel) = 128 Terabytes - 2 Gigabytes
-o no need of any common code change
-o no need to use highmem to handle the 128 Terabytes of RAM
-
-Description:
-
- Userspace is able to modify and it sees only the 3rd/2nd/1st level
- pagetables (pgd_offset() implicitly walks the 1st slot of the 4th
- level pagetable and it returns an entry into the 3rd level pagetable).
- This is where the per-process 512 Gigabytes limit cames from.
-
- The common code pgd is the PDPE, the pmd is the PDE, the
- pte is the PTE. The PML4E remains invisible to the common
- code.
-
- The kernel uses all the first 47 bits of the negative half
- of the virtual address space to build the direct mapping using
- 2 Mbytes page size. The kernel virtual addresses have bit number
- 47 always set to 1 (and in turn also bits 48-63 are set to 1 too,
- due the sign extension). This is where the 128 Terabytes - 2 Gigabytes global
- limit of RAM cames from.
-
- Since the per-process limit is 512 Gigabytes (due to kernel common
- code 3 level pagetable limitation), the higher virtual address mapped
- into userspace is 0x7fffffffff and it makes sense to use it
- as the top of the userspace stack to allow the stack to grow as
- much as possible.
-
- Setting the PAGE_OFFSET to 2^39 (after the last userspace
- virtual address) wouldn't make much difference compared to
- setting PAGE_OFFSET to 0xffff800000000000 because we have an
- hole into the virtual address space. The last byte mapped by the
- 255th slot in the 4th level pagetable is at virtual address
- 0x00007fffffffffff and the first byte mapped by the 256th slot in the
- 4th level pagetable is at address 0xffff800000000000. Due to this
- hole we can't trivially build a direct mapping across all the
- 512 slots of the 4th level pagetable, so we simply use only the
- second (negative) half of the 4th level pagetable for that purpose
- (that provides us 128 Terabytes of contigous virtual addresses).
- Strictly speaking we could build a direct mapping also across the hole
- using some DISCONTIGMEM trick, but we don't need such a large
- direct mapping right now.
-
-Future:
-
- During 2.5.x we can break the 512 Gigabytes per-process limit
- possibly by removing from the common code any knowledge about the
- architectural dependent physical layout of the virtual to physical
- mapping.
-
- Once the 512 Gigabytes limit will be removed the kernel stack will
- be moved (most probably to virtual address 0x00007fffffffffff).
- Nothing will break in userspace due that move, as nothing breaks
- in IA32 compiling the kernel with CONFIG_2G.
-
-Linus agreed on not breaking common code and to live with the 512 Gigabytes
-per-process limitation for the 2.4.x timeframe and he has given me and Andi
-some very useful hints... (thanks! :)
-
-Thanks also to H. Peter Anvin for his interesting and useful suggestions on
-the x86-64-discuss lists!
-
-Other memory management related issues follows:
-
-PAGE_SIZE:
-
- If somebody is wondering why these days we still have a so small
- 4k pagesize (16 or 32 kbytes would be much better for performance
- of course), the PAGE_SIZE have to remain 4k for 32bit apps to
- provide 100% backwards compatible IA32 API (we can't allow silent
- fs corruption or as best a loss of coherency with the page cache
- by allocating MAP_SHARED areas in MAP_ANONYMOUS memory with a
- do_mmap_fake). I think it could be possible to have a dynamic page
- size between 32bit and 64bit apps but it would need extremely
- intrusive changes in the common code as first for page cache and
- we sure don't want to depend on them right now even if the
- hardware would support that.
-
-PAGETABLE SIZE:
-
- In turn we can't afford to have pagetables larger than 4k because
- we could not be able to allocate them due physical memory
- fragmentation, and failing to allocate the kernel stack is a minor
- issue compared to failing the allocation of a pagetable. If we
- fail the allocation of a pagetable the only thing we can do is to
- sched_yield polling the freelist (deadlock prone) or to segfault
- the task (not even the sighandler would be sure to run).
-
-KERNEL STACK:
-
- 1st stage:
-
- The kernel stack will be at first allocated with an order 2 allocation
- (16k) (the utilization of the stack for a 64bit platform really
- isn't exactly the double of a 32bit platform because the local
- variables may not be all 64bit wide, but not much less). This will
- make things even worse than they are right now on IA32 with
- respect of failing fork/clone due memory fragmentation.
-
- 2nd stage:
-
- We'll benchmark if reserving one register as task_struct
- pointer will improve performance of the kernel (instead of
- recalculating the task_struct pointer starting from the stack
- pointer each time). My guess is that recalculating will be faster
- but it worth a try.
-
- If reserving one register for the task_struct pointer
- will be faster we can as well split task_struct and kernel
- stack. task_struct can be a slab allocation or a
- PAGE_SIZEd allocation, and the kernel stack can then be
- allocated in a order 1 allocation. Really this is risky,
- since 8k on a 64bit platform is going to be less than 7k
- on a 32bit platform but we could try it out. This would
- reduce the fragmentation problem of an order of magnitude
- making it equal to the current IA32.
-
- We must also consider the x86-64 seems to provide in hardware a
- per-irq stack that could allow us to remove the irq handler
- footprint from the regular per-process-stack, so it could allow
- us to live with a smaller kernel stack compared to the other
- linux architectures.
-
- 3rd stage:
-
- Before going into production if we still have the order 2
- allocation we can add a sysctl that allows the kernel stack to be
- allocated with vmalloc during memory fragmentation. This have to
- remain turned off during benchmarks :) but it should be ok in real
- life.
-
-Order of PAGE_CACHE_SIZE and other allocations:
-
- On the long run we can increase the PAGE_CACHE_SIZE to be
- an order 2 allocations and also the slab/buffercache etc.ec..
- could be all done with order 2 allocations. To make the above
- to work we should change lots of common code thus it can be done
- only once the basic port will be in a production state. Having
- a working PAGE_CACHE_SIZE would be a benefit also for
- IA32 and other architectures of course.
+<previous description obsolete, deleted>
-Andrea <andrea@suse.de> SuSE
+Virtual memory map with 4 level page tables:
+
+0000000000000000 - 00007fffffffffff (=47bits) user space, different per mm
+hole caused by [48:63] sign extension
+ffff800000000000 - ffff80ffffffffff (=40bits) guard hole
+ffff810000000000 - ffffc0ffffffffff (=46bits) direct mapping of phys. memory
+ffffc10000000000 - ffffc1ffffffffff (=40bits) hole
+ffffc20000000000 - ffffe1ffffffffff (=45bits) vmalloc/ioremap space
+... unused hole ...
+ffffffff80000000 - ffffffff82800000 (=40MB) kernel text mapping, from phys 0
+... unused hole ...
+ffffffff88000000 - fffffffffff00000 (=1919MB) module mapping space
+
+vmalloc space is lazily synchronized into the different PML4 pages of
+the processes using the page fault handler, with init_level4_pgt as
+reference.
+
+Current X86-64 implementations only support 40 bit of address space,
+but we support upto 46bits. This expands into MBZ space in the page tables.
+
+-Andi Kleen, Jul 2004
diff -puN include/asm-x86_64/e820.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/e820.h
--- 25-x86_64/include/asm-x86_64/e820.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/e820.h Fri Dec 10 16:40:04 2004
@@ -26,9 +26,6 @@
#define LOWMEMSIZE() (0x9f000)
-#define MAXMEM (120UL * 1024 * 1024 * 1024 * 1024) /* 120TB */
-
-
#ifndef __ASSEMBLY__
struct e820entry {
u64 addr; /* start of memory segment */
diff -puN include/asm-x86_64/mmu_context.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/mmu_context.h
--- 25-x86_64/include/asm-x86_64/mmu_context.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/mmu_context.h Fri Dec 10 16:40:04 2004
@@ -40,10 +40,7 @@ static inline void switch_mm(struct mm_s
write_pda(active_mm, next);
#endif
set_bit(cpu, &next->cpu_vm_mask);
- /* Re-load page tables */
- *read_pda(level4_pgt) = __pa(next->pgd) | _PAGE_TABLE;
- __flush_tlb();
-
+ asm volatile("movq %0,%%cr3" :: "r" (__pa(next->pml4)) : "memory");
if (unlikely(next->context.ldt != prev->context.ldt))
load_LDT_nolock(&next->context, cpu);
}
diff -puN include/asm-x86_64/page.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/page.h
--- 25-x86_64/include/asm-x86_64/page.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/page.h Fri Dec 10 16:40:04 2004
@@ -67,19 +67,19 @@ extern unsigned long vm_force_exec32;
#define __START_KERNEL 0xffffffff80100000UL
#define __START_KERNEL_map 0xffffffff80000000UL
-#define __PAGE_OFFSET 0x0000010000000000UL /* 1 << 40 */
+#define __PAGE_OFFSET 0xffff810000000000UL
#else
#define __START_KERNEL 0xffffffff80100000
#define __START_KERNEL_map 0xffffffff80000000
-#define __PAGE_OFFSET 0x0000010000000000 /* 1 << 40 */
+#define __PAGE_OFFSET 0xffff810000000000
#endif /* !__ASSEMBLY__ */
/* to align the pointer to the (next) page boundary */
#define PAGE_ALIGN(addr) (((addr)+PAGE_SIZE-1)&PAGE_MASK)
/* See Documentation/x86_64/mm.txt for a description of the memory map. */
-#define __PHYSICAL_MASK_SHIFT 40
+#define __PHYSICAL_MASK_SHIFT 46
#define __PHYSICAL_MASK ((1UL << __PHYSICAL_MASK_SHIFT) - 1)
#define __VIRTUAL_MASK_SHIFT 48
#define __VIRTUAL_MASK ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
diff -puN include/asm-x86_64/pda.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/pda.h
--- 25-x86_64/include/asm-x86_64/pda.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/pda.h Fri Dec 10 16:40:04 2004
@@ -17,7 +17,6 @@ struct x8664_pda {
int irqcount; /* Irq nesting counter. Starts with -1 */
int cpunumber; /* Logical CPU number */
char *irqstackptr; /* top of irqstack */
- unsigned long volatile *level4_pgt; /* Per CPU top level page table */
unsigned int __softirq_pending;
unsigned int __nmi_count; /* number of NMI on this CPUs */
struct mm_struct *active_mm;
diff -puN include/asm-x86_64/pgalloc.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/pgalloc.h
--- 25-x86_64/include/asm-x86_64/pgalloc.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/pgalloc.h Fri Dec 10 16:40:04 2004
@@ -11,6 +11,8 @@
set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)))
#define pgd_populate(mm, pgd, pmd) \
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pmd)))
+#define pml4_populate(mm, pml4, pgd) \
+ set_pml4(pml4, __pml4(_PAGE_TABLE | __pa(pgd)))
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
{
@@ -33,7 +35,7 @@ static inline pmd_t *pmd_alloc_one (stru
return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
}
-static inline pgd_t *pgd_alloc (struct mm_struct *mm)
+static inline pgd_t *pgd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return (pgd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
}
@@ -44,6 +46,31 @@ static inline void pgd_free (pgd_t *pgd)
free_page((unsigned long)pgd);
}
+static inline pml4_t *pml4_alloc(struct mm_struct *mm)
+{
+ unsigned boundary;
+ pml4_t *pml4 = (pml4_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT);
+ if (!pml4)
+ return NULL;
+ /*
+ * Copy kernel pointers in from init.
+ * Could keep a freelist or slab cache of those because the kernel
+ * part never changes.
+ */
+ boundary = pml4_index(__PAGE_OFFSET);
+ memset(pml4, 0, boundary * sizeof(pml4_t));
+ memcpy(pml4 + boundary,
+ init_level4_pgt + boundary,
+ (PTRS_PER_PML4 - boundary) * sizeof(pml4_t));
+ return pml4;
+}
+
+static inline void pml4_free(pml4_t *pml4)
+{
+ BUG_ON((unsigned long)pml4 & (PAGE_SIZE-1));
+ free_page((unsigned long)pml4);
+}
+
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
@@ -73,5 +100,6 @@ extern inline void pte_free(struct page
#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
#define __pmd_free_tlb(tlb,x) pmd_free(x)
+#define __pgd_free_tlb(tlb,x) pgd_free(x)
#endif /* _X86_64_PGALLOC_H */
diff -puN include/asm-x86_64/pgtable.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/pgtable.h
--- 25-x86_64/include/asm-x86_64/pgtable.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/pgtable.h Fri Dec 10 16:41:54 2004
@@ -4,12 +4,6 @@
/*
* This file contains the functions and defines necessary to modify and use
* the x86-64 page table tree.
- *
- * x86-64 has a 4 level table setup. Generic linux MM only supports
- * three levels. The fourth level is currently a single static page that
- * is shared by everybody and just contains a pointer to the current
- * three level page setup on the beginning and some kernel mappings at
- * the end. For more details see Documentation/x86_64/mm.txt
*/
#include <asm/processor.h>
#include <asm/fixmap.h>
@@ -22,10 +16,9 @@ extern pgd_t level3_physmem_pgt[512];
extern pgd_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
extern pml4_t init_level4_pgt[];
-extern pgd_t boot_vmalloc_pgt[];
extern unsigned long __supported_pte_mask;
-#define swapper_pg_dir NULL
+#define swapper_pml4 init_level4_pgt
extern void paging_init(void);
extern void clear_kernel_mapping(unsigned long addr, unsigned long size);
@@ -39,6 +32,9 @@ extern unsigned long pgkern_mask;
extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)];
#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
+/*
+ * 4th top level page
+ */
#define PML4_SHIFT 39
#define PTRS_PER_PML4 512
@@ -66,7 +62,8 @@ extern unsigned long empty_zero_page[PAG
printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e))
#define pgd_ERROR(e) \
printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e))
-
+#define pml4_ERROR(e) \
+ printk("%s:%d: bad pml4 %p(%016lx).\n", __FILE__, __LINE__, &(e), pml4_val(e))
#define pml4_none(x) (!pml4_val(x))
#define pgd_none(x) (!pgd_val(x))
@@ -98,6 +95,11 @@ static inline void set_pml4(pml4_t *dst,
pml4_val(*dst) = pml4_val(val);
}
+extern inline void pml4_clear (pml4_t * pgd)
+{
+ set_pml4(pgd, __pml4(0));
+}
+
#define pgd_page(pgd) \
((unsigned long) __va(pgd_val(pgd) & PHYSICAL_PAGE_MASK))
@@ -111,27 +113,18 @@ static inline void set_pml4(pml4_t *dst,
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE-1))
-#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE)
+#define USER_PTRS_PER_PML4 (TASK_SIZE/PML4_SIZE)
+#define USER_PGDS_IN_LAST_PML4 0
#define FIRST_USER_PGD_NR 0
-#define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
-#define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS)
-
-#define TWOLEVEL_PGDIR_SHIFT 20
-#define BOOT_USER_L4_PTRS 1
-#define BOOT_KERNEL_L4_PTRS 511 /* But we will do it in 4rd level */
-
-
-
#ifndef __ASSEMBLY__
-#define VMALLOC_START 0xffffff0000000000UL
-#define VMALLOC_END 0xffffff7fffffffffUL
-#define MODULES_VADDR 0xffffffffa0000000UL
-#define MODULES_END 0xffffffffafffffffUL
+#define MAXMEM 0x3fffffffffffUL
+#define VMALLOC_START 0xffffc20000000000UL
+#define VMALLOC_END 0xffffe1ffffffffffUL
+#define MODULES_VADDR 0xffffffff88000000
+#define MODULES_END 0xfffffffffff00000
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
-#define IOMAP_START 0xfffffe8000000000UL
-
#define _PAGE_BIT_PRESENT 0
#define _PAGE_BIT_RW 1
#define _PAGE_BIT_USER 2
@@ -222,6 +215,14 @@ static inline unsigned long pgd_bad(pgd_
return val & ~(_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED);
}
+static inline unsigned long pml4_bad(pml4_t pgd)
+{
+ unsigned long val = pml4_val(pgd);
+ val &= ~PTE_MASK;
+ val &= ~(_PAGE_USER | _PAGE_DIRTY);
+ return val & ~(_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED);
+}
+
#define pte_none(x) (!pte_val(x))
#define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE))
#define pte_clear(xp) do { set_pte(xp, __pte(0)); } while (0)
@@ -300,14 +301,14 @@ static inline int pmd_large(pmd_t pte) {
/*
* Level 4 access.
- * Never use these in the common code.
*/
-#define pml4_page(pml4) ((unsigned long) __va(pml4_val(pml4) & PTE_MASK))
-#define pml4_index(address) ((address >> PML4_SHIFT) & (PTRS_PER_PML4-1))
+#define __HAVE_4LEVEL_PAGETABLES 1
+#define pml4_page(pml4) ((unsigned long) __va((unsigned long)pml4_val(pml4) & PTE_MASK))
+#define pml4_index(address) (((address) >> PML4_SHIFT) & (PTRS_PER_PML4-1))
+#define pml4_offset(mm, addr) ((mm)->pml4 + pml4_index(addr))
#define pml4_offset_k(address) (init_level4_pgt + pml4_index(address))
#define pml4_present(pml4) (pml4_val(pml4) & _PAGE_PRESENT)
#define mk_kernel_pml4(address) ((pml4_t){ (address) | _KERNPG_TABLE })
-#define level3_offset_k(dir, address) ((pgd_t *) pml4_page(*(dir)) + pgd_index(address))
/*
* Look up a pgd entry in the gate area. On x86_64, the gate-area resides in
@@ -315,39 +316,17 @@ static inline int pmd_large(pmd_t pte) {
*/
define pgd_offset_gate(mm, addr) pgd_offset_k(addr)
+#define pml4_pgd_offset(pml4, address) ((pgd_t *) pml4_page(*(pml4)) + pgd_index(address))
+#define pml4_pgd_offset_k(pml4, addr) pml4_pgd_offset(pml4, addr)
+
/* PGD - Level3 access */
/* to find an entry in a page-table-directory. */
-#define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
+#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
static inline pgd_t *__pgd_offset_k(pgd_t *pgd, unsigned long address)
{
return pgd + pgd_index(address);
}
-/* Find correct pgd via the hidden fourth level page level: */
-
-/* This accesses the reference page table of the boot cpu.
- Other CPUs get synced lazily via the page fault handler. */
-static inline pgd_t *pgd_offset_k(unsigned long address)
-{
- unsigned long addr;
-
- addr = pml4_val(init_level4_pgt[pml4_index(address)]);
- addr &= PHYSICAL_PAGE_MASK;
- return __pgd_offset_k((pgd_t *)__va(addr), address);
-}
-
-/* Access the pgd of the page table as seen by the current CPU. */
-static inline pgd_t *current_pgd_offset_k(unsigned long address)
-{
- unsigned long addr;
-
- addr = read_pda(level4_pgt)[pml4_index(address)];
- addr &= PHYSICAL_PAGE_MASK;
- return __pgd_offset_k((pgd_t *)__va(addr), address);
-}
-
-#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
-
/* PMD - Level 2 access */
#define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK))
#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
diff -puN include/asm-x86_64/processor.h~4level-architecture-changes-for-x86_64 include/asm-x86_64/processor.h
--- 25-x86_64/include/asm-x86_64/processor.h~4level-architecture-changes-for-x86_64 Fri Dec 10 16:40:04 2004
+++ 25-x86_64-akpm/include/asm-x86_64/processor.h Fri Dec 10 16:40:04 2004
@@ -165,9 +165,9 @@ static inline void clear_in_cr4 (unsigne
/*
- * User space process size: 512GB - 1GB (default).
+ * User space process size. 47bits.
*/
-#define TASK_SIZE (0x0000007fc0000000UL)
+#define TASK_SIZE (0x800000000000)
/* This decides where the kernel will search for a free chunk of vm
* space during mmap's.
_