/* $NetBSD: pmap.h,v 1.67 2015/06/11 15:50:17 matt Exp $ */ /* * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)pmap.h 8.1 (Berkeley) 6/10/93 */ /* * Copyright (c) 1987 Carnegie-Mellon University * * This code is derived from software contributed to Berkeley by * Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)pmap.h 8.1 (Berkeley) 6/10/93 */ #ifndef _MIPS_PMAP_H_ #define _MIPS_PMAP_H_ #ifdef _KERNEL_OPT #include "opt_multiprocessor.h" #endif #include #include #include /* for KSEG0 below */ /* * The user address space is 2Gb (0x0 - 0x80000000). * User programs are laid out in memory as follows: * address * USRTEXT 0x00001000 * USRDATA USRTEXT + text_size * USRSTACK 0x7FFFFFFF * * The user address space is mapped using a two level structure where * virtual address bits 30..22 are used to index into a segment table which * points to a page worth of PTEs (4096 page can hold 1024 PTEs). * Bits 21..12 are then used to index a PTE which describes a page within * a segment. * * The wired entries in the TLB will contain the following: * 0-1 (UPAGES) for curproc user struct and kernel stack. * * Note: The kernel doesn't use the same data structures as user programs. * All the PTE entries are stored in a single array in Sysmap which is * dynamically allocated at boot time. */ #define pmap_trunc_seg(x) ((vaddr_t)(x) & ~SEGOFSET) #define pmap_round_seg(x) (((vaddr_t)(x) + SEGOFSET) & ~SEGOFSET) #define PMAP_INVALID_SEGTAB NULL #ifdef _LP64 #define PMAP_SEGTABSIZE NSEGPG #else #define PMAP_SEGTABSIZE (1 << (31 - SEGSHIFT)) #endif union pt_entry; union pmap_segtab { #ifdef _LP64 union pmap_segtab *seg_seg[PMAP_SEGTABSIZE]; #else union pmap_segtab *seg_seg[1]; #endif union pt_entry *seg_tab[PMAP_SEGTABSIZE]; }; typedef union pmap_segtab pmap_segtab_t; /* * Structure defining an tlb entry data set. */ struct tlb { vaddr_t tlb_hi; /* should be 64 bits */ uint32_t tlb_lo0; /* XXX maybe 64 bits (only 32 really used) */ uint32_t tlb_lo1; /* XXX maybe 64 bits (only 32 really used) */ }; struct tlbmask { vaddr_t tlb_hi; /* should be 64 bits */ uint32_t tlb_lo0; /* XXX maybe 64 bits (only 32 really used) */ uint32_t tlb_lo1; /* XXX maybe 64 bits (only 32 really used) */ uint32_t tlb_mask; }; #ifdef _KERNEL struct pmap; typedef bool (*pte_callback_t)(struct pmap *, vaddr_t, vaddr_t, union pt_entry *, uintptr_t); union pt_entry *pmap_pte_lookup(struct pmap *, vaddr_t); union pt_entry *pmap_pte_reserve(struct pmap *, vaddr_t, int); void pmap_pte_process(struct pmap *, vaddr_t, vaddr_t, pte_callback_t, uintptr_t); void pmap_segtab_activate(struct pmap *, struct lwp *); void pmap_segtab_init(struct pmap *); void pmap_segtab_destroy(struct pmap *, pte_callback_t, uintptr_t); extern kmutex_t pmap_segtab_lock; #endif /* _KERNEL */ /* * Per TLB (normally same as CPU) asid info */ struct pmap_asid_info { LIST_ENTRY(pmap_asid_info) pai_link; uint32_t pai_asid; /* TLB address space tag */ }; #define TLBINFO_LOCK(ti) mutex_spin_enter((ti)->ti_lock) #define TLBINFO_UNLOCK(ti) mutex_spin_exit((ti)->ti_lock) #define PMAP_PAI_ASIDVALID_P(pai, ti) ((pai)->pai_asid != 0) #define PMAP_PAI(pmap, ti) (&(pmap)->pm_pai[tlbinfo_index(ti)]) #define PAI_PMAP(pai, ti) \ ((pmap_t)((intptr_t)(pai) \ - offsetof(struct pmap, pm_pai[tlbinfo_index(ti)]))) /* * Machine dependent pmap structure. */ struct pmap { #ifdef MULTIPROCESSOR kcpuset_t *pm_active; /* pmap was active on ... */ kcpuset_t *pm_onproc; /* pmap is active on ... */ volatile u_int pm_shootdown_pending; #endif pmap_segtab_t *pm_segtab; /* pointers to pages of PTEs */ u_int pm_count; /* pmap reference count */ u_int pm_flags; #define PMAP_DEFERRED_ACTIVATE 0x0001 struct pmap_statistics pm_stats; /* pmap statistics */ vaddr_t pm_minaddr; vaddr_t pm_maxaddr; struct pmap_asid_info pm_pai[1]; }; enum tlb_invalidate_op { TLBINV_NOBODY=0, TLBINV_ONE=1, TLBINV_ALLUSER=2, TLBINV_ALLKERNEL=3, TLBINV_ALL=4 }; struct pmap_tlb_info { char ti_name[8]; uint32_t ti_asid_hint; /* probable next ASID to use */ uint32_t ti_asids_free; /* # of ASIDs free */ #define tlbinfo_noasids_p(ti) ((ti)->ti_asids_free == 0) kmutex_t *ti_lock; u_int ti_wired; /* # of wired TLB entries */ uint32_t ti_asid_mask; uint32_t ti_asid_max; LIST_HEAD(, pmap_asid_info) ti_pais; /* list of active ASIDs */ #ifdef MULTIPROCESSOR pmap_t ti_victim; uint32_t ti_synci_page_bitmap; /* page indices needing a syncicache */ kcpuset_t *ti_kcpuset; /* bitmask of CPUs sharing this TLB */ enum tlb_invalidate_op ti_tlbinvop; u_int ti_index; #define tlbinfo_index(ti) ((ti)->ti_index) struct evcnt ti_evcnt_synci_asts; struct evcnt ti_evcnt_synci_all; struct evcnt ti_evcnt_synci_pages; struct evcnt ti_evcnt_synci_deferred; struct evcnt ti_evcnt_synci_desired; struct evcnt ti_evcnt_synci_duplicate; #else #define tlbinfo_index(ti) (0) #endif struct evcnt ti_evcnt_asid_reinits; u_long ti_asid_bitmap[256 / (sizeof(u_long) * 8)]; }; #ifdef _KERNEL struct pmap_kernel { struct pmap kernel_pmap; #ifdef MULTIPROCESSOR struct pmap_asid_info kernel_pai[MAXCPUS-1]; #endif }; struct pmap_limits { paddr_t avail_start; paddr_t avail_end; vaddr_t virtual_start; vaddr_t virtual_end; }; extern struct pmap_kernel kernel_pmap_store; extern struct pmap_tlb_info pmap_tlb0_info; extern struct pmap_limits pmap_limits; #ifdef MULTIPROCESSOR extern struct pmap_tlb_info *pmap_tlbs[MAXCPUS]; extern u_int pmap_ntlbs; #endif #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) #define pmap_phys_address(x) mips_ptob(x) /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(void); void pmap_remove_all(pmap_t); void pmap_set_modified(paddr_t); void pmap_procwr(struct proc *, vaddr_t, size_t); #define PMAP_NEED_PROCWR #ifdef MULTIPROCESSOR void pmap_tlb_shootdown_process(void); bool pmap_tlb_shootdown_bystanders(pmap_t pmap); void pmap_tlb_info_attach(struct pmap_tlb_info *, struct cpu_info *); void pmap_tlb_syncicache_ast(struct cpu_info *); void pmap_tlb_syncicache_wanted(struct cpu_info *); void pmap_tlb_syncicache(vaddr_t, const kcpuset_t *); #endif void pmap_tlb_info_init(struct pmap_tlb_info *); void pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *); void pmap_tlb_asid_acquire(pmap_t pmap, struct lwp *l); void pmap_tlb_asid_deactivate(pmap_t pmap); void pmap_tlb_asid_check(void); void pmap_tlb_asid_release_all(pmap_t pmap); int pmap_tlb_update_addr(pmap_t pmap, vaddr_t, uint32_t, bool); void pmap_tlb_invalidate_addr(pmap_t pmap, vaddr_t); /* * pmap_prefer() helps reduce virtual-coherency exceptions in * the virtually-indexed cache on mips3 CPUs. */ #ifdef MIPS3_PLUS #define PMAP_PREFER(pa, va, sz, td) pmap_prefer((pa), (va), (sz), (td)) void pmap_prefer(vaddr_t, vaddr_t *, vsize_t, int); #endif /* MIPS3_PLUS */ #define PMAP_STEAL_MEMORY /* enable pmap_steal_memory() */ #define PMAP_ENABLE_PMAP_KMPAGE /* enable the PMAP_KMPAGE flag */ bool pmap_md_direct_mapped_vaddr_p(vaddr_t); /* * Alternate mapping hooks for pool pages. Avoids thrashing the TLB. */ vaddr_t mips_pmap_map_poolpage(paddr_t); paddr_t mips_pmap_unmap_poolpage(vaddr_t); struct vm_page *mips_pmap_alloc_poolpage(int); #define PMAP_ALLOC_POOLPAGE(flags) mips_pmap_alloc_poolpage(flags) #define PMAP_MAP_POOLPAGE(pa) mips_pmap_map_poolpage(pa) #define PMAP_UNMAP_POOLPAGE(va) mips_pmap_unmap_poolpage(va) /* * Other hooks for the pool allocator. */ #ifdef _LP64 #define POOL_VTOPHYS(va) (MIPS_KSEG0_P(va) \ ? MIPS_KSEG0_TO_PHYS(va) \ : MIPS_XKPHYS_TO_PHYS(va)) #define POOL_PHYSTOV(pa) MIPS_PHYS_TO_XKPHYS_CACHED((paddr_t)(pa)) #else #define POOL_VTOPHYS(va) MIPS_KSEG0_TO_PHYS((vaddr_t)(va)) #define POOL_PHYSTOV(pa) MIPS_PHYS_TO_KSEG0_TO_PHYS((paddr_t)(pa)) #endif /* * Select CCA to use for unmanaged pages. */ #define PMAP_CCA_FOR_PA(pa) CCA_UNCACHED /* uncached */ #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64) #define PGC_NOCACHE 0x4000000000000000ULL #define PGC_PREFETCH 0x2000000000000000ULL #endif #define __HAVE_VM_PAGE_MD /* * pmap-specific data stored in the vm_page structure. */ /* * For each struct vm_page, there is a list of all currently valid virtual * mappings of that page. An entry is a pv_entry_t, the list is pv_table. * XXX really should do this as a part of the higher level code. */ typedef struct pv_entry { struct pv_entry *pv_next; /* next pv_entry */ struct pmap *pv_pmap; /* pmap where mapping lies */ vaddr_t pv_va; /* virtual address for mapping */ #define PV_KENTER 0x001 } *pv_entry_t; #define PG_MD_UNCACHED 0x0001 /* page is mapped uncached */ #define PG_MD_MODIFIED 0x0002 /* page has been modified */ #define PG_MD_REFERENCED 0x0004 /* page has been recently referenced */ #define PG_MD_POOLPAGE 0x0008 /* page is used as a poolpage */ #define PG_MD_EXECPAGE 0x0010 /* page is exec mapped */ #define PG_MD_CACHED_P(md) (((md)->pvh_attrs & PG_MD_UNCACHED) == 0) #define PG_MD_UNCACHED_P(md) (((md)->pvh_attrs & PG_MD_UNCACHED) != 0) #define PG_MD_MODIFIED_P(md) (((md)->pvh_attrs & PG_MD_MODIFIED) != 0) #define PG_MD_REFERENCED_P(md) (((md)->pvh_attrs & PG_MD_REFERENCED) != 0) #define PG_MD_POOLPAGE_P(md) (((md)->pvh_attrs & PG_MD_POOLPAGE) != 0) #define PG_MD_EXECPAGE_P(md) (((md)->pvh_attrs & PG_MD_EXECPAGE) != 0) struct vm_page_md { struct pv_entry pvh_first; /* pv_entry first */ #ifdef MULTIPROCESSOR volatile u_int pvh_attrs; /* page attributes */ kmutex_t *pvh_lock; /* pv list lock */ #define PG_MD_PVLIST_LOCK_INIT(md) ((md)->pvh_lock = NULL) #define PG_MD_PVLIST_LOCKED_P(md) (mutex_owner((md)->pvh_lock) != 0) #define PG_MD_PVLIST_LOCK(md, lc) pmap_pvlist_lock((md), (lc)) #define PG_MD_PVLIST_UNLOCK(md) mutex_spin_exit((md)->pvh_lock) #define PG_MD_PVLIST_GEN(md) ((uint16_t)((md)->pvh_attrs >> 16)) #else u_int pvh_attrs; /* page attributes */ #define PG_MD_PVLIST_LOCK_INIT(md) do { } while (/*CONSTCOND*/ 0) #define PG_MD_PVLIST_LOCKED_P(md) true #define PG_MD_PVLIST_LOCK(md, lc) (mutex_spin_enter(&pmap_pvlist_mutex), 0) #define PG_MD_PVLIST_UNLOCK(md) mutex_spin_exit(&pmap_pvlist_mutex) #define PG_MD_PVLIST_GEN(md) (0) #endif }; #define VM_MDPAGE_INIT(pg) \ do { \ struct vm_page_md * const md = VM_PAGE_TO_MD(pg); \ (md)->pvh_first.pv_next = NULL; \ (md)->pvh_first.pv_pmap = NULL; \ (md)->pvh_first.pv_va = VM_PAGE_TO_PHYS(pg); \ PG_MD_PVLIST_LOCK_INIT(md); \ (md)->pvh_attrs = 0; \ } while (/* CONSTCOND */ 0) uint16_t pmap_pvlist_lock(struct vm_page_md *, bool); #endif /* _KERNEL */ #endif /* _MIPS_PMAP_H_ */