243 hat_alloc(struct as *as)
244 {
245 hat_t *hat;
246 htable_t *ht; /* top level htable */
247 uint_t use_vlp;
248 uint_t r;
249 hat_kernel_range_t *rp;
250 uintptr_t va;
251 uintptr_t eva;
252 uint_t start;
253 uint_t cnt;
254 htable_t *src;
255
256 /*
257 * Once we start creating user process HATs we can enable
258 * the htable_steal() code.
259 */
260 if (can_steal_post_boot == 0)
261 can_steal_post_boot = 1;
262
263 ASSERT(AS_WRITE_HELD(as, &as->a_lock));
264 hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
265 hat->hat_as = as;
266 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
267 ASSERT(hat->hat_flags == 0);
268
269 #if defined(__xpv)
270 /*
271 * No VLP stuff on the hypervisor due to the 64-bit split top level
272 * page tables. On 32-bit it's not needed as the hypervisor takes
273 * care of copying the top level PTEs to a below 4Gig page.
274 */
275 use_vlp = 0;
276 #else /* __xpv */
277 /* 32 bit processes uses a VLP style hat when running with PAE */
278 #if defined(__amd64)
279 use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
280 #elif defined(__i386)
281 use_vlp = mmu.pae_hat;
282 #endif
283 #endif /* __xpv */
376 mutex_enter(&hat_list_lock);
377 hat->hat_prev = NULL;
378 hat->hat_next = kas.a_hat->hat_next;
379 if (hat->hat_next)
380 hat->hat_next->hat_prev = hat;
381 else
382 kas.a_hat->hat_prev = hat;
383 kas.a_hat->hat_next = hat;
384 mutex_exit(&hat_list_lock);
385
386 return (hat);
387 }
388
389 /*
390 * process has finished executing but as has not been cleaned up yet.
391 */
392 /*ARGSUSED*/
393 void
394 hat_free_start(hat_t *hat)
395 {
396 ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock));
397
398 /*
399 * If the hat is currently a stealing victim, wait for the stealing
400 * to finish. Once we mark it as HAT_FREEING, htable_steal()
401 * won't look at its pagetables anymore.
402 */
403 mutex_enter(&hat_list_lock);
404 while (hat->hat_flags & HAT_VICTIM)
405 cv_wait(&hat_list_cv, &hat_list_lock);
406 hat->hat_flags |= HAT_FREEING;
407 mutex_exit(&hat_list_lock);
408 }
409
410 /*
411 * An address space is being destroyed, so we destroy the associated hat.
412 */
413 void
414 hat_free_end(hat_t *hat)
415 {
416 kmem_cache_t *cache;
709 NULL, 0, 0);
710
711 hat_hash_cache = kmem_cache_create("HatHash",
712 mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
713 NULL, 0, 0);
714
715 /*
716 * VLP hats can use a smaller hash table size on large memroy machines
717 */
718 if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
719 vlp_hash_cache = hat_hash_cache;
720 } else {
721 vlp_hash_cache = kmem_cache_create("HatVlpHash",
722 mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
723 NULL, 0, 0);
724 }
725
726 /*
727 * Set up the kernel's hat
728 */
729 AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
730 kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
731 mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
732 kas.a_hat->hat_as = &kas;
733 kas.a_hat->hat_flags = 0;
734 AS_LOCK_EXIT(&kas, &kas.a_lock);
735
736 CPUSET_ZERO(khat_cpuset);
737 CPUSET_ADD(khat_cpuset, CPU->cpu_id);
738
739 /*
740 * The kernel hat's next pointer serves as the head of the hat list .
741 * The kernel hat's prev pointer tracks the last hat on the list for
742 * htable_steal() to use.
743 */
744 kas.a_hat->hat_next = NULL;
745 kas.a_hat->hat_prev = NULL;
746
747 /*
748 * Allocate an htable hash bucket for the kernel
749 * XX64 - tune for 64 bit procs
750 */
751 kas.a_hat->hat_num_hash = mmu.hash_cnt;
752 kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
753 bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
754
1140 */
1141 void
1142 hat_swapout(hat_t *hat)
1143 {
1144 uintptr_t vaddr = (uintptr_t)0;
1145 uintptr_t eaddr = _userlimit;
1146 htable_t *ht = NULL;
1147 level_t l;
1148
1149 XPV_DISALLOW_MIGRATE();
1150 /*
1151 * We can't just call hat_unload(hat, 0, _userlimit...) here, because
1152 * seg_spt and shared pagetables can't be swapped out.
1153 * Take a look at segspt_shmswapout() - it's a big no-op.
1154 *
1155 * Instead we'll walk through all the address space and unload
1156 * any mappings which we are sure are not shared, not locked.
1157 */
1158 ASSERT(IS_PAGEALIGNED(vaddr));
1159 ASSERT(IS_PAGEALIGNED(eaddr));
1160 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1161 if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1162 eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1163
1164 while (vaddr < eaddr) {
1165 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1166 if (ht == NULL)
1167 break;
1168
1169 ASSERT(!IN_VA_HOLE(vaddr));
1170
1171 /*
1172 * If the page table is shared skip its entire range.
1173 */
1174 l = ht->ht_level;
1175 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1176 vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1);
1177 htable_release(ht);
1178 ht = NULL;
1179 continue;
1180 }
1421 hat_t *hat,
1422 uintptr_t va,
1423 page_t *pp,
1424 uint_t attr,
1425 uint_t flags,
1426 level_t level,
1427 pfn_t pfn)
1428 {
1429 htable_t *ht;
1430 uint_t entry;
1431 x86pte_t pte;
1432 int rv = 0;
1433
1434 /*
1435 * The number 16 is arbitrary and here to catch a recursion problem
1436 * early before we blow out the kernel stack.
1437 */
1438 ++curthread->t_hatdepth;
1439 ASSERT(curthread->t_hatdepth < 16);
1440
1441 ASSERT(hat == kas.a_hat ||
1442 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1443
1444 if (flags & HAT_LOAD_SHARE)
1445 hat->hat_flags |= HAT_SHARED;
1446
1447 /*
1448 * Find the page table that maps this page if it already exists.
1449 */
1450 ht = htable_lookup(hat, va, level);
1451
1452 /*
1453 * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1454 */
1455 if (pp == NULL)
1456 flags |= HAT_LOAD_NOCONSIST;
1457
1458 if (ht == NULL) {
1459 ht = htable_create(hat, va, level, NULL);
1460 ASSERT(ht != NULL);
1461 }
1462 entry = htable_va2entry(va, ht);
1570 */
1571 static uint_t supported_memload_flags =
1572 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1573 HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1574
1575 void
1576 hat_memload(
1577 hat_t *hat,
1578 caddr_t addr,
1579 page_t *pp,
1580 uint_t attr,
1581 uint_t flags)
1582 {
1583 uintptr_t va = (uintptr_t)addr;
1584 level_t level = 0;
1585 pfn_t pfn = page_pptonum(pp);
1586
1587 XPV_DISALLOW_MIGRATE();
1588 ASSERT(IS_PAGEALIGNED(va));
1589 ASSERT(hat == kas.a_hat || va < _userlimit);
1590 ASSERT(hat == kas.a_hat ||
1591 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1592 ASSERT((flags & supported_memload_flags) == flags);
1593
1594 ASSERT(!IN_VA_HOLE(va));
1595 ASSERT(!PP_ISFREE(pp));
1596
1597 /*
1598 * kernel address special case for performance.
1599 */
1600 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1601 ASSERT(hat == kas.a_hat);
1602 hat_kmap_load(addr, pp, attr, flags);
1603 XPV_ALLOW_MIGRATE();
1604 return;
1605 }
1606
1607 /*
1608 * This is used for memory with normal caching enabled, so
1609 * always set HAT_STORECACHING_OK.
1610 */
1611 attr |= HAT_STORECACHING_OK;
1628 void
1629 hat_memload_array(
1630 hat_t *hat,
1631 caddr_t addr,
1632 size_t len,
1633 page_t **pages,
1634 uint_t attr,
1635 uint_t flags)
1636 {
1637 uintptr_t va = (uintptr_t)addr;
1638 uintptr_t eaddr = va + len;
1639 level_t level;
1640 size_t pgsize;
1641 pgcnt_t pgindx = 0;
1642 pfn_t pfn;
1643 pgcnt_t i;
1644
1645 XPV_DISALLOW_MIGRATE();
1646 ASSERT(IS_PAGEALIGNED(va));
1647 ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1648 ASSERT(hat == kas.a_hat ||
1649 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1650 ASSERT((flags & supported_memload_flags) == flags);
1651
1652 /*
1653 * memload is used for memory with full caching enabled, so
1654 * set HAT_STORECACHING_OK.
1655 */
1656 attr |= HAT_STORECACHING_OK;
1657
1658 /*
1659 * handle all pages using largest possible pagesize
1660 */
1661 while (va < eaddr) {
1662 /*
1663 * decide what level mapping to use (ie. pagesize)
1664 */
1665 pfn = page_pptonum(pages[pgindx]);
1666 for (level = mmu.max_page_level; ; --level) {
1667 pgsize = LEVEL_SIZE(level);
1668 if (level == 0)
1669 break;
1764 void
1765 hat_devload(
1766 hat_t *hat,
1767 caddr_t addr,
1768 size_t len,
1769 pfn_t pfn,
1770 uint_t attr,
1771 int flags)
1772 {
1773 uintptr_t va = ALIGN2PAGE(addr);
1774 uintptr_t eva = va + len;
1775 level_t level;
1776 size_t pgsize;
1777 page_t *pp;
1778 int f; /* per PTE copy of flags - maybe modified */
1779 uint_t a; /* per PTE copy of attr */
1780
1781 XPV_DISALLOW_MIGRATE();
1782 ASSERT(IS_PAGEALIGNED(va));
1783 ASSERT(hat == kas.a_hat || eva <= _userlimit);
1784 ASSERT(hat == kas.a_hat ||
1785 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1786 ASSERT((flags & supported_devload_flags) == flags);
1787
1788 /*
1789 * handle all pages
1790 */
1791 while (va < eva) {
1792
1793 /*
1794 * decide what level mapping to use (ie. pagesize)
1795 */
1796 for (level = mmu.max_page_level; ; --level) {
1797 pgsize = LEVEL_SIZE(level);
1798 if (level == 0)
1799 break;
1800 if (IS_P2ALIGNED(va, pgsize) &&
1801 (eva - va) >= pgsize &&
1802 IS_P2ALIGNED(pfn, mmu_btop(pgsize))) {
1803 #ifdef DEBUG
1804 if (level == 2)
1805 map1gcnt++;
1873 */
1874 void
1875 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1876 {
1877 uintptr_t vaddr = (uintptr_t)addr;
1878 uintptr_t eaddr = vaddr + len;
1879 htable_t *ht = NULL;
1880
1881 /*
1882 * kernel entries are always locked, we don't track lock counts
1883 */
1884 ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1885 ASSERT(IS_PAGEALIGNED(vaddr));
1886 ASSERT(IS_PAGEALIGNED(eaddr));
1887 if (hat == kas.a_hat)
1888 return;
1889 if (eaddr > _userlimit)
1890 panic("hat_unlock() address out of range - above _userlimit");
1891
1892 XPV_DISALLOW_MIGRATE();
1893 ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1894 while (vaddr < eaddr) {
1895 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1896 if (ht == NULL)
1897 break;
1898
1899 ASSERT(!IN_VA_HOLE(vaddr));
1900
1901 if (ht->ht_lock_cnt < 1)
1902 panic("hat_unlock(): lock_cnt < 1, "
1903 "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr);
1904 HTABLE_LOCK_DEC(ht);
1905
1906 vaddr += LEVEL_SIZE(ht->ht_level);
1907 }
1908 if (ht)
1909 htable_release(ht);
1910 XPV_ALLOW_MIGRATE();
1911 }
1912
1913 /* ARGSUSED */
2628 /*
2629 * hat_updateattr() applies the given attribute change to an existing mapping
2630 */
2631 #define HAT_LOAD_ATTR 1
2632 #define HAT_SET_ATTR 2
2633 #define HAT_CLR_ATTR 3
2634
2635 static void
2636 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2637 {
2638 uintptr_t vaddr = (uintptr_t)addr;
2639 uintptr_t eaddr = (uintptr_t)addr + len;
2640 htable_t *ht = NULL;
2641 uint_t entry;
2642 x86pte_t oldpte, newpte;
2643 page_t *pp;
2644
2645 XPV_DISALLOW_MIGRATE();
2646 ASSERT(IS_PAGEALIGNED(vaddr));
2647 ASSERT(IS_PAGEALIGNED(eaddr));
2648 ASSERT(hat == kas.a_hat ||
2649 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2650 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2651 try_again:
2652 oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2653 if (ht == NULL)
2654 break;
2655 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2656 continue;
2657
2658 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2659 if (pp == NULL)
2660 continue;
2661 x86_hm_enter(pp);
2662
2663 newpte = oldpte;
2664 /*
2665 * We found a page table entry in the desired range,
2666 * figure out the new attributes.
2667 */
2668 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2669 if ((attr & PROT_WRITE) &&
2839 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2840 htable_release(ht);
2841 XPV_ALLOW_MIGRATE();
2842 return (pfn);
2843 }
2844
2845 /*
2846 * int hat_probe(hat, addr)
2847 * return 0 if no valid mapping is present. Faster version
2848 * of hat_getattr in certain architectures.
2849 */
2850 int
2851 hat_probe(hat_t *hat, caddr_t addr)
2852 {
2853 uintptr_t vaddr = ALIGN2PAGE(addr);
2854 uint_t entry;
2855 htable_t *ht;
2856 pgcnt_t pg_off;
2857
2858 ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2859 ASSERT(hat == kas.a_hat ||
2860 AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2861 if (IN_VA_HOLE(vaddr))
2862 return (0);
2863
2864 /*
2865 * Most common use of hat_probe is from segmap. We special case it
2866 * for performance.
2867 */
2868 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2869 pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2870 if (mmu.pae_hat)
2871 return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2872 else
2873 return (PTE_ISVALID(
2874 ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2875 }
2876
2877 ht = htable_getpage(hat, vaddr, &entry);
2878 htable_release(ht);
2879 return (ht != NULL);
2880 }
|
243 hat_alloc(struct as *as)
244 {
245 hat_t *hat;
246 htable_t *ht; /* top level htable */
247 uint_t use_vlp;
248 uint_t r;
249 hat_kernel_range_t *rp;
250 uintptr_t va;
251 uintptr_t eva;
252 uint_t start;
253 uint_t cnt;
254 htable_t *src;
255
256 /*
257 * Once we start creating user process HATs we can enable
258 * the htable_steal() code.
259 */
260 if (can_steal_post_boot == 0)
261 can_steal_post_boot = 1;
262
263 ASSERT(AS_WRITE_HELD(as));
264 hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
265 hat->hat_as = as;
266 mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
267 ASSERT(hat->hat_flags == 0);
268
269 #if defined(__xpv)
270 /*
271 * No VLP stuff on the hypervisor due to the 64-bit split top level
272 * page tables. On 32-bit it's not needed as the hypervisor takes
273 * care of copying the top level PTEs to a below 4Gig page.
274 */
275 use_vlp = 0;
276 #else /* __xpv */
277 /* 32 bit processes uses a VLP style hat when running with PAE */
278 #if defined(__amd64)
279 use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
280 #elif defined(__i386)
281 use_vlp = mmu.pae_hat;
282 #endif
283 #endif /* __xpv */
376 mutex_enter(&hat_list_lock);
377 hat->hat_prev = NULL;
378 hat->hat_next = kas.a_hat->hat_next;
379 if (hat->hat_next)
380 hat->hat_next->hat_prev = hat;
381 else
382 kas.a_hat->hat_prev = hat;
383 kas.a_hat->hat_next = hat;
384 mutex_exit(&hat_list_lock);
385
386 return (hat);
387 }
388
389 /*
390 * process has finished executing but as has not been cleaned up yet.
391 */
392 /*ARGSUSED*/
393 void
394 hat_free_start(hat_t *hat)
395 {
396 ASSERT(AS_WRITE_HELD(hat->hat_as));
397
398 /*
399 * If the hat is currently a stealing victim, wait for the stealing
400 * to finish. Once we mark it as HAT_FREEING, htable_steal()
401 * won't look at its pagetables anymore.
402 */
403 mutex_enter(&hat_list_lock);
404 while (hat->hat_flags & HAT_VICTIM)
405 cv_wait(&hat_list_cv, &hat_list_lock);
406 hat->hat_flags |= HAT_FREEING;
407 mutex_exit(&hat_list_lock);
408 }
409
410 /*
411 * An address space is being destroyed, so we destroy the associated hat.
412 */
413 void
414 hat_free_end(hat_t *hat)
415 {
416 kmem_cache_t *cache;
709 NULL, 0, 0);
710
711 hat_hash_cache = kmem_cache_create("HatHash",
712 mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
713 NULL, 0, 0);
714
715 /*
716 * VLP hats can use a smaller hash table size on large memroy machines
717 */
718 if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
719 vlp_hash_cache = hat_hash_cache;
720 } else {
721 vlp_hash_cache = kmem_cache_create("HatVlpHash",
722 mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
723 NULL, 0, 0);
724 }
725
726 /*
727 * Set up the kernel's hat
728 */
729 AS_LOCK_ENTER(&kas, RW_WRITER);
730 kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
731 mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
732 kas.a_hat->hat_as = &kas;
733 kas.a_hat->hat_flags = 0;
734 AS_LOCK_EXIT(&kas);
735
736 CPUSET_ZERO(khat_cpuset);
737 CPUSET_ADD(khat_cpuset, CPU->cpu_id);
738
739 /*
740 * The kernel hat's next pointer serves as the head of the hat list .
741 * The kernel hat's prev pointer tracks the last hat on the list for
742 * htable_steal() to use.
743 */
744 kas.a_hat->hat_next = NULL;
745 kas.a_hat->hat_prev = NULL;
746
747 /*
748 * Allocate an htable hash bucket for the kernel
749 * XX64 - tune for 64 bit procs
750 */
751 kas.a_hat->hat_num_hash = mmu.hash_cnt;
752 kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
753 bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
754
1140 */
1141 void
1142 hat_swapout(hat_t *hat)
1143 {
1144 uintptr_t vaddr = (uintptr_t)0;
1145 uintptr_t eaddr = _userlimit;
1146 htable_t *ht = NULL;
1147 level_t l;
1148
1149 XPV_DISALLOW_MIGRATE();
1150 /*
1151 * We can't just call hat_unload(hat, 0, _userlimit...) here, because
1152 * seg_spt and shared pagetables can't be swapped out.
1153 * Take a look at segspt_shmswapout() - it's a big no-op.
1154 *
1155 * Instead we'll walk through all the address space and unload
1156 * any mappings which we are sure are not shared, not locked.
1157 */
1158 ASSERT(IS_PAGEALIGNED(vaddr));
1159 ASSERT(IS_PAGEALIGNED(eaddr));
1160 ASSERT(AS_LOCK_HELD(hat->hat_as));
1161 if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1162 eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1163
1164 while (vaddr < eaddr) {
1165 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1166 if (ht == NULL)
1167 break;
1168
1169 ASSERT(!IN_VA_HOLE(vaddr));
1170
1171 /*
1172 * If the page table is shared skip its entire range.
1173 */
1174 l = ht->ht_level;
1175 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1176 vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1);
1177 htable_release(ht);
1178 ht = NULL;
1179 continue;
1180 }
1421 hat_t *hat,
1422 uintptr_t va,
1423 page_t *pp,
1424 uint_t attr,
1425 uint_t flags,
1426 level_t level,
1427 pfn_t pfn)
1428 {
1429 htable_t *ht;
1430 uint_t entry;
1431 x86pte_t pte;
1432 int rv = 0;
1433
1434 /*
1435 * The number 16 is arbitrary and here to catch a recursion problem
1436 * early before we blow out the kernel stack.
1437 */
1438 ++curthread->t_hatdepth;
1439 ASSERT(curthread->t_hatdepth < 16);
1440
1441 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
1442
1443 if (flags & HAT_LOAD_SHARE)
1444 hat->hat_flags |= HAT_SHARED;
1445
1446 /*
1447 * Find the page table that maps this page if it already exists.
1448 */
1449 ht = htable_lookup(hat, va, level);
1450
1451 /*
1452 * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1453 */
1454 if (pp == NULL)
1455 flags |= HAT_LOAD_NOCONSIST;
1456
1457 if (ht == NULL) {
1458 ht = htable_create(hat, va, level, NULL);
1459 ASSERT(ht != NULL);
1460 }
1461 entry = htable_va2entry(va, ht);
1569 */
1570 static uint_t supported_memload_flags =
1571 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1572 HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1573
1574 void
1575 hat_memload(
1576 hat_t *hat,
1577 caddr_t addr,
1578 page_t *pp,
1579 uint_t attr,
1580 uint_t flags)
1581 {
1582 uintptr_t va = (uintptr_t)addr;
1583 level_t level = 0;
1584 pfn_t pfn = page_pptonum(pp);
1585
1586 XPV_DISALLOW_MIGRATE();
1587 ASSERT(IS_PAGEALIGNED(va));
1588 ASSERT(hat == kas.a_hat || va < _userlimit);
1589 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
1590 ASSERT((flags & supported_memload_flags) == flags);
1591
1592 ASSERT(!IN_VA_HOLE(va));
1593 ASSERT(!PP_ISFREE(pp));
1594
1595 /*
1596 * kernel address special case for performance.
1597 */
1598 if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1599 ASSERT(hat == kas.a_hat);
1600 hat_kmap_load(addr, pp, attr, flags);
1601 XPV_ALLOW_MIGRATE();
1602 return;
1603 }
1604
1605 /*
1606 * This is used for memory with normal caching enabled, so
1607 * always set HAT_STORECACHING_OK.
1608 */
1609 attr |= HAT_STORECACHING_OK;
1626 void
1627 hat_memload_array(
1628 hat_t *hat,
1629 caddr_t addr,
1630 size_t len,
1631 page_t **pages,
1632 uint_t attr,
1633 uint_t flags)
1634 {
1635 uintptr_t va = (uintptr_t)addr;
1636 uintptr_t eaddr = va + len;
1637 level_t level;
1638 size_t pgsize;
1639 pgcnt_t pgindx = 0;
1640 pfn_t pfn;
1641 pgcnt_t i;
1642
1643 XPV_DISALLOW_MIGRATE();
1644 ASSERT(IS_PAGEALIGNED(va));
1645 ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1646 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
1647 ASSERT((flags & supported_memload_flags) == flags);
1648
1649 /*
1650 * memload is used for memory with full caching enabled, so
1651 * set HAT_STORECACHING_OK.
1652 */
1653 attr |= HAT_STORECACHING_OK;
1654
1655 /*
1656 * handle all pages using largest possible pagesize
1657 */
1658 while (va < eaddr) {
1659 /*
1660 * decide what level mapping to use (ie. pagesize)
1661 */
1662 pfn = page_pptonum(pages[pgindx]);
1663 for (level = mmu.max_page_level; ; --level) {
1664 pgsize = LEVEL_SIZE(level);
1665 if (level == 0)
1666 break;
1761 void
1762 hat_devload(
1763 hat_t *hat,
1764 caddr_t addr,
1765 size_t len,
1766 pfn_t pfn,
1767 uint_t attr,
1768 int flags)
1769 {
1770 uintptr_t va = ALIGN2PAGE(addr);
1771 uintptr_t eva = va + len;
1772 level_t level;
1773 size_t pgsize;
1774 page_t *pp;
1775 int f; /* per PTE copy of flags - maybe modified */
1776 uint_t a; /* per PTE copy of attr */
1777
1778 XPV_DISALLOW_MIGRATE();
1779 ASSERT(IS_PAGEALIGNED(va));
1780 ASSERT(hat == kas.a_hat || eva <= _userlimit);
1781 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
1782 ASSERT((flags & supported_devload_flags) == flags);
1783
1784 /*
1785 * handle all pages
1786 */
1787 while (va < eva) {
1788
1789 /*
1790 * decide what level mapping to use (ie. pagesize)
1791 */
1792 for (level = mmu.max_page_level; ; --level) {
1793 pgsize = LEVEL_SIZE(level);
1794 if (level == 0)
1795 break;
1796 if (IS_P2ALIGNED(va, pgsize) &&
1797 (eva - va) >= pgsize &&
1798 IS_P2ALIGNED(pfn, mmu_btop(pgsize))) {
1799 #ifdef DEBUG
1800 if (level == 2)
1801 map1gcnt++;
1869 */
1870 void
1871 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1872 {
1873 uintptr_t vaddr = (uintptr_t)addr;
1874 uintptr_t eaddr = vaddr + len;
1875 htable_t *ht = NULL;
1876
1877 /*
1878 * kernel entries are always locked, we don't track lock counts
1879 */
1880 ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1881 ASSERT(IS_PAGEALIGNED(vaddr));
1882 ASSERT(IS_PAGEALIGNED(eaddr));
1883 if (hat == kas.a_hat)
1884 return;
1885 if (eaddr > _userlimit)
1886 panic("hat_unlock() address out of range - above _userlimit");
1887
1888 XPV_DISALLOW_MIGRATE();
1889 ASSERT(AS_LOCK_HELD(hat->hat_as));
1890 while (vaddr < eaddr) {
1891 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1892 if (ht == NULL)
1893 break;
1894
1895 ASSERT(!IN_VA_HOLE(vaddr));
1896
1897 if (ht->ht_lock_cnt < 1)
1898 panic("hat_unlock(): lock_cnt < 1, "
1899 "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr);
1900 HTABLE_LOCK_DEC(ht);
1901
1902 vaddr += LEVEL_SIZE(ht->ht_level);
1903 }
1904 if (ht)
1905 htable_release(ht);
1906 XPV_ALLOW_MIGRATE();
1907 }
1908
1909 /* ARGSUSED */
2624 /*
2625 * hat_updateattr() applies the given attribute change to an existing mapping
2626 */
2627 #define HAT_LOAD_ATTR 1
2628 #define HAT_SET_ATTR 2
2629 #define HAT_CLR_ATTR 3
2630
2631 static void
2632 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2633 {
2634 uintptr_t vaddr = (uintptr_t)addr;
2635 uintptr_t eaddr = (uintptr_t)addr + len;
2636 htable_t *ht = NULL;
2637 uint_t entry;
2638 x86pte_t oldpte, newpte;
2639 page_t *pp;
2640
2641 XPV_DISALLOW_MIGRATE();
2642 ASSERT(IS_PAGEALIGNED(vaddr));
2643 ASSERT(IS_PAGEALIGNED(eaddr));
2644 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
2645 for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2646 try_again:
2647 oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2648 if (ht == NULL)
2649 break;
2650 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2651 continue;
2652
2653 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2654 if (pp == NULL)
2655 continue;
2656 x86_hm_enter(pp);
2657
2658 newpte = oldpte;
2659 /*
2660 * We found a page table entry in the desired range,
2661 * figure out the new attributes.
2662 */
2663 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2664 if ((attr & PROT_WRITE) &&
2834 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2835 htable_release(ht);
2836 XPV_ALLOW_MIGRATE();
2837 return (pfn);
2838 }
2839
2840 /*
2841 * int hat_probe(hat, addr)
2842 * return 0 if no valid mapping is present. Faster version
2843 * of hat_getattr in certain architectures.
2844 */
2845 int
2846 hat_probe(hat_t *hat, caddr_t addr)
2847 {
2848 uintptr_t vaddr = ALIGN2PAGE(addr);
2849 uint_t entry;
2850 htable_t *ht;
2851 pgcnt_t pg_off;
2852
2853 ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2854 ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));
2855 if (IN_VA_HOLE(vaddr))
2856 return (0);
2857
2858 /*
2859 * Most common use of hat_probe is from segmap. We special case it
2860 * for performance.
2861 */
2862 if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2863 pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2864 if (mmu.pae_hat)
2865 return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2866 else
2867 return (PTE_ISVALID(
2868 ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2869 }
2870
2871 ht = htable_getpage(hat, vaddr, &entry);
2872 htable_release(ht);
2873 return (ht != NULL);
2874 }
|