Print this page
patch as-lock-macro-simplification


 243 hat_alloc(struct as *as)
 244 {
 245         hat_t                   *hat;
 246         htable_t                *ht;    /* top level htable */
 247         uint_t                  use_vlp;
 248         uint_t                  r;
 249         hat_kernel_range_t      *rp;
 250         uintptr_t               va;
 251         uintptr_t               eva;
 252         uint_t                  start;
 253         uint_t                  cnt;
 254         htable_t                *src;
 255 
 256         /*
 257          * Once we start creating user process HATs we can enable
 258          * the htable_steal() code.
 259          */
 260         if (can_steal_post_boot == 0)
 261                 can_steal_post_boot = 1;
 262 
 263         ASSERT(AS_WRITE_HELD(as, &as->a_lock));
 264         hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
 265         hat->hat_as = as;
 266         mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
 267         ASSERT(hat->hat_flags == 0);
 268 
 269 #if defined(__xpv)
 270         /*
 271          * No VLP stuff on the hypervisor due to the 64-bit split top level
 272          * page tables.  On 32-bit it's not needed as the hypervisor takes
 273          * care of copying the top level PTEs to a below 4Gig page.
 274          */
 275         use_vlp = 0;
 276 #else   /* __xpv */
 277         /* 32 bit processes uses a VLP style hat when running with PAE */
 278 #if defined(__amd64)
 279         use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
 280 #elif defined(__i386)
 281         use_vlp = mmu.pae_hat;
 282 #endif
 283 #endif  /* __xpv */


 376         mutex_enter(&hat_list_lock);
 377         hat->hat_prev = NULL;
 378         hat->hat_next = kas.a_hat->hat_next;
 379         if (hat->hat_next)
 380                 hat->hat_next->hat_prev = hat;
 381         else
 382                 kas.a_hat->hat_prev = hat;
 383         kas.a_hat->hat_next = hat;
 384         mutex_exit(&hat_list_lock);
 385 
 386         return (hat);
 387 }
 388 
 389 /*
 390  * process has finished executing but as has not been cleaned up yet.
 391  */
 392 /*ARGSUSED*/
 393 void
 394 hat_free_start(hat_t *hat)
 395 {
 396         ASSERT(AS_WRITE_HELD(hat->hat_as, &hat->hat_as->a_lock));
 397 
 398         /*
 399          * If the hat is currently a stealing victim, wait for the stealing
 400          * to finish.  Once we mark it as HAT_FREEING, htable_steal()
 401          * won't look at its pagetables anymore.
 402          */
 403         mutex_enter(&hat_list_lock);
 404         while (hat->hat_flags & HAT_VICTIM)
 405                 cv_wait(&hat_list_cv, &hat_list_lock);
 406         hat->hat_flags |= HAT_FREEING;
 407         mutex_exit(&hat_list_lock);
 408 }
 409 
 410 /*
 411  * An address space is being destroyed, so we destroy the associated hat.
 412  */
 413 void
 414 hat_free_end(hat_t *hat)
 415 {
 416         kmem_cache_t *cache;


 709             NULL, 0, 0);
 710 
 711         hat_hash_cache = kmem_cache_create("HatHash",
 712             mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
 713             NULL, 0, 0);
 714 
 715         /*
 716          * VLP hats can use a smaller hash table size on large memroy machines
 717          */
 718         if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
 719                 vlp_hash_cache = hat_hash_cache;
 720         } else {
 721                 vlp_hash_cache = kmem_cache_create("HatVlpHash",
 722                     mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
 723                     NULL, 0, 0);
 724         }
 725 
 726         /*
 727          * Set up the kernel's hat
 728          */
 729         AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
 730         kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
 731         mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
 732         kas.a_hat->hat_as = &kas;
 733         kas.a_hat->hat_flags = 0;
 734         AS_LOCK_EXIT(&kas, &kas.a_lock);
 735 
 736         CPUSET_ZERO(khat_cpuset);
 737         CPUSET_ADD(khat_cpuset, CPU->cpu_id);
 738 
 739         /*
 740          * The kernel hat's next pointer serves as the head of the hat list .
 741          * The kernel hat's prev pointer tracks the last hat on the list for
 742          * htable_steal() to use.
 743          */
 744         kas.a_hat->hat_next = NULL;
 745         kas.a_hat->hat_prev = NULL;
 746 
 747         /*
 748          * Allocate an htable hash bucket for the kernel
 749          * XX64 - tune for 64 bit procs
 750          */
 751         kas.a_hat->hat_num_hash = mmu.hash_cnt;
 752         kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
 753         bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
 754 


1140  */
1141 void
1142 hat_swapout(hat_t *hat)
1143 {
1144         uintptr_t       vaddr = (uintptr_t)0;
1145         uintptr_t       eaddr = _userlimit;
1146         htable_t        *ht = NULL;
1147         level_t         l;
1148 
1149         XPV_DISALLOW_MIGRATE();
1150         /*
1151          * We can't just call hat_unload(hat, 0, _userlimit...)  here, because
1152          * seg_spt and shared pagetables can't be swapped out.
1153          * Take a look at segspt_shmswapout() - it's a big no-op.
1154          *
1155          * Instead we'll walk through all the address space and unload
1156          * any mappings which we are sure are not shared, not locked.
1157          */
1158         ASSERT(IS_PAGEALIGNED(vaddr));
1159         ASSERT(IS_PAGEALIGNED(eaddr));
1160         ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1161         if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1162                 eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1163 
1164         while (vaddr < eaddr) {
1165                 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1166                 if (ht == NULL)
1167                         break;
1168 
1169                 ASSERT(!IN_VA_HOLE(vaddr));
1170 
1171                 /*
1172                  * If the page table is shared skip its entire range.
1173                  */
1174                 l = ht->ht_level;
1175                 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1176                         vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1);
1177                         htable_release(ht);
1178                         ht = NULL;
1179                         continue;
1180                 }


1421         hat_t           *hat,
1422         uintptr_t       va,
1423         page_t          *pp,
1424         uint_t          attr,
1425         uint_t          flags,
1426         level_t         level,
1427         pfn_t           pfn)
1428 {
1429         htable_t        *ht;
1430         uint_t          entry;
1431         x86pte_t        pte;
1432         int             rv = 0;
1433 
1434         /*
1435          * The number 16 is arbitrary and here to catch a recursion problem
1436          * early before we blow out the kernel stack.
1437          */
1438         ++curthread->t_hatdepth;
1439         ASSERT(curthread->t_hatdepth < 16);
1440 
1441         ASSERT(hat == kas.a_hat ||
1442             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1443 
1444         if (flags & HAT_LOAD_SHARE)
1445                 hat->hat_flags |= HAT_SHARED;
1446 
1447         /*
1448          * Find the page table that maps this page if it already exists.
1449          */
1450         ht = htable_lookup(hat, va, level);
1451 
1452         /*
1453          * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1454          */
1455         if (pp == NULL)
1456                 flags |= HAT_LOAD_NOCONSIST;
1457 
1458         if (ht == NULL) {
1459                 ht = htable_create(hat, va, level, NULL);
1460                 ASSERT(ht != NULL);
1461         }
1462         entry = htable_va2entry(va, ht);


1570  */
1571 static uint_t supported_memload_flags =
1572         HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1573         HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1574 
1575 void
1576 hat_memload(
1577         hat_t           *hat,
1578         caddr_t         addr,
1579         page_t          *pp,
1580         uint_t          attr,
1581         uint_t          flags)
1582 {
1583         uintptr_t       va = (uintptr_t)addr;
1584         level_t         level = 0;
1585         pfn_t           pfn = page_pptonum(pp);
1586 
1587         XPV_DISALLOW_MIGRATE();
1588         ASSERT(IS_PAGEALIGNED(va));
1589         ASSERT(hat == kas.a_hat || va < _userlimit);
1590         ASSERT(hat == kas.a_hat ||
1591             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1592         ASSERT((flags & supported_memload_flags) == flags);
1593 
1594         ASSERT(!IN_VA_HOLE(va));
1595         ASSERT(!PP_ISFREE(pp));
1596 
1597         /*
1598          * kernel address special case for performance.
1599          */
1600         if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1601                 ASSERT(hat == kas.a_hat);
1602                 hat_kmap_load(addr, pp, attr, flags);
1603                 XPV_ALLOW_MIGRATE();
1604                 return;
1605         }
1606 
1607         /*
1608          * This is used for memory with normal caching enabled, so
1609          * always set HAT_STORECACHING_OK.
1610          */
1611         attr |= HAT_STORECACHING_OK;


1628 void
1629 hat_memload_array(
1630         hat_t           *hat,
1631         caddr_t         addr,
1632         size_t          len,
1633         page_t          **pages,
1634         uint_t          attr,
1635         uint_t          flags)
1636 {
1637         uintptr_t       va = (uintptr_t)addr;
1638         uintptr_t       eaddr = va + len;
1639         level_t         level;
1640         size_t          pgsize;
1641         pgcnt_t         pgindx = 0;
1642         pfn_t           pfn;
1643         pgcnt_t         i;
1644 
1645         XPV_DISALLOW_MIGRATE();
1646         ASSERT(IS_PAGEALIGNED(va));
1647         ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1648         ASSERT(hat == kas.a_hat ||
1649             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1650         ASSERT((flags & supported_memload_flags) == flags);
1651 
1652         /*
1653          * memload is used for memory with full caching enabled, so
1654          * set HAT_STORECACHING_OK.
1655          */
1656         attr |= HAT_STORECACHING_OK;
1657 
1658         /*
1659          * handle all pages using largest possible pagesize
1660          */
1661         while (va < eaddr) {
1662                 /*
1663                  * decide what level mapping to use (ie. pagesize)
1664                  */
1665                 pfn = page_pptonum(pages[pgindx]);
1666                 for (level = mmu.max_page_level; ; --level) {
1667                         pgsize = LEVEL_SIZE(level);
1668                         if (level == 0)
1669                                 break;


1764 void
1765 hat_devload(
1766         hat_t           *hat,
1767         caddr_t         addr,
1768         size_t          len,
1769         pfn_t           pfn,
1770         uint_t          attr,
1771         int             flags)
1772 {
1773         uintptr_t       va = ALIGN2PAGE(addr);
1774         uintptr_t       eva = va + len;
1775         level_t         level;
1776         size_t          pgsize;
1777         page_t          *pp;
1778         int             f;      /* per PTE copy of flags  - maybe modified */
1779         uint_t          a;      /* per PTE copy of attr */
1780 
1781         XPV_DISALLOW_MIGRATE();
1782         ASSERT(IS_PAGEALIGNED(va));
1783         ASSERT(hat == kas.a_hat || eva <= _userlimit);
1784         ASSERT(hat == kas.a_hat ||
1785             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1786         ASSERT((flags & supported_devload_flags) == flags);
1787 
1788         /*
1789          * handle all pages
1790          */
1791         while (va < eva) {
1792 
1793                 /*
1794                  * decide what level mapping to use (ie. pagesize)
1795                  */
1796                 for (level = mmu.max_page_level; ; --level) {
1797                         pgsize = LEVEL_SIZE(level);
1798                         if (level == 0)
1799                                 break;
1800                         if (IS_P2ALIGNED(va, pgsize) &&
1801                             (eva - va) >= pgsize &&
1802                             IS_P2ALIGNED(pfn, mmu_btop(pgsize))) {
1803 #ifdef DEBUG
1804                                 if (level == 2)
1805                                         map1gcnt++;


1873  */
1874 void
1875 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1876 {
1877         uintptr_t       vaddr = (uintptr_t)addr;
1878         uintptr_t       eaddr = vaddr + len;
1879         htable_t        *ht = NULL;
1880 
1881         /*
1882          * kernel entries are always locked, we don't track lock counts
1883          */
1884         ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1885         ASSERT(IS_PAGEALIGNED(vaddr));
1886         ASSERT(IS_PAGEALIGNED(eaddr));
1887         if (hat == kas.a_hat)
1888                 return;
1889         if (eaddr > _userlimit)
1890                 panic("hat_unlock() address out of range - above _userlimit");
1891 
1892         XPV_DISALLOW_MIGRATE();
1893         ASSERT(AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
1894         while (vaddr < eaddr) {
1895                 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1896                 if (ht == NULL)
1897                         break;
1898 
1899                 ASSERT(!IN_VA_HOLE(vaddr));
1900 
1901                 if (ht->ht_lock_cnt < 1)
1902                         panic("hat_unlock(): lock_cnt < 1, "
1903                             "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr);
1904                 HTABLE_LOCK_DEC(ht);
1905 
1906                 vaddr += LEVEL_SIZE(ht->ht_level);
1907         }
1908         if (ht)
1909                 htable_release(ht);
1910         XPV_ALLOW_MIGRATE();
1911 }
1912 
1913 /* ARGSUSED */


2628 /*
2629  * hat_updateattr() applies the given attribute change to an existing mapping
2630  */
2631 #define HAT_LOAD_ATTR           1
2632 #define HAT_SET_ATTR            2
2633 #define HAT_CLR_ATTR            3
2634 
2635 static void
2636 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2637 {
2638         uintptr_t       vaddr = (uintptr_t)addr;
2639         uintptr_t       eaddr = (uintptr_t)addr + len;
2640         htable_t        *ht = NULL;
2641         uint_t          entry;
2642         x86pte_t        oldpte, newpte;
2643         page_t          *pp;
2644 
2645         XPV_DISALLOW_MIGRATE();
2646         ASSERT(IS_PAGEALIGNED(vaddr));
2647         ASSERT(IS_PAGEALIGNED(eaddr));
2648         ASSERT(hat == kas.a_hat ||
2649             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2650         for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2651 try_again:
2652                 oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2653                 if (ht == NULL)
2654                         break;
2655                 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2656                         continue;
2657 
2658                 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2659                 if (pp == NULL)
2660                         continue;
2661                 x86_hm_enter(pp);
2662 
2663                 newpte = oldpte;
2664                 /*
2665                  * We found a page table entry in the desired range,
2666                  * figure out the new attributes.
2667                  */
2668                 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2669                         if ((attr & PROT_WRITE) &&


2839                 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2840         htable_release(ht);
2841         XPV_ALLOW_MIGRATE();
2842         return (pfn);
2843 }
2844 
2845 /*
2846  * int hat_probe(hat, addr)
2847  *      return 0 if no valid mapping is present.  Faster version
2848  *      of hat_getattr in certain architectures.
2849  */
2850 int
2851 hat_probe(hat_t *hat, caddr_t addr)
2852 {
2853         uintptr_t       vaddr = ALIGN2PAGE(addr);
2854         uint_t          entry;
2855         htable_t        *ht;
2856         pgcnt_t         pg_off;
2857 
2858         ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2859         ASSERT(hat == kas.a_hat ||
2860             AS_LOCK_HELD(hat->hat_as, &hat->hat_as->a_lock));
2861         if (IN_VA_HOLE(vaddr))
2862                 return (0);
2863 
2864         /*
2865          * Most common use of hat_probe is from segmap. We special case it
2866          * for performance.
2867          */
2868         if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2869                 pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2870                 if (mmu.pae_hat)
2871                         return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2872                 else
2873                         return (PTE_ISVALID(
2874                             ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2875         }
2876 
2877         ht = htable_getpage(hat, vaddr, &entry);
2878         htable_release(ht);
2879         return (ht != NULL);
2880 }




 243 hat_alloc(struct as *as)
 244 {
 245         hat_t                   *hat;
 246         htable_t                *ht;    /* top level htable */
 247         uint_t                  use_vlp;
 248         uint_t                  r;
 249         hat_kernel_range_t      *rp;
 250         uintptr_t               va;
 251         uintptr_t               eva;
 252         uint_t                  start;
 253         uint_t                  cnt;
 254         htable_t                *src;
 255 
 256         /*
 257          * Once we start creating user process HATs we can enable
 258          * the htable_steal() code.
 259          */
 260         if (can_steal_post_boot == 0)
 261                 can_steal_post_boot = 1;
 262 
 263         ASSERT(AS_WRITE_HELD(as));
 264         hat = kmem_cache_alloc(hat_cache, KM_SLEEP);
 265         hat->hat_as = as;
 266         mutex_init(&hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
 267         ASSERT(hat->hat_flags == 0);
 268 
 269 #if defined(__xpv)
 270         /*
 271          * No VLP stuff on the hypervisor due to the 64-bit split top level
 272          * page tables.  On 32-bit it's not needed as the hypervisor takes
 273          * care of copying the top level PTEs to a below 4Gig page.
 274          */
 275         use_vlp = 0;
 276 #else   /* __xpv */
 277         /* 32 bit processes uses a VLP style hat when running with PAE */
 278 #if defined(__amd64)
 279         use_vlp = (ttoproc(curthread)->p_model == DATAMODEL_ILP32);
 280 #elif defined(__i386)
 281         use_vlp = mmu.pae_hat;
 282 #endif
 283 #endif  /* __xpv */


 376         mutex_enter(&hat_list_lock);
 377         hat->hat_prev = NULL;
 378         hat->hat_next = kas.a_hat->hat_next;
 379         if (hat->hat_next)
 380                 hat->hat_next->hat_prev = hat;
 381         else
 382                 kas.a_hat->hat_prev = hat;
 383         kas.a_hat->hat_next = hat;
 384         mutex_exit(&hat_list_lock);
 385 
 386         return (hat);
 387 }
 388 
 389 /*
 390  * process has finished executing but as has not been cleaned up yet.
 391  */
 392 /*ARGSUSED*/
 393 void
 394 hat_free_start(hat_t *hat)
 395 {
 396         ASSERT(AS_WRITE_HELD(hat->hat_as));
 397 
 398         /*
 399          * If the hat is currently a stealing victim, wait for the stealing
 400          * to finish.  Once we mark it as HAT_FREEING, htable_steal()
 401          * won't look at its pagetables anymore.
 402          */
 403         mutex_enter(&hat_list_lock);
 404         while (hat->hat_flags & HAT_VICTIM)
 405                 cv_wait(&hat_list_cv, &hat_list_lock);
 406         hat->hat_flags |= HAT_FREEING;
 407         mutex_exit(&hat_list_lock);
 408 }
 409 
 410 /*
 411  * An address space is being destroyed, so we destroy the associated hat.
 412  */
 413 void
 414 hat_free_end(hat_t *hat)
 415 {
 416         kmem_cache_t *cache;


 709             NULL, 0, 0);
 710 
 711         hat_hash_cache = kmem_cache_create("HatHash",
 712             mmu.hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
 713             NULL, 0, 0);
 714 
 715         /*
 716          * VLP hats can use a smaller hash table size on large memroy machines
 717          */
 718         if (mmu.hash_cnt == mmu.vlp_hash_cnt) {
 719                 vlp_hash_cache = hat_hash_cache;
 720         } else {
 721                 vlp_hash_cache = kmem_cache_create("HatVlpHash",
 722                     mmu.vlp_hash_cnt * sizeof (htable_t *), 0, NULL, NULL, NULL,
 723                     NULL, 0, 0);
 724         }
 725 
 726         /*
 727          * Set up the kernel's hat
 728          */
 729         AS_LOCK_ENTER(&kas, RW_WRITER);
 730         kas.a_hat = kmem_cache_alloc(hat_cache, KM_NOSLEEP);
 731         mutex_init(&kas.a_hat->hat_mutex, NULL, MUTEX_DEFAULT, NULL);
 732         kas.a_hat->hat_as = &kas;
 733         kas.a_hat->hat_flags = 0;
 734         AS_LOCK_EXIT(&kas);
 735 
 736         CPUSET_ZERO(khat_cpuset);
 737         CPUSET_ADD(khat_cpuset, CPU->cpu_id);
 738 
 739         /*
 740          * The kernel hat's next pointer serves as the head of the hat list .
 741          * The kernel hat's prev pointer tracks the last hat on the list for
 742          * htable_steal() to use.
 743          */
 744         kas.a_hat->hat_next = NULL;
 745         kas.a_hat->hat_prev = NULL;
 746 
 747         /*
 748          * Allocate an htable hash bucket for the kernel
 749          * XX64 - tune for 64 bit procs
 750          */
 751         kas.a_hat->hat_num_hash = mmu.hash_cnt;
 752         kas.a_hat->hat_ht_hash = kmem_cache_alloc(hat_hash_cache, KM_NOSLEEP);
 753         bzero(kas.a_hat->hat_ht_hash, mmu.hash_cnt * sizeof (htable_t *));
 754 


1140  */
1141 void
1142 hat_swapout(hat_t *hat)
1143 {
1144         uintptr_t       vaddr = (uintptr_t)0;
1145         uintptr_t       eaddr = _userlimit;
1146         htable_t        *ht = NULL;
1147         level_t         l;
1148 
1149         XPV_DISALLOW_MIGRATE();
1150         /*
1151          * We can't just call hat_unload(hat, 0, _userlimit...)  here, because
1152          * seg_spt and shared pagetables can't be swapped out.
1153          * Take a look at segspt_shmswapout() - it's a big no-op.
1154          *
1155          * Instead we'll walk through all the address space and unload
1156          * any mappings which we are sure are not shared, not locked.
1157          */
1158         ASSERT(IS_PAGEALIGNED(vaddr));
1159         ASSERT(IS_PAGEALIGNED(eaddr));
1160         ASSERT(AS_LOCK_HELD(hat->hat_as));
1161         if ((uintptr_t)hat->hat_as->a_userlimit < eaddr)
1162                 eaddr = (uintptr_t)hat->hat_as->a_userlimit;
1163 
1164         while (vaddr < eaddr) {
1165                 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1166                 if (ht == NULL)
1167                         break;
1168 
1169                 ASSERT(!IN_VA_HOLE(vaddr));
1170 
1171                 /*
1172                  * If the page table is shared skip its entire range.
1173                  */
1174                 l = ht->ht_level;
1175                 if (ht->ht_flags & HTABLE_SHARED_PFN) {
1176                         vaddr = ht->ht_vaddr + LEVEL_SIZE(l + 1);
1177                         htable_release(ht);
1178                         ht = NULL;
1179                         continue;
1180                 }


1421         hat_t           *hat,
1422         uintptr_t       va,
1423         page_t          *pp,
1424         uint_t          attr,
1425         uint_t          flags,
1426         level_t         level,
1427         pfn_t           pfn)
1428 {
1429         htable_t        *ht;
1430         uint_t          entry;
1431         x86pte_t        pte;
1432         int             rv = 0;
1433 
1434         /*
1435          * The number 16 is arbitrary and here to catch a recursion problem
1436          * early before we blow out the kernel stack.
1437          */
1438         ++curthread->t_hatdepth;
1439         ASSERT(curthread->t_hatdepth < 16);
1440 
1441         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

1442 
1443         if (flags & HAT_LOAD_SHARE)
1444                 hat->hat_flags |= HAT_SHARED;
1445 
1446         /*
1447          * Find the page table that maps this page if it already exists.
1448          */
1449         ht = htable_lookup(hat, va, level);
1450 
1451         /*
1452          * We must have HAT_LOAD_NOCONSIST if page_t is NULL.
1453          */
1454         if (pp == NULL)
1455                 flags |= HAT_LOAD_NOCONSIST;
1456 
1457         if (ht == NULL) {
1458                 ht = htable_create(hat, va, level, NULL);
1459                 ASSERT(ht != NULL);
1460         }
1461         entry = htable_va2entry(va, ht);


1569  */
1570 static uint_t supported_memload_flags =
1571         HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_ADV | HAT_LOAD_NOCONSIST |
1572         HAT_LOAD_SHARE | HAT_NO_KALLOC | HAT_LOAD_REMAP | HAT_LOAD_TEXT;
1573 
1574 void
1575 hat_memload(
1576         hat_t           *hat,
1577         caddr_t         addr,
1578         page_t          *pp,
1579         uint_t          attr,
1580         uint_t          flags)
1581 {
1582         uintptr_t       va = (uintptr_t)addr;
1583         level_t         level = 0;
1584         pfn_t           pfn = page_pptonum(pp);
1585 
1586         XPV_DISALLOW_MIGRATE();
1587         ASSERT(IS_PAGEALIGNED(va));
1588         ASSERT(hat == kas.a_hat || va < _userlimit);
1589         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

1590         ASSERT((flags & supported_memload_flags) == flags);
1591 
1592         ASSERT(!IN_VA_HOLE(va));
1593         ASSERT(!PP_ISFREE(pp));
1594 
1595         /*
1596          * kernel address special case for performance.
1597          */
1598         if (mmu.kmap_addr <= va && va < mmu.kmap_eaddr) {
1599                 ASSERT(hat == kas.a_hat);
1600                 hat_kmap_load(addr, pp, attr, flags);
1601                 XPV_ALLOW_MIGRATE();
1602                 return;
1603         }
1604 
1605         /*
1606          * This is used for memory with normal caching enabled, so
1607          * always set HAT_STORECACHING_OK.
1608          */
1609         attr |= HAT_STORECACHING_OK;


1626 void
1627 hat_memload_array(
1628         hat_t           *hat,
1629         caddr_t         addr,
1630         size_t          len,
1631         page_t          **pages,
1632         uint_t          attr,
1633         uint_t          flags)
1634 {
1635         uintptr_t       va = (uintptr_t)addr;
1636         uintptr_t       eaddr = va + len;
1637         level_t         level;
1638         size_t          pgsize;
1639         pgcnt_t         pgindx = 0;
1640         pfn_t           pfn;
1641         pgcnt_t         i;
1642 
1643         XPV_DISALLOW_MIGRATE();
1644         ASSERT(IS_PAGEALIGNED(va));
1645         ASSERT(hat == kas.a_hat || va + len <= _userlimit);
1646         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

1647         ASSERT((flags & supported_memload_flags) == flags);
1648 
1649         /*
1650          * memload is used for memory with full caching enabled, so
1651          * set HAT_STORECACHING_OK.
1652          */
1653         attr |= HAT_STORECACHING_OK;
1654 
1655         /*
1656          * handle all pages using largest possible pagesize
1657          */
1658         while (va < eaddr) {
1659                 /*
1660                  * decide what level mapping to use (ie. pagesize)
1661                  */
1662                 pfn = page_pptonum(pages[pgindx]);
1663                 for (level = mmu.max_page_level; ; --level) {
1664                         pgsize = LEVEL_SIZE(level);
1665                         if (level == 0)
1666                                 break;


1761 void
1762 hat_devload(
1763         hat_t           *hat,
1764         caddr_t         addr,
1765         size_t          len,
1766         pfn_t           pfn,
1767         uint_t          attr,
1768         int             flags)
1769 {
1770         uintptr_t       va = ALIGN2PAGE(addr);
1771         uintptr_t       eva = va + len;
1772         level_t         level;
1773         size_t          pgsize;
1774         page_t          *pp;
1775         int             f;      /* per PTE copy of flags  - maybe modified */
1776         uint_t          a;      /* per PTE copy of attr */
1777 
1778         XPV_DISALLOW_MIGRATE();
1779         ASSERT(IS_PAGEALIGNED(va));
1780         ASSERT(hat == kas.a_hat || eva <= _userlimit);
1781         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

1782         ASSERT((flags & supported_devload_flags) == flags);
1783 
1784         /*
1785          * handle all pages
1786          */
1787         while (va < eva) {
1788 
1789                 /*
1790                  * decide what level mapping to use (ie. pagesize)
1791                  */
1792                 for (level = mmu.max_page_level; ; --level) {
1793                         pgsize = LEVEL_SIZE(level);
1794                         if (level == 0)
1795                                 break;
1796                         if (IS_P2ALIGNED(va, pgsize) &&
1797                             (eva - va) >= pgsize &&
1798                             IS_P2ALIGNED(pfn, mmu_btop(pgsize))) {
1799 #ifdef DEBUG
1800                                 if (level == 2)
1801                                         map1gcnt++;


1869  */
1870 void
1871 hat_unlock(hat_t *hat, caddr_t addr, size_t len)
1872 {
1873         uintptr_t       vaddr = (uintptr_t)addr;
1874         uintptr_t       eaddr = vaddr + len;
1875         htable_t        *ht = NULL;
1876 
1877         /*
1878          * kernel entries are always locked, we don't track lock counts
1879          */
1880         ASSERT(hat == kas.a_hat || eaddr <= _userlimit);
1881         ASSERT(IS_PAGEALIGNED(vaddr));
1882         ASSERT(IS_PAGEALIGNED(eaddr));
1883         if (hat == kas.a_hat)
1884                 return;
1885         if (eaddr > _userlimit)
1886                 panic("hat_unlock() address out of range - above _userlimit");
1887 
1888         XPV_DISALLOW_MIGRATE();
1889         ASSERT(AS_LOCK_HELD(hat->hat_as));
1890         while (vaddr < eaddr) {
1891                 (void) htable_walk(hat, &ht, &vaddr, eaddr);
1892                 if (ht == NULL)
1893                         break;
1894 
1895                 ASSERT(!IN_VA_HOLE(vaddr));
1896 
1897                 if (ht->ht_lock_cnt < 1)
1898                         panic("hat_unlock(): lock_cnt < 1, "
1899                             "htable=%p, vaddr=%p\n", (void *)ht, (void *)vaddr);
1900                 HTABLE_LOCK_DEC(ht);
1901 
1902                 vaddr += LEVEL_SIZE(ht->ht_level);
1903         }
1904         if (ht)
1905                 htable_release(ht);
1906         XPV_ALLOW_MIGRATE();
1907 }
1908 
1909 /* ARGSUSED */


2624 /*
2625  * hat_updateattr() applies the given attribute change to an existing mapping
2626  */
2627 #define HAT_LOAD_ATTR           1
2628 #define HAT_SET_ATTR            2
2629 #define HAT_CLR_ATTR            3
2630 
2631 static void
2632 hat_updateattr(hat_t *hat, caddr_t addr, size_t len, uint_t attr, int what)
2633 {
2634         uintptr_t       vaddr = (uintptr_t)addr;
2635         uintptr_t       eaddr = (uintptr_t)addr + len;
2636         htable_t        *ht = NULL;
2637         uint_t          entry;
2638         x86pte_t        oldpte, newpte;
2639         page_t          *pp;
2640 
2641         XPV_DISALLOW_MIGRATE();
2642         ASSERT(IS_PAGEALIGNED(vaddr));
2643         ASSERT(IS_PAGEALIGNED(eaddr));
2644         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

2645         for (; vaddr < eaddr; vaddr += LEVEL_SIZE(ht->ht_level)) {
2646 try_again:
2647                 oldpte = htable_walk(hat, &ht, &vaddr, eaddr);
2648                 if (ht == NULL)
2649                         break;
2650                 if (PTE_GET(oldpte, PT_SOFTWARE) >= PT_NOCONSIST)
2651                         continue;
2652 
2653                 pp = page_numtopp_nolock(PTE2PFN(oldpte, ht->ht_level));
2654                 if (pp == NULL)
2655                         continue;
2656                 x86_hm_enter(pp);
2657 
2658                 newpte = oldpte;
2659                 /*
2660                  * We found a page table entry in the desired range,
2661                  * figure out the new attributes.
2662                  */
2663                 if (what == HAT_SET_ATTR || what == HAT_LOAD_ATTR) {
2664                         if ((attr & PROT_WRITE) &&


2834                 pfn += mmu_btop(vaddr & LEVEL_OFFSET(ht->ht_level));
2835         htable_release(ht);
2836         XPV_ALLOW_MIGRATE();
2837         return (pfn);
2838 }
2839 
2840 /*
2841  * int hat_probe(hat, addr)
2842  *      return 0 if no valid mapping is present.  Faster version
2843  *      of hat_getattr in certain architectures.
2844  */
2845 int
2846 hat_probe(hat_t *hat, caddr_t addr)
2847 {
2848         uintptr_t       vaddr = ALIGN2PAGE(addr);
2849         uint_t          entry;
2850         htable_t        *ht;
2851         pgcnt_t         pg_off;
2852 
2853         ASSERT(hat == kas.a_hat || vaddr <= _userlimit);
2854         ASSERT(hat == kas.a_hat || AS_LOCK_HELD(hat->hat_as));

2855         if (IN_VA_HOLE(vaddr))
2856                 return (0);
2857 
2858         /*
2859          * Most common use of hat_probe is from segmap. We special case it
2860          * for performance.
2861          */
2862         if (mmu.kmap_addr <= vaddr && vaddr < mmu.kmap_eaddr) {
2863                 pg_off = mmu_btop(vaddr - mmu.kmap_addr);
2864                 if (mmu.pae_hat)
2865                         return (PTE_ISVALID(mmu.kmap_ptes[pg_off]));
2866                 else
2867                         return (PTE_ISVALID(
2868                             ((x86pte32_t *)mmu.kmap_ptes)[pg_off]));
2869         }
2870 
2871         ht = htable_getpage(hat, vaddr, &entry);
2872         htable_release(ht);
2873         return (ht != NULL);
2874 }