137
138 hrtime_t red_deep_hires;
139 kthread_t *red_deep_thread;
140
141 uint32_t red_nmapped;
142 uint32_t red_closest = UINT_MAX;
143 uint32_t red_ndoubles;
144
145 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */
146 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */
147
148 static struct seg_ops segkp_ops = {
149 SEGKP_BADOP(int), /* dup */
150 SEGKP_BADOP(int), /* unmap */
151 SEGKP_BADOP(void), /* free */
152 segkp_fault,
153 SEGKP_BADOP(faultcode_t), /* faulta */
154 SEGKP_BADOP(int), /* setprot */
155 segkp_checkprot,
156 segkp_kluster,
157 SEGKP_BADOP(size_t), /* swapout */
158 SEGKP_BADOP(int), /* sync */
159 SEGKP_BADOP(size_t), /* incore */
160 SEGKP_BADOP(int), /* lockop */
161 SEGKP_BADOP(int), /* getprot */
162 SEGKP_BADOP(u_offset_t), /* getoffset */
163 SEGKP_BADOP(int), /* gettype */
164 SEGKP_BADOP(int), /* getvp */
165 SEGKP_BADOP(int), /* advise */
166 segkp_dump, /* dump */
167 segkp_pagelock, /* pagelock */
168 SEGKP_BADOP(int), /* setpgsz */
169 segkp_getmemid, /* getmemid */
170 segkp_getpolicy, /* getpolicy */
171 segkp_capable, /* capable */
172 seg_inherit_notsup /* inherit */
173 };
174
175
176 static void
177 segkp_badop(void)
743 }
744 }
745
746 /* If locked, release physical memory reservation */
747 if (kpd->kp_flags & KPD_LOCKED) {
748 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
749 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
750 atomic_add_long(&anon_segkp_pages_locked, -pages);
751 page_unresv(pages);
752 }
753
754 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
755 kmem_free(kpd, sizeof (struct segkp_data));
756 }
757
758 /*
759 * segkp_map_red() will check the current frame pointer against the
760 * stack base. If the amount of stack remaining is questionable
761 * (less than red_minavail), then segkp_map_red() will map in the redzone
762 * and return 1. Otherwise, it will return 0. segkp_map_red() can
763 * _only_ be called when:
764 *
765 * - it is safe to sleep on page_create_va().
766 * - the caller is non-swappable.
767 *
768 * It is up to the caller to remember whether segkp_map_red() successfully
769 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
770 * time. Note that the caller must _remain_ non-swappable until after
771 * calling segkp_unmap_red().
772 *
773 * Currently, this routine is only called from pagefault() (which necessarily
774 * satisfies the above conditions).
775 */
776 #if defined(STACK_GROWTH_DOWN)
777 int
778 segkp_map_red(void)
779 {
780 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
781 #ifndef _LP64
782 caddr_t stkbase;
783 #endif
784
785 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
786
787 /*
788 * Optimize for the common case where we simply return.
789 */
790 if ((curthread->t_red_pp == NULL) &&
791 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
792 return (0);
793
794 #if defined(_LP64)
795 /*
796 * XXX We probably need something better than this.
797 */
798 panic("kernel stack overflow");
799 /*NOTREACHED*/
800 #else /* _LP64 */
801 if (curthread->t_red_pp == NULL) {
802 page_t *red_pp;
803 struct seg kseg;
804
805 caddr_t red_va = (caddr_t)
806 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
867 red_deep_hires = hrestime.tv_nsec;
868 red_deep_thread = curthread;
869 }
870
871 /*
872 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
873 */
874 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
875 return (0);
876 #endif /* _LP64 */
877 }
878
879 void
880 segkp_unmap_red(void)
881 {
882 page_t *pp;
883 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
884 (uintptr_t)PAGEMASK) - PAGESIZE);
885
886 ASSERT(curthread->t_red_pp != NULL);
887 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
888
889 /*
890 * Because we locked the mapping down, we can't simply rely
891 * on page_destroy() to clean everything up; we need to call
892 * hat_unload() to explicitly unlock the mapping resources.
893 */
894 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
895
896 pp = curthread->t_red_pp;
897
898 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
899
900 /*
901 * Need to upgrade the SE_SHARED lock to SE_EXCL.
902 */
903 if (!page_tryupgrade(pp)) {
904 /*
905 * As there is now wait for upgrade, release the
906 * SE_SHARED lock and wait for SE_EXCL.
907 */
1327 struct segkp_data *kpd;
1328 int i;
1329 int stop;
1330
1331 i = stop = SEGKP_HASH(vaddr);
1332 mutex_enter(&segkp_lock);
1333 do {
1334 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1335 kpd = kpd->kp_next) {
1336 if (vaddr >= kpd->kp_base &&
1337 vaddr < kpd->kp_base + kpd->kp_len) {
1338 mutex_exit(&segkp_lock);
1339 return (kpd);
1340 }
1341 }
1342 if (--i < 0)
1343 i = SEGKP_HASHSZ - 1; /* Wrap */
1344 } while (i != stop);
1345 mutex_exit(&segkp_lock);
1346 return (NULL); /* Not found */
1347 }
1348
1349 /*
1350 * returns size of swappable area.
1351 */
1352 size_t
1353 swapsize(caddr_t v)
1354 {
1355 struct segkp_data *kpd;
1356
1357 if ((kpd = segkp_find(segkp, v)) != NULL)
1358 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
1359 else
1360 return (NULL);
1361 }
1362
1363 /*
1364 * Dump out all the active segkp pages
1365 */
1366 static void
1367 segkp_dump(struct seg *seg)
1368 {
1369 int i;
1370 struct segkp_data *kpd;
1371 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1372
1373 for (i = 0; i < SEGKP_HASHSZ; i++) {
1374 for (kpd = kpsd->kpsd_hash[i];
1375 kpd != NULL; kpd = kpd->kp_next) {
1376 pfn_t pfn;
1377 caddr_t addr;
1378 caddr_t eaddr;
1379
1380 addr = kpd->kp_base;
|
137
138 hrtime_t red_deep_hires;
139 kthread_t *red_deep_thread;
140
141 uint32_t red_nmapped;
142 uint32_t red_closest = UINT_MAX;
143 uint32_t red_ndoubles;
144
145 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */
146 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */
147
148 static struct seg_ops segkp_ops = {
149 SEGKP_BADOP(int), /* dup */
150 SEGKP_BADOP(int), /* unmap */
151 SEGKP_BADOP(void), /* free */
152 segkp_fault,
153 SEGKP_BADOP(faultcode_t), /* faulta */
154 SEGKP_BADOP(int), /* setprot */
155 segkp_checkprot,
156 segkp_kluster,
157 SEGKP_BADOP(int), /* sync */
158 SEGKP_BADOP(size_t), /* incore */
159 SEGKP_BADOP(int), /* lockop */
160 SEGKP_BADOP(int), /* getprot */
161 SEGKP_BADOP(u_offset_t), /* getoffset */
162 SEGKP_BADOP(int), /* gettype */
163 SEGKP_BADOP(int), /* getvp */
164 SEGKP_BADOP(int), /* advise */
165 segkp_dump, /* dump */
166 segkp_pagelock, /* pagelock */
167 SEGKP_BADOP(int), /* setpgsz */
168 segkp_getmemid, /* getmemid */
169 segkp_getpolicy, /* getpolicy */
170 segkp_capable, /* capable */
171 seg_inherit_notsup /* inherit */
172 };
173
174
175 static void
176 segkp_badop(void)
742 }
743 }
744
745 /* If locked, release physical memory reservation */
746 if (kpd->kp_flags & KPD_LOCKED) {
747 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
748 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
749 atomic_add_long(&anon_segkp_pages_locked, -pages);
750 page_unresv(pages);
751 }
752
753 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
754 kmem_free(kpd, sizeof (struct segkp_data));
755 }
756
757 /*
758 * segkp_map_red() will check the current frame pointer against the
759 * stack base. If the amount of stack remaining is questionable
760 * (less than red_minavail), then segkp_map_red() will map in the redzone
761 * and return 1. Otherwise, it will return 0. segkp_map_red() can
762 * _only_ be called when it is safe to sleep on page_create_va().
763 *
764 * It is up to the caller to remember whether segkp_map_red() successfully
765 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
766 * time.
767 *
768 * Currently, this routine is only called from pagefault() (which necessarily
769 * satisfies the above conditions).
770 */
771 #if defined(STACK_GROWTH_DOWN)
772 int
773 segkp_map_red(void)
774 {
775 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
776 #ifndef _LP64
777 caddr_t stkbase;
778 #endif
779
780 /*
781 * Optimize for the common case where we simply return.
782 */
783 if ((curthread->t_red_pp == NULL) &&
784 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
785 return (0);
786
787 #if defined(_LP64)
788 /*
789 * XXX We probably need something better than this.
790 */
791 panic("kernel stack overflow");
792 /*NOTREACHED*/
793 #else /* _LP64 */
794 if (curthread->t_red_pp == NULL) {
795 page_t *red_pp;
796 struct seg kseg;
797
798 caddr_t red_va = (caddr_t)
799 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
860 red_deep_hires = hrestime.tv_nsec;
861 red_deep_thread = curthread;
862 }
863
864 /*
865 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
866 */
867 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
868 return (0);
869 #endif /* _LP64 */
870 }
871
872 void
873 segkp_unmap_red(void)
874 {
875 page_t *pp;
876 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
877 (uintptr_t)PAGEMASK) - PAGESIZE);
878
879 ASSERT(curthread->t_red_pp != NULL);
880
881 /*
882 * Because we locked the mapping down, we can't simply rely
883 * on page_destroy() to clean everything up; we need to call
884 * hat_unload() to explicitly unlock the mapping resources.
885 */
886 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
887
888 pp = curthread->t_red_pp;
889
890 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
891
892 /*
893 * Need to upgrade the SE_SHARED lock to SE_EXCL.
894 */
895 if (!page_tryupgrade(pp)) {
896 /*
897 * As there is now wait for upgrade, release the
898 * SE_SHARED lock and wait for SE_EXCL.
899 */
1319 struct segkp_data *kpd;
1320 int i;
1321 int stop;
1322
1323 i = stop = SEGKP_HASH(vaddr);
1324 mutex_enter(&segkp_lock);
1325 do {
1326 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1327 kpd = kpd->kp_next) {
1328 if (vaddr >= kpd->kp_base &&
1329 vaddr < kpd->kp_base + kpd->kp_len) {
1330 mutex_exit(&segkp_lock);
1331 return (kpd);
1332 }
1333 }
1334 if (--i < 0)
1335 i = SEGKP_HASHSZ - 1; /* Wrap */
1336 } while (i != stop);
1337 mutex_exit(&segkp_lock);
1338 return (NULL); /* Not found */
1339 }
1340
1341 /*
1342 * Dump out all the active segkp pages
1343 */
1344 static void
1345 segkp_dump(struct seg *seg)
1346 {
1347 int i;
1348 struct segkp_data *kpd;
1349 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1350
1351 for (i = 0; i < SEGKP_HASHSZ; i++) {
1352 for (kpd = kpsd->kpsd_hash[i];
1353 kpd != NULL; kpd = kpd->kp_next) {
1354 pfn_t pfn;
1355 caddr_t addr;
1356 caddr_t eaddr;
1357
1358 addr = kpd->kp_base;
|