Print this page
6138 don't abuse atomic_cas_*
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/os/memnode.c
+++ new/usr/src/uts/i86pc/os/memnode.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #include <sys/systm.h>
27 27 #include <sys/sysmacros.h>
28 28 #include <sys/bootconf.h>
29 29 #include <sys/atomic.h>
30 30 #include <sys/lgrp.h>
31 31 #include <sys/memlist.h>
32 32 #include <sys/memnode.h>
33 33 #include <sys/platform_module.h>
34 34 #include <vm/vm_dep.h>
35 35
36 36 int max_mem_nodes = 1;
37 37
38 38 struct mem_node_conf mem_node_config[MAX_MEM_NODES];
39 39 int mem_node_pfn_shift;
40 40 /*
41 41 * num_memnodes should be updated atomically and always >=
42 42 * the number of bits in memnodes_mask or the algorithm may fail.
43 43 */
44 44 uint16_t num_memnodes;
45 45 mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
46 46
47 47 /*
48 48 * If set, mem_node_physalign should be a power of two, and
49 49 * should reflect the minimum address alignment of each node.
50 50 */
51 51 uint64_t mem_node_physalign;
52 52
53 53 /*
54 54 * Platform hooks we will need.
55 55 */
56 56
57 57 #pragma weak plat_build_mem_nodes
58 58 #pragma weak plat_slice_add
59 59 #pragma weak plat_slice_del
60 60
61 61 /*
62 62 * Adjust the memnode config after a DR operation.
63 63 *
|
↓ open down ↓ |
63 lines elided |
↑ open up ↑ |
64 64 * It is rather tricky to do these updates since we can't
65 65 * protect the memnode structures with locks, so we must
66 66 * be mindful of the order in which updates and reads to
67 67 * these values can occur.
68 68 */
69 69
70 70 void
71 71 mem_node_add_slice(pfn_t start, pfn_t end)
72 72 {
73 73 int mnode;
74 - mnodeset_t newmask, oldmask;
75 74
76 75 /*
77 76 * DR will pass us the first pfn that is allocatable.
78 77 * We need to round down to get the real start of
79 78 * the slice.
80 79 */
81 80 if (mem_node_physalign) {
82 81 start &= ~(btop(mem_node_physalign) - 1);
83 82 end = roundup(end, btop(mem_node_physalign)) - 1;
84 83 }
85 84
86 85 mnode = PFN_2_MEM_NODE(start);
87 86 ASSERT(mnode >= 0 && mnode < max_mem_nodes);
88 87
89 88 if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
90 89 /*
|
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
91 90 * Add slice to existing node.
92 91 */
93 92 if (start < mem_node_config[mnode].physbase)
94 93 mem_node_config[mnode].physbase = start;
95 94 if (end > mem_node_config[mnode].physmax)
96 95 mem_node_config[mnode].physmax = end;
97 96 } else {
98 97 mem_node_config[mnode].physbase = start;
99 98 mem_node_config[mnode].physmax = end;
100 99 atomic_inc_16(&num_memnodes);
101 - do {
102 - oldmask = memnodes_mask;
103 - newmask = memnodes_mask | (1ull << mnode);
104 - } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) !=
105 - oldmask);
100 + atomic_or_64(&memnodes_mask, 1ull << mnode);
106 101 }
107 102
108 103 /*
109 104 * Inform the common lgrp framework about the new memory
110 105 */
111 106 lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
112 107 }
113 108
114 109 /*
115 110 * Remove a PFN range from a memnode. On some platforms,
116 111 * the memnode will be created with physbase at the first
117 112 * allocatable PFN, but later deleted with the MC slice
118 113 * base address converted to a PFN, in which case we need
119 114 * to assume physbase and up.
120 115 */
121 116 void
122 117 mem_node_del_slice(pfn_t start, pfn_t end)
123 118 {
124 119 int mnode;
125 120 pgcnt_t delta_pgcnt, node_size;
126 - mnodeset_t omask, nmask;
127 121
128 122 if (mem_node_physalign) {
129 123 start &= ~(btop(mem_node_physalign) - 1);
130 124 end = roundup(end, btop(mem_node_physalign)) - 1;
131 125 }
132 126 mnode = PFN_2_MEM_NODE(start);
133 127
134 128 ASSERT(mnode >= 0 && mnode < max_mem_nodes);
135 129 ASSERT(mem_node_config[mnode].exists == 1);
136 130
137 131 delta_pgcnt = end - start;
138 132 node_size = mem_node_config[mnode].physmax -
139 133 mem_node_config[mnode].physbase;
140 134
141 135 if (node_size > delta_pgcnt) {
142 136 /*
143 137 * Subtract the slice from the memnode.
144 138 */
145 139 if (start <= mem_node_config[mnode].physbase)
146 140 mem_node_config[mnode].physbase = end + 1;
147 141 ASSERT(end <= mem_node_config[mnode].physmax);
148 142 if (end == mem_node_config[mnode].physmax)
149 143 mem_node_config[mnode].physmax = start - 1;
150 144 } else {
151 145 /*
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
152 146 * Let the common lgrp framework know this mnode is
153 147 * leaving
154 148 */
155 149 lgrp_config(LGRP_CONFIG_MEM_DEL,
156 150 mnode, MEM_NODE_2_LGRPHAND(mnode));
157 151
158 152 /*
159 153 * Delete the whole node.
160 154 */
161 155 ASSERT(MNODE_PGCNT(mnode) == 0);
162 - do {
163 - omask = memnodes_mask;
164 - nmask = omask & ~(1ull << mnode);
165 - } while (atomic_cas_64(&memnodes_mask, omask, nmask) != omask);
156 + atomic_and_64(&memnodes_mask, ~(1ull << mnode));
166 157 atomic_dec_16(&num_memnodes);
167 158 mem_node_config[mnode].exists = 0;
168 159 }
169 160 }
170 161
171 162 void
172 163 mem_node_add_range(pfn_t start, pfn_t end)
173 164 {
174 165 if (&plat_slice_add)
175 166 plat_slice_add(start, end);
176 167 else
177 168 mem_node_add_slice(start, end);
178 169 }
179 170
180 171 void
181 172 mem_node_del_range(pfn_t start, pfn_t end)
182 173 {
183 174 if (&plat_slice_del)
184 175 plat_slice_del(start, end);
185 176 else
186 177 mem_node_del_slice(start, end);
187 178 }
188 179
189 180 void
190 181 startup_build_mem_nodes(struct memlist *list)
191 182 {
192 183 pfn_t start, end;
193 184
194 185 /* LINTED: ASSERT will always true or false */
195 186 ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
196 187
197 188 if (&plat_build_mem_nodes) {
198 189 plat_build_mem_nodes(list);
199 190 } else {
200 191 /*
201 192 * Boot install lists are arranged <addr, len>, ...
202 193 */
203 194 while (list) {
204 195 start = list->ml_address >> PAGESHIFT;
205 196 if (start > physmax)
206 197 continue;
207 198 end =
208 199 (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
209 200 if (end > physmax)
210 201 end = physmax;
211 202 mem_node_add_range(start, end);
212 203 list = list->ml_next;
213 204 }
214 205 mem_node_physalign = 0;
215 206 mem_node_pfn_shift = 0;
|
↓ open down ↓ |
40 lines elided |
↑ open up ↑ |
216 207 }
217 208 }
218 209
219 210 /*
220 211 * Allocate an unassigned memnode.
221 212 */
222 213 int
223 214 mem_node_alloc()
224 215 {
225 216 int mnode;
226 - mnodeset_t newmask, oldmask;
227 217
228 218 /*
229 219 * Find an unused memnode. Update it atomically to prevent
230 220 * a first time memnode creation race.
231 221 */
232 222 for (mnode = 0; mnode < max_mem_nodes; mnode++)
233 223 if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists,
234 224 0, 1) == 0)
235 225 break;
236 226
237 227 if (mnode >= max_mem_nodes)
238 228 panic("Out of free memnodes\n");
239 229
240 230 mem_node_config[mnode].physbase = (pfn_t)-1l;
241 231 mem_node_config[mnode].physmax = 0;
242 232 atomic_inc_16(&num_memnodes);
243 - do {
244 - oldmask = memnodes_mask;
245 - newmask = memnodes_mask | (1ull << mnode);
246 - } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask);
233 + atomic_or_64(&memnodes_mask, 1ull << mnode);
247 234
248 235 return (mnode);
249 236 }
250 237
251 238 /*
252 239 * Find the intersection between a memnode and a memlist
253 240 * and returns the number of pages that overlap.
254 241 *
255 242 * Assumes the list is protected from DR operations by
256 243 * the memlist lock.
257 244 */
258 245 pgcnt_t
259 246 mem_node_memlist_pages(int mnode, struct memlist *mlist)
260 247 {
261 248 pfn_t base, end;
262 249 pfn_t cur_base, cur_end;
263 250 pgcnt_t npgs;
264 251 struct memlist *pmem;
265 252
266 253 base = mem_node_config[mnode].physbase;
267 254 end = mem_node_config[mnode].physmax;
268 255 npgs = 0;
269 256
270 257 memlist_read_lock();
271 258
272 259 for (pmem = mlist; pmem; pmem = pmem->ml_next) {
273 260 cur_base = btop(pmem->ml_address);
274 261 cur_end = cur_base + btop(pmem->ml_size) - 1;
275 262 if (end < cur_base || base > cur_end)
276 263 continue;
277 264 npgs = npgs + (MIN(cur_end, end) -
278 265 MAX(cur_base, base)) + 1;
279 266 }
280 267
281 268 memlist_read_unlock();
282 269
283 270 return (npgs);
284 271 }
|
↓ open down ↓ |
28 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX