1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 * Copyright 2015 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 */
28
29 #ifndef _ASM_ATOMIC_H
30 #define _ASM_ATOMIC_H
31
32 #include <sys/ccompile.h>
33 #include <sys/types.h>
34
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38
39 #if !defined(__lint) && defined(__GNUC__)
40
41 /* BEGIN CSTYLED */
42 /*
43 * This file contains a number of static inline functions implementing
44 * various atomic variable functions. Note that these are *not* all of the
45 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
46 * possible atomic_* functions are implemented in usr/src/common/atomic in
47 * pure assembly. In the absence of an identically named function in this
48 * header file, any use of the function will result in the compiler emitting
49 * a function call as usual. On the other hand, if an identically named
50 * function exists in this header as a static inline, the compiler will
51 * inline its contents and the linker never sees the symbol reference. We
52 * use this to avoid implementing some of the more complex and less used
53 * functions and instead falling back to function calls. Note that in some
54 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
55 * but not i386.
56 */
57
58 /*
59 * Instruction suffixes for various operand sizes (assuming AMD64)
60 */
61 #define SUF_8 "b"
62 #define SUF_16 "w"
63 #define SUF_32 "l"
64 #define SUF_64 "q"
65
66 #if defined(__amd64)
67 #define SUF_LONG SUF_64
68 #define SUF_PTR SUF_64
69 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
70 #elif defined(__i386)
71 #define SUF_LONG SUF_32
72 #define SUF_PTR SUF_32
73 #define __ATOMIC_OP64(...)
74 #else
75 #error "port me"
76 #endif
77
78 #if defined(__amd64) || defined(__i386)
79
80 #define __ATOMIC_OPXX(fxn, type, op) \
81 extern __GNU_INLINE void \
82 fxn(volatile type *target) \
83 { \
84 __asm__ __volatile__( \
85 "lock; " op " %0" \
86 : "+m" (*target) \
87 : /* no inputs */ \
88 : "cc"); \
89 }
90
91 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
92 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
93 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
94 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
95 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
96 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
97 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
98 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
99
100 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
101 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
102 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
103 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
104 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
105 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
106 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
107 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
108
109 #undef __ATOMIC_OPXX
110
111 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \
112 extern __GNU_INLINE void \
113 fxn(volatile type1 *target, type2 delta) \
114 { \
115 __asm__ __volatile__( \
116 "lock; " op " %1,%0" \
117 : "+m" (*target) \
118 : "i" reg (delta) \
119 : "cc"); \
120 }
121
122 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q")
123 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r")
124 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r")
125 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r")
126 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q")
127 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r")
128 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r")
129 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r")
130
131 /*
132 * We don't use the above macro here because atomic_add_ptr has an
133 * inconsistent type. The first argument should really be a 'volatile void
134 * **'.
135 */
136 extern __GNU_INLINE void
137 atomic_add_ptr(volatile void *target, ssize_t delta)
138 {
139 volatile void **tmp = (volatile void **)target;
140
141 __asm__ __volatile__(
142 "lock; add" SUF_PTR " %1,%0"
143 : "+m" (*tmp)
144 : "ir" (delta)
145 : "cc");
146 }
147
148 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q")
149 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r")
150 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r")
151 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r")
152 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q")
153 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r")
154 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r")
155 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r")
156
157 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q")
158 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r")
159 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r")
160 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r")
161 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q")
162 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r")
163 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r")
164 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r")
165
166 #undef __ATOMIC_OPXX
167
168 #define __ATOMIC_OPXX(fxn, type, op, reg) \
169 extern __GNU_INLINE type \
170 fxn(volatile type *target, type cmp, type new) \
171 { \
172 type ret; \
173 __asm__ __volatile__( \
174 "lock; " op " %2,%0" \
175 : "+m" (*target), "=a" (ret) \
176 : reg (new), "1" (cmp) \
177 : "cc"); \
178 return (ret); \
179 }
180
181 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q")
182 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r")
183 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r")
184 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r")
185 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q")
186 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r")
187 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r")
188 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r")
189
190 #undef __ATOMIC_OPXX
191
192 /*
193 * We don't use the above macro here because atomic_cas_ptr has an
194 * inconsistent type. The first argument should really be a 'volatile void
195 * **'.
196 */
197 extern __GNU_INLINE void *
198 atomic_cas_ptr(volatile void *target, void *cmp, void *new)
199 {
200 volatile void **tmp = (volatile void **)target;
201 void *ret;
202
203 __asm__ __volatile__(
204 "lock; cmpxchg" SUF_PTR " %2,%0"
205 : "+m" (*tmp), "=a" (ret)
206 : "r" (new), "1" (cmp)
207 : "cc");
208
209 return (ret);
210 }
211
212 #define __ATOMIC_OPXX(fxn, type, op, reg) \
213 extern __GNU_INLINE type \
214 fxn(volatile type *target, type val) \
215 { \
216 __asm__ __volatile__( \
217 op " %1,%0" \
218 : "+m" (*target), "+" reg (val)); \
219 return (val); \
220 }
221
222 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q")
223 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r")
224 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r")
225 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r")
226 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q")
227 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r")
228 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r")
229 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r")
230
231 #undef __ATOMIC_OPXX
232
233 /*
234 * We don't use the above macro here because atomic_swap_ptr has an
235 * inconsistent type. The first argument should really be a 'volatile void
236 * **'.
237 */
238 extern __GNU_INLINE void *
239 atomic_swap_ptr(volatile void *target, void *val)
240 {
241 volatile void **tmp = (volatile void **)target;
242
243 __asm__ __volatile__(
244 "xchg" SUF_PTR " %1,%0"
245 : "+m" (*tmp), "+r" (val));
246
247 return (val);
248 }
249
250 #define __ATOMIC_OPXX(fxn, type1, type2, suf, reg) \
251 extern __GNU_INLINE type1 \
252 fxn(volatile type1 *target, type2 delta) \
253 { \
254 type1 orig; \
255 __asm__ __volatile__( \
256 "lock; xadd" suf " %1, %0" \
257 : "+m" (*target), "=" reg (orig) \
258 : "1" (delta) \
259 : "cc"); \
260 return (orig + delta); \
261 }
262
263 __ATOMIC_OPXX(atomic_add_8_nv, uint8_t, int8_t, SUF_8, "q")
264 __ATOMIC_OPXX(atomic_add_16_nv, uint16_t, int16_t, SUF_16, "r")
265 __ATOMIC_OPXX(atomic_add_32_nv, uint32_t, int32_t, SUF_32, "r")
266 __ATOMIC_OP64(atomic_add_64_nv, uint64_t, int64_t, SUF_64, "r")
267 __ATOMIC_OPXX(atomic_add_char_nv, unsigned char, signed char, SUF_8, "q")
268 __ATOMIC_OPXX(atomic_add_short_nv, ushort_t, short, SUF_16, "r")
269 __ATOMIC_OPXX(atomic_add_int_nv, uint_t, int, SUF_32, "r")
270 __ATOMIC_OPXX(atomic_add_long_nv, ulong_t, long, SUF_LONG, "r")
271
272 #undef __ATOMIC_OPXX
273
274 /*
275 * We don't use the above macro here because atomic_add_ptr_nv has an
276 * inconsistent type. The first argument should really be a 'volatile void
277 * **'.
278 */
279 extern __GNU_INLINE void *
280 atomic_add_ptr_nv(volatile void *target, ssize_t delta)
281 {
282 return ((void *)atomic_add_long_nv((volatile ulong_t *)target, delta));
283 }
284
285 #define __ATOMIC_OPXX(fxn, implfxn, type, c) \
286 extern __GNU_INLINE type \
287 fxn(volatile type *target) \
288 { \
289 return (implfxn(target, c)); \
290 }
291
292 __ATOMIC_OPXX(atomic_inc_8_nv, atomic_add_8_nv, uint8_t, 1)
293 __ATOMIC_OPXX(atomic_inc_16_nv, atomic_add_16_nv, uint16_t, 1)
294 __ATOMIC_OPXX(atomic_inc_32_nv, atomic_add_32_nv, uint32_t, 1)
295 __ATOMIC_OP64(atomic_inc_64_nv, atomic_add_64_nv, uint64_t, 1)
296 __ATOMIC_OPXX(atomic_inc_uchar_nv, atomic_add_char_nv, uchar_t, 1)
297 __ATOMIC_OPXX(atomic_inc_ushort_nv, atomic_add_short_nv, ushort_t, 1)
298 __ATOMIC_OPXX(atomic_inc_uint_nv, atomic_add_int_nv, uint_t, 1)
299 __ATOMIC_OPXX(atomic_inc_ulong_nv, atomic_add_long_nv, ulong_t, 1)
300
301 __ATOMIC_OPXX(atomic_dec_8_nv, atomic_add_8_nv, uint8_t, -1)
302 __ATOMIC_OPXX(atomic_dec_16_nv, atomic_add_16_nv, uint16_t, -1)
303 __ATOMIC_OPXX(atomic_dec_32_nv, atomic_add_32_nv, uint32_t, -1)
304 __ATOMIC_OP64(atomic_dec_64_nv, atomic_add_64_nv, uint64_t, -1)
305 __ATOMIC_OPXX(atomic_dec_uchar_nv, atomic_add_char_nv, uchar_t, -1)
306 __ATOMIC_OPXX(atomic_dec_ushort_nv, atomic_add_short_nv, ushort_t, -1)
307 __ATOMIC_OPXX(atomic_dec_uint_nv, atomic_add_int_nv, uint_t, -1)
308 __ATOMIC_OPXX(atomic_dec_ulong_nv, atomic_add_long_nv, ulong_t, -1)
309
310 #undef __ATOMIC_OPXX
311
312 #define __ATOMIC_OPXX(fxn, cas, op, type) \
313 extern __GNU_INLINE type \
314 fxn(volatile type *target, type delta) \
315 { \
316 type old; \
317 do { \
318 old = *target; \
319 } while (cas(target, old, old op delta) != old); \
320 return (old op delta); \
321 }
322
323 __ATOMIC_OPXX(atomic_or_8_nv, atomic_cas_8, |, uint8_t)
324 __ATOMIC_OPXX(atomic_or_16_nv, atomic_cas_16, |, uint16_t)
325 __ATOMIC_OPXX(atomic_or_32_nv, atomic_cas_32, |, uint32_t)
326 __ATOMIC_OP64(atomic_or_64_nv, atomic_cas_64, |, uint64_t)
327 __ATOMIC_OPXX(atomic_or_uchar_nv, atomic_cas_uchar, |, uchar_t)
328 __ATOMIC_OPXX(atomic_or_ushort_nv, atomic_cas_ushort, |, ushort_t)
329 __ATOMIC_OPXX(atomic_or_uint_nv, atomic_cas_uint, |, uint_t)
330 __ATOMIC_OPXX(atomic_or_ulong_nv, atomic_cas_ulong, |, ulong_t)
331
332 __ATOMIC_OPXX(atomic_and_8_nv, atomic_cas_8, &, uint8_t)
333 __ATOMIC_OPXX(atomic_and_16_nv, atomic_cas_16, &, uint16_t)
334 __ATOMIC_OPXX(atomic_and_32_nv, atomic_cas_32, &, uint32_t)
335 __ATOMIC_OP64(atomic_and_64_nv, atomic_cas_64, &, uint64_t)
336 __ATOMIC_OPXX(atomic_and_uchar_nv, atomic_cas_uchar, &, uchar_t)
337 __ATOMIC_OPXX(atomic_and_ushort_nv, atomic_cas_ushort, &, ushort_t)
338 __ATOMIC_OPXX(atomic_and_uint_nv, atomic_cas_uint, &, uint_t)
339 __ATOMIC_OPXX(atomic_and_ulong_nv, atomic_cas_ulong, &, ulong_t)
340
341 #undef __ATOMIC_OPXX
342
343 #else
344 #error "port me"
345 #endif
346
347 #undef SUF_8
348 #undef SUF_16
349 #undef SUF_32
350 #undef SUF_64
351 #undef SUF_LONG
352 #undef SUF_PTR
353
354 #undef __ATOMIC_OP64
355
356 /* END CSTYLED */
357
358 #endif /* !__lint && __GNUC__ */
359
360 #ifdef __cplusplus
361 }
362 #endif
363
364 #endif /* _ASM_ATOMIC_H */