Print this page
5044 define static inlines for most often used atomic functions


  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #ifndef _ASM_ATOMIC_H
  28 #define _ASM_ATOMIC_H
  29 
  30 #include <sys/ccompile.h>
  31 #include <sys/types.h>
  32 
  33 #ifdef  __cplusplus
  34 extern "C" {
  35 #endif
  36 
  37 #if !defined(__lint) && defined(__GNUC__)
  38 
  39 #if defined(__amd64)






















  40 




  41 #elif defined(__i386)














































































































































































  42 
  43 #else
  44 #error  "port me"
  45 #endif
  46 









  47 #endif /* !__lint && __GNUC__ */
  48 
  49 #ifdef __cplusplus
  50 }
  51 #endif
  52 
  53 #endif  /* _ASM_ATOMIC_H */


  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #ifndef _ASM_ATOMIC_H
  28 #define _ASM_ATOMIC_H
  29 
  30 #include <sys/ccompile.h>
  31 #include <sys/types.h>
  32 
  33 #ifdef  __cplusplus
  34 extern "C" {
  35 #endif
  36 
  37 #if !defined(__lint) && defined(__GNUC__)
  38 
  39 /*
  40  * This file contains a number of static inline functions implementing
  41  * various atomic variable functions.  Note that these are *not* all of the
  42  * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h.  All
  43  * possible atomic_* functions are implemented in usr/src/common/atomic in
  44  * pure assembly.  In the absence of an identically named function in this
  45  * header file, any use of the function will result in the compiler emitting
  46  * a function call as usual.  On the other hand, if an identically named
  47  * function exists in this header as a static inline, the compiler will
  48  * inline its contents and the linker never sees the symbol reference.  We
  49  * use this to avoid implementing some of the more complex and less used
  50  * functions and instead falling back to function calls.  Note that in some
  51  * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
  52  * but not i386.
  53  */
  54 
  55 /*
  56  * Instruction suffixes for various operand sizes (assuming AMD64)
  57  */
  58 #define SUF_8           "b"
  59 #define SUF_16          "w"
  60 #define SUF_32          "l"
  61 #define SUF_64          "q"
  62 
  63 #if defined(__amd64)
  64 #define SUF_LONG        SUF_64
  65 #define SUF_PTR         SUF_64
  66 #define __ATOMIC_OP64(...)      __ATOMIC_OPXX(__VA_ARGS__)
  67 #elif defined(__i386)
  68 #define SUF_LONG        SUF_32
  69 #define SUF_PTR         SUF_32
  70 #define __ATOMIC_OP64(...)
  71 #else
  72 #error "port me"
  73 #endif
  74 
  75 #if defined(__amd64) || defined(__i386)
  76 
  77 #define __ATOMIC_OPXX(fxn, type, op)                                    \
  78 extern __GNU_INLINE void                                                \
  79 fxn(volatile type *target)                                              \
  80 {                                                                       \
  81         __asm__ __volatile__(                                           \
  82             "lock; " op " %0"                                           \
  83             : "+m" (*target));                                          \
  84 }
  85 
  86 __ATOMIC_OPXX(atomic_inc_8,      uint8_t,  "inc" SUF_8)
  87 __ATOMIC_OPXX(atomic_inc_16,     uint16_t, "inc" SUF_16)
  88 __ATOMIC_OPXX(atomic_inc_32,     uint32_t, "inc" SUF_32)
  89 __ATOMIC_OP64(atomic_inc_64,     uint64_t, "inc" SUF_64)
  90 __ATOMIC_OPXX(atomic_inc_uchar,  uchar_t,  "inc" SUF_8)
  91 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
  92 __ATOMIC_OPXX(atomic_inc_uint,   uint_t,   "inc" SUF_32)
  93 __ATOMIC_OPXX(atomic_inc_ulong,  ulong_t,  "inc" SUF_LONG)
  94 
  95 __ATOMIC_OPXX(atomic_dec_8,      uint8_t,  "dec" SUF_8)
  96 __ATOMIC_OPXX(atomic_dec_16,     uint16_t, "dec" SUF_16)
  97 __ATOMIC_OPXX(atomic_dec_32,     uint32_t, "dec" SUF_32)
  98 __ATOMIC_OP64(atomic_dec_64,     uint64_t, "dec" SUF_64)
  99 __ATOMIC_OPXX(atomic_dec_uchar,  uchar_t,  "dec" SUF_8)
 100 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
 101 __ATOMIC_OPXX(atomic_dec_uint,   uint_t,   "dec" SUF_32)
 102 __ATOMIC_OPXX(atomic_dec_ulong,  ulong_t,  "dec" SUF_LONG)
 103 
 104 #undef __ATOMIC_OPXX
 105 
 106 #define __ATOMIC_OPXX(fxn, type1, type2, op)                            \
 107 extern __GNU_INLINE void                                                \
 108 fxn(volatile type1 *target, type2 delta)                                \
 109 {                                                                       \
 110         __asm__ __volatile__(                                           \
 111             "lock; " op " %1,%0"                                        \
 112             : "+m" (*target)                                            \
 113             : "ir" (delta));                                            \
 114 }
 115 
 116 __ATOMIC_OPXX(atomic_add_8,     uint8_t,  int8_t,      "add" SUF_8)
 117 __ATOMIC_OPXX(atomic_add_16,    uint16_t, int16_t,     "add" SUF_16)
 118 __ATOMIC_OPXX(atomic_add_32,    uint32_t, int32_t,     "add" SUF_32)
 119 __ATOMIC_OP64(atomic_add_64,    uint64_t, int64_t,     "add" SUF_64)
 120 __ATOMIC_OPXX(atomic_add_char,  uchar_t,  signed char, "add" SUF_8)
 121 __ATOMIC_OPXX(atomic_add_short, ushort_t, short,       "add" SUF_16)
 122 __ATOMIC_OPXX(atomic_add_int,   uint_t,   int,         "add" SUF_32)
 123 __ATOMIC_OPXX(atomic_add_long,  ulong_t,  long,        "add" SUF_LONG)
 124 
 125 /*
 126  * We don't use the above macro here because atomic_add_ptr has an
 127  * inconsistent type.  The first argument should really be a 'volatile void
 128  * **'.
 129  */
 130 extern __GNU_INLINE void
 131 atomic_add_ptr(volatile void *target, ssize_t delta)
 132 {
 133         volatile void **tmp = (volatile void **)target;
 134 
 135         __asm__ __volatile__(
 136             "lock; add" SUF_PTR " %1,%0"
 137             : "+m" (*tmp)
 138             : "ir" (delta));
 139 }
 140 
 141 __ATOMIC_OPXX(atomic_or_8,       uint8_t,  uint8_t,  "or" SUF_8)
 142 __ATOMIC_OPXX(atomic_or_16,      uint16_t, uint16_t, "or" SUF_16)
 143 __ATOMIC_OPXX(atomic_or_32,      uint32_t, uint32_t, "or" SUF_32)
 144 __ATOMIC_OP64(atomic_or_64,      uint64_t, uint64_t, "or" SUF_64)
 145 __ATOMIC_OPXX(atomic_or_uchar,   uchar_t,  uchar_t,  "or" SUF_8)
 146 __ATOMIC_OPXX(atomic_or_ushort,  ushort_t, ushort_t, "or" SUF_16)
 147 __ATOMIC_OPXX(atomic_or_uint,    uint_t,   uint_t,   "or" SUF_32)
 148 __ATOMIC_OPXX(atomic_or_ulong,   ulong_t,  ulong_t,  "or" SUF_LONG)
 149 
 150 __ATOMIC_OPXX(atomic_and_8,      uint8_t,  uint8_t,  "and" SUF_8)
 151 __ATOMIC_OPXX(atomic_and_16,     uint16_t, uint16_t, "and" SUF_16)
 152 __ATOMIC_OPXX(atomic_and_32,     uint32_t, uint32_t, "and" SUF_32)
 153 __ATOMIC_OP64(atomic_and_64,     uint64_t, uint64_t, "and" SUF_64)
 154 __ATOMIC_OPXX(atomic_and_uchar,  uchar_t,  uchar_t,  "and" SUF_8)
 155 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16)
 156 __ATOMIC_OPXX(atomic_and_uint,   uint_t,   uint_t,   "and" SUF_32)
 157 __ATOMIC_OPXX(atomic_and_ulong,  ulong_t,  ulong_t,  "and" SUF_LONG)
 158 
 159 #undef __ATOMIC_OPXX
 160 
 161 #define __ATOMIC_OPXX(fxn, type, op, reg)                               \
 162 extern __GNU_INLINE type                                                \
 163 fxn(volatile type *target, type cmp, type new)                          \
 164 {                                                                       \
 165         type ret;                                                       \
 166         __asm__ __volatile__(                                           \
 167             "lock; " op " %2,%0"                                        \
 168             : "+m" (*target), "=a" (ret)                                \
 169             : reg (new), "1" (cmp)                                      \
 170             : "cc");                                                    \
 171         return (ret);                                                   \
 172 }
 173 
 174 __ATOMIC_OPXX(atomic_cas_8,      uint8_t,  "cmpxchg" SUF_8,    "q")
 175 __ATOMIC_OPXX(atomic_cas_16,     uint16_t, "cmpxchg" SUF_16,   "r")
 176 __ATOMIC_OPXX(atomic_cas_32,     uint32_t, "cmpxchg" SUF_32,   "r")
 177 __ATOMIC_OP64(atomic_cas_64,     uint64_t, "cmpxchg" SUF_64,   "r")
 178 __ATOMIC_OPXX(atomic_cas_uchar,  uchar_t,  "cmpxchg" SUF_8,    "q")
 179 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16,   "r")
 180 __ATOMIC_OPXX(atomic_cas_uint,   uint_t,   "cmpxchg" SUF_32,   "r")
 181 __ATOMIC_OPXX(atomic_cas_ulong,  ulong_t,  "cmpxchg" SUF_LONG, "r")
 182 
 183 #undef __ATOMIC_OPXX
 184 
 185 /*
 186  * We don't use the above macro here because atomic_cas_ptr has an
 187  * inconsistent type.  The first argument should really be a 'volatile void
 188  * **'.
 189  */
 190 extern __GNU_INLINE void *
 191 atomic_cas_ptr(volatile void *target, void *cmp, void *new)
 192 {
 193         volatile void **tmp = (volatile void **)target;
 194         void *ret;
 195 
 196         __asm__ __volatile__(
 197             "lock; cmpxchg" SUF_PTR " %2,%0"
 198             : "+m" (*tmp), "=a" (ret)
 199             : "r" (new), "1" (cmp)
 200             : "cc");
 201 
 202         return (ret);
 203 }
 204 
 205 #define __ATOMIC_OPXX(fxn, type, op, reg)                               \
 206 extern __GNU_INLINE type                                                \
 207 fxn(volatile type *target, type val)                                    \
 208 {                                                                       \
 209         __asm__ __volatile__(                                           \
 210             op " %1,%0"                                                 \
 211             : "+m" (*target), "+" reg (val));                           \
 212         return (val);                                                   \
 213 }
 214 
 215 __ATOMIC_OPXX(atomic_swap_8,      uint8_t,  "xchg" SUF_8,    "q")
 216 __ATOMIC_OPXX(atomic_swap_16,     uint16_t, "xchg" SUF_16,   "r")
 217 __ATOMIC_OPXX(atomic_swap_32,     uint32_t, "xchg" SUF_32,   "r")
 218 __ATOMIC_OP64(atomic_swap_64,     uint64_t, "xchg" SUF_64,   "r")
 219 __ATOMIC_OPXX(atomic_swap_uchar,  uchar_t,  "xchg" SUF_8,    "q")
 220 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16,   "r")
 221 __ATOMIC_OPXX(atomic_swap_uint,   uint_t,   "xchg" SUF_32,   "r")
 222 __ATOMIC_OPXX(atomic_swap_ulong,  ulong_t,  "xchg" SUF_LONG, "r")
 223 
 224 #undef __ATOMIC_OPXX
 225 
 226 /*
 227  * We don't use the above macro here because atomic_swap_ptr has an
 228  * inconsistent type.  The first argument should really be a 'volatile void
 229  * **'.
 230  */
 231 extern __GNU_INLINE void *
 232 atomic_swap_ptr(volatile void *target, void *val)
 233 {
 234         volatile void **tmp = (volatile void **)target;
 235 
 236         __asm__ __volatile__(
 237             "xchg" SUF_PTR " %1,%0"
 238             : "+m" (*tmp), "+r" (val));
 239 
 240         return (val);
 241 }
 242 
 243 #else
 244 #error  "port me"
 245 #endif
 246 
 247 #undef SUF_8
 248 #undef SUF_16
 249 #undef SUF_32
 250 #undef SUF_64
 251 #undef SUF_LONG
 252 #undef SUF_PTR
 253 
 254 #undef __ATOMIC_OP64
 255 
 256 #endif /* !__lint && __GNUC__ */
 257 
 258 #ifdef __cplusplus
 259 }
 260 #endif
 261 
 262 #endif  /* _ASM_ATOMIC_H */