/usr/include/atomic_ops/sysdeps/gcc/x86.h is in libatomic-ops-dev 7.4.4-3.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 | /*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
*
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*
* Some of the machine specific code was borrowed from our GC distribution.
*/
/* The following really assume we have a 486 or better. Unfortunately */
/* gcc doesn't define a suitable feature test macro based on command */
/* line options. */
/* We should perhaps test dynamically. */
#include "../all_aligned_atomic_load_store.h"
#include "../test_and_set_t_is_char.h"
#if defined(__SSE2__) && !defined(AO_USE_PENTIUM4_INSTRS)
/* "mfence" is a part of SSE2 set (introduced on Intel Pentium 4). */
# define AO_USE_PENTIUM4_INSTRS
#endif
#if defined(AO_USE_PENTIUM4_INSTRS)
AO_INLINE void
AO_nop_full(void)
{
__asm__ __volatile__("mfence" : : : "memory");
}
# define AO_HAVE_nop_full
#else
/* We could use the cpuid instruction. But that seems to be slower */
/* than the default implementation based on test_and_set_full. Thus */
/* we omit that bit of misinformation here. */
#endif /* !AO_USE_PENTIUM4_INSTRS */
/* As far as we can tell, the lfence and sfence instructions are not */
/* currently needed or useful for cached memory accesses. */
/* Really only works for 486 and later */
#ifndef AO_PREFER_GENERALIZED
AO_INLINE AO_t
AO_fetch_and_add_full (volatile AO_t *p, AO_t incr)
{
AO_t result;
__asm__ __volatile__ ("lock; xadd %0, %1" :
"=r" (result), "=m" (*p) : "0" (incr), "m" (*p)
: "memory");
return result;
}
# define AO_HAVE_fetch_and_add_full
#endif /* !AO_PREFER_GENERALIZED */
AO_INLINE unsigned char
AO_char_fetch_and_add_full (volatile unsigned char *p, unsigned char incr)
{
unsigned char result;
__asm__ __volatile__ ("lock; xaddb %0, %1" :
"=q" (result), "=m" (*p) : "0" (incr), "m" (*p)
: "memory");
return result;
}
#define AO_HAVE_char_fetch_and_add_full
AO_INLINE unsigned short
AO_short_fetch_and_add_full (volatile unsigned short *p, unsigned short incr)
{
unsigned short result;
__asm__ __volatile__ ("lock; xaddw %0, %1" :
"=r" (result), "=m" (*p) : "0" (incr), "m" (*p)
: "memory");
return result;
}
#define AO_HAVE_short_fetch_and_add_full
#ifndef AO_PREFER_GENERALIZED
/* Really only works for 486 and later */
AO_INLINE void
AO_and_full (volatile AO_t *p, AO_t value)
{
__asm__ __volatile__ ("lock; and %1, %0" :
"=m" (*p) : "r" (value), "m" (*p)
: "memory");
}
# define AO_HAVE_and_full
AO_INLINE void
AO_or_full (volatile AO_t *p, AO_t value)
{
__asm__ __volatile__ ("lock; or %1, %0" :
"=m" (*p) : "r" (value), "m" (*p)
: "memory");
}
# define AO_HAVE_or_full
AO_INLINE void
AO_xor_full (volatile AO_t *p, AO_t value)
{
__asm__ __volatile__ ("lock; xor %1, %0" :
"=m" (*p) : "r" (value), "m" (*p)
: "memory");
}
# define AO_HAVE_xor_full
/* AO_store_full could be implemented directly using "xchg" but it */
/* could be generalized efficiently as an ordinary store accomplished */
/* with AO_nop_full ("mfence" instruction). */
#endif /* !AO_PREFER_GENERALIZED */
AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t *addr)
{
unsigned char oldval;
/* Note: the "xchg" instruction does not need a "lock" prefix */
__asm__ __volatile__ ("xchgb %0, %1"
: "=q" (oldval), "=m" (*addr)
: "0" ((unsigned char)0xff), "m" (*addr)
: "memory");
return (AO_TS_VAL_t)oldval;
}
#define AO_HAVE_test_and_set_full
#ifndef AO_GENERALIZE_ASM_BOOL_CAS
/* Returns nonzero if the comparison succeeded. */
AO_INLINE int
AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val)
{
# ifdef AO_USE_SYNC_CAS_BUILTIN
return (int)__sync_bool_compare_and_swap(addr, old, new_val
/* empty protection list */);
/* Note: an empty list of variables protected by the */
/* memory barrier should mean all globally accessible */
/* variables are protected. */
# else
char result;
__asm__ __volatile__ ("lock; cmpxchg %3, %0; setz %1"
: "=m" (*addr), "=a" (result)
: "m" (*addr), "r" (new_val), "a" (old)
: "memory");
return (int)result;
# endif
}
# define AO_HAVE_compare_and_swap_full
#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
AO_INLINE AO_t
AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
AO_t new_val)
{
# ifdef AO_USE_SYNC_CAS_BUILTIN
return __sync_val_compare_and_swap(addr, old_val, new_val
/* empty protection list */);
# else
AO_t fetched_val;
__asm__ __volatile__ ("lock; cmpxchg %3, %4"
: "=a" (fetched_val), "=m" (*addr)
: "a" (old_val), "r" (new_val), "m" (*addr)
: "memory");
return fetched_val;
# endif
}
#define AO_HAVE_fetch_compare_and_swap_full
#if !defined(__x86_64__) && !defined(AO_USE_SYNC_CAS_BUILTIN)
# include "../standard_ao_double_t.h"
/* Reading or writing a quadword aligned on a 64-bit boundary is */
/* always carried out atomically on at least a Pentium according to */
/* Chapter 8.1.1 of Volume 3A Part 1 of Intel processor manuals. */
# define AO_ACCESS_double_CHECK_ALIGNED
# include "../loadstore/double_atomic_load_store.h"
/* Returns nonzero if the comparison succeeded. */
/* Really requires at least a Pentium. */
AO_INLINE int
AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
char result;
# ifdef __PIC__
AO_t saved_ebx;
/* If PIC is turned on, we cannot use ebx as it is reserved for the */
/* GOT pointer. We should save and restore ebx. The proposed */
/* solution is not so efficient as the older alternatives using */
/* push ebx or edi as new_val1 (w/o clobbering edi and temporary */
/* local variable usage) but it is more portable (it works even if */
/* ebx is not used as GOT pointer, and it works for the buggy GCC */
/* releases that incorrectly evaluate memory operands offset in the */
/* inline assembly after push). */
# ifdef __OPTIMIZE__
__asm__ __volatile__("mov %%ebx, %2\n\t" /* save ebx */
"lea %0, %%edi\n\t" /* in case addr is in ebx */
"mov %7, %%ebx\n\t" /* load new_val1 */
"lock; cmpxchg8b (%%edi)\n\t"
"mov %2, %%ebx\n\t" /* restore ebx */
"setz %1"
: "=m" (*addr), "=a" (result), "=m" (saved_ebx)
: "m" (*addr), "d" (old_val2), "a" (old_val1),
"c" (new_val2), "m" (new_val1)
: "%edi", "memory");
# else
/* A less-efficient code manually preserving edi if GCC invoked */
/* with -O0 option (otherwise it fails while finding a register */
/* in class 'GENERAL_REGS'). */
AO_t saved_edi;
__asm__ __volatile__("mov %%edi, %3\n\t" /* save edi */
"mov %%ebx, %2\n\t" /* save ebx */
"lea %0, %%edi\n\t" /* in case addr is in ebx */
"mov %8, %%ebx\n\t" /* load new_val1 */
"lock; cmpxchg8b (%%edi)\n\t"
"mov %2, %%ebx\n\t" /* restore ebx */
"mov %3, %%edi\n\t" /* restore edi */
"setz %1"
: "=m" (*addr), "=a" (result),
"=m" (saved_ebx), "=m" (saved_edi)
: "m" (*addr), "d" (old_val2), "a" (old_val1),
"c" (new_val2), "m" (new_val1) : "memory");
# endif
# else
/* For non-PIC mode, this operation could be simplified (and be */
/* faster) by using ebx as new_val1 (GCC would refuse to compile */
/* such code for PIC mode). */
__asm__ __volatile__ ("lock; cmpxchg8b %0; setz %1"
: "=m" (*addr), "=a" (result)
: "m" (*addr), "d" (old_val2), "a" (old_val1),
"c" (new_val2), "b" (new_val1)
: "memory");
# endif
return (int) result;
}
# define AO_HAVE_compare_double_and_swap_double_full
# define AO_T_IS_INT
#elif defined(__ILP32__) || !defined(__x86_64__)
# include "../standard_ao_double_t.h"
/* Reading or writing a quadword aligned on a 64-bit boundary is */
/* always carried out atomically (requires at least a Pentium). */
# define AO_ACCESS_double_CHECK_ALIGNED
# include "../loadstore/double_atomic_load_store.h"
/* X32 has native support for 64-bit integer operations (AO_double_t */
/* is a 64-bit integer and we could use 64-bit cmpxchg). */
/* This primitive is used by compare_double_and_swap_double_full. */
AO_INLINE int
AO_double_compare_and_swap_full(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
/* It is safe to use __sync CAS built-in here. */
return __sync_bool_compare_and_swap(&addr->AO_whole,
old_val.AO_whole, new_val.AO_whole
/* empty protection list */);
}
# define AO_HAVE_double_compare_and_swap_full
# define AO_T_IS_INT
#else /* 64-bit */
AO_INLINE unsigned int
AO_int_fetch_and_add_full (volatile unsigned int *p, unsigned int incr)
{
unsigned int result;
__asm__ __volatile__ ("lock; xaddl %0, %1"
: "=r" (result), "=m" (*p)
: "0" (incr), "m" (*p)
: "memory");
return result;
}
# define AO_HAVE_int_fetch_and_add_full
/* The Intel and AMD Architecture Programmer Manuals state roughly */
/* the following: */
/* - CMPXCHG16B (with a LOCK prefix) can be used to perform 16-byte */
/* atomic accesses in 64-bit mode (with certain alignment */
/* restrictions); */
/* - SSE instructions that access data larger than a quadword (like */
/* MOVDQA) may be implemented using multiple memory accesses; */
/* - LOCK prefix causes an invalid-opcode exception when used with */
/* 128-bit media (SSE) instructions. */
/* Thus, currently, the only way to implement lock-free double_load */
/* and double_store on x86_64 is to use CMPXCHG16B (if available). */
/* TODO: Test some gcc macro to detect presence of cmpxchg16b. */
# ifdef AO_CMPXCHG16B_AVAILABLE
# include "../standard_ao_double_t.h"
/* NEC LE-IT: older AMD Opterons are missing this instruction. */
/* On these machines SIGILL will be thrown. */
/* Define AO_WEAK_DOUBLE_CAS_EMULATION to have an emulated (lock */
/* based) version available. */
/* HB: Changed this to not define either by default. There are */
/* enough machines and tool chains around on which cmpxchg16b */
/* doesn't work. And the emulation is unsafe by our usual rules. */
/* However both are clearly useful in certain cases. */
AO_INLINE int
AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
char result;
__asm__ __volatile__("lock; cmpxchg16b %0; setz %1"
: "=m"(*addr), "=a"(result)
: "m"(*addr), "d" (old_val2), "a" (old_val1),
"c" (new_val2), "b" (new_val1)
: "memory");
return (int) result;
}
# define AO_HAVE_compare_double_and_swap_double_full
# elif defined(AO_WEAK_DOUBLE_CAS_EMULATION)
# include "../standard_ao_double_t.h"
/* This one provides spinlock based emulation of CAS implemented in */
/* atomic_ops.c. We probably do not want to do this here, since it */
/* is not atomic with respect to other kinds of updates of *addr. */
/* On the other hand, this may be a useful facility on occasion. */
int AO_compare_double_and_swap_double_emulation(
volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2);
AO_INLINE int
AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
return AO_compare_double_and_swap_double_emulation(addr,
old_val1, old_val2, new_val1, new_val2);
}
# define AO_HAVE_compare_double_and_swap_double_full
# endif /* AO_WEAK_DOUBLE_CAS_EMULATION && !AO_CMPXCHG16B_AVAILABLE */
#endif /* x86_64 && !ILP32 */
/* Real X86 implementations, except for some old 32-bit WinChips, */
/* appear to enforce ordering between memory operations, EXCEPT that */
/* a later read can pass earlier writes, presumably due to the visible */
/* presence of store buffers. */
/* We ignore both the WinChips and the fact that the official specs */
/* seem to be much weaker (and arguably too weak to be usable). */
#include "../ordered_except_wr.h"
|