Index: configure.ac ================================================================== --- configure.ac +++ configure.ac @@ -732,36 +732,36 @@ ]) ]) atomic_ops="none" - AC_MSG_CHECKING(whether __sync_* works) - AC_TRY_LINK([#include ], [ - int32_t i, j; - if (__sync_add_and_fetch(&i, 1)) - j = __sync_sub_and_fetch(&i, 1); - while (!__sync_bool_compare_and_swap(&i, 0, 1)); - ], [ - AC_MSG_RESULT(yes) - atomic_ops="GCC builtins" - AC_DEFINE(OF_HAVE_GCC_ATOMIC_OPS, 1, - [Whether GCC atomic operations are available]) - ], [ - AC_MSG_RESULT(no) - ]) - AC_MSG_CHECKING(whether we have an atomic ops assembly implementation) AC_EGREP_CPP(yes, [ #if defined(__GNUC__) && (defined(__i386__) || \ defined(__x86_64__) || defined(__amd64__)) || \ defined(__ppc__) yes #endif ], [ + AC_MSG_RESULT(yes) + atomic_ops="assembly implementation" + ], [ + AC_MSG_RESULT(no) + ]) + + AC_MSG_CHECKING(whether __sync_* works) + AC_TRY_LINK([#include ], [ + int32_t i, j; + if (__sync_add_and_fetch(&i, 1)) + j = __sync_sub_and_fetch(&i, 1); + while (!__sync_bool_compare_and_swap(&i, 0, 1)); + ], [ AC_MSG_RESULT(yes) test x"$atomic_ops" = x"none" && \ - atomic_ops="assembly implementation" + atomic_ops="__sync_* builtins" + AC_DEFINE(OF_HAVE_SYNC_BUILTINS, 1, + [Whether __sync_* builtins are available]) ], [ AC_MSG_RESULT(no) ]) AC_CHECK_HEADER(libkern/OSAtomic.h, [ Index: src/atomic.h ================================================================== --- src/atomic.h +++ src/atomic.h @@ -20,1009 +20,18 @@ #ifndef OF_HAVE_ATOMIC_OPS # error No atomic operations available! #endif -#ifdef OF_HAVE_OSATOMIC -# include -#endif - -OF_ASSUME_NONNULL_BEGIN - -static OF_INLINE int -of_atomic_int_add(volatile int *_Nonnull p, int i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p += i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_add_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "lock\n\t" - "xaddl %0, %2\n\t" - "addl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "lock\n\t" - "xaddq %0, %2\n\t" - "addq %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "add %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAdd32(i, p); -#else -# error of_atomic_int_add not implemented! -#endif -} - -static OF_INLINE int32_t -of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p += i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_add_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - __asm__ __volatile__ ( - "lock\n\t" - "xaddl %0, %2\n\t" - "addl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "add %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAdd32(i, p); -#else -# error of_atomic_int32_add not implemented! -#endif -} - -static OF_INLINE void* -of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*(char* volatile*)p += i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_add_and_fetch(p, (void*)i); -#elif defined(OF_X86_64_ASM) - __asm__ __volatile__ ( - "lock\n\t" - "xaddq %0, %2\n\t" - "addq %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return (void*)i; -#elif defined(OF_X86_ASM) - __asm__ __volatile__ ( - "lock\n\t" - "xaddl %0, %2\n\t" - "addl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return (void*)i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "add %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return (void*)i; -#elif defined(OF_HAVE_OSATOMIC) -# ifdef __LP64__ - return (void*)OSAtomicAdd64(i, (int64_t*)p); -# else - return (void*)OSAtomicAdd32(i, (int32_t*)p); -# endif -#else -# error of_atomic_ptr_add not implemented! -#endif -} - -static OF_INLINE int -of_atomic_int_sub(volatile int *_Nonnull p, int i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p -= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_sub_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "negl %0\n\t" - "lock\n\t" - "xaddl %0, %2\n\t" - "subl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "negq %0\n\t" - "lock\n\t" - "xaddq %0, %2\n\t" - "subq %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "sub %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAdd32(-i, p); -#else -# error of_atomic_int_sub not implemented! -#endif -} - -static OF_INLINE int32_t -of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p -= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_sub_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - __asm__ __volatile__ ( - "negl %0\n\t" - "lock\n\t" - "xaddl %0, %2\n\t" - "subl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "sub %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAdd32(-i, p); -#else -# error of_atomic_int32_sub not implemented! -#endif -} - -static OF_INLINE void* -of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*(char* volatile*)p -= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_sub_and_fetch(p, (void*)i); -#elif defined(OF_X86_64_ASM) - __asm__ __volatile__ ( - "negq %0\n\t" - "lock\n\t" - "xaddq %0, %2\n\t" - "subq %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return (void*)i; -#elif defined(OF_X86_ASM) - __asm__ __volatile__ ( - "negl %0\n\t" - "lock\n\t" - "xaddl %0, %2\n\t" - "subl %1, %0" - : "+&r"(i) - : "r"(i), "m"(*p) - ); - - return (void*)i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "sub %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return (void*)i; -#elif defined(OF_HAVE_OSATOMIC) -# ifdef __LP64__ - return (void*)OSAtomicAdd64(-i, (int64_t*)p); -# else - return (void*)OSAtomicAdd32(-i, (int32_t*)p); -# endif -#else -# error of_atomic_ptr_sub not implemented! -#endif -} - -static OF_INLINE int -of_atomic_int_inc(volatile int *_Nonnull p) -{ -#if !defined(OF_HAVE_THREADS) - return ++*p; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_add_and_fetch(p, 1); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int i; - - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "xorl %0, %0\n\t" - "incl %0\n\t" - "lock\n\t" - "xaddl %0, %1\n\t" - "incl %0" - : "=&r"(i) - : "m"(*p) - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "xorq %0, %0\n\t" - "incq %0\n\t" - "lock\n\t" - "xaddq %0, %1\n\t" - "incq %0" - : "=&r"(i) - : "m"(*p) - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - int i; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %1\n\t" - "addi %0, %0, 1\n\t" - "stwcx. %0, 0, %1\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicIncrement32(p); -#else -# error of_atomic_int_inc not implemented! -#endif -} - -static OF_INLINE int32_t -of_atomic_int32_inc(volatile int32_t *_Nonnull p) -{ -#if !defined(OF_HAVE_THREADS) - return ++*p; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_add_and_fetch(p, 1); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int32_t i; - - __asm__ __volatile__ ( - "xorl %0, %0\n\t" - "incl %0\n\t" - "lock\n\t" - "xaddl %0, %1\n\t" - "incl %0" - : "=&r"(i) - : "m"(*p) - ); - - return i; -#elif defined(OF_POWERPC_ASM) - int32_t i; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %1\n\t" - "addi %0, %0, 1\n\t" - "stwcx. %0, 0, %1\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicIncrement32(p); -#else -# error of_atomic_int32_inc not implemented! -#endif -} - -static OF_INLINE int -of_atomic_int_dec(volatile int *_Nonnull p) -{ -#if !defined(OF_HAVE_THREADS) - return --*p; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_sub_and_fetch(p, 1); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int i; - - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "xorl %0, %0\n\t" - "decl %0\n\t" - "lock\n\t" - "xaddl %0, %1\n\t" - "decl %0" - : "=&r"(i) - : "m"(*p) - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "xorq %0, %0\n\t" - "decq %0\n\t" - "lock\n\t" - "xaddq %0, %1\n\t" - "decq %0" - : "=&r"(i) - : "m"(*p) - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - int i; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %1\n\t" - "subi %0, %0, 1\n\t" - "stwcx. %0, 0, %1\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicDecrement32(p); -#else -# error of_atomic_int_dec not implemented! -#endif -} - -static OF_INLINE int32_t -of_atomic_int32_dec(volatile int32_t *_Nonnull p) -{ -#if !defined(OF_HAVE_THREADS) - return --*p; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_sub_and_fetch(p, 1); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int32_t i; - - __asm__ __volatile__ ( - "xorl %0, %0\n\t" - "decl %0\n\t" - "lock\n\t" - "xaddl %0, %1\n\t" - "decl %0" - : "=&r"(i) - : "m"(*p) - ); - - return i; -#elif defined(OF_POWERPC_ASM) - int32_t i; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %1\n\t" - "subi %0, %0, 1\n\t" - "stwcx. %0, 0, %1\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicDecrement32(p); -#else -# error of_atomic_int32_dec not implemented! -#endif -} - -static OF_INLINE unsigned int -of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p |= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_or_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "orl %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "0:\n\t" - "movq %2, %0\n\t" - "movq %0, %%rax\n\t" - "orq %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "rax", "cc" - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "or %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicOr32(i, p); -#else -# error of_atomic_int_or not implemented! -#endif -} - -static OF_INLINE uint32_t -of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p |= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_or_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "orl %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "or %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicOr32(i, p); -#else -# error of_atomic_int32_or not implemented! -#endif -} - -static OF_INLINE unsigned int -of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p &= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_and_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "andl %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "0:\n\t" - "movq %2, %0\n\t" - "movq %0, %%rax\n\t" - "andq %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "rax", "cc" - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "and %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAnd32(i, p); -#else -# error of_atomic_int_and not implemented! -#endif -} - -static OF_INLINE uint32_t -of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p &= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_and_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "andl %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "and %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicAnd32(i, p); -#else -# error of_atomic_int32_and not implemented! -#endif -} - -static OF_INLINE unsigned int -of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p ^= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_xor_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - if (sizeof(int) == 4) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "xorl %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); -# ifdef OF_X86_64_ASM - else if (sizeof(int) == 8) - __asm__ __volatile__ ( - "0:\n\t" - "movq %2, %0\n\t" - "movq %0, %%rax\n\t" - "xorq %1, %0\n\t" - "lock\n\t" - "cmpxchg %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "rax", "cc" - ); -# endif - else - abort(); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "xor %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicXor32(i, p); -#else -# error of_atomic_int_xor not implemented! -#endif -} - -static OF_INLINE uint32_t -of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) -{ -#if !defined(OF_HAVE_THREADS) - return (*p ^= i); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_xor_and_fetch(p, i); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "movl %2, %0\n\t" - "movl %0, %%eax\n\t" - "xorl %1, %0\n\t" - "lock\n\t" - "cmpxchgl %0, %2\n\t" - "jne 0b" - : "=&r"(i) - : "r"(i), "m"(*p) - : "eax", "cc" - ); - - return i; -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %2\n\t" - "xor %0, %0, %1\n\t" - "stwcx. %0, 0, %2\n\t" - "bne- 0b" - : "=&r"(i) - : "r"(i), "r"(p) - ); - - return i; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicXor32(i, p); -#else -# error of_atomic_int32_xor not implemented! -#endif -} - -static OF_INLINE bool -of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) -{ -#if !defined(OF_HAVE_THREADS) - if (*p == o) { - *p = n; - return true; - } - - return false; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_bool_compare_and_swap(p, o, n); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int r; - - __asm__ __volatile__ ( - "lock\n\t" - "cmpxchg %2, %3\n\t" - "sete %b0\n\t" - "movzbl %b0, %0" - : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ - : "r"(n), "m"(*p) - : "cc" - ); - - return r; -#elif defined(OF_POWERPC_ASM) - int r; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %3\n\t" - "cmpw %0, %1\n\t" - "bne 1f\n\t" - "stwcx. %2, 0, %3\n\t" - "bne- 0b\n\t" - "li %0, 1\n\t" - "b 2f\n\t" - "1:\n\t" - "stwcx. %0, 0, %3\n\t" - "li %0, 0\n\t" - "2:" - : "=&r"(r) - : "r"(o), "r"(n), "r"(p) - : "cc" - ); - - return r; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicCompareAndSwapInt(o, n, p); -#else -# error of_atomic_int_cmpswap not implemented! -#endif -} - -static OF_INLINE bool -of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) -{ -#if !defined(OF_HAVE_THREADS) - if (*p == o) { - *p = n; - return true; - } - - return false; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_bool_compare_and_swap(p, o, n); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int r; - - __asm__ __volatile__ ( - "lock\n\t" - "cmpxchg %2, %3\n\t" - "sete %b0\n\t" - "movzbl %b0, %0" - : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ - : "r"(n), "m"(*p) - : "cc" - ); - - return r; -#elif defined(OF_POWERPC_ASM) - int r; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %3\n\t" - "cmpw %0, %1\n\t" - "bne 1f\n\t" - "stwcx. %2, 0, %3\n\t" - "bne- 0b\n\t" - "li %0, 1\n\t" - "b 2f\n\t" - "1:\n\t" - "stwcx. %0, 0, %3\n\t" - "li %0, 0\n\t" - "2:" - : "=&r"(r) - : "r"(o), "r"(n), "r"(p) - : "cc" - ); - - return r; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicCompareAndSwap32(o, n, p); -#else -# error of_atomic_int32_cmpswap not implemented! -#endif -} - -static OF_INLINE bool -of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, - void *_Nullable o, void *_Nullable n) -{ -#if !defined(OF_HAVE_THREADS) - if (*p == o) { - *p = n; - return true; - } - - return false; -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - return __sync_bool_compare_and_swap(p, o, n); -#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) - int r; - - __asm__ __volatile__ ( - "lock\n\t" - "cmpxchg %2, %3\n\t" - "sete %b0\n\t" - "movzbl %b0, %0" - : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ - : "r"(n), "m"(*p) - : "cc" - ); - - return r; -#elif defined(OF_POWERPC_ASM) - int r; - - __asm__ __volatile__ ( - "0:\n\t" - "lwarx %0, 0, %3\n\t" - "cmpw %0, %1\n\t" - "bne 1f\n\t" - "stwcx. %2, 0, %3\n\t" - "bne- 0b\n\t" - "li %0, 1\n\t" - "b 2f\n\t" - "1:\n\t" - "stwcx. %0, 0, %3\n\t" - "li %0, 0\n\t" - "2:" - : "=&r"(r) - : "r"(o), "r"(n), "r"(p) - : "cc" - ); - - return r; -#elif defined(OF_HAVE_OSATOMIC) - return OSAtomicCompareAndSwapPtr(o, n, p); -#else -# error of_atomic_ptr_cmpswap not implemented! -#endif -} - -static OF_INLINE void -of_memory_barrier_sync(void) -{ -#if !defined(OF_HAVE_THREADS) - /* nop */ -#elif defined(OF_X86_64_ASM) - __asm__ __volatile__ ( - "mfence" ::: "memory" - ); -#elif defined(OF_POWERPC_ASM) - __asm__ __volatile__ ( - "sync" ::: "memory" - ); -#elif defined(OF_ARMV7_ASM) || defined(OF_ARM64_ASM) - __asm__ __volatile__ ( - "dmb" ::: "memory" - ); -#elif defined(OF_HAVE_GCC_ATOMIC_OPS) - __sync_synchronize(); -#elif defined(OF_HAVE_OSATOMIC) - OSMemoryBarrier(); -#else -# error of_memory_barrier not implemented! -#endif -} - -static OF_INLINE void -of_memory_barrier_enter(void) -{ - of_memory_barrier_sync(); -} - -static OF_INLINE void -of_memory_barrier_exit(void) -{ - of_memory_barrier_sync(); -} - -static OF_INLINE void -of_memory_barrier_producer(void) -{ -#if defined(OF_X86_64_ASM) - __asm__ __volatile__ ("sfence" ::: "memory"); -#else - of_memory_barrier_sync(); -#endif -} - -static OF_INLINE void -of_memory_barrier_consumer(void) -{ -#if defined(OF_X86_64_ASM) - __asm__ __volatile__ ("lfence" ::: "memory"); -#else - of_memory_barrier_sync(); -#endif -} - -OF_ASSUME_NONNULL_END +#if !defined(OF_HAVE_THREADS) +# import "atomic_no_threads.h" +#elif defined(OF_X86_64_ASM) || defined(OF_X86_ASM) +# import "atomic_x86.h" +#elif defined(OF_POWERPC_ASM) +# import "atomic_powerpc.h" +#elif defined(OF_HAVE_SYNC_BUILTINS) +# import "atomic_sync_builtins.h" +#elif defined(OF_HAVE_OSATOMIC) +# import "atomic_osatomic.h" +#else +# error No atomic operations available! +#endif ADDED src/atomic_no_threads.h Index: src/atomic_no_threads.h ================================================================== --- src/atomic_no_threads.h +++ src/atomic_no_threads.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 + * Jonathan Schleifer + * + * All rights reserved. + * + * This file is part of ObjFW. It may be distributed under the terms of the + * Q Public License 1.0, which can be found in the file LICENSE.QPL included in + * the packaging of this file. + * + * Alternatively, it may be distributed under the terms of the GNU General + * Public License, either version 2 or 3, which can be found in the file + * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this + * file. + */ + +OF_ASSUME_NONNULL_BEGIN + +static OF_INLINE int +of_atomic_int_add(volatile int *_Nonnull p, int i) +{ + return (*p += i); +} + +static OF_INLINE int32_t +of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) +{ + return (*p += i); +} + +static OF_INLINE void* +of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + return (*(char* volatile*)p += i); +} + +static OF_INLINE int +of_atomic_int_sub(volatile int *_Nonnull p, int i) +{ + return (*p -= i); +} + +static OF_INLINE int32_t +of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) +{ + return (*p -= i); +} + +static OF_INLINE void* +of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + return (*(char* volatile*)p -= i); +} + +static OF_INLINE int +of_atomic_int_inc(volatile int *_Nonnull p) +{ + return ++*p; +} + +static OF_INLINE int32_t +of_atomic_int32_inc(volatile int32_t *_Nonnull p) +{ + return ++*p; +} + +static OF_INLINE int +of_atomic_int_dec(volatile int *_Nonnull p) +{ + return --*p; +} + +static OF_INLINE int32_t +of_atomic_int32_dec(volatile int32_t *_Nonnull p) +{ + return --*p; +} + +static OF_INLINE unsigned int +of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return (*p |= i); +} + +static OF_INLINE uint32_t +of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return (*p |= i); +} + +static OF_INLINE unsigned int +of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return (*p &= i); +} + +static OF_INLINE uint32_t +of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return (*p &= i); +} + +static OF_INLINE unsigned int +of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return (*p ^= i); +} + +static OF_INLINE uint32_t +of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return (*p ^= i); +} + +static OF_INLINE bool +of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) +{ + if (*p == o) { + *p = n; + return true; + } + + return false; +} + +static OF_INLINE bool +of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) +{ + if (*p == o) { + *p = n; + return true; + } + + return false; +} + +static OF_INLINE bool +of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, + void *_Nullable o, void *_Nullable n) +{ + if (*p == o) { + *p = n; + return true; + } + + return false; +} + +static OF_INLINE void +of_memory_barrier_sync(void) +{ + /* nop */ +} + +static OF_INLINE void +of_memory_barrier_enter(void) +{ + /* nop */ +} + +static OF_INLINE void +of_memory_barrier_exit(void) +{ + /* nop */ +} + +static OF_INLINE void +of_memory_barrier_producer(void) +{ + /* nop */ +} + +static OF_INLINE void +of_memory_barrier_consumer(void) +{ + /* nop */ +} + +OF_ASSUME_NONNULL_END ADDED src/atomic_osatomic.h Index: src/atomic_osatomic.h ================================================================== --- src/atomic_osatomic.h +++ src/atomic_osatomic.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 + * Jonathan Schleifer + * + * All rights reserved. + * + * This file is part of ObjFW. It may be distributed under the terms of the + * Q Public License 1.0, which can be found in the file LICENSE.QPL included in + * the packaging of this file. + * + * Alternatively, it may be distributed under the terms of the GNU General + * Public License, either version 2 or 3, which can be found in the file + * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this + * file. + */ + +#include + +OF_ASSUME_NONNULL_BEGIN + +static OF_INLINE int +of_atomic_int_add(volatile int *_Nonnull p, int i) +{ + return OSAtomicAdd32(i, p); +} + +static OF_INLINE int32_t +of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) +{ + return OSAtomicAdd32(i, p); +} + +static OF_INLINE void* +of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ +#ifdef __LP64__ + return (void*)OSAtomicAdd64(i, (int64_t*)p); +#else + return (void*)OSAtomicAdd32(i, (int32_t*)p); +#endif +} + +static OF_INLINE int +of_atomic_int_sub(volatile int *_Nonnull p, int i) +{ + return OSAtomicAdd32(-i, p); +} + +static OF_INLINE int32_t +of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) +{ + return OSAtomicAdd32(-i, p); +} + +static OF_INLINE void* +of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ +#ifdef __LP64__ + return (void*)OSAtomicAdd64(-i, (int64_t*)p); +#else + return (void*)OSAtomicAdd32(-i, (int32_t*)p); +#endif +} + +static OF_INLINE int +of_atomic_int_inc(volatile int *_Nonnull p) +{ + return OSAtomicIncrement32(p); +} + +static OF_INLINE int32_t +of_atomic_int32_inc(volatile int32_t *_Nonnull p) +{ + return OSAtomicIncrement32(p); +} + +static OF_INLINE int +of_atomic_int_dec(volatile int *_Nonnull p) +{ + return OSAtomicDecrement32(p); +} + +static OF_INLINE int32_t +of_atomic_int32_dec(volatile int32_t *_Nonnull p) +{ + return OSAtomicDecrement32(p); +} + +static OF_INLINE unsigned int +of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return OSAtomicOr32(i, p); +} + +static OF_INLINE uint32_t +of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return OSAtomicOr32(i, p); +} + +static OF_INLINE unsigned int +of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return OSAtomicAnd32(i, p); +} + +static OF_INLINE uint32_t +of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return OSAtomicAnd32(i, p); +} + +static OF_INLINE unsigned int +of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return OSAtomicXor32(i, p); +} + +static OF_INLINE uint32_t +of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return OSAtomicXor32(i, p); +} + +static OF_INLINE bool +of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) +{ + return OSAtomicCompareAndSwapInt(o, n, p); +} + +static OF_INLINE bool +of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) +{ + return OSAtomicCompareAndSwap32(o, n, p); +} + +static OF_INLINE bool +of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, + void *_Nullable o, void *_Nullable n) +{ + return OSAtomicCompareAndSwapPtr(o, n, p); +} + +static OF_INLINE void +of_memory_barrier_sync(void) +{ + OSMemoryBarrier(); +} + +static OF_INLINE void +of_memory_barrier_enter(void) +{ + OSMemoryBarrier(); +} + +static OF_INLINE void +of_memory_barrier_exit(void) +{ + OSMemoryBarrier(); +} + +static OF_INLINE void +of_memory_barrier_producer(void) +{ + OSMemoryBarrier(); +} + +static OF_INLINE void +of_memory_barrier_consumer(void) +{ + OSMemoryBarrier(); +} + +OF_ASSUME_NONNULL_END ADDED src/atomic_powerpc.h Index: src/atomic_powerpc.h ================================================================== --- src/atomic_powerpc.h +++ src/atomic_powerpc.h @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 + * Jonathan Schleifer + * + * All rights reserved. + * + * This file is part of ObjFW. It may be distributed under the terms of the + * Q Public License 1.0, which can be found in the file LICENSE.QPL included in + * the packaging of this file. + * + * Alternatively, it may be distributed under the terms of the GNU General + * Public License, either version 2 or 3, which can be found in the file + * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this + * file. + */ + +OF_ASSUME_NONNULL_BEGIN + +static OF_INLINE int +of_atomic_int_add(volatile int *_Nonnull p, int i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "add %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "add %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE void* +of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "add %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return (void*)i; +} + +static OF_INLINE int +of_atomic_int_sub(volatile int *_Nonnull p, int i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "sub %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "sub %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE void* +of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "sub %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return (void*)i; +} + +static OF_INLINE int +of_atomic_int_inc(volatile int *_Nonnull p) +{ + int i; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %1\n\t" + "addi %0, %0, 1\n\t" + "stwcx. %0, 0, %1\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(p) + ); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_inc(volatile int32_t *_Nonnull p) +{ + int32_t i; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %1\n\t" + "addi %0, %0, 1\n\t" + "stwcx. %0, 0, %1\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(p) + ); + + return i; +} + +static OF_INLINE int +of_atomic_int_dec(volatile int *_Nonnull p) +{ + int i; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %1\n\t" + "subi %0, %0, 1\n\t" + "stwcx. %0, 0, %1\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(p) + ); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_dec(volatile int32_t *_Nonnull p) +{ + int32_t i; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %1\n\t" + "subi %0, %0, 1\n\t" + "stwcx. %0, 0, %1\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(p) + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "or %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "or %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "and %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "and %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "xor %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %2\n\t" + "xor %0, %0, %1\n\t" + "stwcx. %0, 0, %2\n\t" + "bne- 0b" + : "=&r"(i) + : "r"(i), "r"(p) + ); + + return i; +} + +static OF_INLINE bool +of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) +{ + int r; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %3\n\t" + "cmpw %0, %1\n\t" + "bne 1f\n\t" + "stwcx. %2, 0, %3\n\t" + "bne- 0b\n\t" + "li %0, 1\n\t" + "b 2f\n\t" + "1:\n\t" + "stwcx. %0, 0, %3\n\t" + "li %0, 0\n\t" + "2:" + : "=&r"(r) + : "r"(o), "r"(n), "r"(p) + : "cc" + ); + + return r; +} + +static OF_INLINE bool +of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) +{ + int r; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %3\n\t" + "cmpw %0, %1\n\t" + "bne 1f\n\t" + "stwcx. %2, 0, %3\n\t" + "bne- 0b\n\t" + "li %0, 1\n\t" + "b 2f\n\t" + "1:\n\t" + "stwcx. %0, 0, %3\n\t" + "li %0, 0\n\t" + "2:" + : "=&r"(r) + : "r"(o), "r"(n), "r"(p) + : "cc" + ); + + return r; +} + +static OF_INLINE bool +of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, + void *_Nullable o, void *_Nullable n) +{ + int r; + + __asm__ __volatile__ ( + "0:\n\t" + "lwarx %0, 0, %3\n\t" + "cmpw %0, %1\n\t" + "bne 1f\n\t" + "stwcx. %2, 0, %3\n\t" + "bne- 0b\n\t" + "li %0, 1\n\t" + "b 2f\n\t" + "1:\n\t" + "stwcx. %0, 0, %3\n\t" + "li %0, 0\n\t" + "2:" + : "=&r"(r) + : "r"(o), "r"(n), "r"(p) + : "cc" + ); + + return r; +} + +static OF_INLINE void +of_memory_barrier_sync(void) +{ + __asm__ __volatile__ ( + "sync" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_enter(void) +{ + __asm__ __volatile__ ( + "sync" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_exit(void) +{ + __asm__ __volatile__ ( + "sync" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_producer(void) +{ + __asm__ __volatile__ ( + "sync" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_consumer(void) +{ + __asm__ __volatile__ ( + "sync" ::: "memory" + ); +} + +OF_ASSUME_NONNULL_END ADDED src/atomic_sync_builtins.h Index: src/atomic_sync_builtins.h ================================================================== --- src/atomic_sync_builtins.h +++ src/atomic_sync_builtins.h @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 + * Jonathan Schleifer + * + * All rights reserved. + * + * This file is part of ObjFW. It may be distributed under the terms of the + * Q Public License 1.0, which can be found in the file LICENSE.QPL included in + * the packaging of this file. + * + * Alternatively, it may be distributed under the terms of the GNU General + * Public License, either version 2 or 3, which can be found in the file + * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this + * file. + */ + +OF_ASSUME_NONNULL_BEGIN + +static OF_INLINE int +of_atomic_int_add(volatile int *_Nonnull p, int i) +{ + return __sync_add_and_fetch(p, i); +} + +static OF_INLINE int32_t +of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) +{ + return __sync_add_and_fetch(p, i); +} + +static OF_INLINE void* +of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + return __sync_add_and_fetch(p, (void*)i); +} + +static OF_INLINE int +of_atomic_int_sub(volatile int *_Nonnull p, int i) +{ + return __sync_sub_and_fetch(p, i); +} + +static OF_INLINE int32_t +of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) +{ + return __sync_sub_and_fetch(p, i); +} + +static OF_INLINE void* +of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ + return __sync_sub_and_fetch(p, (void*)i); +} + +static OF_INLINE int +of_atomic_int_inc(volatile int *_Nonnull p) +{ + return __sync_add_and_fetch(p, 1); +} + +static OF_INLINE int32_t +of_atomic_int32_inc(volatile int32_t *_Nonnull p) +{ + return __sync_add_and_fetch(p, 1); +} + +static OF_INLINE int +of_atomic_int_dec(volatile int *_Nonnull p) +{ + return __sync_sub_and_fetch(p, 1); +} + +static OF_INLINE int32_t +of_atomic_int32_dec(volatile int32_t *_Nonnull p) +{ + return __sync_sub_and_fetch(p, 1); +} + +static OF_INLINE unsigned int +of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return __sync_or_and_fetch(p, i); +} + +static OF_INLINE uint32_t +of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return __sync_or_and_fetch(p, i); +} + +static OF_INLINE unsigned int +of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return __sync_and_and_fetch(p, i); +} + +static OF_INLINE uint32_t +of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return __sync_and_and_fetch(p, i); +} + +static OF_INLINE unsigned int +of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) +{ + return __sync_xor_and_fetch(p, i); +} + +static OF_INLINE uint32_t +of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) +{ + return __sync_xor_and_fetch(p, i); +} + +static OF_INLINE bool +of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) +{ + return __sync_bool_compare_and_swap(p, o, n); +} + +static OF_INLINE bool +of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) +{ + return __sync_bool_compare_and_swap(p, o, n); +} + +static OF_INLINE bool +of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, + void *_Nullable o, void *_Nullable n) +{ + return __sync_bool_compare_and_swap(p, o, n); +} + +static OF_INLINE void +of_memory_barrier_sync(void) +{ + __sync_synchronize(); +} + +static OF_INLINE void +of_memory_barrier_enter(void) +{ + __sync_synchronize(); +} + +static OF_INLINE void +of_memory_barrier_exit(void) +{ + __sync_synchronize(); +} + +static OF_INLINE void +of_memory_barrier_producer(void) +{ + __sync_synchronize(); +} + +static OF_INLINE void +of_memory_barrier_consumer(void) +{ + __sync_synchronize(); +} + +OF_ASSUME_NONNULL_END ADDED src/atomic_x86.h Index: src/atomic_x86.h ================================================================== --- src/atomic_x86.h +++ src/atomic_x86.h @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 + * Jonathan Schleifer + * + * All rights reserved. + * + * This file is part of ObjFW. It may be distributed under the terms of the + * Q Public License 1.0, which can be found in the file LICENSE.QPL included in + * the packaging of this file. + * + * Alternatively, it may be distributed under the terms of the GNU General + * Public License, either version 2 or 3, which can be found in the file + * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this + * file. + */ + +OF_ASSUME_NONNULL_BEGIN + +static OF_INLINE int +of_atomic_int_add(volatile int *_Nonnull p, int i) +{ + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "lock\n\t" + "xaddl %0, %2\n\t" + "addl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "lock\n\t" + "xaddq %0, %2\n\t" + "addq %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i) +{ + __asm__ __volatile__ ( + "lock\n\t" + "xaddl %0, %2\n\t" + "addl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return i; +} + +static OF_INLINE void* +of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ +#if defined(OF_X86_64_ASM) + __asm__ __volatile__ ( + "lock\n\t" + "xaddq %0, %2\n\t" + "addq %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return (void*)i; +#elif defined(OF_X86_ASM) + __asm__ __volatile__ ( + "lock\n\t" + "xaddl %0, %2\n\t" + "addl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return (void*)i; +#endif +} + +static OF_INLINE int +of_atomic_int_sub(volatile int *_Nonnull p, int i) +{ + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "negl %0\n\t" + "lock\n\t" + "xaddl %0, %2\n\t" + "subl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "negq %0\n\t" + "lock\n\t" + "xaddq %0, %2\n\t" + "subq %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i) +{ + __asm__ __volatile__ ( + "negl %0\n\t" + "lock\n\t" + "xaddl %0, %2\n\t" + "subl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return i; +} + +static OF_INLINE void* +of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i) +{ +#if defined(OF_X86_64_ASM) + __asm__ __volatile__ ( + "negq %0\n\t" + "lock\n\t" + "xaddq %0, %2\n\t" + "subq %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return (void*)i; +#elif defined(OF_X86_ASM) + __asm__ __volatile__ ( + "negl %0\n\t" + "lock\n\t" + "xaddl %0, %2\n\t" + "subl %1, %0" + : "+&r"(i) + : "r"(i), "m"(*p) + ); + + return (void*)i; +#endif +} + +static OF_INLINE int +of_atomic_int_inc(volatile int *_Nonnull p) +{ + int i; + + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "xorl %0, %0\n\t" + "incl %0\n\t" + "lock\n\t" + "xaddl %0, %1\n\t" + "incl %0" + : "=&r"(i) + : "m"(*p) + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "xorq %0, %0\n\t" + "incq %0\n\t" + "lock\n\t" + "xaddq %0, %1\n\t" + "incq %0" + : "=&r"(i) + : "m"(*p) + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_inc(volatile int32_t *_Nonnull p) +{ + int32_t i; + + __asm__ __volatile__ ( + "xorl %0, %0\n\t" + "incl %0\n\t" + "lock\n\t" + "xaddl %0, %1\n\t" + "incl %0" + : "=&r"(i) + : "m"(*p) + ); + + return i; +} + +static OF_INLINE int +of_atomic_int_dec(volatile int *_Nonnull p) +{ + int i; + + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "xorl %0, %0\n\t" + "decl %0\n\t" + "lock\n\t" + "xaddl %0, %1\n\t" + "decl %0" + : "=&r"(i) + : "m"(*p) + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "xorq %0, %0\n\t" + "decq %0\n\t" + "lock\n\t" + "xaddq %0, %1\n\t" + "decq %0" + : "=&r"(i) + : "m"(*p) + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE int32_t +of_atomic_int32_dec(volatile int32_t *_Nonnull p) +{ + int32_t i; + + __asm__ __volatile__ ( + "xorl %0, %0\n\t" + "decl %0\n\t" + "lock\n\t" + "xaddl %0, %1\n\t" + "decl %0" + : "=&r"(i) + : "m"(*p) + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i) +{ + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "orl %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "0:\n\t" + "movq %2, %0\n\t" + "movq %0, %%rax\n\t" + "orq %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "rax", "cc" + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "orl %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i) +{ + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "andl %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "0:\n\t" + "movq %2, %0\n\t" + "movq %0, %%rax\n\t" + "andq %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "rax", "cc" + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "andl %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); + + return i; +} + +static OF_INLINE unsigned int +of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i) +{ + if (sizeof(int) == 4) + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "xorl %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); +#ifdef OF_X86_64_ASM + else if (sizeof(int) == 8) + __asm__ __volatile__ ( + "0:\n\t" + "movq %2, %0\n\t" + "movq %0, %%rax\n\t" + "xorq %1, %0\n\t" + "lock\n\t" + "cmpxchg %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "rax", "cc" + ); +#endif + else + abort(); + + return i; +} + +static OF_INLINE uint32_t +of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i) +{ + __asm__ __volatile__ ( + "0:\n\t" + "movl %2, %0\n\t" + "movl %0, %%eax\n\t" + "xorl %1, %0\n\t" + "lock\n\t" + "cmpxchgl %0, %2\n\t" + "jne 0b" + : "=&r"(i) + : "r"(i), "m"(*p) + : "eax", "cc" + ); + + return i; +} + +static OF_INLINE bool +of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n) +{ + int r; + + __asm__ __volatile__ ( + "lock\n\t" + "cmpxchg %2, %3\n\t" + "sete %b0\n\t" + "movzbl %b0, %0" + : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ + : "r"(n), "m"(*p) + : "cc" + ); + + return r; +} + +static OF_INLINE bool +of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n) +{ + int r; + + __asm__ __volatile__ ( + "lock\n\t" + "cmpxchg %2, %3\n\t" + "sete %b0\n\t" + "movzbl %b0, %0" + : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ + : "r"(n), "m"(*p) + : "cc" + ); + + return r; +} + +static OF_INLINE bool +of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p, + void *_Nullable o, void *_Nullable n) +{ + int r; + + __asm__ __volatile__ ( + "lock\n\t" + "cmpxchg %2, %3\n\t" + "sete %b0\n\t" + "movzbl %b0, %0" + : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */ + : "r"(n), "m"(*p) + : "cc" + ); + + return r; +} + +static OF_INLINE void +of_memory_barrier_sync(void) +{ + __asm__ __volatile__ ( + "mfence" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_enter(void) +{ + __asm__ __volatile__ ( + "mfence" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_exit(void) +{ + __asm__ __volatile__ ( + "mfence" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_producer(void) +{ + __asm__ __volatile__ ( + "sfence" ::: "memory" + ); +} + +static OF_INLINE void +of_memory_barrier_consumer(void) +{ + __asm__ __volatile__ ( + "lfence" ::: "memory" + ); +} + +OF_ASSUME_NONNULL_END Index: src/objfw-defs.h.in ================================================================== --- src/objfw-defs.h.in +++ src/objfw-defs.h.in @@ -7,11 +7,10 @@ #undef OF_HAVE_BUILTIN_BSWAP64 #undef OF_HAVE_CHMOD #undef OF_HAVE_CHOWN #undef OF_HAVE_FILES #undef OF_HAVE_FORWARDING_TARGET_FOR_SELECTOR -#undef OF_HAVE_GCC_ATOMIC_OPS #undef OF_HAVE_IPV6 #undef OF_HAVE_LINK #undef OF_HAVE_MAX_ALIGN_T #undef OF_HAVE_NETINET_IN_H #undef OF_HAVE_OSATOMIC @@ -19,19 +18,20 @@ #undef OF_HAVE_PIPE #undef OF_HAVE_PLUGINS #undef OF_HAVE_PROCESSES #undef OF_HAVE_PTHREADS #undef OF_HAVE_PTHREAD_SPINLOCKS -#undef OF_HAVE_RECURSIVE_PTHREAD_MUTEXES #undef OF_HAVE_READLINK +#undef OF_HAVE_RECURSIVE_PTHREAD_MUTEXES #undef OF_HAVE_SCHED_YIELD #undef OF_HAVE_SOCKETS #undef OF_HAVE_STDNORETURN #undef OF_HAVE_SYMLINK +#undef OF_HAVE_SYNC_BUILTINS #undef OF_HAVE_SYS_SOCKET_H #undef OF_HAVE_THREADS -#undef OF_HAVE___THREAD #undef OF_HAVE__THREAD_LOCAL +#undef OF_HAVE___THREAD #undef OF_NINTENDO_DS #undef OF_OBJFW_RUNTIME #undef OF_UNIVERSAL #undef SIZE_MAX