ObjFW
atomic_x86.h
1 /*
2  * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017
3  * Jonathan Schleifer <js@heap.zone>
4  *
5  * All rights reserved.
6  *
7  * This file is part of ObjFW. It may be distributed under the terms of the
8  * Q Public License 1.0, which can be found in the file LICENSE.QPL included in
9  * the packaging of this file.
10  *
11  * Alternatively, it may be distributed under the terms of the GNU General
12  * Public License, either version 2 or 3, which can be found in the file
13  * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
14  * file.
15  */
16 
17 OF_ASSUME_NONNULL_BEGIN
18 
19 static OF_INLINE int
20 of_atomic_int_add(volatile int *_Nonnull p, int i)
21 {
22  if (sizeof(int) == 4)
23  __asm__ __volatile__ (
24  "lock\n\t"
25  "xaddl %0, %2\n\t"
26  "addl %1, %0"
27  : "+&r"(i)
28  : "r"(i), "m"(*p)
29  );
30 #ifdef OF_X86_64_ASM
31  else if (sizeof(int) == 8)
32  __asm__ __volatile__ (
33  "lock\n\t"
34  "xaddq %0, %2\n\t"
35  "addq %1, %0"
36  : "+&r"(i)
37  : "r"(i), "m"(*p)
38  );
39 #endif
40  else
41  abort();
42 
43  return i;
44 }
45 
46 static OF_INLINE int32_t
47 of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i)
48 {
49  __asm__ __volatile__ (
50  "lock\n\t"
51  "xaddl %0, %2\n\t"
52  "addl %1, %0"
53  : "+&r"(i)
54  : "r"(i), "m"(*p)
55  );
56 
57  return i;
58 }
59 
60 static OF_INLINE void *_Nullable
61 of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i)
62 {
63 #if defined(OF_X86_64_ASM)
64  __asm__ __volatile__ (
65  "lock\n\t"
66  "xaddq %0, %2\n\t"
67  "addq %1, %0"
68  : "+&r"(i)
69  : "r"(i), "m"(*p)
70  );
71 
72  return (void *)i;
73 #elif defined(OF_X86_ASM)
74  __asm__ __volatile__ (
75  "lock\n\t"
76  "xaddl %0, %2\n\t"
77  "addl %1, %0"
78  : "+&r"(i)
79  : "r"(i), "m"(*p)
80  );
81 
82  return (void *)i;
83 #endif
84 }
85 
86 static OF_INLINE int
87 of_atomic_int_sub(volatile int *_Nonnull p, int i)
88 {
89  if (sizeof(int) == 4)
90  __asm__ __volatile__ (
91  "negl %0\n\t"
92  "lock\n\t"
93  "xaddl %0, %2\n\t"
94  "subl %1, %0"
95  : "+&r"(i)
96  : "r"(i), "m"(*p)
97  );
98 #ifdef OF_X86_64_ASM
99  else if (sizeof(int) == 8)
100  __asm__ __volatile__ (
101  "negq %0\n\t"
102  "lock\n\t"
103  "xaddq %0, %2\n\t"
104  "subq %1, %0"
105  : "+&r"(i)
106  : "r"(i), "m"(*p)
107  );
108 #endif
109  else
110  abort();
111 
112  return i;
113 }
114 
115 static OF_INLINE int32_t
116 of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i)
117 {
118  __asm__ __volatile__ (
119  "negl %0\n\t"
120  "lock\n\t"
121  "xaddl %0, %2\n\t"
122  "subl %1, %0"
123  : "+&r"(i)
124  : "r"(i), "m"(*p)
125  );
126 
127  return i;
128 }
129 
130 static OF_INLINE void *_Nullable
131 of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i)
132 {
133 #if defined(OF_X86_64_ASM)
134  __asm__ __volatile__ (
135  "negq %0\n\t"
136  "lock\n\t"
137  "xaddq %0, %2\n\t"
138  "subq %1, %0"
139  : "+&r"(i)
140  : "r"(i), "m"(*p)
141  );
142 
143  return (void *)i;
144 #elif defined(OF_X86_ASM)
145  __asm__ __volatile__ (
146  "negl %0\n\t"
147  "lock\n\t"
148  "xaddl %0, %2\n\t"
149  "subl %1, %0"
150  : "+&r"(i)
151  : "r"(i), "m"(*p)
152  );
153 
154  return (void *)i;
155 #endif
156 }
157 
158 static OF_INLINE int
159 of_atomic_int_inc(volatile int *_Nonnull p)
160 {
161  int i;
162 
163  if (sizeof(int) == 4)
164  __asm__ __volatile__ (
165  "xorl %0, %0\n\t"
166  "incl %0\n\t"
167  "lock\n\t"
168  "xaddl %0, %1\n\t"
169  "incl %0"
170  : "=&r"(i)
171  : "m"(*p)
172  );
173 #ifdef OF_X86_64_ASM
174  else if (sizeof(int) == 8)
175  __asm__ __volatile__ (
176  "xorq %0, %0\n\t"
177  "incq %0\n\t"
178  "lock\n\t"
179  "xaddq %0, %1\n\t"
180  "incq %0"
181  : "=&r"(i)
182  : "m"(*p)
183  );
184 #endif
185  else
186  abort();
187 
188  return i;
189 }
190 
191 static OF_INLINE int32_t
192 of_atomic_int32_inc(volatile int32_t *_Nonnull p)
193 {
194  int32_t i;
195 
196  __asm__ __volatile__ (
197  "xorl %0, %0\n\t"
198  "incl %0\n\t"
199  "lock\n\t"
200  "xaddl %0, %1\n\t"
201  "incl %0"
202  : "=&r"(i)
203  : "m"(*p)
204  );
205 
206  return i;
207 }
208 
209 static OF_INLINE int
210 of_atomic_int_dec(volatile int *_Nonnull p)
211 {
212  int i;
213 
214  if (sizeof(int) == 4)
215  __asm__ __volatile__ (
216  "xorl %0, %0\n\t"
217  "decl %0\n\t"
218  "lock\n\t"
219  "xaddl %0, %1\n\t"
220  "decl %0"
221  : "=&r"(i)
222  : "m"(*p)
223  );
224 #ifdef OF_X86_64_ASM
225  else if (sizeof(int) == 8)
226  __asm__ __volatile__ (
227  "xorq %0, %0\n\t"
228  "decq %0\n\t"
229  "lock\n\t"
230  "xaddq %0, %1\n\t"
231  "decq %0"
232  : "=&r"(i)
233  : "m"(*p)
234  );
235 #endif
236  else
237  abort();
238 
239  return i;
240 }
241 
242 static OF_INLINE int32_t
243 of_atomic_int32_dec(volatile int32_t *_Nonnull p)
244 {
245  int32_t i;
246 
247  __asm__ __volatile__ (
248  "xorl %0, %0\n\t"
249  "decl %0\n\t"
250  "lock\n\t"
251  "xaddl %0, %1\n\t"
252  "decl %0"
253  : "=&r"(i)
254  : "m"(*p)
255  );
256 
257  return i;
258 }
259 
260 static OF_INLINE unsigned int
261 of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i)
262 {
263  if (sizeof(int) == 4)
264  __asm__ __volatile__ (
265  "0:\n\t"
266  "movl %2, %0\n\t"
267  "movl %0, %%eax\n\t"
268  "orl %1, %0\n\t"
269  "lock\n\t"
270  "cmpxchg %0, %2\n\t"
271  "jne 0b"
272  : "=&r"(i)
273  : "r"(i), "m"(*p)
274  : "eax", "cc"
275  );
276 #ifdef OF_X86_64_ASM
277  else if (sizeof(int) == 8)
278  __asm__ __volatile__ (
279  "0:\n\t"
280  "movq %2, %0\n\t"
281  "movq %0, %%rax\n\t"
282  "orq %1, %0\n\t"
283  "lock\n\t"
284  "cmpxchg %0, %2\n\t"
285  "jne 0b"
286  : "=&r"(i)
287  : "r"(i), "m"(*p)
288  : "rax", "cc"
289  );
290 #endif
291  else
292  abort();
293 
294  return i;
295 }
296 
297 static OF_INLINE uint32_t
298 of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i)
299 {
300  __asm__ __volatile__ (
301  "0:\n\t"
302  "movl %2, %0\n\t"
303  "movl %0, %%eax\n\t"
304  "orl %1, %0\n\t"
305  "lock\n\t"
306  "cmpxchg %0, %2\n\t"
307  "jne 0b"
308  : "=&r"(i)
309  : "r"(i), "m"(*p)
310  : "eax", "cc"
311  );
312 
313  return i;
314 }
315 
316 static OF_INLINE unsigned int
317 of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i)
318 {
319  if (sizeof(int) == 4)
320  __asm__ __volatile__ (
321  "0:\n\t"
322  "movl %2, %0\n\t"
323  "movl %0, %%eax\n\t"
324  "andl %1, %0\n\t"
325  "lock\n\t"
326  "cmpxchg %0, %2\n\t"
327  "jne 0b"
328  : "=&r"(i)
329  : "r"(i), "m"(*p)
330  : "eax", "cc"
331  );
332 #ifdef OF_X86_64_ASM
333  else if (sizeof(int) == 8)
334  __asm__ __volatile__ (
335  "0:\n\t"
336  "movq %2, %0\n\t"
337  "movq %0, %%rax\n\t"
338  "andq %1, %0\n\t"
339  "lock\n\t"
340  "cmpxchg %0, %2\n\t"
341  "jne 0b"
342  : "=&r"(i)
343  : "r"(i), "m"(*p)
344  : "rax", "cc"
345  );
346 #endif
347  else
348  abort();
349 
350  return i;
351 }
352 
353 static OF_INLINE uint32_t
354 of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i)
355 {
356  __asm__ __volatile__ (
357  "0:\n\t"
358  "movl %2, %0\n\t"
359  "movl %0, %%eax\n\t"
360  "andl %1, %0\n\t"
361  "lock\n\t"
362  "cmpxchg %0, %2\n\t"
363  "jne 0b"
364  : "=&r"(i)
365  : "r"(i), "m"(*p)
366  : "eax", "cc"
367  );
368 
369  return i;
370 }
371 
372 static OF_INLINE unsigned int
373 of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i)
374 {
375  if (sizeof(int) == 4)
376  __asm__ __volatile__ (
377  "0:\n\t"
378  "movl %2, %0\n\t"
379  "movl %0, %%eax\n\t"
380  "xorl %1, %0\n\t"
381  "lock\n\t"
382  "cmpxchg %0, %2\n\t"
383  "jne 0b"
384  : "=&r"(i)
385  : "r"(i), "m"(*p)
386  : "eax", "cc"
387  );
388 #ifdef OF_X86_64_ASM
389  else if (sizeof(int) == 8)
390  __asm__ __volatile__ (
391  "0:\n\t"
392  "movq %2, %0\n\t"
393  "movq %0, %%rax\n\t"
394  "xorq %1, %0\n\t"
395  "lock\n\t"
396  "cmpxchg %0, %2\n\t"
397  "jne 0b"
398  : "=&r"(i)
399  : "r"(i), "m"(*p)
400  : "rax", "cc"
401  );
402 #endif
403  else
404  abort();
405 
406  return i;
407 }
408 
409 static OF_INLINE uint32_t
410 of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i)
411 {
412  __asm__ __volatile__ (
413  "0:\n\t"
414  "movl %2, %0\n\t"
415  "movl %0, %%eax\n\t"
416  "xorl %1, %0\n\t"
417  "lock\n\t"
418  "cmpxchgl %0, %2\n\t"
419  "jne 0b"
420  : "=&r"(i)
421  : "r"(i), "m"(*p)
422  : "eax", "cc"
423  );
424 
425  return i;
426 }
427 
428 static OF_INLINE bool
429 of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n)
430 {
431  int r;
432 
433  __asm__ __volatile__ (
434  "lock\n\t"
435  "cmpxchg %2, %3\n\t"
436  "sete %b0\n\t"
437  "movzbl %b0, %0"
438  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
439  : "r"(n), "m"(*p)
440  : "cc"
441  );
442 
443  return r;
444 }
445 
446 static OF_INLINE bool
447 of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n)
448 {
449  int r;
450 
451  __asm__ __volatile__ (
452  "lock\n\t"
453  "cmpxchg %2, %3\n\t"
454  "sete %b0\n\t"
455  "movzbl %b0, %0"
456  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
457  : "r"(n), "m"(*p)
458  : "cc"
459  );
460 
461  return r;
462 }
463 
464 static OF_INLINE bool
465 of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p,
466  void *_Nullable o, void *_Nullable n)
467 {
468  int r;
469 
470  __asm__ __volatile__ (
471  "lock\n\t"
472  "cmpxchg %2, %3\n\t"
473  "sete %b0\n\t"
474  "movzbl %b0, %0"
475  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
476  : "r"(n), "m"(*p)
477  : "cc"
478  );
479 
480  return r;
481 }
482 
483 static OF_INLINE void
484 of_memory_barrier(void)
485 {
486  __asm__ __volatile__ (
487  "mfence" ::: "memory"
488  );
489 }
490 
491 static OF_INLINE void
492 of_memory_barrier_acquire(void)
493 {
494  __asm__ __volatile__ ("" ::: "memory");
495 }
496 
497 static OF_INLINE void
498 of_memory_barrier_release(void)
499 {
500  __asm__ __volatile__ ("" ::: "memory");
501 }
502 
503 OF_ASSUME_NONNULL_END