libatomic_i.h 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. /* Copyright (C) 2012-2015 Free Software Foundation, Inc.
  2. Contributed by Richard Henderson <rth@redhat.com>.
  3. This file is part of the GNU Atomic Library (libatomic).
  4. Libatomic is free software; you can redistribute it and/or modify it
  5. under the terms of the GNU General Public License as published by
  6. the Free Software Foundation; either version 3, or (at your option)
  7. any later version.
  8. Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
  9. WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  10. FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  11. more details.
  12. Under Section 7 of GPL version 3, you are granted additional
  13. permissions described in the GCC Runtime Library Exception, version
  14. 3.1, as published by the Free Software Foundation.
  15. You should have received a copy of the GNU General Public License and
  16. a copy of the GCC Runtime Library Exception along with this program;
  17. see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
  18. <http://www.gnu.org/licenses/>. */
  19. /* This file contains data types and function declarations that are
  20. private to the implementation of libatomic. */
  21. #ifndef LIBATOMIC_H
  22. #define LIBATOMIC_H 1
  23. #include "auto-config.h"
  24. #include <stdbool.h>
  25. #include <stdint.h>
  26. #include <stddef.h>
  27. #include <limits.h>
  28. #include <string.h>
  29. /* Symbol concatenation macros. */
  30. #define C2_(X,Y) X ## Y
  31. #define C2(X,Y) C2_(X,Y)
  32. #define C3_(X,Y,Z) X ## Y ## Z
  33. #define C3(X,Y,Z) C3_(X,Y,Z)
  34. #define C4_(W,X,Y,Z) W ## X ## Y ## Z
  35. #define C4(W,X,Y,Z) C4_(W,X,Y,Z)
  36. /* Stringification macros. */
  37. #define S2(X) #X
  38. #define S(X) S2(X)
  39. /* All of the primitive types on which we operate. */
  40. typedef unsigned U_1 __attribute__((mode(QI)));
  41. #if HAVE_INT2
  42. typedef unsigned U_2 __attribute__((mode(HI)));
  43. #endif
  44. #if HAVE_INT4
  45. typedef unsigned U_4 __attribute__((mode(SI)));
  46. #endif
  47. #if HAVE_INT8
  48. typedef unsigned U_8 __attribute__((mode(DI)));
  49. #endif
  50. #if HAVE_INT16
  51. typedef unsigned U_16 __attribute__((mode(TI)));
  52. #endif
  53. /* The widest type that we support. */
  54. #if HAVE_INT16
  55. # define MAX_SIZE 16
  56. #elif HAVE_INT8
  57. # define MAX_SIZE 8
  58. #elif HAVE_INT4
  59. # define MAX_SIZE 4
  60. #elif HAVE_INT2
  61. # define MAX_SIZE 2
  62. #else
  63. # define MAX_SIZE 1
  64. #endif
  65. typedef C2(U_,MAX_SIZE) U_MAX;
  66. /* Provide dummy fallback types so that stuff is syntactically correct
  67. without having to overdo the ifdefs. The code using these should
  68. always be protected with the HAVE_INT{n} macros. */
  69. #if !HAVE_INT2
  70. typedef U_MAX U_2;
  71. #endif
  72. #if !HAVE_INT4
  73. typedef U_MAX U_4;
  74. #endif
  75. #if !HAVE_INT8
  76. typedef U_MAX U_8;
  77. #endif
  78. #if !HAVE_INT16
  79. typedef U_MAX U_16;
  80. #endif
  81. union max_size_u
  82. {
  83. U_1 b[MAX_SIZE];
  84. U_2 i2;
  85. U_4 i4;
  86. U_8 i8;
  87. U_16 i16;
  88. };
  89. /* The "word" size of the machine. */
  90. typedef unsigned UWORD __attribute__((mode(word)));
  91. /* Macros for handing sub-word sized quantities. */
  92. #define MASK_1 ((UWORD)0xff)
  93. #define MASK_2 ((UWORD)0xffff)
  94. #define MASK_4 ((UWORD)0xffffffff)
  95. #define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
  96. #define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
  97. #define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
  98. /* Most of the files in this library are compiled multiple times with
  99. N defined to be a power of 2 between 1 and 16. The SIZE macro is
  100. then used to append _N to the symbol being manipulated. */
  101. #define SIZE(X) C3(X,_,N)
  102. #define WSIZE(X) C3(X,_,WORDSIZE)
  103. #define PTR(N,X) ((C2(U_,N) *)X)
  104. /* And thus, the type on which this compilation will be operating. */
  105. #define ITYPE SIZE(I)
  106. #define UTYPE SIZE(U)
  107. /* Utility macros for GCC attributes. */
  108. #define UNUSED __attribute__((unused))
  109. #ifdef HAVE_ATTRIBUTE_VISIBILITY
  110. # define HIDDEN __attribute__((visibility("hidden")))
  111. #else
  112. # define HIDDEN
  113. #endif
  114. /* Occasionally we have to play games with internal and external symbol
  115. names, in order to work around builtin functions of the same name.
  116. This macro sets the external name of the function appropriately. */
  117. #define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
  118. /* Locking for a "small" operation. In the bare-metal single processor
  119. cases this could be implemented by disabling interrupts. Thus the extra
  120. word passed between the two functions, saving the interrupt level.
  121. It is assumed that the object being locked does not cross the locking
  122. granularity.
  123. Not actually declared here so that they can be defined static inline
  124. in a target-specfic <host-config.h>.
  125. UWORD protect_start (void *ptr);
  126. void protect_end (void *ptr, UWORD);
  127. */
  128. /* Locking for a "large' operation. This should always be some sort of
  129. test-and-set operation, as we assume that the interrupt latency would
  130. be unreasonably large. */
  131. void libat_lock_n (void *ptr, size_t n);
  132. void libat_unlock_n (void *ptr, size_t n);
  133. /* We'll need to declare all of the sized functions a few times... */
  134. #define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N))
  135. #define DECLARE_ALL_SIZED_(N,T) \
  136. DECLARE_1(T, C2(load_,N), (T *mptr, int)); \
  137. DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \
  138. DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \
  139. DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
  140. DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \
  141. DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \
  142. DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \
  143. DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \
  144. DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \
  145. DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \
  146. DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \
  147. DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \
  148. DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \
  149. DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \
  150. DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \
  151. DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \
  152. DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int))
  153. /* All sized operations are implemented in hidden functions prefixed with
  154. "libat_". These are either renamed or aliased to the expected prefix
  155. of "__atomic". Some amount of renaming is required to avoid hiding or
  156. conflicting with the builtins of the same name, but this additional
  157. use of hidden symbols (where appropriate) avoids unnecessary PLT entries
  158. on relevant targets. */
  159. #if IFUNC_ALT
  160. # define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
  161. #elif defined(HAVE_ATTRIBUTE_ALIAS)
  162. # define MAN(X) HIDDEN
  163. #else
  164. # define MAN(X) ASMNAME(C2(__atomic_,X))
  165. #endif
  166. #if !defined(N) && HAVE_IFUNC
  167. # define DECLARE_1(RET,NAME,ARGS) \
  168. RET C2(libat_,NAME) ARGS MAN(NAME); \
  169. RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
  170. #else
  171. # define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME)
  172. #endif
  173. /* Prefix to use when calling internal, possibly ifunc'ed functions. */
  174. #if HAVE_IFUNC
  175. # define local_ ifunc_
  176. #else
  177. # define local_ libat_
  178. #endif
  179. DECLARE_ALL_SIZED(1);
  180. DECLARE_ALL_SIZED(2);
  181. DECLARE_ALL_SIZED(4);
  182. DECLARE_ALL_SIZED(8);
  183. DECLARE_ALL_SIZED(16);
  184. #undef DECLARE_1
  185. #undef DECLARE_ALL_SIZED
  186. #undef DECLARE_ALL_SIZED_
  187. /* And the generic sized versions. */
  188. void libat_load (size_t, void *, void *, int) MAN(load);
  189. void libat_store (size_t, void *, void *, int) MAN(store);
  190. void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
  191. bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
  192. MAN(compare_exchange);
  193. bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
  194. #undef MAN
  195. #include <host-config.h>
  196. /* We don't have IFUNC_NCOND until after host-config.h. */
  197. #if !HAVE_IFUNC
  198. # define IFUNC_NCOND(N) 0
  199. #endif
  200. #if IFUNC_ALT
  201. # define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */
  202. #elif defined(N) && IFUNC_NCOND(N)
  203. # if IFUNC_NCOND(N) == 1
  204. # define GEN_SELECTOR(X) \
  205. extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
  206. static void * C2(select_,X) (void) \
  207. { \
  208. if (IFUNC_COND_1) \
  209. return C3(libat_,X,_i1); \
  210. return C2(libat_,X); \
  211. }
  212. # elif IFUNC_NCOND(N) == 2
  213. # define GEN_SELECTOR(X) \
  214. extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
  215. extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
  216. static void * C2(select_,X) (void) \
  217. { \
  218. if (IFUNC_COND_1) \
  219. return C3(libat_,X,_i1); \
  220. if (IFUNC_COND_2) \
  221. return C3(libat_,X,_i2); \
  222. return C2(libat_,X); \
  223. }
  224. # elif IFUNC_NCOND(N) == 3
  225. # define GEN_SELECTOR(X) \
  226. extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
  227. extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
  228. extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \
  229. static void * C2(select_,X) (void) \
  230. { \
  231. if (IFUNC_COND_1) \
  232. return C3(libat_,X,_i1); \
  233. if (IFUNC_COND_2) \
  234. return C3(libat_,X,_i2); \
  235. if (IFUNC_COND_3) \
  236. return C3(libat_,X,_i3); \
  237. return C2(libat_,X); \
  238. }
  239. # else
  240. # error "Unsupported number of ifunc alternatives."
  241. # endif
  242. # define EXPORT_ALIAS(X) \
  243. GEN_SELECTOR(X) \
  244. typeof(C2(libat_,X)) C2(ifunc_,X) \
  245. ASMNAME(C2(__atomic_,X)) \
  246. __attribute__((ifunc(S(C2(select_,X)))))
  247. #elif defined(HAVE_ATTRIBUTE_ALIAS)
  248. # define EXPORT_ALIAS(X) \
  249. extern typeof(C2(libat_,X)) C2(export_,X) \
  250. ASMNAME(C2(__atomic_,X)) \
  251. __attribute__((alias(S(C2(libat_,X)))))
  252. #else
  253. # define EXPORT_ALIAS(X) /* original symbol is exported */
  254. #endif
  255. #endif /* LIBATOMIC_H */