tsan_rtl_amd64.S 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323
  1. #include "sanitizer_common/sanitizer_asm.h"
  2. .hidden __tsan_trace_switch
  3. .globl __tsan_trace_switch_thunk
  4. __tsan_trace_switch_thunk:
  5. CFI_STARTPROC
  6. # Save scratch registers.
  7. push %rax
  8. CFI_ADJUST_CFA_OFFSET(8)
  9. CFI_REL_OFFSET(%rax, 0)
  10. push %rcx
  11. CFI_ADJUST_CFA_OFFSET(8)
  12. CFI_REL_OFFSET(%rcx, 0)
  13. push %rdx
  14. CFI_ADJUST_CFA_OFFSET(8)
  15. CFI_REL_OFFSET(%rdx, 0)
  16. push %rsi
  17. CFI_ADJUST_CFA_OFFSET(8)
  18. CFI_REL_OFFSET(%rsi, 0)
  19. push %rdi
  20. CFI_ADJUST_CFA_OFFSET(8)
  21. CFI_REL_OFFSET(%rdi, 0)
  22. push %r8
  23. CFI_ADJUST_CFA_OFFSET(8)
  24. CFI_REL_OFFSET(%r8, 0)
  25. push %r9
  26. CFI_ADJUST_CFA_OFFSET(8)
  27. CFI_REL_OFFSET(%r9, 0)
  28. push %r10
  29. CFI_ADJUST_CFA_OFFSET(8)
  30. CFI_REL_OFFSET(%r10, 0)
  31. push %r11
  32. CFI_ADJUST_CFA_OFFSET(8)
  33. CFI_REL_OFFSET(%r11, 0)
  34. # Align stack frame.
  35. push %rbx # non-scratch
  36. CFI_ADJUST_CFA_OFFSET(8)
  37. CFI_REL_OFFSET(%rbx, 0)
  38. mov %rsp, %rbx # save current rsp
  39. CFI_DEF_CFA_REGISTER(%rbx)
  40. shr $4, %rsp # clear 4 lsb, align to 16
  41. shl $4, %rsp
  42. call __tsan_trace_switch
  43. # Unalign stack frame back.
  44. mov %rbx, %rsp # restore the original rsp
  45. CFI_DEF_CFA_REGISTER(%rsp)
  46. pop %rbx
  47. CFI_ADJUST_CFA_OFFSET(-8)
  48. # Restore scratch registers.
  49. pop %r11
  50. CFI_ADJUST_CFA_OFFSET(-8)
  51. pop %r10
  52. CFI_ADJUST_CFA_OFFSET(-8)
  53. pop %r9
  54. CFI_ADJUST_CFA_OFFSET(-8)
  55. pop %r8
  56. CFI_ADJUST_CFA_OFFSET(-8)
  57. pop %rdi
  58. CFI_ADJUST_CFA_OFFSET(-8)
  59. pop %rsi
  60. CFI_ADJUST_CFA_OFFSET(-8)
  61. pop %rdx
  62. CFI_ADJUST_CFA_OFFSET(-8)
  63. pop %rcx
  64. CFI_ADJUST_CFA_OFFSET(-8)
  65. pop %rax
  66. CFI_ADJUST_CFA_OFFSET(-8)
  67. CFI_RESTORE(%rax)
  68. CFI_RESTORE(%rbx)
  69. CFI_RESTORE(%rcx)
  70. CFI_RESTORE(%rdx)
  71. CFI_RESTORE(%rsi)
  72. CFI_RESTORE(%rdi)
  73. CFI_RESTORE(%r8)
  74. CFI_RESTORE(%r9)
  75. CFI_RESTORE(%r10)
  76. CFI_RESTORE(%r11)
  77. ret
  78. CFI_ENDPROC
  79. .hidden __tsan_report_race
  80. .globl __tsan_report_race_thunk
  81. __tsan_report_race_thunk:
  82. CFI_STARTPROC
  83. # Save scratch registers.
  84. push %rax
  85. CFI_ADJUST_CFA_OFFSET(8)
  86. CFI_REL_OFFSET(%rax, 0)
  87. push %rcx
  88. CFI_ADJUST_CFA_OFFSET(8)
  89. CFI_REL_OFFSET(%rcx, 0)
  90. push %rdx
  91. CFI_ADJUST_CFA_OFFSET(8)
  92. CFI_REL_OFFSET(%rdx, 0)
  93. push %rsi
  94. CFI_ADJUST_CFA_OFFSET(8)
  95. CFI_REL_OFFSET(%rsi, 0)
  96. push %rdi
  97. CFI_ADJUST_CFA_OFFSET(8)
  98. CFI_REL_OFFSET(%rdi, 0)
  99. push %r8
  100. CFI_ADJUST_CFA_OFFSET(8)
  101. CFI_REL_OFFSET(%r8, 0)
  102. push %r9
  103. CFI_ADJUST_CFA_OFFSET(8)
  104. CFI_REL_OFFSET(%r9, 0)
  105. push %r10
  106. CFI_ADJUST_CFA_OFFSET(8)
  107. CFI_REL_OFFSET(%r10, 0)
  108. push %r11
  109. CFI_ADJUST_CFA_OFFSET(8)
  110. CFI_REL_OFFSET(%r11, 0)
  111. # Align stack frame.
  112. push %rbx # non-scratch
  113. CFI_ADJUST_CFA_OFFSET(8)
  114. CFI_REL_OFFSET(%rbx, 0)
  115. mov %rsp, %rbx # save current rsp
  116. CFI_DEF_CFA_REGISTER(%rbx)
  117. shr $4, %rsp # clear 4 lsb, align to 16
  118. shl $4, %rsp
  119. call __tsan_report_race
  120. # Unalign stack frame back.
  121. mov %rbx, %rsp # restore the original rsp
  122. CFI_DEF_CFA_REGISTER(%rsp)
  123. pop %rbx
  124. CFI_ADJUST_CFA_OFFSET(-8)
  125. # Restore scratch registers.
  126. pop %r11
  127. CFI_ADJUST_CFA_OFFSET(-8)
  128. pop %r10
  129. CFI_ADJUST_CFA_OFFSET(-8)
  130. pop %r9
  131. CFI_ADJUST_CFA_OFFSET(-8)
  132. pop %r8
  133. CFI_ADJUST_CFA_OFFSET(-8)
  134. pop %rdi
  135. CFI_ADJUST_CFA_OFFSET(-8)
  136. pop %rsi
  137. CFI_ADJUST_CFA_OFFSET(-8)
  138. pop %rdx
  139. CFI_ADJUST_CFA_OFFSET(-8)
  140. pop %rcx
  141. CFI_ADJUST_CFA_OFFSET(-8)
  142. pop %rax
  143. CFI_ADJUST_CFA_OFFSET(-8)
  144. CFI_RESTORE(%rax)
  145. CFI_RESTORE(%rbx)
  146. CFI_RESTORE(%rcx)
  147. CFI_RESTORE(%rdx)
  148. CFI_RESTORE(%rsi)
  149. CFI_RESTORE(%rdi)
  150. CFI_RESTORE(%r8)
  151. CFI_RESTORE(%r9)
  152. CFI_RESTORE(%r10)
  153. CFI_RESTORE(%r11)
  154. ret
  155. CFI_ENDPROC
  156. .hidden __tsan_setjmp
  157. .comm _ZN14__interception11real_setjmpE,8,8
  158. .globl setjmp
  159. .type setjmp, @function
  160. setjmp:
  161. CFI_STARTPROC
  162. // save env parameter
  163. push %rdi
  164. CFI_ADJUST_CFA_OFFSET(8)
  165. CFI_REL_OFFSET(%rdi, 0)
  166. // obtain %rsp
  167. #if defined(__FreeBSD__)
  168. lea 8(%rsp), %rdi
  169. mov %rdi, %rsi
  170. #else
  171. lea 16(%rsp), %rdi
  172. mov %rdi, %rsi
  173. xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp)
  174. rol $0x11, %rsi
  175. #endif
  176. // call tsan interceptor
  177. call __tsan_setjmp
  178. // restore env parameter
  179. pop %rdi
  180. CFI_ADJUST_CFA_OFFSET(-8)
  181. CFI_RESTORE(%rdi)
  182. // tail jump to libc setjmp
  183. movl $0, %eax
  184. movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
  185. jmp *(%rdx)
  186. CFI_ENDPROC
  187. .size setjmp, .-setjmp
  188. .comm _ZN14__interception12real__setjmpE,8,8
  189. .globl _setjmp
  190. .type _setjmp, @function
  191. _setjmp:
  192. CFI_STARTPROC
  193. // save env parameter
  194. push %rdi
  195. CFI_ADJUST_CFA_OFFSET(8)
  196. CFI_REL_OFFSET(%rdi, 0)
  197. // obtain %rsp
  198. #if defined(__FreeBSD__)
  199. lea 8(%rsp), %rdi
  200. mov %rdi, %rsi
  201. #else
  202. lea 16(%rsp), %rdi
  203. mov %rdi, %rsi
  204. xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp)
  205. rol $0x11, %rsi
  206. #endif
  207. // call tsan interceptor
  208. call __tsan_setjmp
  209. // restore env parameter
  210. pop %rdi
  211. CFI_ADJUST_CFA_OFFSET(-8)
  212. CFI_RESTORE(%rdi)
  213. // tail jump to libc setjmp
  214. movl $0, %eax
  215. movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
  216. jmp *(%rdx)
  217. CFI_ENDPROC
  218. .size _setjmp, .-_setjmp
  219. .comm _ZN14__interception14real_sigsetjmpE,8,8
  220. .globl sigsetjmp
  221. .type sigsetjmp, @function
  222. sigsetjmp:
  223. CFI_STARTPROC
  224. // save env parameter
  225. push %rdi
  226. CFI_ADJUST_CFA_OFFSET(8)
  227. CFI_REL_OFFSET(%rdi, 0)
  228. // save savesigs parameter
  229. push %rsi
  230. CFI_ADJUST_CFA_OFFSET(8)
  231. CFI_REL_OFFSET(%rsi, 0)
  232. // align stack frame
  233. sub $8, %rsp
  234. CFI_ADJUST_CFA_OFFSET(8)
  235. // obtain %rsp
  236. #if defined(__FreeBSD__)
  237. lea 24(%rsp), %rdi
  238. mov %rdi, %rsi
  239. #else
  240. lea 32(%rsp), %rdi
  241. mov %rdi, %rsi
  242. xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp)
  243. rol $0x11, %rsi
  244. #endif
  245. // call tsan interceptor
  246. call __tsan_setjmp
  247. // unalign stack frame
  248. add $8, %rsp
  249. CFI_ADJUST_CFA_OFFSET(-8)
  250. // restore savesigs parameter
  251. pop %rsi
  252. CFI_ADJUST_CFA_OFFSET(-8)
  253. CFI_RESTORE(%rsi)
  254. // restore env parameter
  255. pop %rdi
  256. CFI_ADJUST_CFA_OFFSET(-8)
  257. CFI_RESTORE(%rdi)
  258. // tail jump to libc sigsetjmp
  259. movl $0, %eax
  260. movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
  261. jmp *(%rdx)
  262. CFI_ENDPROC
  263. .size sigsetjmp, .-sigsetjmp
  264. .comm _ZN14__interception16real___sigsetjmpE,8,8
  265. .globl __sigsetjmp
  266. .type __sigsetjmp, @function
  267. __sigsetjmp:
  268. CFI_STARTPROC
  269. // save env parameter
  270. push %rdi
  271. CFI_ADJUST_CFA_OFFSET(8)
  272. CFI_REL_OFFSET(%rdi, 0)
  273. // save savesigs parameter
  274. push %rsi
  275. CFI_ADJUST_CFA_OFFSET(8)
  276. CFI_REL_OFFSET(%rsi, 0)
  277. // align stack frame
  278. sub $8, %rsp
  279. CFI_ADJUST_CFA_OFFSET(8)
  280. // obtain %rsp
  281. #if defined(__FreeBSD__)
  282. lea 24(%rsp), %rdi
  283. mov %rdi, %rsi
  284. #else
  285. lea 32(%rsp), %rdi
  286. mov %rdi, %rsi
  287. xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp)
  288. rol $0x11, %rsi
  289. #endif
  290. // call tsan interceptor
  291. call __tsan_setjmp
  292. // unalign stack frame
  293. add $8, %rsp
  294. CFI_ADJUST_CFA_OFFSET(-8)
  295. // restore savesigs parameter
  296. pop %rsi
  297. CFI_ADJUST_CFA_OFFSET(-8)
  298. CFI_RESTORE(%rsi)
  299. // restore env parameter
  300. pop %rdi
  301. CFI_ADJUST_CFA_OFFSET(-8)
  302. CFI_RESTORE(%rdi)
  303. // tail jump to libc sigsetjmp
  304. movl $0, %eax
  305. movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
  306. jmp *(%rdx)
  307. CFI_ENDPROC
  308. .size __sigsetjmp, .-__sigsetjmp
  309. #if defined(__FreeBSD__) || defined(__linux__)
  310. /* We do not need executable stack. */
  311. .section .note.GNU-stack,"",@progbits
  312. #endif