entry-header.S 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. #include <linux/init.h>
  2. #include <linux/linkage.h>
  3. #include <asm/assembler.h>
  4. #include <asm/asm-offsets.h>
  5. #include <asm/errno.h>
  6. #include <asm/thread_info.h>
  7. #include <asm/v7m.h>
  8. @ Bad Abort numbers
  9. @ -----------------
  10. @
  11. #define BAD_PREFETCH 0
  12. #define BAD_DATA 1
  13. #define BAD_ADDREXCPTN 2
  14. #define BAD_IRQ 3
  15. #define BAD_UNDEFINSTR 4
  16. @
  17. @ Most of the stack format comes from struct pt_regs, but with
  18. @ the addition of 8 bytes for storing syscall args 5 and 6.
  19. @ This _must_ remain a multiple of 8 for EABI.
  20. @
  21. #define S_OFF 8
  22. /*
  23. * The SWI code relies on the fact that R0 is at the bottom of the stack
  24. * (due to slow/fast restore user regs).
  25. */
  26. #if S_R0 != 0
  27. #error "Please fix"
  28. #endif
  29. .macro zero_fp
  30. #ifdef CONFIG_FRAME_POINTER
  31. mov fp, #0
  32. #endif
  33. .endm
  34. #ifdef CONFIG_ALIGNMENT_TRAP
  35. #define ATRAP(x...) x
  36. #else
  37. #define ATRAP(x...)
  38. #endif
  39. .macro alignment_trap, rtmp1, rtmp2, label
  40. #ifdef CONFIG_ALIGNMENT_TRAP
  41. mrc p15, 0, \rtmp2, c1, c0, 0
  42. ldr \rtmp1, \label
  43. ldr \rtmp1, [\rtmp1]
  44. teq \rtmp1, \rtmp2
  45. mcrne p15, 0, \rtmp1, c1, c0, 0
  46. #endif
  47. .endm
  48. #ifdef CONFIG_CPU_V7M
  49. /*
  50. * ARMv7-M exception entry/exit macros.
  51. *
  52. * xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are
  53. * automatically saved on the current stack (32 words) before
  54. * switching to the exception stack (SP_main).
  55. *
  56. * If exception is taken while in user mode, SP_main is
  57. * empty. Otherwise, SP_main is aligned to 64 bit automatically
  58. * (CCR.STKALIGN set).
  59. *
  60. * Linux assumes that the interrupts are disabled when entering an
  61. * exception handler and it may BUG if this is not the case. Interrupts
  62. * are disabled during entry and reenabled in the exit macro.
  63. *
  64. * v7m_exception_slow_exit is used when returning from SVC or PendSV.
  65. * When returning to kernel mode, we don't return from exception.
  66. */
  67. .macro v7m_exception_entry
  68. @ determine the location of the registers saved by the core during
  69. @ exception entry. Depending on the mode the cpu was in when the
  70. @ exception happend that is either on the main or the process stack.
  71. @ Bit 2 of EXC_RETURN stored in the lr register specifies which stack
  72. @ was used.
  73. tst lr, #EXC_RET_STACK_MASK
  74. mrsne r12, psp
  75. moveq r12, sp
  76. @ we cannot rely on r0-r3 and r12 matching the value saved in the
  77. @ exception frame because of tail-chaining. So these have to be
  78. @ reloaded.
  79. ldmia r12!, {r0-r3}
  80. @ Linux expects to have irqs off. Do it here before taking stack space
  81. cpsid i
  82. sub sp, #PT_REGS_SIZE-S_IP
  83. stmdb sp!, {r0-r11}
  84. @ load saved r12, lr, return address and xPSR.
  85. @ r0-r7 are used for signals and never touched from now on. Clobbering
  86. @ r8-r12 is OK.
  87. mov r9, r12
  88. ldmia r9!, {r8, r10-r12}
  89. @ calculate the original stack pointer value.
  90. @ r9 currently points to the memory location just above the auto saved
  91. @ xPSR.
  92. @ The cpu might automatically 8-byte align the stack. Bit 9
  93. @ of the saved xPSR specifies if stack aligning took place. In this case
  94. @ another 32-bit value is included in the stack.
  95. tst r12, V7M_xPSR_FRAMEPTRALIGN
  96. addne r9, r9, #4
  97. @ store saved r12 using str to have a register to hold the base for stm
  98. str r8, [sp, #S_IP]
  99. add r8, sp, #S_SP
  100. @ store r13-r15, xPSR
  101. stmia r8!, {r9-r12}
  102. @ store old_r0
  103. str r0, [r8]
  104. .endm
  105. /*
  106. * PENDSV and SVCALL are configured to have the same exception
  107. * priorities. As a kernel thread runs at SVCALL execution priority it
  108. * can never be preempted and so we will never have to return to a
  109. * kernel thread here.
  110. */
  111. .macro v7m_exception_slow_exit ret_r0
  112. cpsid i
  113. ldr lr, =EXC_RET_THREADMODE_PROCESSSTACK
  114. @ read original r12, sp, lr, pc and xPSR
  115. add r12, sp, #S_IP
  116. ldmia r12, {r1-r5}
  117. @ an exception frame is always 8-byte aligned. To tell the hardware if
  118. @ the sp to be restored is aligned or not set bit 9 of the saved xPSR
  119. @ accordingly.
  120. tst r2, #4
  121. subne r2, r2, #4
  122. orrne r5, V7M_xPSR_FRAMEPTRALIGN
  123. biceq r5, V7M_xPSR_FRAMEPTRALIGN
  124. @ ensure bit 0 is cleared in the PC, otherwise behaviour is
  125. @ unpredictable
  126. bic r4, #1
  127. @ write basic exception frame
  128. stmdb r2!, {r1, r3-r5}
  129. ldmia sp, {r1, r3-r5}
  130. .if \ret_r0
  131. stmdb r2!, {r0, r3-r5}
  132. .else
  133. stmdb r2!, {r1, r3-r5}
  134. .endif
  135. @ restore process sp
  136. msr psp, r2
  137. @ restore original r4-r11
  138. ldmia sp!, {r0-r11}
  139. @ restore main sp
  140. add sp, sp, #PT_REGS_SIZE-S_IP
  141. cpsie i
  142. bx lr
  143. .endm
  144. #endif /* CONFIG_CPU_V7M */
  145. @
  146. @ Store/load the USER SP and LR registers by switching to the SYS
  147. @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not
  148. @ available. Should only be called from SVC mode
  149. @
  150. .macro store_user_sp_lr, rd, rtemp, offset = 0
  151. mrs \rtemp, cpsr
  152. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  153. msr cpsr_c, \rtemp @ switch to the SYS mode
  154. str sp, [\rd, #\offset] @ save sp_usr
  155. str lr, [\rd, #\offset + 4] @ save lr_usr
  156. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  157. msr cpsr_c, \rtemp @ switch back to the SVC mode
  158. .endm
  159. .macro load_user_sp_lr, rd, rtemp, offset = 0
  160. mrs \rtemp, cpsr
  161. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  162. msr cpsr_c, \rtemp @ switch to the SYS mode
  163. ldr sp, [\rd, #\offset] @ load sp_usr
  164. ldr lr, [\rd, #\offset + 4] @ load lr_usr
  165. eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
  166. msr cpsr_c, \rtemp @ switch back to the SVC mode
  167. .endm
  168. .macro svc_exit, rpsr, irq = 0
  169. .if \irq != 0
  170. @ IRQs already off
  171. #ifdef CONFIG_TRACE_IRQFLAGS
  172. @ The parent context IRQs must have been enabled to get here in
  173. @ the first place, so there's no point checking the PSR I bit.
  174. bl trace_hardirqs_on
  175. #endif
  176. .else
  177. @ IRQs off again before pulling preserved data off the stack
  178. disable_irq_notrace
  179. #ifdef CONFIG_TRACE_IRQFLAGS
  180. tst \rpsr, #PSR_I_BIT
  181. bleq trace_hardirqs_on
  182. tst \rpsr, #PSR_I_BIT
  183. blne trace_hardirqs_off
  184. #endif
  185. .endif
  186. ldr r1, [sp, #SVC_ADDR_LIMIT]
  187. uaccess_restore
  188. str r1, [tsk, #TI_ADDR_LIMIT]
  189. #ifndef CONFIG_THUMB2_KERNEL
  190. @ ARM mode SVC restore
  191. msr spsr_cxsf, \rpsr
  192. #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
  193. @ We must avoid clrex due to Cortex-A15 erratum #830321
  194. sub r0, sp, #4 @ uninhabited address
  195. strex r1, r2, [r0] @ clear the exclusive monitor
  196. #endif
  197. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  198. #else
  199. @ Thumb mode SVC restore
  200. ldr lr, [sp, #S_SP] @ top of the stack
  201. ldrd r0, r1, [sp, #S_LR] @ calling lr and pc
  202. @ We must avoid clrex due to Cortex-A15 erratum #830321
  203. strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor
  204. stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context
  205. ldmia sp, {r0 - r12}
  206. mov sp, lr
  207. ldr lr, [sp], #4
  208. rfeia sp!
  209. #endif
  210. .endm
  211. @
  212. @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit
  213. @
  214. @ This macro acts in a similar manner to svc_exit but switches to FIQ
  215. @ mode to restore the final part of the register state.
  216. @
  217. @ We cannot use the normal svc_exit procedure because that would
  218. @ clobber spsr_svc (FIQ could be delivered during the first few
  219. @ instructions of vector_swi meaning its contents have not been
  220. @ saved anywhere).
  221. @
  222. @ Note that, unlike svc_exit, this macro also does not allow a caller
  223. @ supplied rpsr. This is because the FIQ exceptions are not re-entrant
  224. @ and the handlers cannot call into the scheduler (meaning the value
  225. @ on the stack remains correct).
  226. @
  227. .macro svc_exit_via_fiq
  228. ldr r1, [sp, #SVC_ADDR_LIMIT]
  229. uaccess_restore
  230. str r1, [tsk, #TI_ADDR_LIMIT]
  231. #ifndef CONFIG_THUMB2_KERNEL
  232. @ ARM mode restore
  233. mov r0, sp
  234. ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will
  235. @ clobber state restored below)
  236. msr cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
  237. add r8, r0, #S_PC
  238. ldr r9, [r0, #S_PSR]
  239. msr spsr_cxsf, r9
  240. ldr r0, [r0, #S_R0]
  241. ldmia r8, {pc}^
  242. #else
  243. @ Thumb mode restore
  244. add r0, sp, #S_R2
  245. ldr lr, [sp, #S_LR]
  246. ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will
  247. @ clobber state restored below)
  248. ldmia r0, {r2 - r12}
  249. mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
  250. msr cpsr_c, r1
  251. sub r0, #S_R2
  252. add r8, r0, #S_PC
  253. ldmia r0, {r0 - r1}
  254. rfeia r8
  255. #endif
  256. .endm
  257. .macro restore_user_regs, fast = 0, offset = 0
  258. uaccess_enable r1, isb=0
  259. #ifndef CONFIG_THUMB2_KERNEL
  260. @ ARM mode restore
  261. mov r2, sp
  262. ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr
  263. ldr lr, [r2, #\offset + S_PC]! @ get pc
  264. tst r1, #PSR_I_BIT | 0x0f
  265. bne 1f
  266. msr spsr_cxsf, r1 @ save in spsr_svc
  267. #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
  268. @ We must avoid clrex due to Cortex-A15 erratum #830321
  269. strex r1, r2, [r2] @ clear the exclusive monitor
  270. #endif
  271. .if \fast
  272. ldmdb r2, {r1 - lr}^ @ get calling r1 - lr
  273. .else
  274. ldmdb r2, {r0 - lr}^ @ get calling r0 - lr
  275. .endif
  276. mov r0, r0 @ ARMv5T and earlier require a nop
  277. @ after ldm {}^
  278. add sp, sp, #\offset + PT_REGS_SIZE
  279. movs pc, lr @ return & move spsr_svc into cpsr
  280. 1: bug "Returning to usermode but unexpected PSR bits set?", \@
  281. #elif defined(CONFIG_CPU_V7M)
  282. @ V7M restore.
  283. @ Note that we don't need to do clrex here as clearing the local
  284. @ monitor is part of the exception entry and exit sequence.
  285. .if \offset
  286. add sp, #\offset
  287. .endif
  288. v7m_exception_slow_exit ret_r0 = \fast
  289. #else
  290. @ Thumb mode restore
  291. mov r2, sp
  292. load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr
  293. ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr
  294. ldr lr, [sp, #\offset + S_PC] @ get pc
  295. add sp, sp, #\offset + S_SP
  296. tst r1, #PSR_I_BIT | 0x0f
  297. bne 1f
  298. msr spsr_cxsf, r1 @ save in spsr_svc
  299. @ We must avoid clrex due to Cortex-A15 erratum #830321
  300. strex r1, r2, [sp] @ clear the exclusive monitor
  301. .if \fast
  302. ldmdb sp, {r1 - r12} @ get calling r1 - r12
  303. .else
  304. ldmdb sp, {r0 - r12} @ get calling r0 - r12
  305. .endif
  306. add sp, sp, #PT_REGS_SIZE - S_SP
  307. movs pc, lr @ return & move spsr_svc into cpsr
  308. 1: bug "Returning to usermode but unexpected PSR bits set?", \@
  309. #endif /* !CONFIG_THUMB2_KERNEL */
  310. .endm
  311. /*
  312. * Context tracking subsystem. Used to instrument transitions
  313. * between user and kernel mode.
  314. */
  315. .macro ct_user_exit, save = 1
  316. #ifdef CONFIG_CONTEXT_TRACKING
  317. .if \save
  318. stmdb sp!, {r0-r3, ip, lr}
  319. bl context_tracking_user_exit
  320. ldmia sp!, {r0-r3, ip, lr}
  321. .else
  322. bl context_tracking_user_exit
  323. .endif
  324. #endif
  325. .endm
  326. .macro ct_user_enter, save = 1
  327. #ifdef CONFIG_CONTEXT_TRACKING
  328. .if \save
  329. stmdb sp!, {r0-r3, ip, lr}
  330. bl context_tracking_user_enter
  331. ldmia sp!, {r0-r3, ip, lr}
  332. .else
  333. bl context_tracking_user_enter
  334. .endif
  335. #endif
  336. .endm
  337. /*
  338. * These are the registers used in the syscall handler, and allow us to
  339. * have in theory up to 7 arguments to a function - r0 to r6.
  340. *
  341. * r7 is reserved for the system call number for thumb mode.
  342. *
  343. * Note that tbl == why is intentional.
  344. *
  345. * We must set at least "tsk" and "why" when calling ret_with_reschedule.
  346. */
  347. scno .req r7 @ syscall number
  348. tbl .req r8 @ syscall table pointer
  349. why .req r8 @ Linux syscall (!= 0)
  350. tsk .req r9 @ current thread_info