align.c 25 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043
  1. /* align.c - handle alignment exceptions for the Power PC.
  2. *
  3. * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
  4. * Copyright (c) 1998-1999 TiVo, Inc.
  5. * PowerPC 403GCX modifications.
  6. * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
  7. * PowerPC 403GCX/405GP modifications.
  8. * Copyright (c) 2001-2002 PPC64 team, IBM Corp
  9. * 64-bit and Power4 support
  10. * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp
  11. * <benh@kernel.crashing.org>
  12. * Merge ppc32 and ppc64 implementations
  13. *
  14. * This program is free software; you can redistribute it and/or
  15. * modify it under the terms of the GNU General Public License
  16. * as published by the Free Software Foundation; either version
  17. * 2 of the License, or (at your option) any later version.
  18. */
  19. #include <linux/kernel.h>
  20. #include <linux/mm.h>
  21. #include <asm/processor.h>
  22. #include <asm/uaccess.h>
  23. #include <asm/cache.h>
  24. #include <asm/cputable.h>
  25. #include <asm/emulated_ops.h>
  26. #include <asm/switch_to.h>
  27. #include <asm/disassemble.h>
  28. struct aligninfo {
  29. unsigned char len;
  30. unsigned char flags;
  31. };
  32. #define INVALID { 0, 0 }
  33. /* Bits in the flags field */
  34. #define LD 0 /* load */
  35. #define ST 1 /* store */
  36. #define SE 2 /* sign-extend value, or FP ld/st as word */
  37. #define F 4 /* to/from fp regs */
  38. #define U 8 /* update index register */
  39. #define M 0x10 /* multiple load/store */
  40. #define SW 0x20 /* byte swap */
  41. #define S 0x40 /* single-precision fp or... */
  42. #define SX 0x40 /* ... byte count in XER */
  43. #define HARD 0x80 /* string, stwcx. */
  44. #define E4 0x40 /* SPE endianness is word */
  45. #define E8 0x80 /* SPE endianness is double word */
  46. #define SPLT 0x80 /* VSX SPLAT load */
  47. /* DSISR bits reported for a DCBZ instruction: */
  48. #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */
  49. /*
  50. * The PowerPC stores certain bits of the instruction that caused the
  51. * alignment exception in the DSISR register. This array maps those
  52. * bits to information about the operand length and what the
  53. * instruction would do.
  54. */
  55. static struct aligninfo aligninfo[128] = {
  56. { 4, LD }, /* 00 0 0000: lwz / lwarx */
  57. INVALID, /* 00 0 0001 */
  58. { 4, ST }, /* 00 0 0010: stw */
  59. INVALID, /* 00 0 0011 */
  60. { 2, LD }, /* 00 0 0100: lhz */
  61. { 2, LD+SE }, /* 00 0 0101: lha */
  62. { 2, ST }, /* 00 0 0110: sth */
  63. { 4, LD+M }, /* 00 0 0111: lmw */
  64. { 4, LD+F+S }, /* 00 0 1000: lfs */
  65. { 8, LD+F }, /* 00 0 1001: lfd */
  66. { 4, ST+F+S }, /* 00 0 1010: stfs */
  67. { 8, ST+F }, /* 00 0 1011: stfd */
  68. { 16, LD }, /* 00 0 1100: lq */
  69. { 8, LD }, /* 00 0 1101: ld/ldu/lwa */
  70. INVALID, /* 00 0 1110 */
  71. { 8, ST }, /* 00 0 1111: std/stdu */
  72. { 4, LD+U }, /* 00 1 0000: lwzu */
  73. INVALID, /* 00 1 0001 */
  74. { 4, ST+U }, /* 00 1 0010: stwu */
  75. INVALID, /* 00 1 0011 */
  76. { 2, LD+U }, /* 00 1 0100: lhzu */
  77. { 2, LD+SE+U }, /* 00 1 0101: lhau */
  78. { 2, ST+U }, /* 00 1 0110: sthu */
  79. { 4, ST+M }, /* 00 1 0111: stmw */
  80. { 4, LD+F+S+U }, /* 00 1 1000: lfsu */
  81. { 8, LD+F+U }, /* 00 1 1001: lfdu */
  82. { 4, ST+F+S+U }, /* 00 1 1010: stfsu */
  83. { 8, ST+F+U }, /* 00 1 1011: stfdu */
  84. { 16, LD+F }, /* 00 1 1100: lfdp */
  85. INVALID, /* 00 1 1101 */
  86. { 16, ST+F }, /* 00 1 1110: stfdp */
  87. INVALID, /* 00 1 1111 */
  88. { 8, LD }, /* 01 0 0000: ldx */
  89. INVALID, /* 01 0 0001 */
  90. { 8, ST }, /* 01 0 0010: stdx */
  91. INVALID, /* 01 0 0011 */
  92. INVALID, /* 01 0 0100 */
  93. { 4, LD+SE }, /* 01 0 0101: lwax */
  94. INVALID, /* 01 0 0110 */
  95. INVALID, /* 01 0 0111 */
  96. { 4, LD+M+HARD+SX }, /* 01 0 1000: lswx */
  97. { 4, LD+M+HARD }, /* 01 0 1001: lswi */
  98. { 4, ST+M+HARD+SX }, /* 01 0 1010: stswx */
  99. { 4, ST+M+HARD }, /* 01 0 1011: stswi */
  100. INVALID, /* 01 0 1100 */
  101. { 8, LD+U }, /* 01 0 1101: ldu */
  102. INVALID, /* 01 0 1110 */
  103. { 8, ST+U }, /* 01 0 1111: stdu */
  104. { 8, LD+U }, /* 01 1 0000: ldux */
  105. INVALID, /* 01 1 0001 */
  106. { 8, ST+U }, /* 01 1 0010: stdux */
  107. INVALID, /* 01 1 0011 */
  108. INVALID, /* 01 1 0100 */
  109. { 4, LD+SE+U }, /* 01 1 0101: lwaux */
  110. INVALID, /* 01 1 0110 */
  111. INVALID, /* 01 1 0111 */
  112. INVALID, /* 01 1 1000 */
  113. INVALID, /* 01 1 1001 */
  114. INVALID, /* 01 1 1010 */
  115. INVALID, /* 01 1 1011 */
  116. INVALID, /* 01 1 1100 */
  117. INVALID, /* 01 1 1101 */
  118. INVALID, /* 01 1 1110 */
  119. INVALID, /* 01 1 1111 */
  120. INVALID, /* 10 0 0000 */
  121. INVALID, /* 10 0 0001 */
  122. INVALID, /* 10 0 0010: stwcx. */
  123. INVALID, /* 10 0 0011 */
  124. INVALID, /* 10 0 0100 */
  125. INVALID, /* 10 0 0101 */
  126. INVALID, /* 10 0 0110 */
  127. INVALID, /* 10 0 0111 */
  128. { 4, LD+SW }, /* 10 0 1000: lwbrx */
  129. INVALID, /* 10 0 1001 */
  130. { 4, ST+SW }, /* 10 0 1010: stwbrx */
  131. INVALID, /* 10 0 1011 */
  132. { 2, LD+SW }, /* 10 0 1100: lhbrx */
  133. { 4, LD+SE }, /* 10 0 1101 lwa */
  134. { 2, ST+SW }, /* 10 0 1110: sthbrx */
  135. { 16, ST }, /* 10 0 1111: stq */
  136. INVALID, /* 10 1 0000 */
  137. INVALID, /* 10 1 0001 */
  138. INVALID, /* 10 1 0010 */
  139. INVALID, /* 10 1 0011 */
  140. INVALID, /* 10 1 0100 */
  141. INVALID, /* 10 1 0101 */
  142. INVALID, /* 10 1 0110 */
  143. INVALID, /* 10 1 0111 */
  144. INVALID, /* 10 1 1000 */
  145. INVALID, /* 10 1 1001 */
  146. INVALID, /* 10 1 1010 */
  147. INVALID, /* 10 1 1011 */
  148. INVALID, /* 10 1 1100 */
  149. INVALID, /* 10 1 1101 */
  150. INVALID, /* 10 1 1110 */
  151. { 0, ST+HARD }, /* 10 1 1111: dcbz */
  152. { 4, LD }, /* 11 0 0000: lwzx */
  153. INVALID, /* 11 0 0001 */
  154. { 4, ST }, /* 11 0 0010: stwx */
  155. INVALID, /* 11 0 0011 */
  156. { 2, LD }, /* 11 0 0100: lhzx */
  157. { 2, LD+SE }, /* 11 0 0101: lhax */
  158. { 2, ST }, /* 11 0 0110: sthx */
  159. INVALID, /* 11 0 0111 */
  160. { 4, LD+F+S }, /* 11 0 1000: lfsx */
  161. { 8, LD+F }, /* 11 0 1001: lfdx */
  162. { 4, ST+F+S }, /* 11 0 1010: stfsx */
  163. { 8, ST+F }, /* 11 0 1011: stfdx */
  164. { 16, LD+F }, /* 11 0 1100: lfdpx */
  165. { 4, LD+F+SE }, /* 11 0 1101: lfiwax */
  166. { 16, ST+F }, /* 11 0 1110: stfdpx */
  167. { 4, ST+F }, /* 11 0 1111: stfiwx */
  168. { 4, LD+U }, /* 11 1 0000: lwzux */
  169. INVALID, /* 11 1 0001 */
  170. { 4, ST+U }, /* 11 1 0010: stwux */
  171. INVALID, /* 11 1 0011 */
  172. { 2, LD+U }, /* 11 1 0100: lhzux */
  173. { 2, LD+SE+U }, /* 11 1 0101: lhaux */
  174. { 2, ST+U }, /* 11 1 0110: sthux */
  175. INVALID, /* 11 1 0111 */
  176. { 4, LD+F+S+U }, /* 11 1 1000: lfsux */
  177. { 8, LD+F+U }, /* 11 1 1001: lfdux */
  178. { 4, ST+F+S+U }, /* 11 1 1010: stfsux */
  179. { 8, ST+F+U }, /* 11 1 1011: stfdux */
  180. INVALID, /* 11 1 1100 */
  181. { 4, LD+F }, /* 11 1 1101: lfiwzx */
  182. INVALID, /* 11 1 1110 */
  183. INVALID, /* 11 1 1111 */
  184. };
  185. /*
  186. * The dcbz (data cache block zero) instruction
  187. * gives an alignment fault if used on non-cacheable
  188. * memory. We handle the fault mainly for the
  189. * case when we are running with the cache disabled
  190. * for debugging.
  191. */
  192. static int emulate_dcbz(struct pt_regs *regs, unsigned char __user *addr)
  193. {
  194. long __user *p;
  195. int i, size;
  196. #ifdef __powerpc64__
  197. size = ppc64_caches.dline_size;
  198. #else
  199. size = L1_CACHE_BYTES;
  200. #endif
  201. p = (long __user *) (regs->dar & -size);
  202. if (user_mode(regs) && !access_ok(VERIFY_WRITE, p, size))
  203. return -EFAULT;
  204. for (i = 0; i < size / sizeof(long); ++i)
  205. if (__put_user_inatomic(0, p+i))
  206. return -EFAULT;
  207. return 1;
  208. }
  209. /*
  210. * Emulate load & store multiple instructions
  211. * On 64-bit machines, these instructions only affect/use the
  212. * bottom 4 bytes of each register, and the loads clear the
  213. * top 4 bytes of the affected register.
  214. */
  215. #ifdef __BIG_ENDIAN__
  216. #ifdef CONFIG_PPC64
  217. #define REG_BYTE(rp, i) *((u8 *)((rp) + ((i) >> 2)) + ((i) & 3) + 4)
  218. #else
  219. #define REG_BYTE(rp, i) *((u8 *)(rp) + (i))
  220. #endif
  221. #endif
  222. #ifdef __LITTLE_ENDIAN__
  223. #define REG_BYTE(rp, i) (*(((u8 *)((rp) + ((i)>>2)) + ((i)&3))))
  224. #endif
  225. #define SWIZ_PTR(p) ((unsigned char __user *)((p) ^ swiz))
  226. static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr,
  227. unsigned int reg, unsigned int nb,
  228. unsigned int flags, unsigned int instr,
  229. unsigned long swiz)
  230. {
  231. unsigned long *rptr;
  232. unsigned int nb0, i, bswiz;
  233. unsigned long p;
  234. /*
  235. * We do not try to emulate 8 bytes multiple as they aren't really
  236. * available in our operating environments and we don't try to
  237. * emulate multiples operations in kernel land as they should never
  238. * be used/generated there at least not on unaligned boundaries
  239. */
  240. if (unlikely((nb > 4) || !user_mode(regs)))
  241. return 0;
  242. /* lmw, stmw, lswi/x, stswi/x */
  243. nb0 = 0;
  244. if (flags & HARD) {
  245. if (flags & SX) {
  246. nb = regs->xer & 127;
  247. if (nb == 0)
  248. return 1;
  249. } else {
  250. unsigned long pc = regs->nip ^ (swiz & 4);
  251. if (__get_user_inatomic(instr,
  252. (unsigned int __user *)pc))
  253. return -EFAULT;
  254. if (swiz == 0 && (flags & SW))
  255. instr = cpu_to_le32(instr);
  256. nb = (instr >> 11) & 0x1f;
  257. if (nb == 0)
  258. nb = 32;
  259. }
  260. if (nb + reg * 4 > 128) {
  261. nb0 = nb + reg * 4 - 128;
  262. nb = 128 - reg * 4;
  263. }
  264. #ifdef __LITTLE_ENDIAN__
  265. /*
  266. * String instructions are endian neutral but the code
  267. * below is not. Force byte swapping on so that the
  268. * effects of swizzling are undone in the load/store
  269. * loops below.
  270. */
  271. flags ^= SW;
  272. #endif
  273. } else {
  274. /* lwm, stmw */
  275. nb = (32 - reg) * 4;
  276. }
  277. if (!access_ok((flags & ST ? VERIFY_WRITE: VERIFY_READ), addr, nb+nb0))
  278. return -EFAULT; /* bad address */
  279. rptr = &regs->gpr[reg];
  280. p = (unsigned long) addr;
  281. bswiz = (flags & SW)? 3: 0;
  282. if (!(flags & ST)) {
  283. /*
  284. * This zeroes the top 4 bytes of the affected registers
  285. * in 64-bit mode, and also zeroes out any remaining
  286. * bytes of the last register for lsw*.
  287. */
  288. memset(rptr, 0, ((nb + 3) / 4) * sizeof(unsigned long));
  289. if (nb0 > 0)
  290. memset(&regs->gpr[0], 0,
  291. ((nb0 + 3) / 4) * sizeof(unsigned long));
  292. for (i = 0; i < nb; ++i, ++p)
  293. if (__get_user_inatomic(REG_BYTE(rptr, i ^ bswiz),
  294. SWIZ_PTR(p)))
  295. return -EFAULT;
  296. if (nb0 > 0) {
  297. rptr = &regs->gpr[0];
  298. addr += nb;
  299. for (i = 0; i < nb0; ++i, ++p)
  300. if (__get_user_inatomic(REG_BYTE(rptr,
  301. i ^ bswiz),
  302. SWIZ_PTR(p)))
  303. return -EFAULT;
  304. }
  305. } else {
  306. for (i = 0; i < nb; ++i, ++p)
  307. if (__put_user_inatomic(REG_BYTE(rptr, i ^ bswiz),
  308. SWIZ_PTR(p)))
  309. return -EFAULT;
  310. if (nb0 > 0) {
  311. rptr = &regs->gpr[0];
  312. addr += nb;
  313. for (i = 0; i < nb0; ++i, ++p)
  314. if (__put_user_inatomic(REG_BYTE(rptr,
  315. i ^ bswiz),
  316. SWIZ_PTR(p)))
  317. return -EFAULT;
  318. }
  319. }
  320. return 1;
  321. }
  322. /*
  323. * Emulate floating-point pair loads and stores.
  324. * Only POWER6 has these instructions, and it does true little-endian,
  325. * so we don't need the address swizzling.
  326. */
  327. static int emulate_fp_pair(unsigned char __user *addr, unsigned int reg,
  328. unsigned int flags)
  329. {
  330. char *ptr0 = (char *) &current->thread.TS_FPR(reg);
  331. char *ptr1 = (char *) &current->thread.TS_FPR(reg+1);
  332. int i, ret, sw = 0;
  333. if (reg & 1)
  334. return 0; /* invalid form: FRS/FRT must be even */
  335. if (flags & SW)
  336. sw = 7;
  337. ret = 0;
  338. for (i = 0; i < 8; ++i) {
  339. if (!(flags & ST)) {
  340. ret |= __get_user(ptr0[i^sw], addr + i);
  341. ret |= __get_user(ptr1[i^sw], addr + i + 8);
  342. } else {
  343. ret |= __put_user(ptr0[i^sw], addr + i);
  344. ret |= __put_user(ptr1[i^sw], addr + i + 8);
  345. }
  346. }
  347. if (ret)
  348. return -EFAULT;
  349. return 1; /* exception handled and fixed up */
  350. }
  351. #ifdef CONFIG_PPC64
  352. static int emulate_lq_stq(struct pt_regs *regs, unsigned char __user *addr,
  353. unsigned int reg, unsigned int flags)
  354. {
  355. char *ptr0 = (char *)&regs->gpr[reg];
  356. char *ptr1 = (char *)&regs->gpr[reg+1];
  357. int i, ret, sw = 0;
  358. if (reg & 1)
  359. return 0; /* invalid form: GPR must be even */
  360. if (flags & SW)
  361. sw = 7;
  362. ret = 0;
  363. for (i = 0; i < 8; ++i) {
  364. if (!(flags & ST)) {
  365. ret |= __get_user(ptr0[i^sw], addr + i);
  366. ret |= __get_user(ptr1[i^sw], addr + i + 8);
  367. } else {
  368. ret |= __put_user(ptr0[i^sw], addr + i);
  369. ret |= __put_user(ptr1[i^sw], addr + i + 8);
  370. }
  371. }
  372. if (ret)
  373. return -EFAULT;
  374. return 1; /* exception handled and fixed up */
  375. }
  376. #endif /* CONFIG_PPC64 */
  377. #ifdef CONFIG_SPE
  378. static struct aligninfo spe_aligninfo[32] = {
  379. { 8, LD+E8 }, /* 0 00 00: evldd[x] */
  380. { 8, LD+E4 }, /* 0 00 01: evldw[x] */
  381. { 8, LD }, /* 0 00 10: evldh[x] */
  382. INVALID, /* 0 00 11 */
  383. { 2, LD }, /* 0 01 00: evlhhesplat[x] */
  384. INVALID, /* 0 01 01 */
  385. { 2, LD }, /* 0 01 10: evlhhousplat[x] */
  386. { 2, LD+SE }, /* 0 01 11: evlhhossplat[x] */
  387. { 4, LD }, /* 0 10 00: evlwhe[x] */
  388. INVALID, /* 0 10 01 */
  389. { 4, LD }, /* 0 10 10: evlwhou[x] */
  390. { 4, LD+SE }, /* 0 10 11: evlwhos[x] */
  391. { 4, LD+E4 }, /* 0 11 00: evlwwsplat[x] */
  392. INVALID, /* 0 11 01 */
  393. { 4, LD }, /* 0 11 10: evlwhsplat[x] */
  394. INVALID, /* 0 11 11 */
  395. { 8, ST+E8 }, /* 1 00 00: evstdd[x] */
  396. { 8, ST+E4 }, /* 1 00 01: evstdw[x] */
  397. { 8, ST }, /* 1 00 10: evstdh[x] */
  398. INVALID, /* 1 00 11 */
  399. INVALID, /* 1 01 00 */
  400. INVALID, /* 1 01 01 */
  401. INVALID, /* 1 01 10 */
  402. INVALID, /* 1 01 11 */
  403. { 4, ST }, /* 1 10 00: evstwhe[x] */
  404. INVALID, /* 1 10 01 */
  405. { 4, ST }, /* 1 10 10: evstwho[x] */
  406. INVALID, /* 1 10 11 */
  407. { 4, ST+E4 }, /* 1 11 00: evstwwe[x] */
  408. INVALID, /* 1 11 01 */
  409. { 4, ST+E4 }, /* 1 11 10: evstwwo[x] */
  410. INVALID, /* 1 11 11 */
  411. };
  412. #define EVLDD 0x00
  413. #define EVLDW 0x01
  414. #define EVLDH 0x02
  415. #define EVLHHESPLAT 0x04
  416. #define EVLHHOUSPLAT 0x06
  417. #define EVLHHOSSPLAT 0x07
  418. #define EVLWHE 0x08
  419. #define EVLWHOU 0x0A
  420. #define EVLWHOS 0x0B
  421. #define EVLWWSPLAT 0x0C
  422. #define EVLWHSPLAT 0x0E
  423. #define EVSTDD 0x10
  424. #define EVSTDW 0x11
  425. #define EVSTDH 0x12
  426. #define EVSTWHE 0x18
  427. #define EVSTWHO 0x1A
  428. #define EVSTWWE 0x1C
  429. #define EVSTWWO 0x1E
  430. /*
  431. * Emulate SPE loads and stores.
  432. * Only Book-E has these instructions, and it does true little-endian,
  433. * so we don't need the address swizzling.
  434. */
  435. static int emulate_spe(struct pt_regs *regs, unsigned int reg,
  436. unsigned int instr)
  437. {
  438. int ret;
  439. union {
  440. u64 ll;
  441. u32 w[2];
  442. u16 h[4];
  443. u8 v[8];
  444. } data, temp;
  445. unsigned char __user *p, *addr;
  446. unsigned long *evr = &current->thread.evr[reg];
  447. unsigned int nb, flags;
  448. instr = (instr >> 1) & 0x1f;
  449. /* DAR has the operand effective address */
  450. addr = (unsigned char __user *)regs->dar;
  451. nb = spe_aligninfo[instr].len;
  452. flags = spe_aligninfo[instr].flags;
  453. /* Verify the address of the operand */
  454. if (unlikely(user_mode(regs) &&
  455. !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ),
  456. addr, nb)))
  457. return -EFAULT;
  458. /* userland only */
  459. if (unlikely(!user_mode(regs)))
  460. return 0;
  461. flush_spe_to_thread(current);
  462. /* If we are loading, get the data from user space, else
  463. * get it from register values
  464. */
  465. if (flags & ST) {
  466. data.ll = 0;
  467. switch (instr) {
  468. case EVSTDD:
  469. case EVSTDW:
  470. case EVSTDH:
  471. data.w[0] = *evr;
  472. data.w[1] = regs->gpr[reg];
  473. break;
  474. case EVSTWHE:
  475. data.h[2] = *evr >> 16;
  476. data.h[3] = regs->gpr[reg] >> 16;
  477. break;
  478. case EVSTWHO:
  479. data.h[2] = *evr & 0xffff;
  480. data.h[3] = regs->gpr[reg] & 0xffff;
  481. break;
  482. case EVSTWWE:
  483. data.w[1] = *evr;
  484. break;
  485. case EVSTWWO:
  486. data.w[1] = regs->gpr[reg];
  487. break;
  488. default:
  489. return -EINVAL;
  490. }
  491. } else {
  492. temp.ll = data.ll = 0;
  493. ret = 0;
  494. p = addr;
  495. switch (nb) {
  496. case 8:
  497. ret |= __get_user_inatomic(temp.v[0], p++);
  498. ret |= __get_user_inatomic(temp.v[1], p++);
  499. ret |= __get_user_inatomic(temp.v[2], p++);
  500. ret |= __get_user_inatomic(temp.v[3], p++);
  501. case 4:
  502. ret |= __get_user_inatomic(temp.v[4], p++);
  503. ret |= __get_user_inatomic(temp.v[5], p++);
  504. case 2:
  505. ret |= __get_user_inatomic(temp.v[6], p++);
  506. ret |= __get_user_inatomic(temp.v[7], p++);
  507. if (unlikely(ret))
  508. return -EFAULT;
  509. }
  510. switch (instr) {
  511. case EVLDD:
  512. case EVLDW:
  513. case EVLDH:
  514. data.ll = temp.ll;
  515. break;
  516. case EVLHHESPLAT:
  517. data.h[0] = temp.h[3];
  518. data.h[2] = temp.h[3];
  519. break;
  520. case EVLHHOUSPLAT:
  521. case EVLHHOSSPLAT:
  522. data.h[1] = temp.h[3];
  523. data.h[3] = temp.h[3];
  524. break;
  525. case EVLWHE:
  526. data.h[0] = temp.h[2];
  527. data.h[2] = temp.h[3];
  528. break;
  529. case EVLWHOU:
  530. case EVLWHOS:
  531. data.h[1] = temp.h[2];
  532. data.h[3] = temp.h[3];
  533. break;
  534. case EVLWWSPLAT:
  535. data.w[0] = temp.w[1];
  536. data.w[1] = temp.w[1];
  537. break;
  538. case EVLWHSPLAT:
  539. data.h[0] = temp.h[2];
  540. data.h[1] = temp.h[2];
  541. data.h[2] = temp.h[3];
  542. data.h[3] = temp.h[3];
  543. break;
  544. default:
  545. return -EINVAL;
  546. }
  547. }
  548. if (flags & SW) {
  549. switch (flags & 0xf0) {
  550. case E8:
  551. data.ll = swab64(data.ll);
  552. break;
  553. case E4:
  554. data.w[0] = swab32(data.w[0]);
  555. data.w[1] = swab32(data.w[1]);
  556. break;
  557. /* Its half word endian */
  558. default:
  559. data.h[0] = swab16(data.h[0]);
  560. data.h[1] = swab16(data.h[1]);
  561. data.h[2] = swab16(data.h[2]);
  562. data.h[3] = swab16(data.h[3]);
  563. break;
  564. }
  565. }
  566. if (flags & SE) {
  567. data.w[0] = (s16)data.h[1];
  568. data.w[1] = (s16)data.h[3];
  569. }
  570. /* Store result to memory or update registers */
  571. if (flags & ST) {
  572. ret = 0;
  573. p = addr;
  574. switch (nb) {
  575. case 8:
  576. ret |= __put_user_inatomic(data.v[0], p++);
  577. ret |= __put_user_inatomic(data.v[1], p++);
  578. ret |= __put_user_inatomic(data.v[2], p++);
  579. ret |= __put_user_inatomic(data.v[3], p++);
  580. case 4:
  581. ret |= __put_user_inatomic(data.v[4], p++);
  582. ret |= __put_user_inatomic(data.v[5], p++);
  583. case 2:
  584. ret |= __put_user_inatomic(data.v[6], p++);
  585. ret |= __put_user_inatomic(data.v[7], p++);
  586. }
  587. if (unlikely(ret))
  588. return -EFAULT;
  589. } else {
  590. *evr = data.w[0];
  591. regs->gpr[reg] = data.w[1];
  592. }
  593. return 1;
  594. }
  595. #endif /* CONFIG_SPE */
  596. #ifdef CONFIG_VSX
  597. /*
  598. * Emulate VSX instructions...
  599. */
  600. static int emulate_vsx(unsigned char __user *addr, unsigned int reg,
  601. unsigned int areg, struct pt_regs *regs,
  602. unsigned int flags, unsigned int length,
  603. unsigned int elsize)
  604. {
  605. char *ptr;
  606. unsigned long *lptr;
  607. int ret = 0;
  608. int sw = 0;
  609. int i, j;
  610. /* userland only */
  611. if (unlikely(!user_mode(regs)))
  612. return 0;
  613. flush_vsx_to_thread(current);
  614. if (reg < 32)
  615. ptr = (char *) &current->thread.fp_state.fpr[reg][0];
  616. else
  617. ptr = (char *) &current->thread.vr_state.vr[reg - 32];
  618. lptr = (unsigned long *) ptr;
  619. #ifdef __LITTLE_ENDIAN__
  620. if (flags & SW) {
  621. elsize = length;
  622. sw = length-1;
  623. } else {
  624. /*
  625. * The elements are BE ordered, even in LE mode, so process
  626. * them in reverse order.
  627. */
  628. addr += length - elsize;
  629. /* 8 byte memory accesses go in the top 8 bytes of the VR */
  630. if (length == 8)
  631. ptr += 8;
  632. }
  633. #else
  634. if (flags & SW)
  635. sw = elsize-1;
  636. #endif
  637. for (j = 0; j < length; j += elsize) {
  638. for (i = 0; i < elsize; ++i) {
  639. if (flags & ST)
  640. ret |= __put_user(ptr[i^sw], addr + i);
  641. else
  642. ret |= __get_user(ptr[i^sw], addr + i);
  643. }
  644. ptr += elsize;
  645. #ifdef __LITTLE_ENDIAN__
  646. addr -= elsize;
  647. #else
  648. addr += elsize;
  649. #endif
  650. }
  651. #ifdef __BIG_ENDIAN__
  652. #define VSX_HI 0
  653. #define VSX_LO 1
  654. #else
  655. #define VSX_HI 1
  656. #define VSX_LO 0
  657. #endif
  658. if (!ret) {
  659. if (flags & U)
  660. regs->gpr[areg] = regs->dar;
  661. /* Splat load copies the same data to top and bottom 8 bytes */
  662. if (flags & SPLT)
  663. lptr[VSX_LO] = lptr[VSX_HI];
  664. /* For 8 byte loads, zero the low 8 bytes */
  665. else if (!(flags & ST) && (8 == length))
  666. lptr[VSX_LO] = 0;
  667. } else
  668. return -EFAULT;
  669. return 1;
  670. }
  671. #endif
  672. /*
  673. * Called on alignment exception. Attempts to fixup
  674. *
  675. * Return 1 on success
  676. * Return 0 if unable to handle the interrupt
  677. * Return -EFAULT if data address is bad
  678. */
  679. int fix_alignment(struct pt_regs *regs)
  680. {
  681. unsigned int instr, nb, flags, instruction = 0;
  682. unsigned int reg, areg;
  683. unsigned int dsisr;
  684. unsigned char __user *addr;
  685. unsigned long p, swiz;
  686. int ret, i;
  687. union data {
  688. u64 ll;
  689. double dd;
  690. unsigned char v[8];
  691. struct {
  692. #ifdef __LITTLE_ENDIAN__
  693. int low32;
  694. unsigned hi32;
  695. #else
  696. unsigned hi32;
  697. int low32;
  698. #endif
  699. } x32;
  700. struct {
  701. #ifdef __LITTLE_ENDIAN__
  702. short low16;
  703. unsigned char hi48[6];
  704. #else
  705. unsigned char hi48[6];
  706. short low16;
  707. #endif
  708. } x16;
  709. } data;
  710. /*
  711. * We require a complete register set, if not, then our assembly
  712. * is broken
  713. */
  714. CHECK_FULL_REGS(regs);
  715. dsisr = regs->dsisr;
  716. /* Some processors don't provide us with a DSISR we can use here,
  717. * let's make one up from the instruction
  718. */
  719. if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) {
  720. unsigned long pc = regs->nip;
  721. if (cpu_has_feature(CPU_FTR_PPC_LE) && (regs->msr & MSR_LE))
  722. pc ^= 4;
  723. if (unlikely(__get_user_inatomic(instr,
  724. (unsigned int __user *)pc)))
  725. return -EFAULT;
  726. if (cpu_has_feature(CPU_FTR_REAL_LE) && (regs->msr & MSR_LE))
  727. instr = cpu_to_le32(instr);
  728. dsisr = make_dsisr(instr);
  729. instruction = instr;
  730. }
  731. /* extract the operation and registers from the dsisr */
  732. reg = (dsisr >> 5) & 0x1f; /* source/dest register */
  733. areg = dsisr & 0x1f; /* register to update */
  734. #ifdef CONFIG_SPE
  735. if ((instr >> 26) == 0x4) {
  736. PPC_WARN_ALIGNMENT(spe, regs);
  737. return emulate_spe(regs, reg, instr);
  738. }
  739. #endif
  740. instr = (dsisr >> 10) & 0x7f;
  741. instr |= (dsisr >> 13) & 0x60;
  742. /* Lookup the operation in our table */
  743. nb = aligninfo[instr].len;
  744. flags = aligninfo[instr].flags;
  745. /* ldbrx/stdbrx overlap lfs/stfs in the DSISR unfortunately */
  746. if (IS_XFORM(instruction) && ((instruction >> 1) & 0x3ff) == 532) {
  747. nb = 8;
  748. flags = LD+SW;
  749. } else if (IS_XFORM(instruction) &&
  750. ((instruction >> 1) & 0x3ff) == 660) {
  751. nb = 8;
  752. flags = ST+SW;
  753. }
  754. /* Byteswap little endian loads and stores */
  755. swiz = 0;
  756. if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) {
  757. flags ^= SW;
  758. #ifdef __BIG_ENDIAN__
  759. /*
  760. * So-called "PowerPC little endian" mode works by
  761. * swizzling addresses rather than by actually doing
  762. * any byte-swapping. To emulate this, we XOR each
  763. * byte address with 7. We also byte-swap, because
  764. * the processor's address swizzling depends on the
  765. * operand size (it xors the address with 7 for bytes,
  766. * 6 for halfwords, 4 for words, 0 for doublewords) but
  767. * we will xor with 7 and load/store each byte separately.
  768. */
  769. if (cpu_has_feature(CPU_FTR_PPC_LE))
  770. swiz = 7;
  771. #endif
  772. }
  773. /* DAR has the operand effective address */
  774. addr = (unsigned char __user *)regs->dar;
  775. #ifdef CONFIG_VSX
  776. if ((instruction & 0xfc00003e) == 0x7c000018) {
  777. unsigned int elsize;
  778. /* Additional register addressing bit (64 VSX vs 32 FPR/GPR) */
  779. reg |= (instruction & 0x1) << 5;
  780. /* Simple inline decoder instead of a table */
  781. /* VSX has only 8 and 16 byte memory accesses */
  782. nb = 8;
  783. if (instruction & 0x200)
  784. nb = 16;
  785. /* Vector stores in little-endian mode swap individual
  786. elements, so process them separately */
  787. elsize = 4;
  788. if (instruction & 0x80)
  789. elsize = 8;
  790. flags = 0;
  791. if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE))
  792. flags |= SW;
  793. if (instruction & 0x100)
  794. flags |= ST;
  795. if (instruction & 0x040)
  796. flags |= U;
  797. /* splat load needs a special decoder */
  798. if ((instruction & 0x400) == 0){
  799. flags |= SPLT;
  800. nb = 8;
  801. }
  802. PPC_WARN_ALIGNMENT(vsx, regs);
  803. return emulate_vsx(addr, reg, areg, regs, flags, nb, elsize);
  804. }
  805. #endif
  806. /* A size of 0 indicates an instruction we don't support, with
  807. * the exception of DCBZ which is handled as a special case here
  808. */
  809. if (instr == DCBZ) {
  810. PPC_WARN_ALIGNMENT(dcbz, regs);
  811. return emulate_dcbz(regs, addr);
  812. }
  813. if (unlikely(nb == 0))
  814. return 0;
  815. /* Load/Store Multiple instructions are handled in their own
  816. * function
  817. */
  818. if (flags & M) {
  819. PPC_WARN_ALIGNMENT(multiple, regs);
  820. return emulate_multiple(regs, addr, reg, nb,
  821. flags, instr, swiz);
  822. }
  823. /* Verify the address of the operand */
  824. if (unlikely(user_mode(regs) &&
  825. !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ),
  826. addr, nb)))
  827. return -EFAULT;
  828. /* Force the fprs into the save area so we can reference them */
  829. if (flags & F) {
  830. /* userland only */
  831. if (unlikely(!user_mode(regs)))
  832. return 0;
  833. flush_fp_to_thread(current);
  834. }
  835. if (nb == 16) {
  836. if (flags & F) {
  837. /* Special case for 16-byte FP loads and stores */
  838. PPC_WARN_ALIGNMENT(fp_pair, regs);
  839. return emulate_fp_pair(addr, reg, flags);
  840. } else {
  841. #ifdef CONFIG_PPC64
  842. /* Special case for 16-byte loads and stores */
  843. PPC_WARN_ALIGNMENT(lq_stq, regs);
  844. return emulate_lq_stq(regs, addr, reg, flags);
  845. #else
  846. return 0;
  847. #endif
  848. }
  849. }
  850. PPC_WARN_ALIGNMENT(unaligned, regs);
  851. /* If we are loading, get the data from user space, else
  852. * get it from register values
  853. */
  854. if (!(flags & ST)) {
  855. unsigned int start = 0;
  856. switch (nb) {
  857. case 4:
  858. start = offsetof(union data, x32.low32);
  859. break;
  860. case 2:
  861. start = offsetof(union data, x16.low16);
  862. break;
  863. }
  864. data.ll = 0;
  865. ret = 0;
  866. p = (unsigned long)addr;
  867. for (i = 0; i < nb; i++)
  868. ret |= __get_user_inatomic(data.v[start + i],
  869. SWIZ_PTR(p++));
  870. if (unlikely(ret))
  871. return -EFAULT;
  872. } else if (flags & F) {
  873. data.ll = current->thread.TS_FPR(reg);
  874. if (flags & S) {
  875. /* Single-precision FP store requires conversion... */
  876. #ifdef CONFIG_PPC_FPU
  877. preempt_disable();
  878. enable_kernel_fp();
  879. cvt_df(&data.dd, (float *)&data.x32.low32);
  880. preempt_enable();
  881. #else
  882. return 0;
  883. #endif
  884. }
  885. } else
  886. data.ll = regs->gpr[reg];
  887. if (flags & SW) {
  888. switch (nb) {
  889. case 8:
  890. data.ll = swab64(data.ll);
  891. break;
  892. case 4:
  893. data.x32.low32 = swab32(data.x32.low32);
  894. break;
  895. case 2:
  896. data.x16.low16 = swab16(data.x16.low16);
  897. break;
  898. }
  899. }
  900. /* Perform other misc operations like sign extension
  901. * or floating point single precision conversion
  902. */
  903. switch (flags & ~(U|SW)) {
  904. case LD+SE: /* sign extending integer loads */
  905. case LD+F+SE: /* sign extend for lfiwax */
  906. if ( nb == 2 )
  907. data.ll = data.x16.low16;
  908. else /* nb must be 4 */
  909. data.ll = data.x32.low32;
  910. break;
  911. /* Single-precision FP load requires conversion... */
  912. case LD+F+S:
  913. #ifdef CONFIG_PPC_FPU
  914. preempt_disable();
  915. enable_kernel_fp();
  916. cvt_fd((float *)&data.x32.low32, &data.dd);
  917. preempt_enable();
  918. #else
  919. return 0;
  920. #endif
  921. break;
  922. }
  923. /* Store result to memory or update registers */
  924. if (flags & ST) {
  925. unsigned int start = 0;
  926. switch (nb) {
  927. case 4:
  928. start = offsetof(union data, x32.low32);
  929. break;
  930. case 2:
  931. start = offsetof(union data, x16.low16);
  932. break;
  933. }
  934. ret = 0;
  935. p = (unsigned long)addr;
  936. for (i = 0; i < nb; i++)
  937. ret |= __put_user_inatomic(data.v[start + i],
  938. SWIZ_PTR(p++));
  939. if (unlikely(ret))
  940. return -EFAULT;
  941. } else if (flags & F)
  942. current->thread.TS_FPR(reg) = data.ll;
  943. else
  944. regs->gpr[reg] = data.ll;
  945. /* Update RA as needed */
  946. if (flags & U)
  947. regs->gpr[areg] = regs->dar;
  948. return 1;
  949. }