cpu_errata.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. /*
  2. * Contains CPU specific errata definitions
  3. *
  4. * Copyright (C) 2014 ARM Ltd.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <linux/types.h>
  19. #include <asm/cachetype.h>
  20. #include <asm/cpu.h>
  21. #include <asm/cputype.h>
  22. #include <asm/cpufeature.h>
  23. static bool __maybe_unused
  24. is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  25. {
  26. WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  27. return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model,
  28. entry->midr_range_min,
  29. entry->midr_range_max);
  30. }
  31. static bool
  32. has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
  33. int scope)
  34. {
  35. u64 mask = CTR_CACHE_MINLINE_MASK;
  36. /* Skip matching the min line sizes for cache type check */
  37. if (entry->capability == ARM64_MISMATCHED_CACHE_TYPE)
  38. mask ^= arm64_ftr_reg_ctrel0.strict_mask;
  39. WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  40. return (read_cpuid_cachetype() & mask) !=
  41. (arm64_ftr_reg_ctrel0.sys_val & mask);
  42. }
  43. static int cpu_enable_trap_ctr_access(void *__unused)
  44. {
  45. /* Clear SCTLR_EL1.UCT */
  46. config_sctlr_el1(SCTLR_EL1_UCT, 0);
  47. return 0;
  48. }
  49. #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
  50. #include <asm/mmu_context.h>
  51. #include <asm/cacheflush.h>
  52. DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
  53. #ifdef CONFIG_KVM
  54. extern char __smccc_workaround_1_smc_start[];
  55. extern char __smccc_workaround_1_smc_end[];
  56. extern char __smccc_workaround_1_hvc_start[];
  57. extern char __smccc_workaround_1_hvc_end[];
  58. static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
  59. const char *hyp_vecs_end)
  60. {
  61. void *dst = __bp_harden_hyp_vecs_start + slot * SZ_2K;
  62. int i;
  63. for (i = 0; i < SZ_2K; i += 0x80)
  64. memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
  65. flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
  66. }
  67. static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
  68. const char *hyp_vecs_start,
  69. const char *hyp_vecs_end)
  70. {
  71. static int last_slot = -1;
  72. static DEFINE_SPINLOCK(bp_lock);
  73. int cpu, slot = -1;
  74. spin_lock(&bp_lock);
  75. for_each_possible_cpu(cpu) {
  76. if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
  77. slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
  78. break;
  79. }
  80. }
  81. if (slot == -1) {
  82. last_slot++;
  83. BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
  84. / SZ_2K) <= last_slot);
  85. slot = last_slot;
  86. __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
  87. }
  88. __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
  89. __this_cpu_write(bp_hardening_data.fn, fn);
  90. spin_unlock(&bp_lock);
  91. }
  92. #else
  93. #define __smccc_workaround_1_smc_start NULL
  94. #define __smccc_workaround_1_smc_end NULL
  95. #define __smccc_workaround_1_hvc_start NULL
  96. #define __smccc_workaround_1_hvc_end NULL
  97. static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
  98. const char *hyp_vecs_start,
  99. const char *hyp_vecs_end)
  100. {
  101. __this_cpu_write(bp_hardening_data.fn, fn);
  102. }
  103. #endif /* CONFIG_KVM */
  104. static void install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
  105. bp_hardening_cb_t fn,
  106. const char *hyp_vecs_start,
  107. const char *hyp_vecs_end)
  108. {
  109. u64 pfr0;
  110. if (!entry->matches(entry, SCOPE_LOCAL_CPU))
  111. return;
  112. pfr0 = read_cpuid(ID_AA64PFR0_EL1);
  113. if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
  114. return;
  115. __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
  116. }
  117. #include <uapi/linux/psci.h>
  118. #include <linux/arm-smccc.h>
  119. #include <linux/psci.h>
  120. static void call_smc_arch_workaround_1(void)
  121. {
  122. arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
  123. }
  124. static void call_hvc_arch_workaround_1(void)
  125. {
  126. arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
  127. }
  128. static int enable_smccc_arch_workaround_1(void *data)
  129. {
  130. const struct arm64_cpu_capabilities *entry = data;
  131. bp_hardening_cb_t cb;
  132. void *smccc_start, *smccc_end;
  133. struct arm_smccc_res res;
  134. if (!entry->matches(entry, SCOPE_LOCAL_CPU))
  135. return 0;
  136. if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
  137. return 0;
  138. switch (psci_ops.conduit) {
  139. case PSCI_CONDUIT_HVC:
  140. arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
  141. ARM_SMCCC_ARCH_WORKAROUND_1, &res);
  142. if ((int)res.a0 < 0)
  143. return 0;
  144. cb = call_hvc_arch_workaround_1;
  145. smccc_start = __smccc_workaround_1_hvc_start;
  146. smccc_end = __smccc_workaround_1_hvc_end;
  147. break;
  148. case PSCI_CONDUIT_SMC:
  149. arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
  150. ARM_SMCCC_ARCH_WORKAROUND_1, &res);
  151. if ((int)res.a0 < 0)
  152. return 0;
  153. cb = call_smc_arch_workaround_1;
  154. smccc_start = __smccc_workaround_1_smc_start;
  155. smccc_end = __smccc_workaround_1_smc_end;
  156. break;
  157. default:
  158. return 0;
  159. }
  160. install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
  161. return 0;
  162. }
  163. #endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */
  164. #ifdef CONFIG_ARM64_SSBD
  165. DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
  166. int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
  167. static const struct ssbd_options {
  168. const char *str;
  169. int state;
  170. } ssbd_options[] = {
  171. { "force-on", ARM64_SSBD_FORCE_ENABLE, },
  172. { "force-off", ARM64_SSBD_FORCE_DISABLE, },
  173. { "kernel", ARM64_SSBD_KERNEL, },
  174. };
  175. static int __init ssbd_cfg(char *buf)
  176. {
  177. int i;
  178. if (!buf || !buf[0])
  179. return -EINVAL;
  180. for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
  181. int len = strlen(ssbd_options[i].str);
  182. if (strncmp(buf, ssbd_options[i].str, len))
  183. continue;
  184. ssbd_state = ssbd_options[i].state;
  185. return 0;
  186. }
  187. return -EINVAL;
  188. }
  189. early_param("ssbd", ssbd_cfg);
  190. void __init arm64_update_smccc_conduit(struct alt_instr *alt,
  191. __le32 *origptr, __le32 *updptr,
  192. int nr_inst)
  193. {
  194. u32 insn;
  195. BUG_ON(nr_inst != 1);
  196. switch (psci_ops.conduit) {
  197. case PSCI_CONDUIT_HVC:
  198. insn = aarch64_insn_get_hvc_value();
  199. break;
  200. case PSCI_CONDUIT_SMC:
  201. insn = aarch64_insn_get_smc_value();
  202. break;
  203. default:
  204. return;
  205. }
  206. *updptr = cpu_to_le32(insn);
  207. }
  208. void __init arm64_enable_wa2_handling(struct alt_instr *alt,
  209. __le32 *origptr, __le32 *updptr,
  210. int nr_inst)
  211. {
  212. BUG_ON(nr_inst != 1);
  213. /*
  214. * Only allow mitigation on EL1 entry/exit and guest
  215. * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
  216. * be flipped.
  217. */
  218. if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
  219. *updptr = cpu_to_le32(aarch64_insn_gen_nop());
  220. }
  221. void arm64_set_ssbd_mitigation(bool state)
  222. {
  223. switch (psci_ops.conduit) {
  224. case PSCI_CONDUIT_HVC:
  225. arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
  226. break;
  227. case PSCI_CONDUIT_SMC:
  228. arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
  229. break;
  230. default:
  231. WARN_ON_ONCE(1);
  232. break;
  233. }
  234. }
  235. static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
  236. int scope)
  237. {
  238. struct arm_smccc_res res;
  239. bool required = true;
  240. s32 val;
  241. WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  242. if (psci_ops.smccc_version == SMCCC_VERSION_1_0) {
  243. ssbd_state = ARM64_SSBD_UNKNOWN;
  244. return false;
  245. }
  246. switch (psci_ops.conduit) {
  247. case PSCI_CONDUIT_HVC:
  248. arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
  249. ARM_SMCCC_ARCH_WORKAROUND_2, &res);
  250. break;
  251. case PSCI_CONDUIT_SMC:
  252. arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
  253. ARM_SMCCC_ARCH_WORKAROUND_2, &res);
  254. break;
  255. default:
  256. ssbd_state = ARM64_SSBD_UNKNOWN;
  257. return false;
  258. }
  259. val = (s32)res.a0;
  260. switch (val) {
  261. case SMCCC_RET_NOT_SUPPORTED:
  262. ssbd_state = ARM64_SSBD_UNKNOWN;
  263. return false;
  264. case SMCCC_RET_NOT_REQUIRED:
  265. pr_info_once("%s mitigation not required\n", entry->desc);
  266. ssbd_state = ARM64_SSBD_MITIGATED;
  267. return false;
  268. case SMCCC_RET_SUCCESS:
  269. required = true;
  270. break;
  271. case 1: /* Mitigation not required on this CPU */
  272. required = false;
  273. break;
  274. default:
  275. WARN_ON(1);
  276. return false;
  277. }
  278. switch (ssbd_state) {
  279. case ARM64_SSBD_FORCE_DISABLE:
  280. pr_info_once("%s disabled from command-line\n", entry->desc);
  281. arm64_set_ssbd_mitigation(false);
  282. required = false;
  283. break;
  284. case ARM64_SSBD_KERNEL:
  285. if (required) {
  286. __this_cpu_write(arm64_ssbd_callback_required, 1);
  287. arm64_set_ssbd_mitigation(true);
  288. }
  289. break;
  290. case ARM64_SSBD_FORCE_ENABLE:
  291. pr_info_once("%s forced from command-line\n", entry->desc);
  292. arm64_set_ssbd_mitigation(true);
  293. required = true;
  294. break;
  295. default:
  296. WARN_ON(1);
  297. break;
  298. }
  299. return required;
  300. }
  301. #endif /* CONFIG_ARM64_SSBD */
  302. #define MIDR_RANGE(model, min, max) \
  303. .def_scope = SCOPE_LOCAL_CPU, \
  304. .matches = is_affected_midr_range, \
  305. .midr_model = model, \
  306. .midr_range_min = min, \
  307. .midr_range_max = max
  308. #define MIDR_ALL_VERSIONS(model) \
  309. .def_scope = SCOPE_LOCAL_CPU, \
  310. .matches = is_affected_midr_range, \
  311. .midr_model = model, \
  312. .midr_range_min = 0, \
  313. .midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK)
  314. const struct arm64_cpu_capabilities arm64_errata[] = {
  315. #if defined(CONFIG_ARM64_ERRATUM_826319) || \
  316. defined(CONFIG_ARM64_ERRATUM_827319) || \
  317. defined(CONFIG_ARM64_ERRATUM_824069)
  318. {
  319. /* Cortex-A53 r0p[012] */
  320. .desc = "ARM errata 826319, 827319, 824069",
  321. .capability = ARM64_WORKAROUND_CLEAN_CACHE,
  322. MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02),
  323. .enable = cpu_enable_cache_maint_trap,
  324. },
  325. #endif
  326. #ifdef CONFIG_ARM64_ERRATUM_819472
  327. {
  328. /* Cortex-A53 r0p[01] */
  329. .desc = "ARM errata 819472",
  330. .capability = ARM64_WORKAROUND_CLEAN_CACHE,
  331. MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01),
  332. .enable = cpu_enable_cache_maint_trap,
  333. },
  334. #endif
  335. #ifdef CONFIG_ARM64_ERRATUM_832075
  336. {
  337. /* Cortex-A57 r0p0 - r1p2 */
  338. .desc = "ARM erratum 832075",
  339. .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
  340. MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
  341. (1 << MIDR_VARIANT_SHIFT) | 2),
  342. },
  343. #endif
  344. #ifdef CONFIG_ARM64_ERRATUM_834220
  345. {
  346. /* Cortex-A57 r0p0 - r1p2 */
  347. .desc = "ARM erratum 834220",
  348. .capability = ARM64_WORKAROUND_834220,
  349. MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
  350. (1 << MIDR_VARIANT_SHIFT) | 2),
  351. },
  352. #endif
  353. #ifdef CONFIG_ARM64_ERRATUM_845719
  354. {
  355. /* Cortex-A53 r0p[01234] */
  356. .desc = "ARM erratum 845719",
  357. .capability = ARM64_WORKAROUND_845719,
  358. MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
  359. },
  360. #endif
  361. #ifdef CONFIG_CAVIUM_ERRATUM_23154
  362. {
  363. /* Cavium ThunderX, pass 1.x */
  364. .desc = "Cavium erratum 23154",
  365. .capability = ARM64_WORKAROUND_CAVIUM_23154,
  366. MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
  367. },
  368. #endif
  369. #ifdef CONFIG_CAVIUM_ERRATUM_27456
  370. {
  371. /* Cavium ThunderX, T88 pass 1.x - 2.1 */
  372. .desc = "Cavium erratum 27456",
  373. .capability = ARM64_WORKAROUND_CAVIUM_27456,
  374. MIDR_RANGE(MIDR_THUNDERX, 0x00,
  375. (1 << MIDR_VARIANT_SHIFT) | 1),
  376. },
  377. {
  378. /* Cavium ThunderX, T81 pass 1.0 */
  379. .desc = "Cavium erratum 27456",
  380. .capability = ARM64_WORKAROUND_CAVIUM_27456,
  381. MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00),
  382. },
  383. #endif
  384. {
  385. .desc = "Mismatched cache line size",
  386. .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
  387. .matches = has_mismatched_cache_type,
  388. .def_scope = SCOPE_LOCAL_CPU,
  389. .enable = cpu_enable_trap_ctr_access,
  390. },
  391. {
  392. .desc = "Mismatched cache type",
  393. .capability = ARM64_MISMATCHED_CACHE_TYPE,
  394. .matches = has_mismatched_cache_type,
  395. .def_scope = SCOPE_LOCAL_CPU,
  396. .enable = cpu_enable_trap_ctr_access,
  397. },
  398. #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
  399. {
  400. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  401. MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
  402. .enable = enable_smccc_arch_workaround_1,
  403. },
  404. {
  405. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  406. MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
  407. .enable = enable_smccc_arch_workaround_1,
  408. },
  409. {
  410. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  411. MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
  412. .enable = enable_smccc_arch_workaround_1,
  413. },
  414. {
  415. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  416. MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
  417. .enable = enable_smccc_arch_workaround_1,
  418. },
  419. {
  420. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  421. MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
  422. .enable = enable_smccc_arch_workaround_1,
  423. },
  424. {
  425. .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
  426. MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
  427. .enable = enable_smccc_arch_workaround_1,
  428. },
  429. #endif
  430. #ifdef CONFIG_ARM64_SSBD
  431. {
  432. .desc = "Speculative Store Bypass Disable",
  433. .def_scope = SCOPE_LOCAL_CPU,
  434. .capability = ARM64_SSBD,
  435. .matches = has_ssbd_mitigation,
  436. },
  437. #endif
  438. {
  439. }
  440. };
  441. /*
  442. * The CPU Errata work arounds are detected and applied at boot time
  443. * and the related information is freed soon after. If the new CPU requires
  444. * an errata not detected at boot, fail this CPU.
  445. */
  446. void verify_local_cpu_errata_workarounds(void)
  447. {
  448. const struct arm64_cpu_capabilities *caps = arm64_errata;
  449. for (; caps->matches; caps++) {
  450. if (cpus_have_cap(caps->capability)) {
  451. if (caps->enable)
  452. caps->enable((void *)caps);
  453. } else if (caps->matches(caps, SCOPE_LOCAL_CPU)) {
  454. pr_crit("CPU%d: Requires work around for %s, not detected"
  455. " at boot time\n",
  456. smp_processor_id(),
  457. caps->desc ? : "an erratum");
  458. cpu_die_early();
  459. }
  460. }
  461. }
  462. void update_cpu_errata_workarounds(void)
  463. {
  464. update_cpu_capabilities(arm64_errata, "enabling workaround for");
  465. }
  466. void __init enable_errata_workarounds(void)
  467. {
  468. enable_cpu_capabilities(arm64_errata);
  469. }