mcpm_entry.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458
  1. /*
  2. * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
  3. *
  4. * Created by: Nicolas Pitre, March 2012
  5. * Copyright: (C) 2012-2013 Linaro Limited
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as
  9. * published by the Free Software Foundation.
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/init.h>
  13. #include <linux/irqflags.h>
  14. #include <linux/cpu_pm.h>
  15. #include <asm/mcpm.h>
  16. #include <asm/cacheflush.h>
  17. #include <asm/idmap.h>
  18. #include <asm/cputype.h>
  19. #include <asm/suspend.h>
  20. /*
  21. * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
  22. * For a comprehensive description of the main algorithm used here, please
  23. * see Documentation/arm/cluster-pm-race-avoidance.txt.
  24. */
  25. struct sync_struct mcpm_sync;
  26. /*
  27. * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
  28. * This must be called at the point of committing to teardown of a CPU.
  29. * The CPU cache (SCTRL.C bit) is expected to still be active.
  30. */
  31. static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
  32. {
  33. mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
  34. sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
  35. }
  36. /*
  37. * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
  38. * cluster can be torn down without disrupting this CPU.
  39. * To avoid deadlocks, this must be called before a CPU is powered down.
  40. * The CPU cache (SCTRL.C bit) is expected to be off.
  41. * However L2 cache might or might not be active.
  42. */
  43. static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
  44. {
  45. dmb();
  46. mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
  47. sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
  48. sev();
  49. }
  50. /*
  51. * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
  52. * @state: the final state of the cluster:
  53. * CLUSTER_UP: no destructive teardown was done and the cluster has been
  54. * restored to the previous state (CPU cache still active); or
  55. * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
  56. * (CPU cache disabled, L2 cache either enabled or disabled).
  57. */
  58. static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
  59. {
  60. dmb();
  61. mcpm_sync.clusters[cluster].cluster = state;
  62. sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
  63. sev();
  64. }
  65. /*
  66. * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
  67. * This function should be called by the last man, after local CPU teardown
  68. * is complete. CPU cache expected to be active.
  69. *
  70. * Returns:
  71. * false: the critical section was not entered because an inbound CPU was
  72. * observed, or the cluster is already being set up;
  73. * true: the critical section was entered: it is now safe to tear down the
  74. * cluster.
  75. */
  76. static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
  77. {
  78. unsigned int i;
  79. struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
  80. /* Warn inbound CPUs that the cluster is being torn down: */
  81. c->cluster = CLUSTER_GOING_DOWN;
  82. sync_cache_w(&c->cluster);
  83. /* Back out if the inbound cluster is already in the critical region: */
  84. sync_cache_r(&c->inbound);
  85. if (c->inbound == INBOUND_COMING_UP)
  86. goto abort;
  87. /*
  88. * Wait for all CPUs to get out of the GOING_DOWN state, so that local
  89. * teardown is complete on each CPU before tearing down the cluster.
  90. *
  91. * If any CPU has been woken up again from the DOWN state, then we
  92. * shouldn't be taking the cluster down at all: abort in that case.
  93. */
  94. sync_cache_r(&c->cpus);
  95. for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
  96. int cpustate;
  97. if (i == cpu)
  98. continue;
  99. while (1) {
  100. cpustate = c->cpus[i].cpu;
  101. if (cpustate != CPU_GOING_DOWN)
  102. break;
  103. wfe();
  104. sync_cache_r(&c->cpus[i].cpu);
  105. }
  106. switch (cpustate) {
  107. case CPU_DOWN:
  108. continue;
  109. default:
  110. goto abort;
  111. }
  112. }
  113. return true;
  114. abort:
  115. __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
  116. return false;
  117. }
  118. static int __mcpm_cluster_state(unsigned int cluster)
  119. {
  120. sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
  121. return mcpm_sync.clusters[cluster].cluster;
  122. }
  123. extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
  124. void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
  125. {
  126. unsigned long val = ptr ? virt_to_phys(ptr) : 0;
  127. mcpm_entry_vectors[cluster][cpu] = val;
  128. sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
  129. }
  130. extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
  131. void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
  132. unsigned long poke_phys_addr, unsigned long poke_val)
  133. {
  134. unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
  135. poke[0] = poke_phys_addr;
  136. poke[1] = poke_val;
  137. __sync_cache_range_w(poke, 2 * sizeof(*poke));
  138. }
  139. static const struct mcpm_platform_ops *platform_ops;
  140. int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
  141. {
  142. if (platform_ops)
  143. return -EBUSY;
  144. platform_ops = ops;
  145. return 0;
  146. }
  147. bool mcpm_is_available(void)
  148. {
  149. return (platform_ops) ? true : false;
  150. }
  151. /*
  152. * We can't use regular spinlocks. In the switcher case, it is possible
  153. * for an outbound CPU to call power_down() after its inbound counterpart
  154. * is already live using the same logical CPU number which trips lockdep
  155. * debugging.
  156. */
  157. static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  158. static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
  159. static inline bool mcpm_cluster_unused(unsigned int cluster)
  160. {
  161. int i, cnt;
  162. for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
  163. cnt |= mcpm_cpu_use_count[cluster][i];
  164. return !cnt;
  165. }
  166. int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
  167. {
  168. bool cpu_is_down, cluster_is_down;
  169. int ret = 0;
  170. pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
  171. if (!platform_ops)
  172. return -EUNATCH; /* try not to shadow power_up errors */
  173. might_sleep();
  174. /*
  175. * Since this is called with IRQs enabled, and no arch_spin_lock_irq
  176. * variant exists, we need to disable IRQs manually here.
  177. */
  178. local_irq_disable();
  179. arch_spin_lock(&mcpm_lock);
  180. cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
  181. cluster_is_down = mcpm_cluster_unused(cluster);
  182. mcpm_cpu_use_count[cluster][cpu]++;
  183. /*
  184. * The only possible values are:
  185. * 0 = CPU down
  186. * 1 = CPU (still) up
  187. * 2 = CPU requested to be up before it had a chance
  188. * to actually make itself down.
  189. * Any other value is a bug.
  190. */
  191. BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
  192. mcpm_cpu_use_count[cluster][cpu] != 2);
  193. if (cluster_is_down)
  194. ret = platform_ops->cluster_powerup(cluster);
  195. if (cpu_is_down && !ret)
  196. ret = platform_ops->cpu_powerup(cpu, cluster);
  197. arch_spin_unlock(&mcpm_lock);
  198. local_irq_enable();
  199. return ret;
  200. }
  201. typedef void (*phys_reset_t)(unsigned long);
  202. void mcpm_cpu_power_down(void)
  203. {
  204. unsigned int mpidr, cpu, cluster;
  205. bool cpu_going_down, last_man;
  206. phys_reset_t phys_reset;
  207. mpidr = read_cpuid_mpidr();
  208. cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  209. cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  210. pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
  211. if (WARN_ON_ONCE(!platform_ops))
  212. return;
  213. BUG_ON(!irqs_disabled());
  214. setup_mm_for_reboot();
  215. __mcpm_cpu_going_down(cpu, cluster);
  216. arch_spin_lock(&mcpm_lock);
  217. BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
  218. mcpm_cpu_use_count[cluster][cpu]--;
  219. BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
  220. mcpm_cpu_use_count[cluster][cpu] != 1);
  221. cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
  222. last_man = mcpm_cluster_unused(cluster);
  223. if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
  224. platform_ops->cpu_powerdown_prepare(cpu, cluster);
  225. platform_ops->cluster_powerdown_prepare(cluster);
  226. arch_spin_unlock(&mcpm_lock);
  227. platform_ops->cluster_cache_disable();
  228. __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
  229. } else {
  230. if (cpu_going_down)
  231. platform_ops->cpu_powerdown_prepare(cpu, cluster);
  232. arch_spin_unlock(&mcpm_lock);
  233. /*
  234. * If cpu_going_down is false here, that means a power_up
  235. * request raced ahead of us. Even if we do not want to
  236. * shut this CPU down, the caller still expects execution
  237. * to return through the system resume entry path, like
  238. * when the WFI is aborted due to a new IRQ or the like..
  239. * So let's continue with cache cleaning in all cases.
  240. */
  241. platform_ops->cpu_cache_disable();
  242. }
  243. __mcpm_cpu_down(cpu, cluster);
  244. /* Now we are prepared for power-down, do it: */
  245. if (cpu_going_down)
  246. wfi();
  247. /*
  248. * It is possible for a power_up request to happen concurrently
  249. * with a power_down request for the same CPU. In this case the
  250. * CPU might not be able to actually enter a powered down state
  251. * with the WFI instruction if the power_up request has removed
  252. * the required reset condition. We must perform a re-entry in
  253. * the kernel as if the power_up method just had deasserted reset
  254. * on the CPU.
  255. */
  256. phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
  257. phys_reset(virt_to_phys(mcpm_entry_point));
  258. /* should never get here */
  259. BUG();
  260. }
  261. int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
  262. {
  263. int ret;
  264. if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
  265. return -EUNATCH;
  266. ret = platform_ops->wait_for_powerdown(cpu, cluster);
  267. if (ret)
  268. pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
  269. __func__, cpu, cluster, ret);
  270. return ret;
  271. }
  272. void mcpm_cpu_suspend(void)
  273. {
  274. if (WARN_ON_ONCE(!platform_ops))
  275. return;
  276. /* Some platforms might have to enable special resume modes, etc. */
  277. if (platform_ops->cpu_suspend_prepare) {
  278. unsigned int mpidr = read_cpuid_mpidr();
  279. unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  280. unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  281. arch_spin_lock(&mcpm_lock);
  282. platform_ops->cpu_suspend_prepare(cpu, cluster);
  283. arch_spin_unlock(&mcpm_lock);
  284. }
  285. mcpm_cpu_power_down();
  286. }
  287. int mcpm_cpu_powered_up(void)
  288. {
  289. unsigned int mpidr, cpu, cluster;
  290. bool cpu_was_down, first_man;
  291. unsigned long flags;
  292. if (!platform_ops)
  293. return -EUNATCH;
  294. mpidr = read_cpuid_mpidr();
  295. cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  296. cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  297. local_irq_save(flags);
  298. arch_spin_lock(&mcpm_lock);
  299. cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
  300. first_man = mcpm_cluster_unused(cluster);
  301. if (first_man && platform_ops->cluster_is_up)
  302. platform_ops->cluster_is_up(cluster);
  303. if (cpu_was_down)
  304. mcpm_cpu_use_count[cluster][cpu] = 1;
  305. if (platform_ops->cpu_is_up)
  306. platform_ops->cpu_is_up(cpu, cluster);
  307. arch_spin_unlock(&mcpm_lock);
  308. local_irq_restore(flags);
  309. return 0;
  310. }
  311. #ifdef CONFIG_ARM_CPU_SUSPEND
  312. static int __init nocache_trampoline(unsigned long _arg)
  313. {
  314. void (*cache_disable)(void) = (void *)_arg;
  315. unsigned int mpidr = read_cpuid_mpidr();
  316. unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  317. unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  318. phys_reset_t phys_reset;
  319. mcpm_set_entry_vector(cpu, cluster, cpu_resume);
  320. setup_mm_for_reboot();
  321. __mcpm_cpu_going_down(cpu, cluster);
  322. BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
  323. cache_disable();
  324. __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
  325. __mcpm_cpu_down(cpu, cluster);
  326. phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
  327. phys_reset(virt_to_phys(mcpm_entry_point));
  328. BUG();
  329. }
  330. int __init mcpm_loopback(void (*cache_disable)(void))
  331. {
  332. int ret;
  333. /*
  334. * We're going to soft-restart the current CPU through the
  335. * low-level MCPM code by leveraging the suspend/resume
  336. * infrastructure. Let's play it safe by using cpu_pm_enter()
  337. * in case the CPU init code path resets the VFP or similar.
  338. */
  339. local_irq_disable();
  340. local_fiq_disable();
  341. ret = cpu_pm_enter();
  342. if (!ret) {
  343. ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
  344. cpu_pm_exit();
  345. }
  346. local_fiq_enable();
  347. local_irq_enable();
  348. if (ret)
  349. pr_err("%s returned %d\n", __func__, ret);
  350. return ret;
  351. }
  352. #endif
  353. extern unsigned long mcpm_power_up_setup_phys;
  354. int __init mcpm_sync_init(
  355. void (*power_up_setup)(unsigned int affinity_level))
  356. {
  357. unsigned int i, j, mpidr, this_cluster;
  358. BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
  359. BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
  360. /*
  361. * Set initial CPU and cluster states.
  362. * Only one cluster is assumed to be active at this point.
  363. */
  364. for (i = 0; i < MAX_NR_CLUSTERS; i++) {
  365. mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
  366. mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
  367. for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
  368. mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
  369. }
  370. mpidr = read_cpuid_mpidr();
  371. this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  372. for_each_online_cpu(i) {
  373. mcpm_cpu_use_count[this_cluster][i] = 1;
  374. mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
  375. }
  376. mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
  377. sync_cache_w(&mcpm_sync);
  378. if (power_up_setup) {
  379. mcpm_power_up_setup_phys = virt_to_phys(power_up_setup);
  380. sync_cache_w(&mcpm_power_up_setup_phys);
  381. }
  382. return 0;
  383. }