xcall.c 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. /*
  2. * Copyright (c) 2014-2018 Richard Braun.
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. */
  17. #include <assert.h>
  18. #include <stdalign.h>
  19. #include <stddef.h>
  20. #include <stdio.h>
  21. #include <kern/atomic.h>
  22. #include <kern/clock.h>
  23. #include <kern/init.h>
  24. #include <kern/log.h>
  25. #include <kern/macros.h>
  26. #include <kern/percpu.h>
  27. #include <kern/spinlock.h>
  28. #include <kern/syscnt.h>
  29. #include <kern/thread.h>
  30. #include <kern/xcall.h>
  31. #include <machine/cpu.h>
  32. struct xcall
  33. {
  34. xcall_fn_t fn;
  35. void *arg;
  36. };
  37. /*
  38. * Per-CPU data.
  39. *
  40. * The lock is used to serialize cross-calls from different processors
  41. * to the same processor. It is held during the complete cross-call
  42. * sequence. Inside the critical section, accesses to the receive call
  43. * are used to enforce release-acquire ordering between the sending
  44. * and receiving processors.
  45. *
  46. * Locking keys :
  47. * (a) atomic
  48. * (c) cpu_data
  49. */
  50. struct xcall_cpu_data
  51. {
  52. __cacheline_aligned struct spinlock lock;
  53. struct xcall *recv_call; // (c)
  54. struct syscnt sc_sent; // (a)
  55. struct syscnt sc_received; // (a)
  56. };
  57. static struct xcall_cpu_data xcall_cpu_data __percpu;
  58. static struct xcall_cpu_data*
  59. xcall_get_local_cpu_data (void)
  60. {
  61. return (cpu_local_ptr (xcall_cpu_data));
  62. }
  63. static struct xcall_cpu_data*
  64. xcall_get_cpu_data (uint32_t cpu)
  65. {
  66. return (percpu_ptr (xcall_cpu_data, cpu));
  67. }
  68. static void
  69. xcall_init (struct xcall *call, xcall_fn_t fn, void *arg)
  70. {
  71. call->fn = fn;
  72. call->arg = arg;
  73. }
  74. static void
  75. xcall_process (struct xcall *call)
  76. {
  77. call->fn (call->arg);
  78. }
  79. static void
  80. xcall_cpu_data_init (struct xcall_cpu_data *cpu_data, uint32_t cpu)
  81. {
  82. char name[SYSCNT_NAME_SIZE];
  83. snprintf (name, sizeof (name), "xcall_sent/%u", cpu);
  84. syscnt_register (&cpu_data->sc_sent, name);
  85. snprintf (name, sizeof (name), "xcall_received/%u", cpu);
  86. syscnt_register (&cpu_data->sc_received, name);
  87. cpu_data->recv_call = NULL;
  88. spinlock_init (&cpu_data->lock);
  89. }
  90. static struct xcall*
  91. xcall_cpu_data_get_recv_call (const struct xcall_cpu_data *cpu_data)
  92. {
  93. return (atomic_load_acq (&cpu_data->recv_call));
  94. }
  95. static void
  96. xcall_cpu_data_set_recv_call (struct xcall_cpu_data *cpu_data,
  97. struct xcall *call)
  98. {
  99. atomic_store_rel (&cpu_data->recv_call, call);
  100. }
  101. static void
  102. xcall_cpu_data_clear_recv_call (struct xcall_cpu_data *cpu_data)
  103. {
  104. xcall_cpu_data_set_recv_call (cpu_data, NULL);
  105. }
  106. static int __init
  107. xcall_setup (void)
  108. {
  109. for (uint32_t i = 0; i < cpu_count (); i++)
  110. xcall_cpu_data_init (xcall_get_cpu_data (i), i);
  111. return (0);
  112. }
  113. INIT_OP_DEFINE (xcall_setup,
  114. INIT_OP_DEP (cpu_mp_probe, true),
  115. INIT_OP_DEP (thread_bootstrap, true),
  116. INIT_OP_DEP (spinlock_setup, true),
  117. INIT_OP_DEP (syscnt_setup, true));
  118. void
  119. xcall_call (xcall_fn_t fn, void *arg, uint32_t cpu)
  120. {
  121. assert (cpu_intr_enabled ());
  122. assert (fn);
  123. struct xcall call;
  124. xcall_init (&call, fn, arg);
  125. _Auto cpu_data = xcall_get_cpu_data (cpu);
  126. spinlock_lock (&cpu_data->lock);
  127. // Enforce release ordering on the receive call.
  128. xcall_cpu_data_set_recv_call (cpu_data, &call);
  129. cpu_send_xcall (cpu);
  130. // Enforce acquire ordering on the receive call.
  131. while (xcall_cpu_data_get_recv_call (cpu_data))
  132. cpu_pause ();
  133. spinlock_unlock (&cpu_data->lock);
  134. syscnt_inc (&cpu_data->sc_sent);
  135. }
  136. void
  137. xcall_intr (void)
  138. {
  139. assert (thread_check_intr_context ());
  140. _Auto cpu_data = xcall_get_local_cpu_data ();
  141. // Enforce acquire ordering on the receive call.
  142. _Auto call = xcall_cpu_data_get_recv_call (cpu_data);
  143. if (call)
  144. xcall_process (call);
  145. else
  146. log_err ("xcall: spurious interrupt on cpu%u", cpu_id ());
  147. syscnt_inc (&cpu_data->sc_received);
  148. // Enforce release ordering on the receive call.
  149. xcall_cpu_data_clear_recv_call (cpu_data);
  150. }
  151. static void
  152. xcall_async_work (struct work *work)
  153. {
  154. _Auto async = structof (work, struct xcall_async, work);
  155. xcall_call (async->fn, async->arg, async->cpu);
  156. SPINLOCK_GUARD (&async->lock);
  157. async->done = true;
  158. if (async->waiter)
  159. thread_wakeup (async->waiter);
  160. }
  161. void
  162. xcall_async_init (struct xcall_async *async,
  163. xcall_fn_t fn, void *arg, uint32_t cpu)
  164. {
  165. async->fn = fn;
  166. async->arg = arg;
  167. async->cpu = cpu;
  168. spinlock_init (&async->lock);
  169. async->waiter = NULL;
  170. async->done = false;
  171. work_init (&async->work, xcall_async_work);
  172. }
  173. void
  174. xcall_async_call (struct xcall_async *async)
  175. {
  176. work_schedule (&async->work, 0);
  177. }
  178. void
  179. xcall_async_wait (struct xcall_async *async)
  180. {
  181. SPINLOCK_INTR_GUARD (&async->lock);
  182. if (!async->done)
  183. {
  184. async->waiter = thread_self ();
  185. thread_sleep (&async->lock, &async->work, "asyncx");
  186. }
  187. }
  188. int
  189. xcall_async_timedwait (struct xcall_async *async, uint64_t ticks, bool abs)
  190. {
  191. int ret = 0;
  192. SPINLOCK_INTR_GUARD (&async->lock);
  193. if (!async->done)
  194. {
  195. async->waiter = thread_self ();
  196. ret = thread_timedsleep (&async->lock, &async->work, "asyncx",
  197. ticks + (abs ? 0 : clock_get_time () + 1));
  198. }
  199. return (ret);
  200. }