thread.c 75 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837
  1. /*
  2. * Copyright (c) 2012-2018 Richard Braun.
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. *
  17. *
  18. * The scheduling algorithm implemented by this module, named Distributed
  19. * Group Ratio Round-Robin (DGR3), is based on the following papers :
  20. * - "Group Ratio Round-Robin: O(1) Proportional Share Scheduling for
  21. * Uniprocessor and Multiprocessor Systems" by Bogdan Caprita, Wong Chun
  22. * Chan, Jason Nieh, Clifford Stein and Haoqiang Zheng.
  23. * - "Efficient and Scalable Multiprocessor Fair Scheduling Using Distributed
  24. * Weighted Round-Robin" by Tong li, Dan Baumberger and Scott Hahn.
  25. *
  26. * Note that the Group Ratio Round-Robin (GR3) paper offers a multiprocessor
  27. * extension, but based on a single global queue, which strongly limits its
  28. * scalability on systems with many processors. That extension isn't used in
  29. * this implementation.
  30. *
  31. * The basic idea is to use GR3 for processor-local scheduling, and Distributed
  32. * Weighted Round-Robin (DWRR) for inter-processor load balancing. These
  33. * algorithms were chosen for several reasons. To begin with, they provide
  34. * fair scheduling, a very desirable property for a modern scheduler. Next,
  35. * being based on round-robin, their algorithmic complexity is very low (GR3
  36. * has O(1) scheduling complexity, and O(g) complexity on thread addition
  37. * or removal, g being the number of groups, with one group per priority, a
  38. * low number in practice). Finally, they're very simple to implement, making
  39. * them easy to adjust and maintain.
  40. *
  41. * Both algorithms are actually slightly modified for efficiency. First, this
  42. * version of GR3 is simplified by mapping exactly one group to one priority,
  43. * and in turn, one weight. This is possible because priorities are intended
  44. * to match Unix nice values, and systems commonly provide a constant, small
  45. * set of nice values. This removes the need for accounting deficit. Next,
  46. * round tracking is used to improve the handling of dynamic events : work
  47. * scaling is performed only on thread addition, and not when a thread that
  48. * was removed is added again during the same round. In addition, since GR3
  49. * is itself a round-robin algorithm, it already provides the feature required
  50. * from local scheduling by DWRR, namely round slicing. Consequently, DWRR
  51. * doesn't sit "on top" of GR3, but is actually merged with it. The result is
  52. * an algorithm that shares the same data for both local scheduling and load
  53. * balancing.
  54. *
  55. * A few terms are used by both papers with slightly different meanings. Here
  56. * are the definitions used in this implementation :
  57. * - The time unit is the system timer period (1 / tick frequency)
  58. * - Work is the amount of execution time units consumed
  59. * - Weight is the amount of execution time units allocated
  60. * - A round is the shortest period during which all threads in a run queue
  61. * consume their allocated time (i.e. their work reaches their weight)
  62. *
  63. * TODO Sub-tick accounting.
  64. *
  65. * TODO Take into account the underlying CPU topology (and adjust load
  66. * balancing to access the global highest round less frequently on large
  67. * processor groups, perhaps by applying the load balancing algorithm in a
  68. * bottom-up fashion with one highest round per processor group).
  69. *
  70. * TODO For now, interactivity can not be experimented. The current strategy
  71. * is to always add threads in front of their group queue and track rounds
  72. * so that they don't get more time than they should. A direct consequence
  73. * is that continually spawning threads at short intervals is likely to cause
  74. * starvation. This could be fixed by adding newly created threads at the back
  75. * of their group queue. For now, don't overengineer, and wait until all this
  76. * can actually be tested.
  77. *
  78. * TODO Review weight computation (it may be more appropriate to determine
  79. * weights in a smoother way than a raw scaling).
  80. */
  81. #include <assert.h>
  82. #include <errno.h>
  83. #include <stdalign.h>
  84. #include <stdbool.h>
  85. #include <stddef.h>
  86. #include <stdint.h>
  87. #include <stdio.h>
  88. #include <stdnoreturn.h>
  89. #include <string.h>
  90. #include <kern/atomic.h>
  91. #include <kern/capability.h>
  92. #include <kern/clock.h>
  93. #include <kern/cpumap.h>
  94. #include <kern/init.h>
  95. #include <kern/kmem.h>
  96. #include <kern/list.h>
  97. #include <kern/macros.h>
  98. #include <kern/panic.h>
  99. #include <kern/percpu.h>
  100. #include <kern/perfmon.h>
  101. #include <kern/rcu.h>
  102. #include <kern/shell.h>
  103. #include <kern/sleepq.h>
  104. #include <kern/spinlock.h>
  105. #include <kern/syscnt.h>
  106. #include <kern/task.h>
  107. #include <kern/thread.h>
  108. #include <kern/timer.h>
  109. #include <kern/turnstile.h>
  110. #include <kern/types.h>
  111. #include <kern/user.h>
  112. #include <kern/work.h>
  113. #include <machine/cpu.h>
  114. #include <machine/page.h>
  115. #include <machine/pmap.h>
  116. #include <machine/tcb.h>
  117. #include <vm/kmem.h>
  118. #include <vm/map.h>
  119. /*
  120. * Preemption level of a suspended thread.
  121. *
  122. * The expected interrupt, preemption and run queue lock state when
  123. * dispatching a thread is :
  124. * - interrupts disabled
  125. * - preemption disabled
  126. * - run queue locked
  127. *
  128. * Locking the run queue increases the preemption level once more,
  129. * making its value 2.
  130. */
  131. #define THREAD_SUSPEND_PREEMPT_LEVEL 2
  132. /*
  133. * Scheduling classes.
  134. *
  135. * Classes are sorted by order of priority (lower indexes first). The same
  136. * class can apply to several policies.
  137. *
  138. * The idle class is reserved for the per-CPU idle threads.
  139. */
  140. #define THREAD_SCHED_CLASS_RT 0
  141. #define THREAD_SCHED_CLASS_FS 1
  142. #define THREAD_SCHED_CLASS_IDLE 2
  143. #define THREAD_NR_SCHED_CLASSES 3
  144. /*
  145. * Global priority bases for each scheduling class.
  146. *
  147. * Global priorities are only used to determine which of two threads
  148. * has the higher priority, and should only matter for priority
  149. * inheritance.
  150. *
  151. * In the current configuration, all fair-scheduling threads have the
  152. * same global priority.
  153. */
  154. #define THREAD_SCHED_GLOBAL_PRIO_RT 2
  155. #define THREAD_SCHED_GLOBAL_PRIO_FS 1
  156. #define THREAD_SCHED_GLOBAL_PRIO_IDLE 0
  157. // Default time slice for real-time round-robin scheduling.
  158. #define THREAD_DEFAULT_RR_TIME_SLICE (CLOCK_FREQ / 10)
  159. /*
  160. * Maximum number of threads which can be pulled from a remote run queue
  161. * while interrupts are disabled.
  162. */
  163. #define THREAD_MAX_MIGRATIONS 16
  164. // Delay (in ticks) between two balance attempts when a run queue is idle.
  165. #define THREAD_IDLE_BALANCE_TICKS (CLOCK_FREQ / 2)
  166. // Run queue properties for real-time threads.
  167. struct thread_rt_runq
  168. {
  169. uint32_t bitmap;
  170. struct list threads[THREAD_SCHED_RT_PRIO_MAX + 1];
  171. };
  172. /*
  173. * Initial value of the highest round.
  174. *
  175. * Set to a high value to make sure overflows are correctly handled.
  176. */
  177. #define THREAD_FS_INITIAL_ROUND ((unsigned long)-10)
  178. // Round slice base unit for fair-scheduling threads.
  179. #define THREAD_FS_ROUND_SLICE_BASE (CLOCK_FREQ / 10)
  180. // Group of threads sharing the same weight.
  181. struct thread_fs_group
  182. {
  183. struct list node;
  184. struct list threads;
  185. uint32_t weight;
  186. uint32_t work;
  187. };
  188. /*
  189. * Run queue properties for fair-scheduling threads.
  190. *
  191. * The current group pointer has a valid address only when the run queue isn't
  192. * empty.
  193. */
  194. struct thread_fs_runq
  195. {
  196. struct thread_fs_group group_array[THREAD_SCHED_FS_PRIO_MAX + 1];
  197. struct list groups;
  198. struct list threads;
  199. struct thread_fs_group *current;
  200. uint32_t nr_threads;
  201. uint32_t weight;
  202. uint32_t work;
  203. };
  204. /*
  205. * Per processor run queue.
  206. *
  207. * Locking multiple run queues is done in the ascending order of their CPU
  208. * identifier. Interrupts must be disabled whenever locking a run queue, even
  209. * a remote one, otherwise an interrupt (which invokes the scheduler on its
  210. * return path) may violate the locking order.
  211. */
  212. struct thread_runq
  213. {
  214. __cacheline_aligned struct spinlock lock;
  215. uint32_t cpu;
  216. uint32_t nr_threads;
  217. struct thread *current;
  218. // Real-time related members.
  219. struct thread_rt_runq rt_runq;
  220. /*
  221. * Fair-scheduling related members.
  222. *
  223. * The current round is set when the active run queue becomes non-empty.
  224. * It's not reset when both run queues become empty. As a result, the
  225. * current round has a meaningful value only when at least one thread is
  226. * present, i.e. the global weight isn't zero.
  227. */
  228. size_t fs_round;
  229. uint32_t fs_weight;
  230. struct thread_fs_runq fs_runqs[2];
  231. struct thread_fs_runq *fs_runq_active;
  232. struct thread_fs_runq *fs_runq_expired;
  233. struct thread *balancer;
  234. struct thread *idler;
  235. // Ticks before the next balancing attempt when a run queue is idle.
  236. uint32_t idle_balance_ticks;
  237. struct syscnt sc_schedule_intrs;
  238. struct syscnt sc_boosts;
  239. };
  240. // Operations of a scheduling class.
  241. struct thread_sched_ops
  242. {
  243. struct thread_runq* (*select_runq) (struct thread *);
  244. void (*add) (struct thread_runq *, struct thread *);
  245. void (*remove) (struct thread_runq *, struct thread *);
  246. void (*put_prev) (struct thread_runq *, struct thread *);
  247. struct thread* (*get_next) (struct thread_runq *);
  248. void (*reset_priority) (struct thread *, uint16_t);
  249. void (*update_priority) (struct thread *, uint16_t);
  250. uint32_t (*get_global_priority) (uint16_t);
  251. void (*set_next) (struct thread_runq *, struct thread *);
  252. void (*tick) (struct thread_runq *, struct thread *);
  253. };
  254. static struct thread_runq thread_runq __percpu;
  255. /*
  256. * Statically allocated fake threads that provide thread context to processors
  257. * during bootstrap.
  258. */
  259. static struct thread thread_booters[CONFIG_MAX_CPUS] __initdata;
  260. static struct kmem_cache thread_cache;
  261. #ifndef CONFIG_THREAD_STACK_GUARD
  262. static struct kmem_cache thread_stack_cache;
  263. #endif
  264. static const uint8_t thread_policy_table[THREAD_NR_SCHED_POLICIES] =
  265. {
  266. [THREAD_SCHED_POLICY_FIFO] = THREAD_SCHED_CLASS_RT,
  267. [THREAD_SCHED_POLICY_RR] = THREAD_SCHED_CLASS_RT,
  268. [THREAD_SCHED_POLICY_FS] = THREAD_SCHED_CLASS_FS,
  269. [THREAD_SCHED_POLICY_IDLE] = THREAD_SCHED_CLASS_IDLE,
  270. };
  271. static const struct thread_sched_ops thread_sched_ops[THREAD_NR_SCHED_CLASSES];
  272. // Map of run queues for which a processor is running.
  273. static struct cpumap thread_active_runqs;
  274. /*
  275. * Map of idle run queues.
  276. *
  277. * Access to this map isn't synchronized. It is merely used as a fast hint
  278. * to find run queues that are likely to be idle.
  279. */
  280. static struct cpumap thread_idle_runqs;
  281. /*
  282. * System-wide value of the current highest round.
  283. *
  284. * This global variable is accessed without any synchronization. Its value
  285. * being slightly inaccurate doesn't harm the fairness properties of the
  286. * scheduling and load balancing algorithms.
  287. *
  288. * There can be moderate bouncing on this word so give it its own cache line.
  289. */
  290. static struct
  291. {
  292. __cacheline_aligned volatile size_t value;
  293. } thread_fs_highest_round_struct;
  294. #define thread_fs_highest_round (thread_fs_highest_round_struct.value)
  295. /*
  296. * Number of processors which have requested the scheduler to run.
  297. *
  298. * This value is used to implement a global barrier across the entire
  299. * system at boot time, so that inter-processor requests may not be
  300. * lost in case a processor is slower to initialize.
  301. */
  302. static uint32_t thread_nr_boot_cpus __initdata;
  303. struct thread_zombie
  304. {
  305. struct work work;
  306. struct thread *thread;
  307. };
  308. struct thread_runq_guard_t
  309. {
  310. struct thread_runq *runq;
  311. cpu_flags_t flags;
  312. bool preempt_disabled;
  313. };
  314. static uint8_t
  315. thread_policy_to_class (uint8_t policy)
  316. {
  317. assert (policy < ARRAY_SIZE (thread_policy_table));
  318. return (thread_policy_table[policy]);
  319. }
  320. static void
  321. thread_set_wchan (struct thread *thread, const void *wchan_addr,
  322. const char *wchan_desc)
  323. {
  324. assert (wchan_addr && wchan_desc);
  325. thread->wchan_addr = wchan_addr;
  326. thread->wchan_desc = wchan_desc;
  327. }
  328. static void
  329. thread_clear_wchan (struct thread *thread)
  330. {
  331. thread->wchan_addr = NULL;
  332. thread->wchan_desc = NULL;
  333. }
  334. static const struct thread_sched_ops*
  335. thread_get_sched_ops (uint8_t sched_class)
  336. {
  337. assert (sched_class < ARRAY_SIZE (thread_sched_ops));
  338. return (&thread_sched_ops[sched_class]);
  339. }
  340. static const struct thread_sched_ops*
  341. thread_get_user_sched_ops (const struct thread *thread)
  342. {
  343. return (thread_get_sched_ops (thread_user_sched_class (thread)));
  344. }
  345. static const struct thread_sched_ops*
  346. thread_get_real_sched_ops (const struct thread *thread)
  347. {
  348. return (thread_get_sched_ops (thread_real_sched_class (thread)));
  349. }
  350. static void __init
  351. thread_runq_init_rt (struct thread_runq *runq)
  352. {
  353. runq->rt_runq.bitmap = 0;
  354. for (size_t i = 0; i < ARRAY_SIZE (runq->rt_runq.threads); i++)
  355. list_init (&runq->rt_runq.threads[i]);
  356. }
  357. static void __init
  358. thread_fs_group_init (struct thread_fs_group *group)
  359. {
  360. list_init (&group->threads);
  361. group->weight = 0;
  362. group->work = 0;
  363. }
  364. static void __init
  365. thread_fs_runq_init (struct thread_fs_runq *fs_runq)
  366. {
  367. for (size_t i = 0; i < ARRAY_SIZE (fs_runq->group_array); i++)
  368. thread_fs_group_init (&fs_runq->group_array[i]);
  369. list_init (&fs_runq->groups);
  370. list_init (&fs_runq->threads);
  371. fs_runq->nr_threads = 0;
  372. fs_runq->weight = 0;
  373. fs_runq->work = 0;
  374. }
  375. static void __init
  376. thread_runq_init_fs (struct thread_runq *runq)
  377. {
  378. runq->fs_weight = 0;
  379. runq->fs_runq_active = &runq->fs_runqs[0];
  380. runq->fs_runq_expired = &runq->fs_runqs[1];
  381. thread_fs_runq_init (runq->fs_runq_active);
  382. thread_fs_runq_init (runq->fs_runq_expired);
  383. }
  384. static void __init
  385. thread_runq_init (struct thread_runq *runq, uint32_t cpu,
  386. struct thread *booter)
  387. {
  388. char name[SYSCNT_NAME_SIZE];
  389. spinlock_init (&runq->lock);
  390. runq->cpu = cpu;
  391. runq->nr_threads = 0;
  392. runq->current = booter;
  393. thread_runq_init_rt (runq);
  394. thread_runq_init_fs (runq);
  395. runq->balancer = NULL;
  396. runq->idler = NULL;
  397. runq->idle_balance_ticks = (uint32_t)-1;
  398. snprintf (name, sizeof (name), "thread_schedule_intrs/%u", cpu);
  399. syscnt_register (&runq->sc_schedule_intrs, name);
  400. snprintf (name, sizeof (name), "thread_boosts/%u", cpu);
  401. syscnt_register (&runq->sc_boosts, name);
  402. }
  403. static inline struct thread_runq*
  404. thread_runq_local (void)
  405. {
  406. assert (!thread_preempt_enabled () || thread_pinned ());
  407. return (cpu_local_ptr (thread_runq));
  408. }
  409. static inline uint32_t
  410. thread_runq_cpu (struct thread_runq *runq)
  411. {
  412. return (runq->cpu);
  413. }
  414. static void
  415. thread_runq_add (struct thread_runq *runq, struct thread *thread)
  416. {
  417. assert (!cpu_intr_enabled ());
  418. assert (spinlock_locked (&runq->lock));
  419. assert (!thread->in_runq);
  420. const _Auto ops = thread_get_real_sched_ops (thread);
  421. ops->add (runq, thread);
  422. if (runq->nr_threads == 0)
  423. cpumap_clear_atomic (&thread_idle_runqs, thread_runq_cpu (runq));
  424. ++runq->nr_threads;
  425. if (thread_real_sched_class (thread) <
  426. thread_real_sched_class (runq->current))
  427. thread_set_flag (runq->current, THREAD_YIELD);
  428. atomic_store_rlx (&thread->runq, runq);
  429. thread->in_runq = true;
  430. }
  431. static void
  432. thread_runq_remove (struct thread_runq *runq, struct thread *thread)
  433. {
  434. assert (!cpu_intr_enabled ());
  435. assert (spinlock_locked (&runq->lock));
  436. assert (thread->in_runq);
  437. if (--runq->nr_threads == 0)
  438. cpumap_set_atomic (&thread_idle_runqs, thread_runq_cpu (runq));
  439. const _Auto ops = thread_get_real_sched_ops (thread);
  440. ops->remove (runq, thread);
  441. thread->in_runq = false;
  442. }
  443. static void
  444. thread_runq_put_prev (struct thread_runq *runq, struct thread *thread)
  445. {
  446. assert (!cpu_intr_enabled ());
  447. assert (spinlock_locked (&runq->lock));
  448. const _Auto ops = thread_get_real_sched_ops (thread);
  449. if (ops->put_prev)
  450. ops->put_prev (runq, thread);
  451. }
  452. static struct thread*
  453. thread_runq_get_next (struct thread_runq *runq)
  454. {
  455. assert (!cpu_intr_enabled ());
  456. assert (spinlock_locked (&runq->lock));
  457. for (size_t i = 0; i < ARRAY_SIZE (thread_sched_ops); i++)
  458. {
  459. struct thread *thread = thread_sched_ops[i].get_next (runq);
  460. if (thread)
  461. {
  462. atomic_store_rlx (&runq->current, thread);
  463. return (thread);
  464. }
  465. }
  466. // The idle class should never be empty.
  467. panic ("thread: unable to find next thread");
  468. }
  469. static void
  470. thread_runq_set_next (struct thread_runq *runq, struct thread *thread)
  471. {
  472. const _Auto ops = thread_get_real_sched_ops (thread);
  473. if (ops->set_next)
  474. ops->set_next (runq, thread);
  475. atomic_store_rlx (&runq->current, thread);
  476. }
  477. static void
  478. thread_runq_wakeup (struct thread_runq *runq, struct thread *thread)
  479. {
  480. assert (!cpu_intr_enabled ());
  481. assert (spinlock_locked (&runq->lock));
  482. assert (thread->state == THREAD_RUNNING);
  483. thread_runq_add (runq, thread);
  484. if (runq != thread_runq_local () &&
  485. thread_test_flag (runq->current, THREAD_YIELD))
  486. cpu_send_thread_schedule (thread_runq_cpu (runq));
  487. }
  488. static void
  489. thread_runq_wakeup_balancer (struct thread_runq *runq)
  490. {
  491. if (runq->balancer->state == THREAD_RUNNING)
  492. return;
  493. thread_clear_wchan (runq->balancer);
  494. atomic_store_rlx (&runq->balancer->state, THREAD_RUNNING);
  495. thread_runq_wakeup (runq, runq->balancer);
  496. }
  497. static void
  498. thread_runq_schedule_load (struct thread *thread)
  499. {
  500. pmap_load (thread->xtask->map->pmap);
  501. #ifdef CONFIG_PERFMON
  502. perfmon_td_load (thread_get_perfmon_td (thread));
  503. #endif
  504. }
  505. static void
  506. thread_runq_schedule_unload (struct thread *thread __unused)
  507. {
  508. #ifdef CONFIG_PERFMON
  509. perfmon_td_unload (thread_get_perfmon_td (thread));
  510. #endif
  511. }
  512. static struct thread_runq*
  513. thread_lock_runq (struct thread *thread, cpu_flags_t *flags)
  514. {
  515. while (1)
  516. {
  517. _Auto runq = atomic_load_rlx (&thread->runq);
  518. spinlock_lock_intr_save (&runq->lock, flags);
  519. if (likely (runq == atomic_load_rlx (&thread->runq)))
  520. return (runq);
  521. spinlock_unlock_intr_restore (&runq->lock, *flags);
  522. }
  523. }
  524. static void
  525. thread_unlock_runq (struct thread_runq *runq, cpu_flags_t flags)
  526. {
  527. spinlock_unlock_intr_restore (&runq->lock, flags);
  528. }
  529. static struct thread_runq_guard_t
  530. thread_runq_guard_make (struct thread *thread, bool disable_preempt)
  531. {
  532. struct thread_runq_guard_t ret;
  533. ret.preempt_disabled = disable_preempt;
  534. if (ret.preempt_disabled)
  535. thread_preempt_disable ();
  536. ret.runq = thread_lock_runq (thread, &ret.flags);
  537. return (ret);
  538. }
  539. static void
  540. thread_runq_guard_fini (struct thread_runq_guard_t *guard)
  541. {
  542. thread_unlock_runq (guard->runq, guard->flags);
  543. if (guard->preempt_disabled)
  544. thread_preempt_enable ();
  545. }
  546. #define thread_runq_guard \
  547. thread_runq_guard_t CLEANUP (thread_runq_guard_fini) __unused
  548. static struct thread_runq*
  549. thread_runq_schedule (struct thread_runq *runq)
  550. {
  551. struct thread *prev = thread_self ();
  552. assert (prev->cur_port ||
  553. (__builtin_frame_address (0) >= prev->stack &&
  554. __builtin_frame_address (0) < prev->stack + TCB_STACK_SIZE));
  555. assert (prev->preempt_level == THREAD_SUSPEND_PREEMPT_LEVEL);
  556. assert (!cpu_intr_enabled ());
  557. assert (spinlock_locked (&runq->lock));
  558. thread_clear_flag (prev, THREAD_YIELD);
  559. thread_runq_put_prev (runq, prev);
  560. if (prev->suspend)
  561. {
  562. prev->state = THREAD_SUSPENDED;
  563. prev->suspend = false;
  564. }
  565. if (prev->state != THREAD_RUNNING)
  566. {
  567. thread_runq_remove (runq, prev);
  568. if (!runq->nr_threads && prev != runq->balancer)
  569. thread_runq_wakeup_balancer (runq);
  570. }
  571. struct thread *next = thread_runq_get_next (runq);
  572. assert (next != runq->idler || !runq->nr_threads);
  573. assert (next->preempt_level == THREAD_SUSPEND_PREEMPT_LEVEL);
  574. if (likely (prev != next))
  575. {
  576. thread_runq_schedule_unload (prev);
  577. rcu_report_context_switch (thread_rcu_reader (prev));
  578. pmap_ipc_pte_context_switch (prev->pmap_data, next->pmap_data);
  579. spinlock_transfer_owner (&runq->lock, next);
  580. /*
  581. * This is where the true context switch occurs. The next thread must
  582. * unlock the run queue and reenable preemption. Note that unlocking
  583. * and locking the run queue again is equivalent to a full memory
  584. * barrier.
  585. */
  586. tcb_switch (&prev->tcb, &next->tcb);
  587. /*
  588. * The thread is dispatched on a processor once again.
  589. *
  590. * Keep in mind the system state may have changed a lot since this
  591. * function was called. In particular :
  592. * - The next thread may have been destroyed, and must not be
  593. * referenced any more.
  594. * - The current thread may have been migrated to another processor.
  595. */
  596. barrier ();
  597. thread_runq_schedule_load (prev);
  598. runq = thread_runq_local ();
  599. }
  600. assert (prev->preempt_level == THREAD_SUSPEND_PREEMPT_LEVEL);
  601. assert (!cpu_intr_enabled ());
  602. assert (spinlock_locked (&runq->lock));
  603. return (runq);
  604. }
  605. static void
  606. thread_runq_double_lock (struct thread_runq *a, struct thread_runq *b)
  607. {
  608. assert (!cpu_intr_enabled ());
  609. assert (!thread_preempt_enabled ());
  610. assert (a != b);
  611. if (a->cpu < b->cpu)
  612. {
  613. spinlock_lock (&a->lock);
  614. spinlock_lock (&b->lock);
  615. }
  616. else
  617. {
  618. spinlock_lock (&b->lock);
  619. spinlock_lock (&a->lock);
  620. }
  621. }
  622. static struct thread_runq*
  623. thread_sched_rt_select_runq (struct thread *thread)
  624. {
  625. /*
  626. * Real-time tasks are commonly configured to run on one specific
  627. * processor only.
  628. */
  629. int i = cpumap_find_first (&thread->cpumap);
  630. assert (i >= 0);
  631. assert (cpumap_test (&thread_active_runqs, i));
  632. struct thread_runq *runq = percpu_ptr (thread_runq, i);
  633. spinlock_lock (&runq->lock);
  634. return (runq);
  635. }
  636. static void
  637. thread_sched_rt_add (struct thread_runq *runq, struct thread *thread)
  638. {
  639. struct thread_rt_runq *rt_runq = &runq->rt_runq;
  640. struct list *threads = &rt_runq->threads[thread_real_priority (thread)];
  641. list_insert_tail (threads, &thread->rt_data.node);
  642. if (list_singular (threads))
  643. rt_runq->bitmap |= (1ULL << thread_real_priority (thread));
  644. if (thread_real_sched_class (thread) ==
  645. thread_real_sched_class (runq->current) &&
  646. thread_real_priority (thread) > thread_real_priority (runq->current))
  647. thread_set_flag (runq->current, THREAD_YIELD);
  648. }
  649. static void
  650. thread_sched_rt_remove (struct thread_runq *runq, struct thread *thread)
  651. {
  652. struct thread_rt_runq *rt_runq = &runq->rt_runq;
  653. assert (thread_real_priority (thread) < ARRAY_SIZE (rt_runq->threads));
  654. struct list *threads = &rt_runq->threads[thread_real_priority (thread)];
  655. list_remove (&thread->rt_data.node);
  656. if (list_empty (threads))
  657. rt_runq->bitmap &= ~(1ULL << thread_real_priority (thread));
  658. }
  659. static void
  660. thread_sched_rt_put_prev (struct thread_runq *runq, struct thread *thread)
  661. {
  662. thread_sched_rt_add (runq, thread);
  663. }
  664. static struct thread*
  665. thread_sched_rt_get_next (struct thread_runq *runq)
  666. {
  667. struct thread_rt_runq *rt_runq = &runq->rt_runq;
  668. if (!rt_runq->bitmap)
  669. return (NULL);
  670. uint32_t priority = THREAD_SCHED_RT_PRIO_MAX -
  671. __builtin_clz (rt_runq->bitmap);
  672. assert (priority < ARRAY_SIZE (rt_runq->threads));
  673. struct list *threads = &rt_runq->threads[priority];
  674. assert (!list_empty (threads));
  675. _Auto thread = list_first_entry (threads, struct thread, rt_data.node);
  676. thread_sched_rt_remove (runq, thread);
  677. return (thread);
  678. }
  679. static void
  680. thread_sched_rt_reset_priority (struct thread *thread, uint16_t priority)
  681. {
  682. assert (priority <= THREAD_SCHED_RT_PRIO_MAX);
  683. thread->rt_data.time_slice = THREAD_DEFAULT_RR_TIME_SLICE;
  684. }
  685. static uint32_t
  686. thread_sched_rt_get_global_priority (uint16_t priority)
  687. {
  688. return (THREAD_SCHED_GLOBAL_PRIO_RT + priority);
  689. }
  690. static void
  691. thread_sched_rt_set_next (struct thread_runq *runq, struct thread *thread)
  692. {
  693. thread_sched_rt_remove (runq, thread);
  694. }
  695. static void
  696. thread_sched_rt_tick (struct thread_runq *runq __unused, struct thread *thread)
  697. {
  698. if (thread_real_sched_policy (thread) != THREAD_SCHED_POLICY_RR ||
  699. --thread->rt_data.time_slice > 0)
  700. return;
  701. thread->rt_data.time_slice = THREAD_DEFAULT_RR_TIME_SLICE;
  702. thread_set_flag (thread, THREAD_YIELD);
  703. }
  704. static inline uint16_t
  705. thread_sched_fs_prio2weight (uint16_t priority)
  706. {
  707. return ((priority + 1) * THREAD_FS_ROUND_SLICE_BASE);
  708. }
  709. static struct thread_runq*
  710. thread_sched_fs_select_runq (struct thread *thread)
  711. {
  712. struct thread_runq *runq;
  713. cpumap_for_each (&thread_idle_runqs, i)
  714. {
  715. if (!cpumap_test (&thread->cpumap, i))
  716. continue;
  717. runq = percpu_ptr (thread_runq, i);
  718. spinlock_lock (&runq->lock);
  719. // The run queue really is idle, return it.
  720. if (runq->current == runq->idler)
  721. return (runq);
  722. spinlock_unlock (&runq->lock);
  723. }
  724. runq = NULL;
  725. cpumap_for_each (&thread_active_runqs, i)
  726. {
  727. if (!cpumap_test (&thread->cpumap, i))
  728. continue;
  729. _Auto tmp = percpu_ptr (thread_runq, i);
  730. spinlock_lock (&tmp->lock);
  731. if (! runq)
  732. {
  733. runq = tmp;
  734. continue;
  735. }
  736. // A run queue may have become idle.
  737. if (tmp->current == tmp->idler)
  738. {
  739. spinlock_unlock (&runq->lock);
  740. return (tmp);
  741. }
  742. /*
  743. * The run queue isn't idle, but there are no fair-scheduling thread,
  744. * which means there are real-time threads.
  745. */
  746. if (tmp->fs_weight == 0)
  747. {
  748. spinlock_unlock (&tmp->lock);
  749. continue;
  750. }
  751. ssize_t delta = (ssize_t)(tmp->fs_round - runq->fs_round);
  752. // Look for the least loaded of the run queues in the highest round.
  753. if (delta > 0 ||
  754. (!delta && tmp->fs_weight < runq->fs_weight))
  755. {
  756. spinlock_unlock (&runq->lock);
  757. runq = tmp;
  758. continue;
  759. }
  760. spinlock_unlock (&tmp->lock);
  761. }
  762. assert (runq);
  763. return (runq);
  764. }
  765. static uint32_t
  766. thread_sched_fs_enqueue_scale (uint32_t work, uint32_t old_weight,
  767. uint32_t new_weight)
  768. {
  769. assert (old_weight);
  770. #ifndef __LP64__
  771. if (likely (work < 0x10000 && new_weight < 0x10000))
  772. return ((work * new_weight) / old_weight);
  773. #endif
  774. return ((uint32_t)(((uint64_t)work * new_weight) / old_weight));
  775. }
  776. static void
  777. thread_sched_fs_enqueue (struct thread_fs_runq *fs_runq, size_t round,
  778. struct thread *thread)
  779. {
  780. assert (!thread->fs_data.fs_runq);
  781. assert (thread->fs_data.work <= thread->fs_data.weight);
  782. _Auto group = &fs_runq->group_array[thread_real_priority (thread)];
  783. uint32_t group_weight = group->weight + thread->fs_data.weight,
  784. total_weight = fs_runq->weight + thread->fs_data.weight;
  785. struct list *node = group->weight ?
  786. list_prev (&group->node) : list_last (&fs_runq->groups);
  787. struct list *init_node = node;
  788. while (!list_end (&fs_runq->groups, node))
  789. {
  790. _Auto tmp = list_entry (node, struct thread_fs_group, node);
  791. if (tmp->weight >= group_weight)
  792. break;
  793. node = list_prev (node);
  794. }
  795. if (!group->weight)
  796. list_insert_after (&group->node, node);
  797. else if (node != init_node)
  798. {
  799. list_remove (&group->node);
  800. list_insert_after (&group->node, node);
  801. }
  802. /*
  803. * XXX Unfairness can occur if the run queue round wraps around and the
  804. * thread is "lucky" enough to have the same round value. This should be
  805. * rare and harmless otherwise.
  806. */
  807. if (thread->fs_data.round == round)
  808. {
  809. fs_runq->work += thread->fs_data.work;
  810. group->work += thread->fs_data.work;
  811. }
  812. else
  813. {
  814. uint32_t group_work, thread_work;
  815. if (!fs_runq->weight)
  816. thread_work = 0;
  817. else
  818. {
  819. group_work = group->weight == 0 ?
  820. thread_sched_fs_enqueue_scale (fs_runq->work,
  821. fs_runq->weight,
  822. thread->fs_data.weight) :
  823. thread_sched_fs_enqueue_scale (group->work,
  824. group->weight,
  825. group_weight);
  826. thread_work = group_work - group->work;
  827. fs_runq->work += thread_work;
  828. group->work = group_work;
  829. }
  830. thread->fs_data.round = round;
  831. thread->fs_data.work = thread_work;
  832. }
  833. ++fs_runq->nr_threads;
  834. fs_runq->weight = total_weight;
  835. group->weight = group_weight;
  836. // Insert at the front of the group to improve interactivity.
  837. list_insert_head (&group->threads, &thread->fs_data.group_node);
  838. list_insert_tail (&fs_runq->threads, &thread->fs_data.runq_node);
  839. thread->fs_data.fs_runq = fs_runq;
  840. }
  841. static void
  842. thread_sched_fs_restart (struct thread_runq *runq)
  843. {
  844. _Auto fs_runq = runq->fs_runq_active;
  845. struct list *node = list_first (&fs_runq->groups);
  846. assert (node);
  847. fs_runq->current = list_entry (node, struct thread_fs_group, node);
  848. if (thread_real_sched_class (runq->current) == THREAD_SCHED_CLASS_FS)
  849. thread_set_flag (runq->current, THREAD_YIELD);
  850. }
  851. static void
  852. thread_sched_fs_add (struct thread_runq *runq, struct thread *thread)
  853. {
  854. if (!runq->fs_weight)
  855. runq->fs_round = thread_fs_highest_round;
  856. uint32_t total_weight = runq->fs_weight + thread->fs_data.weight;
  857. // TODO Limit the maximum number of threads to prevent this situation.
  858. if (total_weight < runq->fs_weight)
  859. panic ("thread: weight overflow");
  860. runq->fs_weight = total_weight;
  861. thread_sched_fs_enqueue (runq->fs_runq_active, runq->fs_round, thread);
  862. thread_sched_fs_restart (runq);
  863. }
  864. static void
  865. thread_sched_fs_dequeue (struct thread *thread)
  866. {
  867. assert (thread->fs_data.fs_runq);
  868. _Auto fs_runq = thread->fs_data.fs_runq;
  869. _Auto group = &fs_runq->group_array[thread_real_priority (thread)];
  870. thread->fs_data.fs_runq = NULL;
  871. list_remove (&thread->fs_data.runq_node);
  872. list_remove (&thread->fs_data.group_node);
  873. fs_runq->work -= thread->fs_data.work;
  874. group->work -= thread->fs_data.work;
  875. fs_runq->weight -= thread->fs_data.weight;
  876. group->weight -= thread->fs_data.weight;
  877. --fs_runq->nr_threads;
  878. if (!group->weight)
  879. list_remove (&group->node);
  880. else
  881. {
  882. struct list *node = list_next (&group->node),
  883. *init_node = node;
  884. while (!list_end (&fs_runq->groups, node))
  885. {
  886. _Auto tmp = list_entry (node, struct thread_fs_group, node);
  887. if (tmp->weight <= group->weight)
  888. break;
  889. node = list_next (node);
  890. }
  891. if (node != init_node)
  892. {
  893. list_remove (&group->node);
  894. list_insert_before (&group->node, node);
  895. }
  896. }
  897. }
  898. static void
  899. thread_sched_fs_remove (struct thread_runq *runq, struct thread *thread)
  900. {
  901. runq->fs_weight -= thread->fs_data.weight;
  902. _Auto fs_runq = thread->fs_data.fs_runq;
  903. thread_sched_fs_dequeue (thread);
  904. if (fs_runq != runq->fs_runq_active)
  905. ;
  906. else if (!fs_runq->nr_threads)
  907. thread_runq_wakeup_balancer (runq);
  908. else
  909. thread_sched_fs_restart (runq);
  910. }
  911. static void
  912. thread_sched_fs_deactivate (struct thread_runq *runq, struct thread *thread)
  913. {
  914. assert (thread->fs_data.fs_runq == runq->fs_runq_active);
  915. assert (thread->fs_data.round == runq->fs_round);
  916. thread_sched_fs_dequeue (thread);
  917. ++thread->fs_data.round;
  918. thread->fs_data.work -= thread->fs_data.weight;
  919. thread_sched_fs_enqueue (runq->fs_runq_expired, runq->fs_round + 1, thread);
  920. if (!runq->fs_runq_active->nr_threads)
  921. thread_runq_wakeup_balancer (runq);
  922. }
  923. static void
  924. thread_sched_fs_put_prev (struct thread_runq *runq, struct thread *thread)
  925. {
  926. _Auto fs_runq = runq->fs_runq_active;
  927. assert (thread_real_priority (thread) < ARRAY_SIZE (fs_runq->group_array));
  928. _Auto group = &fs_runq->group_array[thread_real_priority (thread)];
  929. list_insert_tail (&group->threads, &thread->fs_data.group_node);
  930. if (thread->fs_data.work >= thread->fs_data.weight)
  931. thread_sched_fs_deactivate (runq, thread);
  932. }
  933. static int
  934. thread_sched_fs_ratio_exceeded (struct thread_fs_group *current,
  935. struct thread_fs_group *next)
  936. {
  937. #ifndef __LP64__
  938. if (likely (current->weight < 0x10000 && next->weight < 0x10000))
  939. {
  940. uint32_t ia = (current->work + 1) * next->weight,
  941. ib = (next->work + 1) * current->weight;
  942. return (ia > ib);
  943. }
  944. #endif
  945. uint64_t a = ((uint64_t)current->work + 1) * next->weight,
  946. b = ((uint64_t)next->work + 1) * current->weight;
  947. return (a > b);
  948. }
  949. static struct thread*
  950. thread_sched_fs_get_next (struct thread_runq *runq)
  951. {
  952. _Auto fs_runq = runq->fs_runq_active;
  953. if (!fs_runq->nr_threads)
  954. return (NULL);
  955. _Auto group = fs_runq->current;
  956. struct list *node = list_next (&group->node);
  957. if (list_end (&fs_runq->groups, node))
  958. group = list_entry (list_first (&fs_runq->groups),
  959. struct thread_fs_group, node);
  960. else
  961. {
  962. _Auto next = list_entry (node, struct thread_fs_group, node);
  963. group = thread_sched_fs_ratio_exceeded (group, next) ?
  964. next : list_entry (list_first (&fs_runq->groups),
  965. struct thread_fs_group, node);
  966. }
  967. fs_runq->current = group;
  968. return (list_pop (&group->threads, struct thread, fs_data.group_node));
  969. }
  970. static void
  971. thread_sched_fs_reset_priority (struct thread *thread, uint16_t priority)
  972. {
  973. assert (priority <= THREAD_SCHED_FS_PRIO_MAX);
  974. thread->fs_data.fs_runq = NULL;
  975. thread->fs_data.round = 0;
  976. thread->fs_data.weight = thread_sched_fs_prio2weight (priority);
  977. thread->fs_data.work = 0;
  978. }
  979. static void
  980. thread_sched_fs_update_priority (struct thread *thread, uint16_t priority)
  981. {
  982. assert (priority <= THREAD_SCHED_FS_PRIO_MAX);
  983. thread->fs_data.weight = thread_sched_fs_prio2weight (priority);
  984. if (thread->fs_data.work >= thread->fs_data.weight)
  985. thread->fs_data.work = thread->fs_data.weight;
  986. }
  987. static uint32_t
  988. thread_sched_fs_get_global_priority (uint16_t priority __unused)
  989. {
  990. return (THREAD_SCHED_GLOBAL_PRIO_FS);
  991. }
  992. static void
  993. thread_sched_fs_set_next (struct thread_runq *rq __unused, struct thread *thr)
  994. {
  995. list_remove (&thr->fs_data.group_node);
  996. }
  997. static void
  998. thread_sched_fs_tick (struct thread_runq *runq, struct thread *thread)
  999. {
  1000. _Auto fs_runq = runq->fs_runq_active;
  1001. ++fs_runq->work;
  1002. _Auto group = &fs_runq->group_array[thread_real_priority (thread)];
  1003. ++group->work;
  1004. thread_set_flag (thread, THREAD_YIELD);
  1005. ++thread->fs_data.work;
  1006. }
  1007. static void
  1008. thread_sched_fs_start_next_round (struct thread_runq *runq)
  1009. {
  1010. _Auto tmp = runq->fs_runq_expired;
  1011. runq->fs_runq_expired = runq->fs_runq_active;
  1012. runq->fs_runq_active = tmp;
  1013. if (runq->fs_runq_active->nr_threads)
  1014. {
  1015. ++runq->fs_round;
  1016. ssize_t delta = (ssize_t)(runq->fs_round - thread_fs_highest_round);
  1017. if (delta > 0)
  1018. thread_fs_highest_round = runq->fs_round;
  1019. thread_sched_fs_restart (runq);
  1020. }
  1021. }
  1022. // Check that a remote run queue satisfies the minimum migration requirements.
  1023. static int
  1024. thread_sched_fs_balance_eligible (struct thread_runq *runq,
  1025. size_t highest_round)
  1026. {
  1027. if (!runq->fs_weight ||
  1028. (runq->fs_round != highest_round &&
  1029. runq->fs_round != highest_round - 1))
  1030. return (0);
  1031. uint32_t nr_threads = runq->fs_runq_active->nr_threads +
  1032. runq->fs_runq_expired->nr_threads;
  1033. if (! nr_threads ||
  1034. (nr_threads == 1 &&
  1035. thread_real_sched_class (runq->current) == THREAD_SCHED_CLASS_FS))
  1036. return (0);
  1037. return (1);
  1038. }
  1039. // Try to find the most suitable run queue from which to pull threads.
  1040. static struct thread_runq*
  1041. thread_sched_fs_balance_scan (struct thread_runq *runq,
  1042. size_t highest_round)
  1043. {
  1044. struct thread_runq *remote_runq = NULL;
  1045. cpu_flags_t flags;
  1046. thread_preempt_disable_intr_save (&flags);
  1047. cpumap_for_each (&thread_active_runqs, i)
  1048. {
  1049. _Auto tmp = percpu_ptr (thread_runq, i);
  1050. if (tmp == runq)
  1051. continue;
  1052. spinlock_lock (&tmp->lock);
  1053. if (!thread_sched_fs_balance_eligible (tmp, highest_round))
  1054. {
  1055. spinlock_unlock (&tmp->lock);
  1056. continue;
  1057. }
  1058. else if (! remote_runq)
  1059. {
  1060. remote_runq = tmp;
  1061. continue;
  1062. }
  1063. else if (tmp->fs_weight > remote_runq->fs_weight)
  1064. {
  1065. spinlock_unlock (&remote_runq->lock);
  1066. remote_runq = tmp;
  1067. continue;
  1068. }
  1069. spinlock_unlock (&tmp->lock);
  1070. }
  1071. if (remote_runq)
  1072. spinlock_unlock (&remote_runq->lock);
  1073. thread_preempt_enable_intr_restore (flags);
  1074. return (remote_runq);
  1075. }
  1076. static uint32_t
  1077. thread_sched_fs_balance_pull (struct thread_runq *runq,
  1078. struct thread_runq *remote_runq,
  1079. struct thread_fs_runq *fs_runq,
  1080. uint32_t nr_pulls)
  1081. {
  1082. int cpu = thread_runq_cpu (runq);
  1083. struct thread *thread, *tmp;
  1084. list_for_each_entry_safe (&fs_runq->threads, thread, tmp,
  1085. fs_data.runq_node)
  1086. {
  1087. if (thread == remote_runq->current)
  1088. continue;
  1089. /*
  1090. * The pin level is changed without explicit synchronization.
  1091. * However, it can only be changed by its owning thread. As threads
  1092. * currently running aren't considered for migration, the thread had
  1093. * to be preempted and invoke the scheduler. Since balancer threads
  1094. * acquire the run queue lock, there is strong ordering between
  1095. * changing the pin level and setting the current thread of a
  1096. * run queue.
  1097. *
  1098. * TODO Review comment.
  1099. */
  1100. if (thread->pin_level || !cpumap_test (&thread->cpumap, cpu))
  1101. continue;
  1102. /*
  1103. * Make sure at least one thread is pulled if possible. If one or more
  1104. * thread has already been pulled, take weights into account.
  1105. */
  1106. if (nr_pulls &&
  1107. runq->fs_weight + thread->fs_data.weight >
  1108. remote_runq->fs_weight - thread->fs_data.weight)
  1109. break;
  1110. thread_runq_remove (remote_runq, thread);
  1111. // Don't discard the work already accounted for.
  1112. thread->fs_data.round = runq->fs_round;
  1113. thread_runq_add (runq, thread);
  1114. if (++nr_pulls == THREAD_MAX_MIGRATIONS)
  1115. break;
  1116. }
  1117. return (nr_pulls);
  1118. }
  1119. static uint32_t
  1120. thread_sched_fs_balance_migrate (struct thread_runq *runq,
  1121. struct thread_runq *remote_runq,
  1122. size_t highest_round)
  1123. {
  1124. uint32_t nr_pulls = 0;
  1125. if (!thread_sched_fs_balance_eligible (remote_runq, highest_round))
  1126. return (nr_pulls);
  1127. nr_pulls = thread_sched_fs_balance_pull (runq, remote_runq,
  1128. remote_runq->fs_runq_active, 0);
  1129. if (nr_pulls == THREAD_MAX_MIGRATIONS)
  1130. return (nr_pulls);
  1131. /*
  1132. * Threads in the expired queue of a processor in round highest are
  1133. * actually in round highest + 1.
  1134. */
  1135. if (remote_runq->fs_round != highest_round)
  1136. nr_pulls = thread_sched_fs_balance_pull (runq, remote_runq,
  1137. remote_runq->fs_runq_expired,
  1138. nr_pulls);
  1139. return (nr_pulls);
  1140. }
  1141. /*
  1142. * Inter-processor load balancing for fair-scheduling threads.
  1143. *
  1144. * Preemption must be disabled, and the local run queue must be locked when
  1145. * calling this function. If balancing actually occurs, the lock will be
  1146. * released and preemption enabled when needed.
  1147. */
  1148. static void
  1149. thread_sched_fs_balance (struct thread_runq *runq, cpu_flags_t *flags)
  1150. {
  1151. /*
  1152. * Grab the highest round now and only use the copy so the value is stable
  1153. * during the balancing operation.
  1154. */
  1155. size_t highest_round = thread_fs_highest_round;
  1156. if (runq->fs_round != highest_round &&
  1157. runq->fs_runq_expired->nr_threads)
  1158. goto no_migration;
  1159. spinlock_unlock_intr_restore (&runq->lock, *flags);
  1160. thread_preempt_enable ();
  1161. uint32_t nr_migrations;
  1162. _Auto remote_runq = thread_sched_fs_balance_scan (runq, highest_round);
  1163. if (remote_runq)
  1164. {
  1165. thread_preempt_disable_intr_save (flags);
  1166. thread_runq_double_lock (runq, remote_runq);
  1167. nr_migrations = thread_sched_fs_balance_migrate (runq, remote_runq,
  1168. highest_round);
  1169. spinlock_unlock (&remote_runq->lock);
  1170. if (nr_migrations)
  1171. return;
  1172. spinlock_unlock_intr_restore (&runq->lock, *flags);
  1173. thread_preempt_enable ();
  1174. }
  1175. /*
  1176. * The scan or the migration failed. As a fallback, make another, simpler
  1177. * pass on every run queue, and stop as soon as at least one thread could
  1178. * be successfully pulled.
  1179. */
  1180. cpumap_for_each (&thread_active_runqs, i)
  1181. {
  1182. remote_runq = percpu_ptr (thread_runq, i);
  1183. if (remote_runq == runq)
  1184. continue;
  1185. thread_preempt_disable_intr_save (flags);
  1186. thread_runq_double_lock (runq, remote_runq);
  1187. nr_migrations = thread_sched_fs_balance_migrate (runq, remote_runq,
  1188. highest_round);
  1189. spinlock_unlock (&remote_runq->lock);
  1190. if (nr_migrations != 0)
  1191. return;
  1192. spinlock_unlock_intr_restore (&runq->lock, *flags);
  1193. thread_preempt_enable ();
  1194. }
  1195. thread_preempt_disable ();
  1196. spinlock_lock_intr_save (&runq->lock, flags);
  1197. no_migration:
  1198. /*
  1199. * No thread could be migrated. Check the active run queue, as another
  1200. * processor might have added threads while the balancer was running.
  1201. * If the run queue is still empty, switch to the next round. The run
  1202. * queue lock must remain held until the next scheduling decision to
  1203. * prevent a remote balancer thread from stealing active threads.
  1204. */
  1205. if (!runq->fs_runq_active->nr_threads)
  1206. thread_sched_fs_start_next_round (runq);
  1207. }
  1208. static struct thread_runq*
  1209. thread_sched_idle_select_runq (struct thread *thread __unused)
  1210. {
  1211. panic ("thread: idler threads cannot be awoken");
  1212. }
  1213. static noreturn void
  1214. thread_sched_idle_panic (void)
  1215. {
  1216. panic ("thread: only idle threads are allowed in the idle class");
  1217. }
  1218. static void
  1219. thread_sched_idle_add (struct thread_runq *runq __unused,
  1220. struct thread *thread __unused)
  1221. {
  1222. thread_sched_idle_panic ();
  1223. }
  1224. #define thread_sched_idle_remove thread_sched_idle_add
  1225. static struct thread*
  1226. thread_sched_idle_get_next (struct thread_runq *runq)
  1227. {
  1228. return (runq->idler);
  1229. }
  1230. static uint32_t
  1231. thread_sched_idle_get_global_priority (uint16_t priority __unused)
  1232. {
  1233. return (THREAD_SCHED_GLOBAL_PRIO_IDLE);
  1234. }
  1235. static const struct thread_sched_ops thread_sched_ops[THREAD_NR_SCHED_CLASSES] =
  1236. {
  1237. [THREAD_SCHED_CLASS_RT] =
  1238. {
  1239. .select_runq = thread_sched_rt_select_runq,
  1240. .add = thread_sched_rt_add,
  1241. .remove = thread_sched_rt_remove,
  1242. .put_prev = thread_sched_rt_put_prev,
  1243. .get_next = thread_sched_rt_get_next,
  1244. .reset_priority = thread_sched_rt_reset_priority,
  1245. .update_priority = NULL,
  1246. .get_global_priority = thread_sched_rt_get_global_priority,
  1247. .set_next = thread_sched_rt_set_next,
  1248. .tick = thread_sched_rt_tick,
  1249. },
  1250. [THREAD_SCHED_CLASS_FS] =
  1251. {
  1252. .select_runq = thread_sched_fs_select_runq,
  1253. .add = thread_sched_fs_add,
  1254. .remove = thread_sched_fs_remove,
  1255. .put_prev = thread_sched_fs_put_prev,
  1256. .get_next = thread_sched_fs_get_next,
  1257. .reset_priority = thread_sched_fs_reset_priority,
  1258. .update_priority = thread_sched_fs_update_priority,
  1259. .get_global_priority = thread_sched_fs_get_global_priority,
  1260. .set_next = thread_sched_fs_set_next,
  1261. .tick = thread_sched_fs_tick,
  1262. },
  1263. [THREAD_SCHED_CLASS_IDLE] =
  1264. {
  1265. .select_runq = thread_sched_idle_select_runq,
  1266. .add = thread_sched_idle_add,
  1267. .remove = thread_sched_idle_remove,
  1268. .put_prev = NULL,
  1269. .get_next = thread_sched_idle_get_next,
  1270. .reset_priority = NULL,
  1271. .update_priority = NULL,
  1272. .get_global_priority = thread_sched_idle_get_global_priority,
  1273. .set_next = NULL,
  1274. .tick = NULL,
  1275. },
  1276. };
  1277. static void
  1278. thread_set_user_sched_policy (struct thread *thread, uint8_t sched_policy)
  1279. {
  1280. thread->user_sched_data.sched_policy = sched_policy;
  1281. }
  1282. static void
  1283. thread_set_user_sched_class (struct thread *thread, uint8_t sched_class)
  1284. {
  1285. thread->user_sched_data.sched_class = sched_class;
  1286. }
  1287. static void
  1288. thread_set_user_priority (struct thread *thread, uint16_t prio)
  1289. {
  1290. const _Auto ops = thread_get_user_sched_ops (thread);
  1291. thread->user_sched_data.priority = prio;
  1292. thread->user_sched_data.global_priority = ops->get_global_priority (prio);
  1293. }
  1294. static void
  1295. thread_update_user_priority (struct thread *thread, uint16_t priority)
  1296. {
  1297. thread_set_user_priority (thread, priority);
  1298. }
  1299. static void
  1300. thread_set_real_sched_policy (struct thread *thread, uint8_t sched_policy)
  1301. {
  1302. thread->real_sched_data.sched_policy = sched_policy;
  1303. }
  1304. static void
  1305. thread_set_real_sched_class (struct thread *thread, uint8_t sched_class)
  1306. {
  1307. thread->real_sched_data.sched_class = sched_class;
  1308. }
  1309. static void
  1310. thread_set_real_priority (struct thread *thread, uint16_t prio)
  1311. {
  1312. const _Auto ops = thread_get_real_sched_ops (thread);
  1313. thread->real_sched_data.priority = prio;
  1314. thread->real_sched_data.global_priority = ops->get_global_priority (prio);
  1315. if (ops->reset_priority)
  1316. ops->reset_priority (thread, prio);
  1317. }
  1318. static void
  1319. thread_update_real_priority (struct thread *thread, uint16_t prio)
  1320. {
  1321. const _Auto ops = thread_get_real_sched_ops (thread);
  1322. thread->real_sched_data.priority = prio;
  1323. thread->real_sched_data.global_priority = ops->get_global_priority (prio);
  1324. if (ops->update_priority)
  1325. ops->update_priority (thread, prio);
  1326. }
  1327. static void
  1328. thread_reset_real_priority (struct thread *thread)
  1329. {
  1330. thread->real_sched_data = thread->user_sched_data;
  1331. thread->boosted = false;
  1332. const _Auto ops = thread_get_user_sched_ops (thread);
  1333. if (ops->reset_priority)
  1334. ops->reset_priority (thread, thread->real_sched_data.priority);
  1335. }
  1336. static void __init
  1337. thread_init_booter (uint32_t cpu)
  1338. {
  1339. // Initialize only what's needed during bootstrap.
  1340. struct thread *booter = &thread_booters[cpu];
  1341. booter->kuid.id = 0;
  1342. booter->kuid.nr_refs = 0; // Make sure booters aren't destroyed.
  1343. booter->flags = 0;
  1344. booter->intr_level = 0;
  1345. booter->preempt_level = 1;
  1346. booter->pagefault_level = 0;
  1347. rcu_reader_init (&booter->rcu_reader);
  1348. cpumap_fill (&booter->cpumap);
  1349. thread_set_user_sched_policy (booter, THREAD_SCHED_POLICY_IDLE);
  1350. thread_set_user_sched_class (booter, THREAD_SCHED_CLASS_IDLE);
  1351. thread_set_user_priority (booter, 0);
  1352. thread_reset_real_priority (booter);
  1353. booter->task = booter->xtask = task_get_kernel_task ();
  1354. snprintf (booter->name, sizeof (booter->name),
  1355. THREAD_KERNEL_PREFIX "thread_boot/%u", cpu);
  1356. }
  1357. static int __init
  1358. thread_setup_booter (void)
  1359. {
  1360. tcb_set_current (&thread_booters[0].tcb);
  1361. thread_init_booter (0);
  1362. return (0);
  1363. }
  1364. INIT_OP_DEFINE (thread_setup_booter,
  1365. INIT_OP_DEP (tcb_setup, true));
  1366. static int __init
  1367. thread_bootstrap (void)
  1368. {
  1369. cpumap_zero (&thread_active_runqs);
  1370. cpumap_zero (&thread_idle_runqs);
  1371. thread_fs_highest_round = THREAD_FS_INITIAL_ROUND;
  1372. cpumap_set (&thread_active_runqs, 0);
  1373. thread_runq_init (cpu_local_ptr (thread_runq), 0, &thread_booters[0]);
  1374. return (0);
  1375. }
  1376. INIT_OP_DEFINE (thread_bootstrap,
  1377. INIT_OP_DEP (syscnt_setup, true),
  1378. INIT_OP_DEP (thread_setup_booter, true));
  1379. void
  1380. thread_main (void (*fn) (void *), void *arg)
  1381. {
  1382. assert (!cpu_intr_enabled ());
  1383. assert (!thread_preempt_enabled ());
  1384. struct thread *thread = thread_self ();
  1385. thread_runq_schedule_load (thread);
  1386. spinlock_unlock (&thread_runq_local()->lock);
  1387. cpu_intr_enable ();
  1388. thread_preempt_enable ();
  1389. fn (arg);
  1390. thread_exit ();
  1391. }
  1392. static int
  1393. thread_init (struct thread *thread, void *stack,
  1394. const struct thread_attr *attr,
  1395. void (*fn) (void *), void *arg)
  1396. {
  1397. struct thread *caller = thread_self ();
  1398. struct task *task = attr->task ?: caller->task;
  1399. struct cpumap *cpumap = attr->cpumap ?: &caller->cpumap;
  1400. assert (attr->policy < ARRAY_SIZE (thread_policy_table));
  1401. kuid_head_init (&thread->kuid);
  1402. thread->flags = 0;
  1403. thread->runq = NULL;
  1404. thread->in_runq = false;
  1405. thread_set_wchan (thread, thread, "init");
  1406. thread->state = THREAD_SLEEPING;
  1407. thread->priv_sleepq = sleepq_create ();
  1408. int error;
  1409. if (!thread->priv_sleepq)
  1410. {
  1411. error = ENOMEM;
  1412. goto error_sleepq;
  1413. }
  1414. thread->priv_turnstile = turnstile_create ();
  1415. if (!thread->priv_turnstile)
  1416. {
  1417. error = ENOMEM;
  1418. goto error_turnstile;
  1419. }
  1420. turnstile_td_init (&thread->turnstile_td);
  1421. thread->propagate_priority = false;
  1422. thread->suspend = false;
  1423. thread->preempt_level = THREAD_SUSPEND_PREEMPT_LEVEL;
  1424. thread->pin_level = 0;
  1425. thread->intr_level = 0;
  1426. thread->pagefault_level = 0;
  1427. rcu_reader_init (&thread->rcu_reader);
  1428. cpumap_copy (&thread->cpumap, cpumap);
  1429. thread_set_user_sched_policy (thread, attr->policy);
  1430. thread_set_user_sched_class (thread, thread_policy_to_class (attr->policy));
  1431. thread_set_user_priority (thread, attr->priority);
  1432. thread_reset_real_priority (thread);
  1433. thread->join_waiter = NULL;
  1434. spinlock_init (&thread->join_lock);
  1435. thread->terminating = false;
  1436. thread->task = thread->xtask = task;
  1437. thread->stack = stack;
  1438. strlcpy (thread->name, attr->name, sizeof (thread->name));
  1439. thread->fixup = NULL;
  1440. thread->cur_port = NULL;
  1441. thread->futex_td = NULL;
  1442. bulletin_init (&thread->dead_subs);
  1443. for (size_t i = 0; i < THREAD_NR_PMAP_DATA; ++i)
  1444. pmap_ipc_pte_init (&thread->pmap_data[i]);
  1445. #ifdef CONFIG_PERFMON
  1446. perfmon_td_init (thread_get_perfmon_td (thread));
  1447. #endif
  1448. if (attr->flags & THREAD_ATTR_DETACHED)
  1449. thread->flags |= THREAD_DETACHED;
  1450. error = tcb_build (&thread->tcb, stack, fn, arg);
  1451. if (error)
  1452. goto error_tcb;
  1453. else if (thread->task != task_get_kernel_task ())
  1454. {
  1455. error = kuid_alloc (&thread->kuid, KUID_THREAD);
  1456. if (error)
  1457. goto error_kuid;
  1458. }
  1459. task_add_thread (task, thread);
  1460. return (0);
  1461. error_kuid:
  1462. tcb_cleanup (&thread->tcb);
  1463. error_tcb:
  1464. turnstile_destroy (thread->priv_turnstile);
  1465. error_turnstile:
  1466. sleepq_destroy (thread->priv_sleepq);
  1467. error_sleepq:
  1468. return (error);
  1469. }
  1470. #ifdef CONFIG_THREAD_STACK_GUARD
  1471. #include <machine/pmap.h>
  1472. #include <vm/kmem.h>
  1473. #include <vm/page.h>
  1474. static void*
  1475. thread_alloc_stack (void)
  1476. {
  1477. _Auto kernel_pmap = pmap_get_kernel_pmap ();
  1478. size_t stack_size = vm_page_round (TCB_STACK_SIZE);
  1479. void *mem = vm_kmem_alloc ((PAGE_SIZE * 2) + stack_size);
  1480. if (! mem)
  1481. return (NULL);
  1482. uintptr_t va = (uintptr_t)mem;
  1483. /*
  1484. * TODO Until memory protection is implemented, use the pmap system
  1485. * to remove mappings.
  1486. */
  1487. phys_addr_t first_pa, last_pa;
  1488. int error = pmap_kextract (va, &first_pa);
  1489. assert (! error);
  1490. error = pmap_kextract (va + PAGE_SIZE + stack_size, &last_pa);
  1491. assert (! error);
  1492. _Auto first_page = vm_page_lookup (first_pa);
  1493. assert (first_page);
  1494. _Auto last_page = vm_page_lookup (last_pa);
  1495. assert (last_page);
  1496. pmap_remove (kernel_pmap, va, PMAP_PEF_GLOBAL);
  1497. pmap_remove (kernel_pmap, va + PAGE_SIZE + stack_size,
  1498. PMAP_PEF_GLOBAL | PMAP_IGNORE_ERRORS);
  1499. pmap_update (kernel_pmap);
  1500. return ((char *)va + PAGE_SIZE);
  1501. }
  1502. static void
  1503. thread_free_stack (void *stack)
  1504. {
  1505. size_t stack_size = vm_page_round (TCB_STACK_SIZE);
  1506. void *va = (char *)stack - PAGE_SIZE;
  1507. vm_kmem_free (va, (PAGE_SIZE * 2) + stack_size);
  1508. }
  1509. #else // CONFIG_THREAD_STACK_GUARD
  1510. static void*
  1511. thread_alloc_stack (void)
  1512. {
  1513. return (kmem_cache_alloc (&thread_stack_cache));
  1514. }
  1515. static void
  1516. thread_free_stack (void *stack)
  1517. {
  1518. kmem_cache_free (&thread_stack_cache, stack);
  1519. }
  1520. #endif
  1521. static void
  1522. thread_destroy (struct thread *thread)
  1523. {
  1524. assert (thread != thread_self ());
  1525. assert (thread->state == THREAD_DEAD);
  1526. // See task_info().
  1527. task_remove_thread (thread->task, thread);
  1528. turnstile_destroy (thread->priv_turnstile);
  1529. sleepq_destroy (thread->priv_sleepq);
  1530. thread_free_stack (thread->stack);
  1531. tcb_cleanup (&thread->tcb);
  1532. kmem_cache_free (&thread_cache, thread);
  1533. }
  1534. static void
  1535. thread_join_common (struct thread *thread)
  1536. {
  1537. struct thread *self = thread_self ();
  1538. assert (thread != self);
  1539. spinlock_lock (&thread->join_lock);
  1540. assert (!thread->join_waiter);
  1541. thread->join_waiter = self;
  1542. while (!thread->terminating)
  1543. thread_sleep (&thread->join_lock, thread, "exit");
  1544. spinlock_unlock (&thread->join_lock);
  1545. uint32_t state;
  1546. do
  1547. {
  1548. struct thread_runq_guard g = thread_runq_guard_make (thread, false);
  1549. state = thread->state;
  1550. }
  1551. while (state != THREAD_DEAD);
  1552. thread_destroy (thread);
  1553. }
  1554. void
  1555. thread_terminate (struct thread *thread)
  1556. {
  1557. SPINLOCK_GUARD (&thread->join_lock);
  1558. thread->terminating = true;
  1559. cap_notify_dead (&thread->dead_subs);
  1560. kuid_remove (&thread->kuid, KUID_THREAD);
  1561. thread_wakeup (thread->join_waiter);
  1562. }
  1563. static void
  1564. thread_balance_idle_tick (struct thread_runq *runq)
  1565. {
  1566. assert (runq->idle_balance_ticks != 0);
  1567. /*
  1568. * Interrupts can occur early, at a time the balancer thread hasn't been
  1569. * created yet.
  1570. */
  1571. if (runq->balancer &&
  1572. --runq->idle_balance_ticks == 0)
  1573. thread_runq_wakeup_balancer (runq);
  1574. }
  1575. static void
  1576. thread_balance (void *arg)
  1577. {
  1578. struct thread_runq *runq = arg;
  1579. struct thread *self = runq->balancer;
  1580. assert (self == runq->balancer);
  1581. thread_preempt_disable ();
  1582. cpu_flags_t flags;
  1583. spinlock_lock_intr_save (&runq->lock, &flags);
  1584. while (1)
  1585. {
  1586. runq->idle_balance_ticks = THREAD_IDLE_BALANCE_TICKS;
  1587. thread_set_wchan (self, runq, "runq");
  1588. atomic_store_rlx (&self->state, THREAD_SLEEPING);
  1589. runq = thread_runq_schedule (runq);
  1590. assert (runq == arg);
  1591. /*
  1592. * This function may temporarily enable preemption and release the
  1593. * run queue lock, but on return, the lock must remain held until this
  1594. * balancer thread sleeps.
  1595. */
  1596. thread_sched_fs_balance (runq, &flags);
  1597. }
  1598. }
  1599. static void __init
  1600. thread_setup_balancer (struct thread_runq *runq)
  1601. {
  1602. struct cpumap *cpumap;
  1603. if (cpumap_create (&cpumap) != 0)
  1604. panic ("thread: unable to create balancer thread CPU map");
  1605. cpumap_zero (cpumap);
  1606. cpumap_set (cpumap, thread_runq_cpu (runq));
  1607. char name[THREAD_NAME_SIZE];
  1608. snprintf (name, sizeof (name), THREAD_KERNEL_PREFIX "thread_balance/%u",
  1609. thread_runq_cpu (runq));
  1610. struct thread_attr attr;
  1611. thread_attr_init (&attr, name);
  1612. thread_attr_set_cpumap (&attr, cpumap);
  1613. thread_attr_set_policy (&attr, THREAD_SCHED_POLICY_FIFO);
  1614. thread_attr_set_priority (&attr, THREAD_SCHED_RT_PRIO_MIN);
  1615. int error = thread_create (&runq->balancer, &attr, thread_balance, runq);
  1616. cpumap_destroy (cpumap);
  1617. if (error)
  1618. panic ("thread: unable to create balancer thread");
  1619. }
  1620. static void
  1621. thread_idle (void *arg __unused)
  1622. {
  1623. struct thread *self = thread_self ();
  1624. while (1)
  1625. {
  1626. thread_preempt_disable ();
  1627. while (1)
  1628. {
  1629. cpu_intr_disable ();
  1630. if (thread_test_flag (self, THREAD_YIELD))
  1631. {
  1632. cpu_intr_enable ();
  1633. break;
  1634. }
  1635. cpu_idle ();
  1636. }
  1637. thread_preempt_enable ();
  1638. }
  1639. }
  1640. static void __init
  1641. thread_setup_idler (struct thread_runq *runq)
  1642. {
  1643. struct cpumap *cpumap;
  1644. if (cpumap_create (&cpumap) != 0)
  1645. panic ("thread: unable to allocate idler thread CPU map");
  1646. cpumap_zero (cpumap);
  1647. cpumap_set (cpumap, thread_runq_cpu (runq));
  1648. struct thread *idler = kmem_cache_alloc (&thread_cache);
  1649. if (! idler)
  1650. panic ("thread: unable to allocate idler thread");
  1651. void *stack = thread_alloc_stack ();
  1652. if (! stack)
  1653. panic ("thread: unable to allocate idler thread stack");
  1654. char name[THREAD_NAME_SIZE];
  1655. snprintf (name, sizeof (name), THREAD_KERNEL_PREFIX "thread_idle/%u",
  1656. thread_runq_cpu (runq));
  1657. struct thread_attr attr;
  1658. thread_attr_init (&attr, name);
  1659. thread_attr_set_cpumap (&attr, cpumap);
  1660. thread_attr_set_policy (&attr, THREAD_SCHED_POLICY_IDLE);
  1661. if (thread_init (idler, stack, &attr, thread_idle, NULL) != 0)
  1662. panic ("thread: unable to initialize idler thread");
  1663. cpumap_destroy (cpumap);
  1664. // An idler thread needs special tuning.
  1665. thread_clear_wchan (idler);
  1666. idler->state = THREAD_RUNNING;
  1667. idler->runq = runq;
  1668. runq->idler = idler;
  1669. }
  1670. static void __init
  1671. thread_setup_runq (struct thread_runq *runq)
  1672. {
  1673. thread_setup_balancer (runq);
  1674. thread_setup_idler (runq);
  1675. }
  1676. #ifdef CONFIG_SHELL
  1677. /*
  1678. * This function is meant for debugging only. As a result, it uses a weak
  1679. * locking policy which allows tracing threads which state may mutate during
  1680. * tracing.
  1681. */
  1682. static void
  1683. thread_shell_trace (struct shell *shell, int argc, char **argv)
  1684. {
  1685. if (argc != 3)
  1686. {
  1687. stream_puts (shell->stream, "usage: thread_trace task thread\n");
  1688. return;
  1689. }
  1690. const char *task_name = argv[1], *thread_name = argv[2];
  1691. struct task *task = task_lookup (task_name);
  1692. if (! task)
  1693. {
  1694. fmt_xprintf (shell->stream, "thread_trace: task not found: %s\n",
  1695. task_name);
  1696. return;
  1697. }
  1698. struct thread *thread = task_lookup_thread (task, thread_name);
  1699. task_unref (task);
  1700. if (! thread)
  1701. {
  1702. fmt_xprintf (shell->stream, "thread_trace: thread not found: %s\n",
  1703. thread_name);
  1704. return;
  1705. }
  1706. cpu_flags_t flags;
  1707. _Auto runq = thread_lock_runq (thread, &flags);
  1708. if (thread == runq->current)
  1709. stream_puts (shell->stream, "thread_trace: thread is running\n");
  1710. else
  1711. tcb_trace (&thread->tcb);
  1712. thread_unlock_runq (runq, flags);
  1713. thread_unref (thread);
  1714. }
  1715. static struct shell_cmd thread_shell_cmds[] =
  1716. {
  1717. SHELL_CMD_INITIALIZER ("thread_trace", thread_shell_trace,
  1718. "thread_trace <task_name> <thread_name>",
  1719. "display the stack trace of a given thread"),
  1720. };
  1721. static int __init
  1722. thread_setup_shell (void)
  1723. {
  1724. SHELL_REGISTER_CMDS (thread_shell_cmds, shell_get_main_cmd_set ());
  1725. return (0);
  1726. }
  1727. INIT_OP_DEFINE (thread_setup_shell,
  1728. INIT_OP_DEP (printf_setup, true),
  1729. INIT_OP_DEP (shell_setup, true),
  1730. INIT_OP_DEP (task_setup, true),
  1731. INIT_OP_DEP (thread_setup, true));
  1732. #endif
  1733. static void __init
  1734. thread_setup_common (uint32_t cpu)
  1735. {
  1736. assert (cpu);
  1737. cpumap_set (&thread_active_runqs, cpu);
  1738. thread_init_booter (cpu);
  1739. thread_runq_init (percpu_ptr (thread_runq, cpu), cpu, &thread_booters[cpu]);
  1740. }
  1741. static int __init
  1742. thread_setup (void)
  1743. {
  1744. for (uint32_t cpu = 1; cpu < cpu_count (); ++cpu)
  1745. thread_setup_common (cpu);
  1746. kmem_cache_init (&thread_cache, "thread", sizeof (struct thread),
  1747. CPU_L1_SIZE, NULL, 0);
  1748. #ifndef CONFIG_THREAD_STACK_GUARD
  1749. kmem_cache_init (&thread_stack_cache, "thread_stack", TCB_STACK_SIZE,
  1750. CPU_DATA_ALIGN, NULL, 0);
  1751. #endif
  1752. cpumap_for_each (&thread_active_runqs, cpu)
  1753. thread_setup_runq (percpu_ptr (thread_runq, cpu));
  1754. return (0);
  1755. }
  1756. #ifdef CONFIG_THREAD_STACK_GUARD
  1757. #define THREAD_STACK_GUARD_INIT_OP_DEPS \
  1758. INIT_OP_DEP (vm_kmem_setup, true), \
  1759. INIT_OP_DEP (vm_map_setup, true), \
  1760. INIT_OP_DEP (vm_page_setup, true),
  1761. #else
  1762. #define THREAD_STACK_GUARD_INIT_OP_DEPS
  1763. #endif
  1764. #ifdef CONFIG_PERFMON
  1765. #define THREAD_PERFMON_INIT_OP_DEPS INIT_OP_DEP (perfmon_bootstrap, true),
  1766. #else
  1767. #define THREAD_PERFMON_INIT_OP_DEPS
  1768. #endif
  1769. INIT_OP_DEFINE (thread_setup,
  1770. INIT_OP_DEP (cpumap_setup, true),
  1771. INIT_OP_DEP (kmem_setup, true),
  1772. INIT_OP_DEP (pmap_setup, true),
  1773. INIT_OP_DEP (sleepq_setup, true),
  1774. INIT_OP_DEP (task_setup, true),
  1775. INIT_OP_DEP (thread_bootstrap, true),
  1776. INIT_OP_DEP (turnstile_setup, true),
  1777. THREAD_STACK_GUARD_INIT_OP_DEPS
  1778. THREAD_PERFMON_INIT_OP_DEPS);
  1779. void __init
  1780. thread_ap_setup (void)
  1781. {
  1782. tcb_set_current (&thread_booters[cpu_id ()].tcb);
  1783. }
  1784. int
  1785. thread_create (struct thread **threadp, const struct thread_attr *attr,
  1786. void (*fn) (void *), void *arg)
  1787. {
  1788. int error;
  1789. if (attr->cpumap)
  1790. {
  1791. error = cpumap_check (attr->cpumap);
  1792. if (error)
  1793. return (error);
  1794. }
  1795. struct thread *thread = kmem_cache_alloc (&thread_cache);
  1796. if (! thread)
  1797. {
  1798. error = ENOMEM;
  1799. goto error_thread;
  1800. }
  1801. void *stack = thread_alloc_stack ();
  1802. if (! stack)
  1803. {
  1804. error = ENOMEM;
  1805. goto error_stack;
  1806. }
  1807. error = thread_init (thread, stack, attr, fn, arg);
  1808. if (error)
  1809. goto error_init;
  1810. /*
  1811. * The new thread address must be written before the thread is started
  1812. * in case it's passed to it.
  1813. */
  1814. if (threadp)
  1815. *threadp = thread;
  1816. thread_wakeup (thread);
  1817. return (0);
  1818. error_init:
  1819. thread_free_stack (stack);
  1820. error_stack:
  1821. kmem_cache_free (&thread_cache, thread);
  1822. error_thread:
  1823. return (error);
  1824. }
  1825. static void
  1826. thread_reap (struct work *work)
  1827. {
  1828. _Auto zombie = structof (work, struct thread_zombie, work);
  1829. thread_join_common (zombie->thread);
  1830. }
  1831. void
  1832. thread_exit (void)
  1833. {
  1834. struct thread_zombie zombie;
  1835. struct thread *thread = thread_self ();
  1836. if (likely (thread->task != task_get_kernel_task ()))
  1837. turnstile_td_exit (&thread->turnstile_td);
  1838. futex_td_exit (thread->futex_td);
  1839. if (thread_test_flag (thread, THREAD_DETACHED))
  1840. {
  1841. zombie.thread = thread;
  1842. work_init (&zombie.work, thread_reap);
  1843. work_schedule (&zombie.work, 0);
  1844. }
  1845. /*
  1846. * Disable preemption before dropping the reference, as this may
  1847. * trigger the active state poll of the join operation. Doing so
  1848. * keeps the duration of that active wait minimum.
  1849. */
  1850. thread_preempt_disable ();
  1851. thread_unref (thread);
  1852. _Auto runq = thread_runq_local ();
  1853. cpu_flags_t flags;
  1854. spinlock_lock_intr_save (&runq->lock, &flags);
  1855. atomic_store_rlx (&thread->state, THREAD_DEAD);
  1856. thread_runq_schedule (runq);
  1857. panic ("thread: dead thread walking");
  1858. }
  1859. void
  1860. thread_join (struct thread *thread)
  1861. {
  1862. assert (!thread_test_flag (thread, THREAD_DETACHED));
  1863. thread_join_common (thread);
  1864. }
  1865. static int
  1866. thread_wakeup_common (struct thread *thread, int error, bool resume)
  1867. {
  1868. if (!thread || thread == thread_self ())
  1869. return (EINVAL);
  1870. /*
  1871. * There is at most one reference on threads that were never dispatched,
  1872. * in which case there is no need to lock anything.
  1873. */
  1874. struct thread_runq *runq;
  1875. cpu_flags_t flags;
  1876. if (!thread->runq)
  1877. {
  1878. assert (thread->state != THREAD_RUNNING);
  1879. thread_clear_wchan (thread);
  1880. thread->state = THREAD_RUNNING;
  1881. }
  1882. else
  1883. {
  1884. runq = thread_lock_runq (thread, &flags);
  1885. if (thread->state == THREAD_RUNNING ||
  1886. (thread->state == THREAD_SUSPENDED && !resume))
  1887. {
  1888. thread_unlock_runq (runq, flags);
  1889. return (EINVAL);
  1890. }
  1891. thread_clear_wchan (thread);
  1892. atomic_store_rlx (&thread->state, THREAD_RUNNING);
  1893. thread_unlock_runq (runq, flags);
  1894. }
  1895. thread_preempt_disable_intr_save (&flags);
  1896. if (!thread->pin_level)
  1897. runq = thread_get_real_sched_ops(thread)->select_runq (thread);
  1898. else
  1899. {
  1900. /*
  1901. * This access doesn't need to be atomic, as the current thread is
  1902. * the only one which may update the member.
  1903. */
  1904. runq = thread->runq;
  1905. spinlock_lock (&runq->lock);
  1906. }
  1907. thread->wakeup_error = error;
  1908. thread_runq_wakeup (runq, thread);
  1909. spinlock_unlock (&runq->lock);
  1910. thread_preempt_enable_intr_restore (flags);
  1911. return (0);
  1912. }
  1913. int
  1914. thread_wakeup (struct thread *thread)
  1915. {
  1916. return (thread_wakeup_common (thread, 0, false));
  1917. }
  1918. struct thread_timeout_waiter
  1919. {
  1920. struct thread *thread;
  1921. struct timer timer;
  1922. };
  1923. static void
  1924. thread_timeout (struct timer *timer)
  1925. {
  1926. _Auto waiter = structof (timer, struct thread_timeout_waiter, timer);
  1927. thread_wakeup_common (waiter->thread, ETIMEDOUT, false);
  1928. }
  1929. static int
  1930. thread_sleep_common (struct spinlock *interlock, const void *wchan_addr,
  1931. const char *wchan_desc, bool timed, uint64_t ticks)
  1932. {
  1933. struct thread *thread = thread_self ();
  1934. struct thread_timeout_waiter waiter;
  1935. if (timed)
  1936. {
  1937. waiter.thread = thread;
  1938. timer_init (&waiter.timer, thread_timeout, TIMER_INTR);
  1939. timer_schedule (&waiter.timer, ticks);
  1940. }
  1941. _Auto runq = thread_runq_local ();
  1942. cpu_flags_t flags;
  1943. spinlock_lock_intr_save (&runq->lock, &flags);
  1944. if (interlock)
  1945. {
  1946. thread_preempt_disable ();
  1947. spinlock_unlock (interlock);
  1948. }
  1949. thread_set_wchan (thread, wchan_addr, wchan_desc);
  1950. atomic_store_rlx (&thread->state, THREAD_SLEEPING);
  1951. runq = thread_runq_schedule (runq);
  1952. assert (thread->state == THREAD_RUNNING);
  1953. spinlock_unlock_intr_restore (&runq->lock, flags);
  1954. if (timed)
  1955. timer_cancel (&waiter.timer);
  1956. if (interlock)
  1957. {
  1958. spinlock_lock (interlock);
  1959. thread_preempt_enable_no_resched ();
  1960. }
  1961. return (thread->wakeup_error);
  1962. }
  1963. void
  1964. thread_sleep (struct spinlock *lock, const void *wchan_addr,
  1965. const char *wchan_desc)
  1966. {
  1967. int error = thread_sleep_common (lock, wchan_addr, wchan_desc, false, 0);
  1968. assert (! error);
  1969. }
  1970. int
  1971. thread_timedsleep (struct spinlock *lock, const void *wchan_addr,
  1972. const char *wchan_desc, uint64_t ticks)
  1973. {
  1974. return (thread_sleep_common (lock, wchan_addr, wchan_desc, true, ticks));
  1975. }
  1976. int
  1977. thread_suspend (struct thread *thread)
  1978. {
  1979. if (! thread)
  1980. return (EINVAL);
  1981. struct thread_runq_guard g = thread_runq_guard_make (thread, true);
  1982. if (thread == g.runq->idler ||
  1983. thread == g.runq->balancer ||
  1984. thread->state == THREAD_DEAD)
  1985. return (EINVAL);
  1986. else if (thread->state == THREAD_SUSPENDED || thread->suspend)
  1987. return (0);
  1988. else if (thread->state == THREAD_SLEEPING)
  1989. {
  1990. thread->state = THREAD_SUSPENDED;
  1991. return (0);
  1992. }
  1993. assert (thread->state == THREAD_RUNNING);
  1994. if (thread != g.runq->current)
  1995. {
  1996. thread->state = THREAD_SUSPENDED;
  1997. thread_runq_remove (g.runq, thread);
  1998. }
  1999. else
  2000. {
  2001. thread->suspend = true;
  2002. if (g.runq == thread_runq_local ())
  2003. g.runq = thread_runq_schedule (g.runq);
  2004. else
  2005. {
  2006. thread_set_flag (thread, THREAD_YIELD);
  2007. cpu_send_thread_schedule (thread_runq_cpu (g.runq));
  2008. }
  2009. }
  2010. return (0);
  2011. }
  2012. int
  2013. thread_resume (struct thread *thread)
  2014. {
  2015. return (thread_wakeup_common (thread, 0, true));
  2016. }
  2017. void
  2018. thread_delay (uint64_t ticks, bool absolute)
  2019. {
  2020. thread_preempt_disable ();
  2021. if (! absolute)
  2022. // Add a tick to avoid quantization errors.
  2023. ticks += clock_get_time () + 1;
  2024. thread_timedsleep (NULL, thread_self (), "delay", ticks);
  2025. thread_preempt_enable ();
  2026. }
  2027. static void __init
  2028. thread_boot_barrier_wait (void)
  2029. {
  2030. assert (!cpu_intr_enabled ());
  2031. atomic_add_rlx (&thread_nr_boot_cpus, 1);
  2032. while (atomic_load_seq (&thread_nr_boot_cpus) != cpu_count ())
  2033. cpu_pause ();
  2034. }
  2035. void __init
  2036. thread_run_scheduler (void)
  2037. {
  2038. assert (!cpu_intr_enabled ());
  2039. thread_boot_barrier_wait ();
  2040. _Auto runq = thread_runq_local ();
  2041. struct thread *thread = thread_self ();
  2042. assert (thread == runq->current);
  2043. assert (thread->preempt_level == THREAD_SUSPEND_PREEMPT_LEVEL - 1);
  2044. spinlock_lock (&runq->lock);
  2045. thread = thread_runq_get_next (thread_runq_local ());
  2046. spinlock_transfer_owner (&runq->lock, thread);
  2047. tcb_load (&thread->tcb);
  2048. }
  2049. void
  2050. thread_yield (void)
  2051. {
  2052. struct thread *thread = thread_self ();
  2053. if (!thread_preempt_enabled ())
  2054. return;
  2055. do
  2056. {
  2057. thread_preempt_disable ();
  2058. _Auto runq = thread_runq_local ();
  2059. cpu_flags_t flags;
  2060. spinlock_lock_intr_save (&runq->lock, &flags);
  2061. runq = thread_runq_schedule (runq);
  2062. spinlock_unlock_intr_restore (&runq->lock, flags);
  2063. thread_preempt_enable_no_resched ();
  2064. }
  2065. while (thread_test_flag (thread, THREAD_YIELD));
  2066. }
  2067. void
  2068. thread_schedule (void)
  2069. {
  2070. if (unlikely (thread_test_flag (thread_self (), THREAD_YIELD)))
  2071. thread_yield ();
  2072. }
  2073. void
  2074. thread_schedule_intr (void)
  2075. {
  2076. assert (thread_check_intr_context ());
  2077. syscnt_inc (&thread_runq_local()->sc_schedule_intrs);
  2078. }
  2079. void
  2080. thread_report_periodic_event (void)
  2081. {
  2082. assert (thread_check_intr_context ());
  2083. _Auto runq = thread_runq_local ();
  2084. struct thread *thread = thread_self ();
  2085. spinlock_lock (&runq->lock);
  2086. if (!runq->nr_threads)
  2087. thread_balance_idle_tick (runq);
  2088. const _Auto ops = thread_get_real_sched_ops (thread);
  2089. if (ops->tick)
  2090. ops->tick (runq, thread);
  2091. spinlock_unlock (&runq->lock);
  2092. }
  2093. char
  2094. thread_state_to_chr (uint32_t state)
  2095. {
  2096. switch (state)
  2097. {
  2098. case THREAD_RUNNING:
  2099. return ('R');
  2100. case THREAD_SLEEPING:
  2101. return ('S');
  2102. case THREAD_DEAD:
  2103. return ('Z');
  2104. case THREAD_SUSPENDED:
  2105. return ('T');
  2106. default:
  2107. panic ("thread: unknown state");
  2108. }
  2109. }
  2110. const char*
  2111. thread_sched_class_to_str (uint8_t sched_class)
  2112. {
  2113. switch (sched_class)
  2114. {
  2115. case THREAD_SCHED_CLASS_RT:
  2116. return ("rt");
  2117. case THREAD_SCHED_CLASS_FS:
  2118. return ("fs");
  2119. case THREAD_SCHED_CLASS_IDLE:
  2120. return ("idle");
  2121. default:
  2122. panic ("thread: unknown scheduling class");
  2123. }
  2124. }
  2125. static void
  2126. thread_setsched_impl (struct thread *thread, uint8_t policy,
  2127. uint16_t priority)
  2128. {
  2129. struct thread_runq_guard g = thread_runq_guard_make (thread, false);
  2130. if (thread_user_sched_policy (thread) == policy &&
  2131. thread_user_priority (thread) == priority)
  2132. return;
  2133. bool current, requeue = thread->in_runq;
  2134. if (! requeue)
  2135. current = false;
  2136. else
  2137. {
  2138. if (thread != g.runq->current)
  2139. current = false;
  2140. else
  2141. {
  2142. thread_runq_put_prev (g.runq, thread);
  2143. current = true;
  2144. }
  2145. thread_runq_remove (g.runq, thread);
  2146. }
  2147. bool update = true;
  2148. if (thread_user_sched_policy (thread) == policy)
  2149. thread_update_user_priority (thread, priority);
  2150. else
  2151. {
  2152. thread_set_user_sched_policy (thread, policy);
  2153. thread_set_user_sched_class (thread, thread_policy_to_class (policy));
  2154. thread_set_user_priority (thread, priority);
  2155. update = false;
  2156. }
  2157. if (thread->boosted)
  2158. {
  2159. if (thread_user_global_priority (thread) >=
  2160. thread_real_global_priority (thread))
  2161. thread_reset_real_priority (thread);
  2162. }
  2163. else if (update)
  2164. thread_update_real_priority (thread, priority);
  2165. else
  2166. {
  2167. thread_set_real_sched_policy (thread, policy);
  2168. thread_set_real_sched_class (thread, thread_policy_to_class (policy));
  2169. thread_set_real_priority (thread, priority);
  2170. }
  2171. if (requeue)
  2172. {
  2173. thread_runq_add (g.runq, thread);
  2174. if (current)
  2175. thread_runq_set_next (g.runq, thread);
  2176. }
  2177. }
  2178. void
  2179. thread_setscheduler (struct thread *thread, uint8_t policy, uint16_t prio)
  2180. {
  2181. _Auto td = thread_turnstile_td (thread);
  2182. turnstile_td_lock (td);
  2183. thread_setsched_impl (thread, policy, prio);
  2184. turnstile_td_unlock (td);
  2185. turnstile_td_propagate_priority (td);
  2186. }
  2187. static void
  2188. thread_pi_setsched_impl (struct thread_runq *runq, struct thread *thread,
  2189. uint8_t policy, uint16_t prio)
  2190. {
  2191. assert (turnstile_td_locked (thread_turnstile_td (thread)));
  2192. if (thread_real_sched_policy (thread) == policy &&
  2193. thread_real_priority (thread) == prio)
  2194. return;
  2195. const _Auto ops = thread_get_sched_ops (thread_policy_to_class (policy));
  2196. uint32_t global_prio = ops->get_global_priority (prio);
  2197. bool current, requeue = thread->in_runq;
  2198. if (! requeue)
  2199. current = false;
  2200. else
  2201. {
  2202. if (thread != runq->current)
  2203. current = false;
  2204. else
  2205. {
  2206. thread_runq_put_prev (runq, thread);
  2207. current = true;
  2208. }
  2209. thread_runq_remove (runq, thread);
  2210. }
  2211. if (global_prio <= thread_user_global_priority (thread))
  2212. thread_reset_real_priority (thread);
  2213. else
  2214. {
  2215. if (thread_real_sched_policy (thread) == policy)
  2216. thread_update_real_priority (thread, prio);
  2217. else
  2218. {
  2219. thread_set_real_sched_policy (thread, policy);
  2220. thread_set_real_sched_class (thread,
  2221. thread_policy_to_class (policy));
  2222. thread_set_real_priority (thread, prio);
  2223. }
  2224. thread->boosted = true;
  2225. syscnt_inc (&runq->sc_boosts);
  2226. }
  2227. if (requeue)
  2228. {
  2229. thread_runq_add (runq, thread);
  2230. if (current)
  2231. thread_runq_set_next (runq, thread);
  2232. }
  2233. }
  2234. void
  2235. thread_pi_setscheduler (struct thread *thread, uint8_t policy, uint16_t prio)
  2236. {
  2237. struct thread_runq_guard g = thread_runq_guard_make (thread, false);
  2238. thread_pi_setsched_impl (g.runq, thread, policy, prio);
  2239. }
  2240. void
  2241. thread_propagate_priority (void)
  2242. {
  2243. /*
  2244. * Although it's possible to propagate priority with preemption
  2245. * disabled, the operation can be too expensive to allow it.
  2246. */
  2247. if (!thread_preempt_enabled ())
  2248. {
  2249. thread_set_priority_propagation_needed ();
  2250. return;
  2251. }
  2252. struct thread *thread = thread_self ();
  2253. // Clear before propagation to avoid infinite recursion.
  2254. thread->propagate_priority = false;
  2255. turnstile_td_propagate_priority (thread_turnstile_td (thread));
  2256. }
  2257. uint32_t
  2258. thread_cpu (const struct thread *thread)
  2259. {
  2260. const _Auto runq = atomic_load_rlx (&thread->runq);
  2261. return (runq->cpu);
  2262. }
  2263. uint32_t
  2264. thread_state (const struct thread *thread)
  2265. {
  2266. return (atomic_load_rlx (&thread->state));
  2267. }
  2268. bool
  2269. thread_is_running (const struct thread *thread)
  2270. {
  2271. const _Auto runq = atomic_load_rlx (&thread->runq);
  2272. return (runq && atomic_load_rlx (&runq->current) == thread);
  2273. }
  2274. int
  2275. thread_get_affinity (const struct thread *thr, struct cpumap *cpumap)
  2276. {
  2277. if (! thr)
  2278. return (EINVAL);
  2279. struct thread_runq_guard g =
  2280. thread_runq_guard_make ((struct thread *)thr, true);
  2281. cpumap_copy (cpumap, &thr->cpumap);
  2282. return (0);
  2283. }
  2284. int
  2285. thread_set_affinity (struct thread *thread, const struct cpumap *cpumap)
  2286. {
  2287. if (! thread)
  2288. return (EINVAL);
  2289. struct thread_runq_guard g = thread_runq_guard_make (thread, true);
  2290. if (thread == g.runq->idler ||
  2291. thread == g.runq->balancer ||
  2292. thread->state == THREAD_DEAD)
  2293. return (EINVAL);
  2294. else if (cpumap_intersects (&thread->cpumap, cpumap))
  2295. { // The desired CPU map intersects the current one.
  2296. cpumap_copy (&thread->cpumap, cpumap);
  2297. return (0);
  2298. }
  2299. else if (thread->pin_level != 0)
  2300. // The thread is pinned, and cannot be migrated to a different CPU.
  2301. return (EAGAIN);
  2302. // At this point, we know the thread must be migrated.
  2303. cpumap_copy (&thread->cpumap, cpumap);
  2304. if (thread == g.runq->current)
  2305. {
  2306. if (g.runq == thread_runq_local ())
  2307. g.runq = thread_runq_schedule (g.runq);
  2308. else
  2309. {
  2310. thread_set_flag (thread, THREAD_YIELD);
  2311. cpu_send_thread_schedule (thread_runq_cpu (g.runq));
  2312. }
  2313. }
  2314. return (0);
  2315. }
  2316. static ssize_t
  2317. thread_name_impl (struct thread *thread, char *name, bool set)
  2318. {
  2319. SPINLOCK_GUARD (&thread->task->lock);
  2320. if (set)
  2321. memcpy (thread->name, name, sizeof (thread->name));
  2322. else
  2323. memcpy (name, thread->name, sizeof (thread->name));
  2324. return (0);
  2325. }
  2326. static ssize_t
  2327. thread_ipc_affinity_impl (struct thread *thread, void *map,
  2328. uint32_t size, bool set)
  2329. {
  2330. if (! size)
  2331. return (-EINVAL);
  2332. struct cpumap *cpumap;
  2333. if (cpumap_create (&cpumap) != 0)
  2334. return (-ENOMEM);
  2335. cpumap_zero (cpumap);
  2336. size = MIN (size, sizeof (cpumap->cpus));
  2337. int error = user_copy_from (cpumap->cpus, map, size);
  2338. if (error)
  2339. ;
  2340. else if (set)
  2341. error = thread_set_affinity (thread, cpumap);
  2342. else if ((error = thread_get_affinity (thread, cpumap)) == 0)
  2343. error = user_copy_to (map, cpumap->cpus, size);
  2344. cpumap_destroy (cpumap);
  2345. return (-error);
  2346. }
  2347. #define THREAD_IPC_NEEDS_COPY \
  2348. ((1u << THREAD_IPC_GET_NAME) | (1u << THREAD_IPC_GET_AFFINITY) | \
  2349. (1u << THREAD_IPC_GET_ID))
  2350. ssize_t
  2351. thread_handle_msg (struct thread *thr, struct cap_iters *src,
  2352. struct cap_iters *dst, struct ipc_msg_data *data)
  2353. {
  2354. struct thread_ipc_msg tmsg;
  2355. struct ipc_iov_iter k_it;
  2356. ipc_iov_iter_init_buf (&k_it, &tmsg, sizeof (tmsg));
  2357. ssize_t rv = user_copyv_from (&k_it, &src->iov);
  2358. if (rv < 0)
  2359. return (rv);
  2360. switch (tmsg.op)
  2361. {
  2362. case THREAD_IPC_GET_NAME:
  2363. case THREAD_IPC_SET_NAME:
  2364. rv = thread_name_impl (thr, tmsg.name,
  2365. tmsg.op == THREAD_IPC_SET_NAME);
  2366. break;
  2367. case THREAD_IPC_GET_AFFINITY:
  2368. case THREAD_IPC_SET_AFFINITY:
  2369. rv = thread_ipc_affinity_impl (thr, tmsg.cpumap.map, tmsg.cpumap.size,
  2370. tmsg.op == THREAD_IPC_SET_AFFINITY);
  2371. break;
  2372. case THREAD_IPC_GET_ID:
  2373. tmsg.id = thread_id (thr);
  2374. break;
  2375. default:
  2376. return (-EINVAL);
  2377. }
  2378. if (rv == 0 && ((1u << tmsg.op) & THREAD_IPC_NEEDS_COPY))
  2379. {
  2380. ipc_iov_iter_init_buf (&k_it, &tmsg, sizeof (tmsg));
  2381. rv = user_copyv_to (&dst->iov, &k_it);
  2382. }
  2383. (void)data;
  2384. return (rv < 0 ? rv : 0);
  2385. }