profile.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564
  1. /*
  2. * linux/kernel/profile.c
  3. * Simple profiling. Manages a direct-mapped profile hit count buffer,
  4. * with configurable resolution, support for restricting the cpus on
  5. * which profiling is done, and switching between cpu time and
  6. * schedule() calls via kernel command line parameters passed at boot.
  7. *
  8. * Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
  9. * Red Hat, July 2004
  10. * Consolidation of architecture support code for profiling,
  11. * Nadia Yvette Chambers, Oracle, July 2004
  12. * Amortized hit count accounting via per-cpu open-addressed hashtables
  13. * to resolve timer interrupt livelocks, Nadia Yvette Chambers,
  14. * Oracle, 2004
  15. */
  16. #include <linux/export.h>
  17. #include <linux/profile.h>
  18. #include <linux/bootmem.h>
  19. #include <linux/notifier.h>
  20. #include <linux/mm.h>
  21. #include <linux/cpumask.h>
  22. #include <linux/cpu.h>
  23. #include <linux/highmem.h>
  24. #include <linux/mutex.h>
  25. #include <linux/slab.h>
  26. #include <linux/vmalloc.h>
  27. #include <asm/sections.h>
  28. #include <asm/irq_regs.h>
  29. #include <asm/ptrace.h>
  30. struct profile_hit {
  31. u32 pc, hits;
  32. };
  33. #define PROFILE_GRPSHIFT 3
  34. #define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
  35. #define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
  36. #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
  37. static atomic_t *prof_buffer;
  38. static unsigned long prof_len, prof_shift;
  39. int prof_on __read_mostly;
  40. EXPORT_SYMBOL_GPL(prof_on);
  41. static cpumask_var_t prof_cpu_mask;
  42. #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
  43. static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
  44. static DEFINE_PER_CPU(int, cpu_profile_flip);
  45. static DEFINE_MUTEX(profile_flip_mutex);
  46. #endif /* CONFIG_SMP */
  47. int profile_setup(char *str)
  48. {
  49. static const char schedstr[] = "schedule";
  50. static const char sleepstr[] = "sleep";
  51. static const char kvmstr[] = "kvm";
  52. int par;
  53. if (!strncmp(str, sleepstr, strlen(sleepstr))) {
  54. #ifdef CONFIG_SCHEDSTATS
  55. force_schedstat_enabled();
  56. prof_on = SLEEP_PROFILING;
  57. if (str[strlen(sleepstr)] == ',')
  58. str += strlen(sleepstr) + 1;
  59. if (get_option(&str, &par))
  60. prof_shift = par;
  61. pr_info("kernel sleep profiling enabled (shift: %ld)\n",
  62. prof_shift);
  63. #else
  64. pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
  65. #endif /* CONFIG_SCHEDSTATS */
  66. } else if (!strncmp(str, schedstr, strlen(schedstr))) {
  67. prof_on = SCHED_PROFILING;
  68. if (str[strlen(schedstr)] == ',')
  69. str += strlen(schedstr) + 1;
  70. if (get_option(&str, &par))
  71. prof_shift = par;
  72. pr_info("kernel schedule profiling enabled (shift: %ld)\n",
  73. prof_shift);
  74. } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
  75. prof_on = KVM_PROFILING;
  76. if (str[strlen(kvmstr)] == ',')
  77. str += strlen(kvmstr) + 1;
  78. if (get_option(&str, &par))
  79. prof_shift = par;
  80. pr_info("kernel KVM profiling enabled (shift: %ld)\n",
  81. prof_shift);
  82. } else if (get_option(&str, &par)) {
  83. prof_shift = par;
  84. prof_on = CPU_PROFILING;
  85. pr_info("kernel profiling enabled (shift: %ld)\n",
  86. prof_shift);
  87. }
  88. return 1;
  89. }
  90. __setup("profile=", profile_setup);
  91. int __ref profile_init(void)
  92. {
  93. int buffer_bytes;
  94. if (!prof_on)
  95. return 0;
  96. /* only text is profiled */
  97. prof_len = (_etext - _stext) >> prof_shift;
  98. buffer_bytes = prof_len*sizeof(atomic_t);
  99. if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
  100. return -ENOMEM;
  101. cpumask_copy(prof_cpu_mask, cpu_possible_mask);
  102. prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
  103. if (prof_buffer)
  104. return 0;
  105. prof_buffer = alloc_pages_exact(buffer_bytes,
  106. GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
  107. if (prof_buffer)
  108. return 0;
  109. prof_buffer = vzalloc(buffer_bytes);
  110. if (prof_buffer)
  111. return 0;
  112. free_cpumask_var(prof_cpu_mask);
  113. return -ENOMEM;
  114. }
  115. /* Profile event notifications */
  116. static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
  117. static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
  118. static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
  119. void profile_task_exit(struct task_struct *task)
  120. {
  121. blocking_notifier_call_chain(&task_exit_notifier, 0, task);
  122. }
  123. int profile_handoff_task(struct task_struct *task)
  124. {
  125. int ret;
  126. ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
  127. return (ret == NOTIFY_OK) ? 1 : 0;
  128. }
  129. void profile_munmap(unsigned long addr)
  130. {
  131. blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
  132. }
  133. int task_handoff_register(struct notifier_block *n)
  134. {
  135. return atomic_notifier_chain_register(&task_free_notifier, n);
  136. }
  137. EXPORT_SYMBOL_GPL(task_handoff_register);
  138. int task_handoff_unregister(struct notifier_block *n)
  139. {
  140. return atomic_notifier_chain_unregister(&task_free_notifier, n);
  141. }
  142. EXPORT_SYMBOL_GPL(task_handoff_unregister);
  143. int profile_event_register(enum profile_type type, struct notifier_block *n)
  144. {
  145. int err = -EINVAL;
  146. switch (type) {
  147. case PROFILE_TASK_EXIT:
  148. err = blocking_notifier_chain_register(
  149. &task_exit_notifier, n);
  150. break;
  151. case PROFILE_MUNMAP:
  152. err = blocking_notifier_chain_register(
  153. &munmap_notifier, n);
  154. break;
  155. }
  156. return err;
  157. }
  158. EXPORT_SYMBOL_GPL(profile_event_register);
  159. int profile_event_unregister(enum profile_type type, struct notifier_block *n)
  160. {
  161. int err = -EINVAL;
  162. switch (type) {
  163. case PROFILE_TASK_EXIT:
  164. err = blocking_notifier_chain_unregister(
  165. &task_exit_notifier, n);
  166. break;
  167. case PROFILE_MUNMAP:
  168. err = blocking_notifier_chain_unregister(
  169. &munmap_notifier, n);
  170. break;
  171. }
  172. return err;
  173. }
  174. EXPORT_SYMBOL_GPL(profile_event_unregister);
  175. #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
  176. /*
  177. * Each cpu has a pair of open-addressed hashtables for pending
  178. * profile hits. read_profile() IPI's all cpus to request them
  179. * to flip buffers and flushes their contents to prof_buffer itself.
  180. * Flip requests are serialized by the profile_flip_mutex. The sole
  181. * use of having a second hashtable is for avoiding cacheline
  182. * contention that would otherwise happen during flushes of pending
  183. * profile hits required for the accuracy of reported profile hits
  184. * and so resurrect the interrupt livelock issue.
  185. *
  186. * The open-addressed hashtables are indexed by profile buffer slot
  187. * and hold the number of pending hits to that profile buffer slot on
  188. * a cpu in an entry. When the hashtable overflows, all pending hits
  189. * are accounted to their corresponding profile buffer slots with
  190. * atomic_add() and the hashtable emptied. As numerous pending hits
  191. * may be accounted to a profile buffer slot in a hashtable entry,
  192. * this amortizes a number of atomic profile buffer increments likely
  193. * to be far larger than the number of entries in the hashtable,
  194. * particularly given that the number of distinct profile buffer
  195. * positions to which hits are accounted during short intervals (e.g.
  196. * several seconds) is usually very small. Exclusion from buffer
  197. * flipping is provided by interrupt disablement (note that for
  198. * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
  199. * process context).
  200. * The hash function is meant to be lightweight as opposed to strong,
  201. * and was vaguely inspired by ppc64 firmware-supported inverted
  202. * pagetable hash functions, but uses a full hashtable full of finite
  203. * collision chains, not just pairs of them.
  204. *
  205. * -- nyc
  206. */
  207. static void __profile_flip_buffers(void *unused)
  208. {
  209. int cpu = smp_processor_id();
  210. per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
  211. }
  212. static void profile_flip_buffers(void)
  213. {
  214. int i, j, cpu;
  215. mutex_lock(&profile_flip_mutex);
  216. j = per_cpu(cpu_profile_flip, get_cpu());
  217. put_cpu();
  218. on_each_cpu(__profile_flip_buffers, NULL, 1);
  219. for_each_online_cpu(cpu) {
  220. struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
  221. for (i = 0; i < NR_PROFILE_HIT; ++i) {
  222. if (!hits[i].hits) {
  223. if (hits[i].pc)
  224. hits[i].pc = 0;
  225. continue;
  226. }
  227. atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
  228. hits[i].hits = hits[i].pc = 0;
  229. }
  230. }
  231. mutex_unlock(&profile_flip_mutex);
  232. }
  233. static void profile_discard_flip_buffers(void)
  234. {
  235. int i, cpu;
  236. mutex_lock(&profile_flip_mutex);
  237. i = per_cpu(cpu_profile_flip, get_cpu());
  238. put_cpu();
  239. on_each_cpu(__profile_flip_buffers, NULL, 1);
  240. for_each_online_cpu(cpu) {
  241. struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
  242. memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
  243. }
  244. mutex_unlock(&profile_flip_mutex);
  245. }
  246. static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
  247. {
  248. unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
  249. int i, j, cpu;
  250. struct profile_hit *hits;
  251. pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
  252. i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
  253. secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
  254. cpu = get_cpu();
  255. hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
  256. if (!hits) {
  257. put_cpu();
  258. return;
  259. }
  260. /*
  261. * We buffer the global profiler buffer into a per-CPU
  262. * queue and thus reduce the number of global (and possibly
  263. * NUMA-alien) accesses. The write-queue is self-coalescing:
  264. */
  265. local_irq_save(flags);
  266. do {
  267. for (j = 0; j < PROFILE_GRPSZ; ++j) {
  268. if (hits[i + j].pc == pc) {
  269. hits[i + j].hits += nr_hits;
  270. goto out;
  271. } else if (!hits[i + j].hits) {
  272. hits[i + j].pc = pc;
  273. hits[i + j].hits = nr_hits;
  274. goto out;
  275. }
  276. }
  277. i = (i + secondary) & (NR_PROFILE_HIT - 1);
  278. } while (i != primary);
  279. /*
  280. * Add the current hit(s) and flush the write-queue out
  281. * to the global buffer:
  282. */
  283. atomic_add(nr_hits, &prof_buffer[pc]);
  284. for (i = 0; i < NR_PROFILE_HIT; ++i) {
  285. atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
  286. hits[i].pc = hits[i].hits = 0;
  287. }
  288. out:
  289. local_irq_restore(flags);
  290. put_cpu();
  291. }
  292. static int profile_dead_cpu(unsigned int cpu)
  293. {
  294. struct page *page;
  295. int i;
  296. if (prof_cpu_mask != NULL)
  297. cpumask_clear_cpu(cpu, prof_cpu_mask);
  298. for (i = 0; i < 2; i++) {
  299. if (per_cpu(cpu_profile_hits, cpu)[i]) {
  300. page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
  301. per_cpu(cpu_profile_hits, cpu)[i] = NULL;
  302. __free_page(page);
  303. }
  304. }
  305. return 0;
  306. }
  307. static int profile_prepare_cpu(unsigned int cpu)
  308. {
  309. int i, node = cpu_to_mem(cpu);
  310. struct page *page;
  311. per_cpu(cpu_profile_flip, cpu) = 0;
  312. for (i = 0; i < 2; i++) {
  313. if (per_cpu(cpu_profile_hits, cpu)[i])
  314. continue;
  315. page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
  316. if (!page) {
  317. profile_dead_cpu(cpu);
  318. return -ENOMEM;
  319. }
  320. per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
  321. }
  322. return 0;
  323. }
  324. static int profile_online_cpu(unsigned int cpu)
  325. {
  326. if (prof_cpu_mask != NULL)
  327. cpumask_set_cpu(cpu, prof_cpu_mask);
  328. return 0;
  329. }
  330. #else /* !CONFIG_SMP */
  331. #define profile_flip_buffers() do { } while (0)
  332. #define profile_discard_flip_buffers() do { } while (0)
  333. static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
  334. {
  335. unsigned long pc;
  336. pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
  337. atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
  338. }
  339. #endif /* !CONFIG_SMP */
  340. void profile_hits(int type, void *__pc, unsigned int nr_hits)
  341. {
  342. if (prof_on != type || !prof_buffer)
  343. return;
  344. do_profile_hits(type, __pc, nr_hits);
  345. }
  346. EXPORT_SYMBOL_GPL(profile_hits);
  347. void profile_tick(int type)
  348. {
  349. struct pt_regs *regs = get_irq_regs();
  350. if (!user_mode(regs) && prof_cpu_mask != NULL &&
  351. cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
  352. profile_hit(type, (void *)profile_pc(regs));
  353. }
  354. #ifdef CONFIG_PROC_FS
  355. #include <linux/proc_fs.h>
  356. #include <linux/seq_file.h>
  357. #include <asm/uaccess.h>
  358. static int prof_cpu_mask_proc_show(struct seq_file *m, void *v)
  359. {
  360. seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
  361. return 0;
  362. }
  363. static int prof_cpu_mask_proc_open(struct inode *inode, struct file *file)
  364. {
  365. return single_open(file, prof_cpu_mask_proc_show, NULL);
  366. }
  367. static ssize_t prof_cpu_mask_proc_write(struct file *file,
  368. const char __user *buffer, size_t count, loff_t *pos)
  369. {
  370. cpumask_var_t new_value;
  371. int err;
  372. if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
  373. return -ENOMEM;
  374. err = cpumask_parse_user(buffer, count, new_value);
  375. if (!err) {
  376. cpumask_copy(prof_cpu_mask, new_value);
  377. err = count;
  378. }
  379. free_cpumask_var(new_value);
  380. return err;
  381. }
  382. static const struct file_operations prof_cpu_mask_proc_fops = {
  383. .open = prof_cpu_mask_proc_open,
  384. .read = seq_read,
  385. .llseek = seq_lseek,
  386. .release = single_release,
  387. .write = prof_cpu_mask_proc_write,
  388. };
  389. void create_prof_cpu_mask(void)
  390. {
  391. /* create /proc/irq/prof_cpu_mask */
  392. proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_fops);
  393. }
  394. /*
  395. * This function accesses profiling information. The returned data is
  396. * binary: the sampling step and the actual contents of the profile
  397. * buffer. Use of the program readprofile is recommended in order to
  398. * get meaningful info out of these data.
  399. */
  400. static ssize_t
  401. read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  402. {
  403. unsigned long p = *ppos;
  404. ssize_t read;
  405. char *pnt;
  406. unsigned int sample_step = 1 << prof_shift;
  407. profile_flip_buffers();
  408. if (p >= (prof_len+1)*sizeof(unsigned int))
  409. return 0;
  410. if (count > (prof_len+1)*sizeof(unsigned int) - p)
  411. count = (prof_len+1)*sizeof(unsigned int) - p;
  412. read = 0;
  413. while (p < sizeof(unsigned int) && count > 0) {
  414. if (put_user(*((char *)(&sample_step)+p), buf))
  415. return -EFAULT;
  416. buf++; p++; count--; read++;
  417. }
  418. pnt = (char *)prof_buffer + p - sizeof(atomic_t);
  419. if (copy_to_user(buf, (void *)pnt, count))
  420. return -EFAULT;
  421. read += count;
  422. *ppos += read;
  423. return read;
  424. }
  425. /*
  426. * Writing to /proc/profile resets the counters
  427. *
  428. * Writing a 'profiling multiplier' value into it also re-sets the profiling
  429. * interrupt frequency, on architectures that support this.
  430. */
  431. static ssize_t write_profile(struct file *file, const char __user *buf,
  432. size_t count, loff_t *ppos)
  433. {
  434. #ifdef CONFIG_SMP
  435. extern int setup_profiling_timer(unsigned int multiplier);
  436. if (count == sizeof(int)) {
  437. unsigned int multiplier;
  438. if (copy_from_user(&multiplier, buf, sizeof(int)))
  439. return -EFAULT;
  440. if (setup_profiling_timer(multiplier))
  441. return -EINVAL;
  442. }
  443. #endif
  444. profile_discard_flip_buffers();
  445. memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
  446. return count;
  447. }
  448. static const struct file_operations proc_profile_operations = {
  449. .read = read_profile,
  450. .write = write_profile,
  451. .llseek = default_llseek,
  452. };
  453. int __ref create_proc_profile(void)
  454. {
  455. struct proc_dir_entry *entry;
  456. #ifdef CONFIG_SMP
  457. enum cpuhp_state online_state;
  458. #endif
  459. int err = 0;
  460. if (!prof_on)
  461. return 0;
  462. #ifdef CONFIG_SMP
  463. err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
  464. profile_prepare_cpu, profile_dead_cpu);
  465. if (err)
  466. return err;
  467. err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
  468. profile_online_cpu, NULL);
  469. if (err < 0)
  470. goto err_state_prep;
  471. online_state = err;
  472. err = 0;
  473. #endif
  474. entry = proc_create("profile", S_IWUSR | S_IRUGO,
  475. NULL, &proc_profile_operations);
  476. if (!entry)
  477. goto err_state_onl;
  478. proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
  479. return err;
  480. err_state_onl:
  481. #ifdef CONFIG_SMP
  482. cpuhp_remove_state(online_state);
  483. err_state_prep:
  484. cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
  485. #endif
  486. return err;
  487. }
  488. subsys_initcall(create_proc_profile);
  489. #endif /* CONFIG_PROC_FS */