blk-softirq.c 4.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180
  1. /*
  2. * Functions related to softirq rq completions
  3. */
  4. #include <linux/kernel.h>
  5. #include <linux/module.h>
  6. #include <linux/init.h>
  7. #include <linux/bio.h>
  8. #include <linux/blkdev.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/cpu.h>
  11. #include <linux/sched.h>
  12. #include "blk.h"
  13. static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
  14. /*
  15. * Softirq action handler - move entries to local list and loop over them
  16. * while passing them to the queue registered handler.
  17. */
  18. static __latent_entropy void blk_done_softirq(struct softirq_action *h)
  19. {
  20. struct list_head *cpu_list, local_list;
  21. local_irq_disable();
  22. cpu_list = this_cpu_ptr(&blk_cpu_done);
  23. list_replace_init(cpu_list, &local_list);
  24. local_irq_enable();
  25. while (!list_empty(&local_list)) {
  26. struct request *rq;
  27. rq = list_entry(local_list.next, struct request, ipi_list);
  28. list_del_init(&rq->ipi_list);
  29. rq->q->softirq_done_fn(rq);
  30. }
  31. }
  32. #ifdef CONFIG_SMP
  33. static void trigger_softirq(void *data)
  34. {
  35. struct request *rq = data;
  36. unsigned long flags;
  37. struct list_head *list;
  38. local_irq_save(flags);
  39. list = this_cpu_ptr(&blk_cpu_done);
  40. list_add_tail(&rq->ipi_list, list);
  41. if (list->next == &rq->ipi_list)
  42. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  43. local_irq_restore(flags);
  44. }
  45. /*
  46. * Setup and invoke a run of 'trigger_softirq' on the given cpu.
  47. */
  48. static int raise_blk_irq(int cpu, struct request *rq)
  49. {
  50. if (cpu_online(cpu)) {
  51. struct call_single_data *data = &rq->csd;
  52. data->func = trigger_softirq;
  53. data->info = rq;
  54. data->flags = 0;
  55. smp_call_function_single_async(cpu, data);
  56. return 0;
  57. }
  58. return 1;
  59. }
  60. #else /* CONFIG_SMP */
  61. static int raise_blk_irq(int cpu, struct request *rq)
  62. {
  63. return 1;
  64. }
  65. #endif
  66. static int blk_softirq_cpu_dead(unsigned int cpu)
  67. {
  68. /*
  69. * If a CPU goes away, splice its entries to the current CPU
  70. * and trigger a run of the softirq
  71. */
  72. local_irq_disable();
  73. list_splice_init(&per_cpu(blk_cpu_done, cpu),
  74. this_cpu_ptr(&blk_cpu_done));
  75. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  76. local_irq_enable();
  77. return 0;
  78. }
  79. void __blk_complete_request(struct request *req)
  80. {
  81. int ccpu, cpu;
  82. struct request_queue *q = req->q;
  83. unsigned long flags;
  84. bool shared = false;
  85. BUG_ON(!q->softirq_done_fn);
  86. local_irq_save(flags);
  87. cpu = smp_processor_id();
  88. /*
  89. * Select completion CPU
  90. */
  91. if (req->cpu != -1) {
  92. ccpu = req->cpu;
  93. if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
  94. shared = cpus_share_cache(cpu, ccpu);
  95. } else
  96. ccpu = cpu;
  97. /*
  98. * If current CPU and requested CPU share a cache, run the softirq on
  99. * the current CPU. One might concern this is just like
  100. * QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is
  101. * running in interrupt handler, and currently I/O controller doesn't
  102. * support multiple interrupts, so current CPU is unique actually. This
  103. * avoids IPI sending from current CPU to the first CPU of a group.
  104. */
  105. if (ccpu == cpu || shared) {
  106. struct list_head *list;
  107. do_local:
  108. list = this_cpu_ptr(&blk_cpu_done);
  109. list_add_tail(&req->ipi_list, list);
  110. /*
  111. * if the list only contains our just added request,
  112. * signal a raise of the softirq. If there are already
  113. * entries there, someone already raised the irq but it
  114. * hasn't run yet.
  115. */
  116. if (list->next == &req->ipi_list)
  117. raise_softirq_irqoff(BLOCK_SOFTIRQ);
  118. } else if (raise_blk_irq(ccpu, req))
  119. goto do_local;
  120. local_irq_restore(flags);
  121. }
  122. /**
  123. * blk_complete_request - end I/O on a request
  124. * @req: the request being processed
  125. *
  126. * Description:
  127. * Ends all I/O on a request. It does not handle partial completions,
  128. * unless the driver actually implements this in its completion callback
  129. * through requeueing. The actual completion happens out-of-order,
  130. * through a softirq handler. The user must have registered a completion
  131. * callback through blk_queue_softirq_done().
  132. **/
  133. void blk_complete_request(struct request *req)
  134. {
  135. if (unlikely(blk_should_fake_timeout(req->q)))
  136. return;
  137. if (!blk_mark_rq_complete(req))
  138. __blk_complete_request(req);
  139. }
  140. EXPORT_SYMBOL(blk_complete_request);
  141. static __init int blk_softirq_init(void)
  142. {
  143. int i;
  144. for_each_possible_cpu(i)
  145. INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
  146. open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
  147. cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
  148. "block/softirq:dead", NULL,
  149. blk_softirq_cpu_dead);
  150. return 0;
  151. }
  152. subsys_initcall(blk_softirq_init);