kern/capability.c

@@ -163,6 +163,45 @@ cap_thread_create (struct cap_thread **outp, struct thread *thread)
   return (0);
 }
 
+struct cap_flow_guard
+{
+  struct spinlock *lock;
+  cpu_flags_t flags;
+  bool use_flags;
+};
+
+static void
+cap_flow_guard_lock (struct cap_flow_guard *guard)
+{
+  if (guard->use_flags)
+    spinlock_lock_intr_save (guard->lock, &guard->flags);
+  else
+    spinlock_lock (guard->lock);
+}
+
+static struct cap_flow_guard
+cap_flow_guard_make (struct cap_flow *flow)
+{
+  struct cap_flow_guard guard = { .lock = &flow->lock };
+  guard.use_flags = (flow->base.flags & CAP_FLOW_HANDLE_INTR) != 0;
+  cap_flow_guard_lock (&guard);
+  return (guard);
+}
+
+static void
+cap_flow_guard_fini (void *p)
+{
+  struct cap_flow_guard *guard = p;
+  if (guard->use_flags)
+    spinlock_unlock_intr_restore (guard->lock, guard->flags);
+  else
+    spinlock_unlock (guard->lock);
+}
+
+#define CAP_FLOW_LOCK_GUARD(flow)   \
+  CLEANUP (cap_flow_guard_fini) _Auto __unused UNIQ (cfg) =   \
+    cap_flow_guard_make (flow)
+
 static void cap_recv_wakeup_fast (struct cap_flow *);
 
 static void
@@ -179,18 +218,18 @@ cap_channel_fini (struct sref_counter *sref)
 
   uintptr_t tag = chp->tag;
   // Mutate the type.
-  struct cap_alert *alert = (void *)chp;
+  struct cap_alert *alert __attribute__ ((may_alias)) = (void *)chp;
 
   alert->k_alert.type = cap_alert_type(alert) = CAP_ALERT_CHAN_CLOSED;
   alert->k_alert.tag = tag;
   pqueue_node_init (&alert->pnode, CAP_ALERT_CHANNEL_PRIO);
   hlist_node_init (&alert->hnode);
 
-  spinlock_lock (&flow->lock);
+  _Auto guard = cap_flow_guard_make (flow);
   hlist_insert_head (&flow->alloc_alerts, &alert->hnode);
   pqueue_insert (&flow->pending_alerts, &alert->pnode);
   cap_recv_wakeup_fast (flow);
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (&guard);
 
   cap_base_rel (flow);
 }
@@ -273,7 +312,7 @@ cap_flow_fini (struct sref_counter *sref)
   struct cap_alert *alert, *tmp;
   pqueue_for_each_entry_safe (&flow->pending_alerts, alert, tmp, pnode)
     if (cap_alert_type (alert) == CAP_ALERT_USER)
-      kmem_free (alert, sizeof (*alert));
+      kmem_cache_free (&cap_misc_cache, alert);
 
   hlist_for_each_entry_safe (&flow->alloc_alerts, alert, tmp, hnode)
     cap_alert_free (alert);
@@ -285,10 +324,15 @@ cap_flow_fini (struct sref_counter *sref)
   kmem_cache_free (&cap_flow_cache, flow);
 }
 
+#define CAP_FLOW_VALID_FLAGS   (CAP_FLOW_HANDLE_INTR | CAP_FLOW_EXT_PAGER)
+
 int
 cap_flow_create (struct cap_flow **outp, uint32_t flags,
                  uintptr_t tag, uintptr_t entry)
 {
+  if (flags & ~CAP_FLOW_VALID_FLAGS)
+    return (EINVAL);
+
   struct cap_flow *ret = kmem_cache_alloc (&cap_flow_cache);
   if (! ret)
     return (ENOMEM);
@@ -300,7 +344,7 @@ cap_flow_create (struct cap_flow **outp, uint32_t flags,
   slist_init (&ret->ports);
   hlist_init (&ret->alloc_alerts);
   pqueue_init (&ret->pending_alerts);
-  ret->flags = flags;
+  ret->base.flags = flags;
   ret->tag = tag;
   ret->entry = entry;
 
@@ -425,12 +469,12 @@ cap_transfer_iters (struct task *task, struct cap_iters *r_it,
 }
 
 static struct cap_alert*
-cap_flow_alloc_alert (struct cap_flow *flow, uint32_t flg)
+cap_flow_alloc_alert (struct cap_flow_guard *guard, uint32_t flg)
 {
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (guard);
   void *ptr = kmem_cache_alloc2 (&cap_misc_cache, (flg & CAP_ALERT_NONBLOCK) ?
                                                   0 : KMEM_ALLOC_SLEEP);
-  spinlock_lock (&flow->lock);
+  cap_flow_guard_lock (guard);
   return (ptr);
 }
 
@@ -457,13 +501,14 @@ cap_recv_wakeup_fast (struct cap_flow *flow)
 
 static struct cap_alert*
 cap_recv_pop_alert (struct cap_flow *flow, void *buf, uint32_t flags,
-                    struct ipc_msg_data *mdata, int *outp)
+                    struct ipc_msg_data *mdata, int *outp,
+                    struct cap_flow_guard *guard)
 {
   if (!pqueue_empty (&flow->pending_alerts))
     return (pqueue_pop_entry (&flow->pending_alerts, struct cap_alert, pnode));
   else if (flags & CAP_ALERT_NONBLOCK)
     {
-      spinlock_unlock (&flow->lock);
+      cap_flow_guard_fini (guard);
       *outp = EAGAIN;
       return (NULL);
     }
@@ -478,7 +523,7 @@ cap_recv_pop_alert (struct cap_flow *flow, void *buf, uint32_t flags,
   if (recv.spurious)
     return (pqueue_pop_entry (&flow->pending_alerts, struct cap_alert, pnode));
 
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (guard);
   if (recv.mdata.bytes_recv >= 0 && mdata)
     {
       recv.mdata.bytes_recv = CAP_ALERT_SIZE;
@@ -495,10 +540,10 @@ cap_recv_alert (struct cap_flow *flow, void *buf,
 {
   uint32_t ids[2] = { 0, 0 };
   uintptr_t tag = 0;
-  spinlock_lock (&flow->lock);
+  _Auto guard = cap_flow_guard_make (flow);
 
   int error;
-  _Auto entry = cap_recv_pop_alert (flow, buf, flags, mdata, &error);
+  _Auto entry = cap_recv_pop_alert (flow, buf, flags, mdata, &error, &guard);
 
   if (! entry)
     return (error);
@@ -522,11 +567,11 @@ cap_recv_alert (struct cap_flow *flow, void *buf,
     }
 
   pqueue_inc (&flow->pending_alerts, 1);
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (&guard);
 
   if (unlikely (user_copy_to (buf, payload, CAP_ALERT_SIZE) != 0))
     {
-      SPINLOCK_GUARD (&flow->lock);
+      cap_flow_guard_lock (&guard);
       pqueue_insert (&flow->pending_alerts, &entry->pnode);
 
       if (type == CAP_ALERT_INTR)
@@ -536,6 +581,7 @@ cap_recv_alert (struct cap_flow *flow, void *buf,
         hlist_insert_head (&flow->alloc_alerts, &entry->hnode);
 
       cap_recv_wakeup_fast (flow);
+      cap_flow_guard_fini (&guard);
       return (EFAULT);
     }
   else if (mdata)
@@ -592,10 +638,10 @@ int
   struct cap_receiver *recv;
 
   {
-    SPINLOCK_GUARD (&flow->lock);
+    CLEANUP (cap_flow_guard_fini) _Auto guard = cap_flow_guard_make (flow);
     if (list_empty (&flow->receivers))
       {
-        _Auto alert = cap_flow_alloc_alert (flow, flags);
+        _Auto alert = cap_flow_alloc_alert (&guard, flags);
         if (! alert)
           return (ENOMEM);
 
@@ -664,7 +710,7 @@ cap_ipc_msg_data_init (struct ipc_msg_data *data, uintptr_t tag)
 static void
 cap_flow_push_port (struct cap_flow *flow, struct cap_port *port)
 {
-  SPINLOCK_GUARD (&flow->lock);
+  CAP_FLOW_LOCK_GUARD (flow);
   slist_insert_head (&flow->ports, &port->snode);
 
   if (list_empty (&flow->waiters))
@@ -677,7 +723,7 @@ cap_flow_push_port (struct cap_flow *flow, struct cap_port *port)
 static struct cap_port*
 cap_pop_port (struct cap_flow *flow, struct thread *self)
 {
-  SPINLOCK_GUARD (&flow->lock);
+  CAP_FLOW_LOCK_GUARD (flow);
   if (slist_empty (&flow->ports))
     {
       struct cap_sender sender;
@@ -910,7 +956,7 @@ cap_flow_add_port (struct cap_flow *flow, void *stack, size_t size,
 int
 cap_flow_rem_port (struct cap_flow *flow, uintptr_t stack)
 {
-  spinlock_lock (&flow->lock);
+  _Auto guard = cap_flow_guard_make (flow);
   struct cap_port *entry;
   struct slist_node *prev = NULL;
 
@@ -925,12 +971,12 @@ cap_flow_rem_port (struct cap_flow *flow, uintptr_t stack)
 
   if (! entry)
     {
-      spinlock_unlock (&flow->lock);
+      cap_flow_guard_fini (&guard);
       return (ESRCH);
     }
 
   slist_remove (&flow->ports, prev);
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (&guard);
 
   // Unmap the stack if the user didn't specify one.
   int error = stack != ~(uintptr_t)0 ? 0 :
@@ -955,7 +1001,7 @@ cap_handle_intr (void *arg)
   list_rcu_for_each (list, tmp)
     {
       _Auto alert = list_entry (tmp, struct cap_alert_async, xlink);
-      SPINLOCK_GUARD (&alert->flow->lock);
+      SPINLOCK_INTR_GUARD (&alert->flow->lock);
       if (++alert->base.k_alert.intr.count == 1)
         {
           pqueue_insert (&alert->flow->pending_alerts, &alert->base.pnode);
@@ -1043,17 +1089,17 @@ cap_intr_register (struct cap_flow *flow, uint32_t irq)
       return (error);
     }
 
-  spinlock_lock (&flow->lock);
+  _Auto guard = cap_flow_guard_make (flow);
   if (unlikely (cap_alert_async_find (flow, CAP_ALERT_INTR, irq)))
     {
-      spinlock_unlock (&flow->lock);
+      cap_flow_guard_fini (&guard);
       cap_intr_rem (irq, &ap->xlink);
       kmem_cache_free (&cap_misc_cache, ap);
       return (EALREADY);
     }
 
   hlist_insert_head (&flow->alloc_alerts, &ap->base.hnode);
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (&guard);
   return (0);
 }
 
@@ -1061,7 +1107,7 @@ static int
 cap_unregister_impl (struct cap_flow *flow, int type,
                      uint32_t id, struct cap_alert_async **outp)
 {
-  SPINLOCK_GUARD (&flow->lock);
+  CAP_FLOW_LOCK_GUARD (flow);
   _Auto entry = cap_alert_async_find (flow, type, id);
 
   if (! entry)
@@ -1116,10 +1162,10 @@ cap_register_task_thread (struct cap_flow *flow, struct kuid_head *kuid,
       .any_id = kuid->id
     };
 
-  spinlock_lock (&flow->lock);
+  _Auto guard = cap_flow_guard_make (flow);
   if (unlikely (cap_alert_async_find (flow, type, kuid->id)))
     {
-      spinlock_unlock (&flow->lock);
+      cap_flow_guard_fini (&guard);
       kmem_cache_free (&cap_misc_cache, ap);
       return (EALREADY);
     }
@@ -1130,7 +1176,7 @@ cap_register_task_thread (struct cap_flow *flow, struct kuid_head *kuid,
   list_insert_tail (&outp->subs, &ap->xlink);
 
   spinlock_unlock (&outp->lock);
-  spinlock_unlock (&flow->lock);
+  cap_flow_guard_fini (&guard);
   return (0);
 }
 
@@ -1207,7 +1253,7 @@ cap_notify_dead (struct bulletin *bulletin)
     {
       _Auto flow = ap->flow;
 
-      SPINLOCK_GUARD (&flow->lock);
+      CAP_FLOW_LOCK_GUARD (flow);
       if (!pqueue_node_unlinked (&ap->base.pnode))
         continue;
 

kern/capability.h

@@ -90,7 +90,8 @@ static_assert (OFFSETOF (struct cap_kern_alert, intr.irq) ==
 
 struct cap_base
 {
-  unsigned int type;
+  unsigned char type;
+  unsigned int flags:24;
   struct sref_counter sref;
 };
 
@@ -109,6 +110,9 @@ struct cap_thread_info
 
 #define CAPABILITY   struct cap_base base
 
+#define CAP_FLOW_HANDLE_INTR   0x01   // Flow can handle interrupts.
+#define CAP_FLOW_EXT_PAGER     0x02   // Flow is an external pager.
+
 struct cap_flow
 {
   CAPABILITY;
@@ -119,7 +123,6 @@ struct cap_flow
   struct pqueue pending_alerts;
   uintptr_t tag;
   uintptr_t entry;
-  uint32_t flags;
 #if CONFIG_MAX_CPUS > 1
   char pad[CPU_L1_SIZE];
 #endif