xen: Fix dispatching of event handlers on multi-CPU platforms

The Xen interrupt handler is invoking the handlers of all pending events
whose port numbers belong to the same `evtchn_pending` array element as the
element for which there is a pending event for the current vCPU. This means
that the handler for an event bound to a given vCPU can potentially be
executed on a different vCPU. This creates issues in event handlers (such
as `xennet_event_handler()` that assume they are always run sequentially
and cannot run in parallel on different vCPUs.
Fix the Xen interrupt handler so that it only invokes handlers for events
bound to the current vCPU. The issue with seemingly spurious interrupts
mentioned in a comment in the interrupt handler function cannot be
reproduced (tested on t2.nano, t2.medium and t2.2xlarge AWS instances),
thus the `evtchn_mask` array is no longer used.
In addition, allow device drivers to request that their event handler is
bound to a specific vCPU (via the newly defined xen_bind_evtchn()
function); this allows spreading the interrupt load among different vCPUs.

Author: Francesco Lavra

src/xen/xen.c

@@ -41,6 +41,7 @@ typedef struct xen_platform_info {
     evtchn_port_t xenstore_evtchn;
     struct spinlock xenstore_lock;
 
+    u64 vcpu_evtchn_enable[XEN_LEGACY_MAX_VCPUS][sizeof(u64) * 8];
     struct vector evtchn_handlers;
 
     closure_struct(xenstore_watch_handler, watch_handler);
@@ -87,6 +88,7 @@ closure_function(0, 0, void, xen_interrupt)
     int vcpu = current_cpu()->id;
     volatile struct shared_info *si = xen_info->shared_info;
     volatile struct vcpu_info *vci = &si->vcpu_info[vcpu];
+    u64 *evtchn_enable = xen_info->vcpu_evtchn_enable[vcpu];
 
     xenint_debug("xen_interrupt enter");
     assert(vci->evtchn_upcall_pending);
@@ -97,24 +99,16 @@ closure_function(0, 0, void, xen_interrupt)
         /* this may not process in the right order, or it might not matter - care later */
         bitmap_word_foreach_set(l1_pending, bit1, i1, 0) {
             (void)i1;
-            /* TODO: any per-cpu event mask would be also applied here... */
-            u64 l2_pending = si->evtchn_pending[bit1] & ~si->evtchn_mask[bit1];
+            u64 l2_pending = si->evtchn_pending[bit1] & evtchn_enable[bit1];
             xenint_debug("pending 0x%lx, mask 0x%lx, masked 0x%lx",
-                         si->evtchn_pending[bit1], si->evtchn_mask[bit1], l2_pending);
+                         si->evtchn_pending[bit1], ~evtchn_enable[bit1], l2_pending);
             __sync_and_and_fetch(&si->evtchn_pending[bit1], ~l2_pending);
             u64 l2_offset = bit1 << 6;
             bitmap_word_foreach_set(l2_pending, bit2, i2, l2_offset) {
                 (void)i2;
                 xenint_debug("  int %d pending", i2);
                 thunk handler = vector_get(&xen_info->evtchn_handlers, i2);
-                if (handler) {
-                    xenint_debug("  evtchn %d: applying handler %p", i2, handler);
-                    apply(handler);
-                } else {
-                    /* XXX we have an issue with seemingly spurious interrupts at evtchn >= 2048... */
-                    xenint_debug("  evtchn %d: spurious interrupt", i2);
-                    si->evtchn_mask[bit1] |= U64_FROM_BIT(bit2);
-                }
+                apply(handler);
             }
         }
         vci->evtchn_upcall_mask = 0;
@@ -127,17 +121,30 @@ void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler)
     assert(vector_set(&xen_info->evtchn_handlers, evtchn, handler));
 }
 
-int xen_unmask_evtchn(evtchn_port_t evtchn)
+int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu)
+{
+    evtchn_op_t eop;
+    eop.cmd = EVTCHNOP_bind_vcpu;
+    eop.u.bind_vcpu.port = evtchn;
+    eop.u.bind_vcpu.vcpu = vcpu;
+    return HYPERVISOR_event_channel_op(&eop);
+}
+
+int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu)
 {
     assert(evtchn > 0 && evtchn < EVTCHN_2L_NR_CHANNELS);
     evtchn_op_t eop;
     eop.cmd = EVTCHNOP_unmask;
     eop.u.unmask.port = evtchn;
-    return HYPERVISOR_event_channel_op(&eop);
+    int rv = HYPERVISOR_event_channel_op(&eop);
+    if (rv == 0)
+        atomic_set_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6));
+    return rv;
 }
 
-int xen_close_evtchn(evtchn_port_t evtchn)
+int xen_close_evtchn(evtchn_port_t evtchn, int vcpu)
 {
+    atomic_clear_bit(&xen_info->vcpu_evtchn_enable[vcpu][evtchn >> 6], evtchn & MASK(6));
     vector_set(&xen_info->evtchn_handlers, evtchn, 0);
     evtchn_op_t eop;
     eop.cmd = EVTCHNOP_close;
@@ -357,7 +364,7 @@ static int xen_setup_vcpu(int vcpu, u64 shared_info_phys)
     xen_debug("cpu %d timer event channel %d", vcpu, timer_evtchn);
     xen_register_evtchn_handler(timer_evtchn,
                                 closure_func(xen_info->h, thunk, xen_runloop_timer_handler));
-    assert(xen_unmask_evtchn(timer_evtchn) == 0);
+    assert(xen_unmask_evtchn(timer_evtchn, vcpu) == 0);
     return 0;
 }
 
@@ -540,7 +547,7 @@ boolean xen_detect(kernel_heaps kh)
     spin_lock_init(&xen_info->xenstore_lock);
     xen_register_evtchn_handler(xen_info->xenstore_evtchn,
                                 closure_func(h, thunk, xenstore_evtchn_handler));
-    assert(xen_unmask_evtchn(xen_info->xenstore_evtchn) == 0);
+    assert(xen_unmask_evtchn(xen_info->xenstore_evtchn, 0) == 0);
 
     if (!xen_grant_init(kh, features)) {
         msg_err("xen: failed to set up grant tables");

src/xen/xen_internal.h

@@ -38,8 +38,9 @@ typedef struct xen_dev {
 status xen_allocate_evtchn(domid_t other_id, evtchn_port_t *evtchn);
 void xen_register_evtchn_handler(evtchn_port_t evtchn, thunk handler);
 int xen_notify_evtchn(evtchn_port_t evtchn);
-int xen_unmask_evtchn(evtchn_port_t evtchn);
-int xen_close_evtchn(evtchn_port_t evtchn);
+int xen_bind_evtchn(evtchn_port_t evtchn, int vcpu);
+int xen_unmask_evtchn(evtchn_port_t evtchn, int vcpu);
+int xen_close_evtchn(evtchn_port_t evtchn, int vcpu);
 
 grant_ref_t xen_grant_page_access(u16 domid, u64 phys, boolean readonly);
 void xen_revoke_page_access(grant_ref_t ref);

src/xen/xenblk.c

@@ -39,6 +39,7 @@ struct xenblk_dev {
     blkif_front_ring_t ring;
     grant_ref_t ring_gntref;
     evtchn_port_t evtchn;
+    s32 evtchn_cpu;
     closure_struct(xenblk_io, read);
     closure_struct(xenblk_io, write);
     closure_struct(storage_simple_req_handler, req_handler);
@@ -275,7 +276,12 @@ static void xenblk_remove(xenblk_dev xbd)
     }
     xen_driver_unbind(xbd->meta);
     xenbus_set_state(0, xbd->dev.frontend, XenbusStateClosed);
-    xen_close_evtchn(xbd->evtchn);
+    s32 evtchn_cpu = xbd->evtchn_cpu;
+    if (evtchn_cpu >= 0)
+        irq_put_target_cpu(evtchn_cpu);
+    else
+        evtchn_cpu = 0;
+    xen_close_evtchn(xbd->evtchn, evtchn_cpu);
     xen_revoke_page_access(xbd->ring_gntref);
     deallocate(xbd->contiguous, xbd->ring.sring, PAGESIZE);
     deallocate_vector(xbd->rreqs);
@@ -336,7 +342,18 @@ closure_func_basic(thunk, void, xenblk_watch_service)
             timm_dealloc(s);
             goto remove;
         }
-        int rv = xen_unmask_evtchn(xbd->evtchn);
+        u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0));
+        int rv = xen_bind_evtchn(xbd->evtchn, evtchn_cpu);
+        if (rv < 0) {
+            msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xbd->evtchn,
+                     evtchn_cpu, rv);
+            irq_put_target_cpu(evtchn_cpu);
+            xbd->evtchn_cpu = -1;
+            evtchn_cpu = 0;
+        } else {
+            xbd->evtchn_cpu = evtchn_cpu;
+        }
+        rv = xen_unmask_evtchn(xbd->evtchn, evtchn_cpu);
         if (rv < 0) {
             msg_err("%s: failed to unmask event channel %d: rv %d", func_ss, xbd->evtchn, rv);
             goto remove;
@@ -472,7 +489,7 @@ static status xenblk_enable(xenblk_dev xbd)
     }
     return STATUS_OK;
   out_evtchn:
-    xen_close_evtchn(xbd->evtchn);
+    xen_close_evtchn(xbd->evtchn, 0);
   out_revoke:
     xen_revoke_page_access(xbd->ring_gntref);
   out_dealloc:

src/xen/xennet.c

@@ -724,7 +724,15 @@ static status xennet_enable(xennet_dev xd)
     /* we're kind of always up ... start rx now */
     xd->rx_ring.sring->rsp_event = xd->rx_ring.rsp_cons + 1;
     write_barrier();
-    int rv = xen_unmask_evtchn(xd->evtchn);
+    u32 evtchn_cpu = irq_get_target_cpu(irange(0, 0));
+    int rv = xen_bind_evtchn(xd->evtchn, evtchn_cpu);
+    if (rv < 0) {
+        msg_warn("%s: failed to bind event channel %d to vCPU %d: rv %d", func_ss, xd->evtchn,
+                 evtchn_cpu, rv);
+        irq_put_target_cpu(evtchn_cpu);
+        evtchn_cpu = 0;
+    }
+    rv = xen_unmask_evtchn(xd->evtchn, evtchn_cpu);
     if (rv < 0) {
         s = timm("result", "failed to unmask event channel %d: rv %d", xd->evtchn, rv);
         goto out_dealloc_rx_buffers;