Handle IPIs and timer interrupts more quickly

machine/mentry.S

@@ -18,11 +18,7 @@ trap_table:
   .word mcall_trap
   .word bad_trap
   .word bad_trap
-#define SOFTWARE_INTERRUPT_VECTOR 12
-  .word software_interrupt
-#define TIMER_INTERRUPT_VECTOR 13
-  .word timer_interrupt
-#define TRAP_FROM_MACHINE_MODE_VECTOR 14
+#define TRAP_FROM_MACHINE_MODE_VECTOR 12
   .word __trap_from_machine_mode
 
   .option norvc
@@ -31,10 +27,6 @@ trap_table:
 reset_vector:
   j do_reset
 
-nmi_vector:
-.Lunhandleable_trap:
-  j bad_trap
-
 trap_vector:
   csrrw sp, mscratch, sp
   beqz sp, .Ltrap_from_machine_mode
@@ -51,8 +43,12 @@ trap_vector:
   # Is it a machine timer interrupt?
   li a0, IRQ_M_TIMER * 2
   bne a0, a1, 1f
-  li a1, TIMER_INTERRUPT_VECTOR
-  j .Lhandle_trap_in_machine_mode
+
+  # Yes.  Simply clear MSIE and raise SSIP.
+  li a0, MIP_MTIP
+  csrc mie, a0
+  li a0, MIP_STIP
+  csrs mip, a0
 
 .Lmret:
   # Go back whence we came.
@@ -64,9 +60,35 @@ trap_vector:
 1:
   # Is it an IPI?
   li a0, IRQ_M_SOFT * 2
-  bne a0, a1, .Lunhandleable_trap
-  li a1, SOFTWARE_INTERRUPT_VECTOR
-  j .Lhandle_trap_in_machine_mode
+  bne a0, a1, bad_trap
+
+  # Yes.  First, clear the MIPI bit.
+  LOAD a0, MENTRY_IPI_OFFSET(sp)
+  STORE x0, (a0)
+  fence
+
+  # Now, decode the cause(s).
+#ifdef __riscv_atomic
+  addi a0, sp, MENTRY_IPI_PENDING_OFFSET
+  amoswap.w a0, x0, (a0)
+#else
+  lw a0, MENTRY_IPI_PENDING_OFFSET(a0)
+  sw x0, MENTRY_IPI_PENDING_OFFSET(a0)
+#endif
+  and a1, a0, IPI_SOFT
+  beqz a1, 1f
+  csrs mip, MIP_SSIP
+1:
+  andi a1, a0, IPI_FENCE_I
+  beqz a1, 1f
+  fence.i
+1:
+  andi a1, a0, IPI_SFENCE_VM
+  beqz a1, 1f
+  sfence.vma
+1:
+  j .Lmret
+
 
 .Lhandle_trap_in_machine_mode:
   # Preserve the registers.  Compute the address of the trap handler.

machine/mtrap.c

@@ -4,6 +4,7 @@
 #include "atomic.h"
 #include "bits.h"
 #include "vm.h"
+#include "unprivileged_memory.h"
 #include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -13,15 +14,6 @@ void __attribute__((noreturn)) bad_trap()
   die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc));
 }
 
-uintptr_t timer_interrupt()
-{
-  // just send the timer interrupt to the supervisor
-  clear_csr(mie, MIP_MTIP);
-  set_csr(mip, MIP_STIP);
-
-  return 0;
-}
-
 static uintptr_t mcall_console_putchar(uint8_t ch)
 {
   htif_console_putchar(ch);
@@ -59,15 +51,6 @@ static void send_ipi(uintptr_t recipient, int event)
   }
 }
 
-static uintptr_t mcall_send_ipi(uintptr_t recipient)
-{
-  if (recipient >= num_harts)
-    return -1;
-
-  send_ipi(recipient, IPI_SOFT);
-  return 0;
-}
-
 static uintptr_t mcall_console_getchar()
 {
   return htif_console_getchar();
@@ -75,9 +58,7 @@ static uintptr_t mcall_console_getchar()
 
 static uintptr_t mcall_clear_ipi()
 {
-  clear_csr(mip, MIP_SSIP);
-  mb();
-  return atomic_swap(&HLS()->sipi_pending, 0);
+  return clear_csr(mip, MIP_SSIP) & MIP_SSIP;
 }
 
 static uintptr_t mcall_shutdown()
@@ -93,31 +74,12 @@ static uintptr_t mcall_set_timer(uint64_t when)
   return 0;
 }
 
-void software_interrupt()
-{
-  *HLS()->ipi = 0;
-  mb();
-  int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0);
-  mb();
-
-  if (ipi_pending & IPI_SOFT) {
-    HLS()->sipi_pending = 1;
-    set_csr(mip, MIP_SSIP);
-  }
-
-  if (ipi_pending & IPI_FENCE_I)
-    asm volatile ("fence.i");
-
-  if (ipi_pending & IPI_SFENCE_VM)
-    flush_tlb();
-}
-
 static void send_ipi_many(uintptr_t* pmask, int event)
 {
   _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits");
   uintptr_t mask = -1;
   if (pmask)
-    mask = *pmask;
+    mask = load_uintptr_t(pmask, read_csr(mepc));
 
   // send IPIs to everyone
   for (ssize_t i = num_harts-1; i >= 0; i--)
@@ -125,14 +87,21 @@ static void send_ipi_many(uintptr_t* pmask, int event)
       send_ipi(i, event);
 
   // wait until all events have been handled.
-  // prevent deadlock while spinning by handling any IPIs from other harts.
+  // prevent deadlock by consuming incoming IPIs.
+  uint32_t incoming_ipi = 0;
   for (ssize_t i = num_harts-1; i >= 0; i--)
     if ((mask >> i) & 1)
-      while (OTHER_HLS(i)->mipi_pending & event)
-        software_interrupt();
+      while (OTHER_HLS(i)->ipi)
+        incoming_ipi |= atomic_swap(HLS()->ipi, 0);
+
+  // if we got an IPI, restore it; it will be taken after returning
+  if (incoming_ipi) {
+    *HLS()->ipi = incoming_ipi;
+    mb();
+  }
 }
 
-static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid)
+static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask)
 {
   // ignore the ASID and do a global flush.
   // this allows us to avoid queueing a message.
@@ -149,6 +118,8 @@ static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask)
 void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
 {
   uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval;
+  uintptr_t ipi_type = 0;
+
   switch (n)
   {
     case MCALL_CONSOLE_PUTCHAR:
@@ -158,7 +129,13 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
       retval = mcall_console_getchar();
       break;
     case MCALL_SEND_IPI:
-      retval = mcall_send_ipi(arg0);
+      ipi_type = IPI_SOFT;
+    case MCALL_REMOTE_SFENCE_VM:
+      ipi_type = IPI_SFENCE_VM;
+    case MCALL_REMOTE_FENCE_I:
+      ipi_type = IPI_FENCE_I;
+      send_ipi_many((uintptr_t*)arg0, ipi_type);
+      retval = 0;
       break;
     case MCALL_CLEAR_IPI:
       retval = mcall_clear_ipi();
@@ -173,12 +150,6 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
       retval = mcall_set_timer(arg0);
 #endif
       break;
-    case MCALL_REMOTE_SFENCE_VM:
-      retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1);
-      break;
-    case MCALL_REMOTE_FENCE_I:
-      retval = mcall_remote_fence_i((uintptr_t*)arg0);
-      break;
     default:
       retval = -ENOSYS;
       break;

machine/mtrap.h

@@ -36,10 +36,10 @@ extern volatile uint32_t* plic_priorities;
 extern size_t plic_ndevs;
 
 typedef struct {
-  uint64_t* timecmp;
   uint32_t* ipi;
   volatile int mipi_pending;
-  volatile int sipi_pending;
+
+  uint64_t* timecmp;
 
   volatile uint32_t* plic_m_thresh;
   volatile uintptr_t* plic_m_ie;
@@ -47,10 +47,6 @@ typedef struct {
   volatile uintptr_t* plic_s_ie;
 } hls_t;
 
-#define IPI_SOFT      0x1
-#define IPI_FENCE_I   0x2
-#define IPI_SFENCE_VM 0x4
-
 #define MACHINE_STACK_TOP() ({ \
   register uintptr_t sp asm ("sp"); \
   (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); })
@@ -79,9 +75,15 @@ static inline void wfi()
 
 #endif // !__ASSEMBLER__
 
+#define IPI_SOFT      0x1
+#define IPI_FENCE_I   0x2
+#define IPI_SFENCE_VM 0x4
+
 #define MACHINE_STACK_SIZE RISCV_PGSIZE
-#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \
-                           + HLS_SIZE)
+#define MENTRY_HLS_OFFSET (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE)
+#define MENTRY_FRAME_SIZE (MENTRY_HLS_OFFSET + HLS_SIZE)
+#define MENTRY_IPI_OFFSET (MENTRY_HLS_OFFSET)
+#define MENTRY_IPI_PENDING_OFFSET (MENTRY_HLS_OFFSET + REGBYTES)
 
 #ifdef __riscv_flen
 # define SOFT_FLOAT_CONTEXT_SIZE 0

machine/unprivileged_memory.h

@@ -43,8 +43,10 @@ DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw)
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu)
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld)
 DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, ld)
 #else
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, lw)
 
 static inline uint64_t load_uint64_t(const uint64_t* addr, uintptr_t mepc)
 {