Merge branch 'plct-cmo-upstream' of https://github.com/plctlab/plct-spike into plctlab-plct-cmo-upstream

README.md

@@ -33,6 +33,7 @@ Spike supports the following RISC-V ISA features:
   - Svnapot extension, v1.0
   - Svpbmt extension, v1.0
   - Svinval extension, v1.0
+  - CMO extension, v1.0
   - Debug v0.14
 
 As a Spike extension, the remainder of the proposed

disasm/disasm.cc

@@ -2003,6 +2003,16 @@ void disassembler_t::add_instructions(isa_parser_t* isa)
       DEFINE_R3TYPE(fsrw);
     }
   }
+
+  if (isa->extension_enabled(EXT_ZICBOM)) {
+    DISASM_INSN("cbo.clean", cbo_clean, 0, {&xrs1});
+    DISASM_INSN("cbo.flush", cbo_flush, 0, {&xrs1});
+    DISASM_INSN("cbo.inval", cbo_inval, 0, {&xrs1});
+  }
+
+  if (isa->extension_enabled(EXT_ZICBOZ)) {
+    DISASM_INSN("cbo.zero", cbo_zero, 0, {&xrs1});
+  }
 }
 
 disassembler_t::disassembler_t(isa_parser_t* isa)

riscv/cachesim.cc

@@ -159,6 +159,31 @@ void cache_sim_t::access(uint64_t addr, size_t bytes, bool store)
     *check_tag(addr) |= DIRTY;
 }
 
+void cache_sim_t::clean_invalidate(uint64_t addr, size_t bytes, bool clean, bool inval)
+{
+  uint64_t start_addr = addr & ~(linesz-1);
+  uint64_t end_addr = (addr + bytes + linesz-1) & ~(linesz-1);
+  uint64_t cur_addr = start_addr;
+  while (cur_addr < end_addr) {
+    uint64_t* hit_way = check_tag(cur_addr);
+    if (likely(hit_way != NULL))
+    {
+      if (clean) {
+        if (*hit_way & DIRTY) {
+          writebacks++;
+          *hit_way &= ~DIRTY;
+        }
+      }
+
+      if (inval)
+        *hit_way &= ~VALID;
+    }
+    cur_addr += linesz;
+  }
+  if (miss_handler)
+    miss_handler->clean_invalidate(addr, bytes, clean, inval);
+}
+
 fa_cache_sim_t::fa_cache_sim_t(size_t ways, size_t linesz, const char* name)
   : cache_sim_t(1, ways, linesz, name)
 {

riscv/cachesim.h

@@ -27,6 +27,7 @@ class cache_sim_t
   virtual ~cache_sim_t();
 
   void access(uint64_t addr, size_t bytes, bool store);
+  void clean_invalidate(uint64_t addr, size_t bytes, bool clean, bool inval);
   void print_stats();
   void set_miss_handler(cache_sim_t* mh) { miss_handler = mh; }
   void set_log(bool _log) { log = _log; }
@@ -90,6 +91,10 @@ class cache_memtracer_t : public memtracer_t
   {
     cache->set_miss_handler(mh);
   }
+  void clean_invalidate(uint64_t addr, size_t bytes, bool clean, bool inval)
+  {
+    cache->clean_invalidate(addr, bytes, clean, inval);
+  }
   void set_log(bool log)
   {
     cache->set_log(log);

riscv/decode.h

@@ -278,6 +278,15 @@ private:
 #define require_noover_widen(astart, asize, bstart, bsize) \
   require(!is_overlapped_widen(astart, asize, bstart, bsize))
 #define require_vm do { if (insn.v_vm() == 0) require(insn.rd() != 0);} while(0);
+#define require_envcfg(field) \
+  do { \
+    if (((STATE.prv != PRV_M) && (m##field == 0)) || \
+        ((STATE.prv == PRV_U && !STATE.v) && (s##field == 0))) \
+      throw trap_illegal_instruction(insn.bits()); \
+    else if (STATE.v && ((h##field == 0) || \
+                        ((STATE.prv == PRV_U) && (s##field == 0)))) \
+      throw trap_virtual_instruction(insn.bits()); \
+  } while(0);
 
 #define set_fp_exceptions ({ if (softfloat_exceptionFlags) { \
                                STATE.fflags->write(STATE.fflags->read() | softfloat_exceptionFlags); \
@@ -3038,6 +3047,11 @@ reg_t index[P.VU.vlmax]; \
     WRITE_RD(sext_xlen(rd)); \
   }
 
+#define DECLARE_XENVCFG_VARS(field) \
+  reg_t m##field = get_field(STATE.menvcfg->read(), MENVCFG_##field); \
+  reg_t s##field = get_field(STATE.senvcfg->read(), SENVCFG_##field); \
+  reg_t h##field = get_field(STATE.henvcfg->read(), HENVCFG_##field)
+
 #define DEBUG_START             0x0
 #define DEBUG_END               (0x1000 - 1)
 

riscv/insns/cbo_clean.h

@@ -0,0 +1,4 @@
+require_extension(EXT_ZICBOM);
+DECLARE_XENVCFG_VARS(CBCFE);
+require_envcfg(CBCFE);
+MMU.clean_inval(RS1, true, false);

riscv/insns/cbo_flush.h

@@ -0,0 +1,4 @@
+require_extension(EXT_ZICBOM);
+DECLARE_XENVCFG_VARS(CBCFE);
+require_envcfg(CBCFE);
+MMU.clean_inval(RS1, true, true);

riscv/insns/cbo_inval.h

@@ -0,0 +1,9 @@
+require_extension(EXT_ZICBOM);
+DECLARE_XENVCFG_VARS(CBIE);
+require_envcfg(CBIE);
+if (((STATE.prv != PRV_M) && (mCBIE == 1)) ||
+    ((!STATE.v && (STATE.prv == PRV_U)) && (sCBIE = 1)) ||
+    (STATE.v && ((hCBIE == 1) || ((STATE.prv == PRV_U) && (sCBIE== 0)))))
+  MMU.clean_inval(RS1, true, true);
+else
+  MMU.clean_inval(RS1, false, true);

riscv/insns/cbo_zero.h

@@ -0,0 +1,4 @@
+require_extension(EXT_ZICBOZ);
+DECLARE_XENVCFG_VARS(CBZE);
+require_envcfg(CBZE);
+MMU.cbo_zero(RS1);

riscv/insns/ori.h

@@ -1 +1,2 @@
+// prefetch.i/r/w hint when rd = 0 and i_imm[4:0] = 0/1/3
 WRITE_RD(insn.i_imm() | RS1);

riscv/memtracer.h

@@ -21,6 +21,7 @@ class memtracer_t
 
   virtual bool interested_in_range(uint64_t begin, uint64_t end, access_type type) = 0;
   virtual void trace(uint64_t addr, size_t bytes, access_type type) = 0;
+  virtual void clean_invalidate(uint64_t addr, size_t bytes, bool clean, bool inval) = 0;
 };
 
 class memtracer_list_t : public memtracer_t
@@ -29,15 +30,20 @@ class memtracer_list_t : public memtracer_t
   bool empty() { return list.empty(); }
   bool interested_in_range(uint64_t begin, uint64_t end, access_type type)
   {
-    for (std::vector<memtracer_t*>::iterator it = list.begin(); it != list.end(); ++it)
-      if ((*it)->interested_in_range(begin, end, type))
+    for (auto it: list)
+      if (it->interested_in_range(begin, end, type))
         return true;
     return false;
   }
   void trace(uint64_t addr, size_t bytes, access_type type)
   {
-    for (std::vector<memtracer_t*>::iterator it = list.begin(); it != list.end(); ++it)
-      (*it)->trace(addr, bytes, type);
+    for (auto it: list)
+      it->trace(addr, bytes, type);
+  }
+  void clean_invalidate(uint64_t addr, size_t bytes, bool clean, bool inval)
+  {
+    for (auto it: list)
+      it->clean_invalidate(addr, bytes, clean, inval);
   }
   void hook(memtracer_t* h)
   {

riscv/mmu.h

@@ -184,27 +184,30 @@ public:
       } \
   }
 
+  // AMO/Zicbom faults should be reported as store faults
+  #define convert_load_traps_to_store_traps(BODY) \
+    try { \
+      BODY \
+    } catch (trap_load_address_misaligned& t) { \
+      /* Misaligned fault will not be triggered by Zicbom */ \
+      throw trap_store_address_misaligned(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
+    } catch (trap_load_page_fault& t) { \
+      throw trap_store_page_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
+    } catch (trap_load_access_fault& t) { \
+      throw trap_store_access_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
+    } catch (trap_load_guest_page_fault& t) { \
+      throw trap_store_guest_page_fault(t.get_tval(), t.get_tval2(), t.get_tinst()); \
+    }
+
   // template for functions that perform an atomic memory operation
   #define amo_func(type) \
     template<typename op> \
     type##_t amo_##type(reg_t addr, op f) { \
-      try { \
+      convert_load_traps_to_store_traps({ \
         auto lhs = load_##type(addr, true); \
         store_##type(addr, f(lhs)); \
         return lhs; \
-      } catch (trap_load_address_misaligned& t) { \
-        /* AMO faults should be reported as store faults */ \
-        throw trap_store_address_misaligned(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
-      } catch (trap_load_page_fault& t) { \
-        /* AMO faults should be reported as store faults */ \
-        throw trap_store_page_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
-      } catch (trap_load_access_fault& t) { \
-        /* AMO faults should be reported as store faults */ \
-        throw trap_store_access_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
-      } catch (trap_load_guest_page_fault& t) { \
-        /* AMO faults should be reported as store faults */ \
-        throw trap_store_guest_page_fault(t.get_tval(), t.get_tval2(), t.get_tinst()); \
-      } \
+      }) \
     }
 
   void store_float128(reg_t addr, float128_t val)
@@ -242,6 +245,25 @@ public:
   amo_func(uint32)
   amo_func(uint64)
 
+  void cbo_zero(reg_t addr) {
+    auto base = addr & ~(blocksz - 1);
+    for (size_t offset = 0; offset < blocksz; offset += 1)
+      store_uint8(base + offset, 0);
+  }
+
+  void clean_inval(reg_t addr, bool clean, bool inval) {
+    convert_load_traps_to_store_traps({
+      reg_t paddr = addr & ~(blocksz - 1);
+      paddr = translate(paddr, blocksz, LOAD, 0);
+      if (auto host_addr = sim->addr_to_mem(paddr)) {
+        if (tracer.interested_in_range(paddr, paddr + PGSIZE, LOAD))
+          tracer.clean_invalidate(paddr, blocksz, clean, inval);
+      } else {
+        throw trap_store_access_fault((proc) ? proc->state.v : false, addr, 0, 0);
+      }
+    })
+  }
+
   inline void yield_load_reservation()
   {
     load_reservation_address = (reg_t)-1;
@@ -389,12 +411,18 @@ public:
     return target_big_endian? target_endian<T>::to_be(n) : target_endian<T>::to_le(n);
   }
 
+  void set_cache_blocksz(uint64_t size)
+  {
+    blocksz = size;
+  }
+
 private:
   simif_t* sim;
   processor_t* proc;
   memtracer_list_t tracer;
   reg_t load_reservation_address;
   uint16_t fetch_temp;
+  uint64_t blocksz;
 
   // implement an instruction cache for simulator performance
   icache_entry_t icache[ICACHE_ENTRIES];

riscv/processor.cc

@@ -338,6 +338,11 @@ isa_parser_t::isa_parser_t(const char* str)
       extension_table[EXT_SVPBMT] = true;
     } else if (ext_str == "svinval") {
       extension_table[EXT_SVINVAL] = true;
+    } else if (ext_str == "zicbom") {
+      extension_table[EXT_ZICBOM] = true;
+    } else if (ext_str == "zicboz") {
+       extension_table[EXT_ZICBOZ] = true;
+    } else if (ext_str == "zicbop") {
     } else if (ext_str[0] == 'x') {
       max_isa |= 1L << ('x' - 'a');
       extension_table[toupper('x')] = true;
@@ -569,6 +574,15 @@ void state_t::reset(processor_t* const proc, reg_t max_isa)
   csrmap[CSR_MIMPID] = std::make_shared<const_csr_t>(proc, CSR_MIMPID, 0);
   csrmap[CSR_MVENDORID] = std::make_shared<const_csr_t>(proc, CSR_MVENDORID, 0);
   csrmap[CSR_MHARTID] = std::make_shared<const_csr_t>(proc, CSR_MHARTID, proc->get_id());
+  const reg_t menvcfg_mask = (proc->extension_enabled(EXT_ZICBOM) ? MENVCFG_CBCFE | MENVCFG_CBIE: 0) |
+                             (proc->extension_enabled(EXT_ZICBOZ) ? MENVCFG_CBZE: 0);
+  csrmap[CSR_MENVCFG] = menvcfg = std::make_shared<masked_csr_t>(proc, CSR_MENVCFG, menvcfg_mask, 0);
+  const reg_t senvcfg_mask = (proc->extension_enabled(EXT_ZICBOM) ? SENVCFG_CBCFE | SENVCFG_CBIE: 0) |
+                             (proc->extension_enabled(EXT_ZICBOZ) ? SENVCFG_CBZE: 0);
+  csrmap[CSR_SENVCFG] = senvcfg = std::make_shared<masked_csr_t>(proc, CSR_SENVCFG, senvcfg_mask, 0);
+  const reg_t henvcfg_mask = (proc->extension_enabled(EXT_ZICBOM) ? HENVCFG_CBCFE | HENVCFG_CBIE: 0) |
+                             (proc->extension_enabled(EXT_ZICBOZ) ? HENVCFG_CBZE: 0);
+  csrmap[CSR_HENVCFG] = henvcfg = std::make_shared<masked_csr_t>(proc, CSR_HENVCFG, henvcfg_mask, 0);
 
   serialized = false;
 

riscv/processor.h

@@ -218,6 +218,11 @@ struct state_t
 
   csr_t_p fflags;
   csr_t_p frm;
+
+  csr_t_p menvcfg;
+  csr_t_p senvcfg;
+  csr_t_p henvcfg;
+
   bool serialized; // whether timer CSRs are in a well-defined state
 
   // When true, execute a single instruction and then enter debug mode.  This
@@ -273,6 +278,8 @@ typedef enum {
   EXT_ZHINX,
   EXT_ZHINXMIN,
   EXT_XBITMANIP,
+  EXT_ZICBOM,
+  EXT_ZICBOZ,
 } isa_extension_t;
 
 typedef enum {

riscv/riscv.mk.in

@@ -1260,6 +1260,12 @@ riscv_insn_svinval = \
 	hinval_vvma \
 	hinval_gvma \
 
+riscv_insn_ext_cmo = \
+	cbo_clean \
+	cbo_flush \
+	cbo_inval \
+	cbo_zero \
+
 riscv_insn_list = \
 	$(riscv_insn_ext_a) \
 	$(riscv_insn_ext_c) \
@@ -1276,6 +1282,7 @@ riscv_insn_list = \
 	$(riscv_insn_ext_p) \
 	$(riscv_insn_priv) \
 	$(riscv_insn_svinval) \
+	$(riscv_insn_ext_cmo) \
 
 riscv_gen_srcs = \
 	$(addsuffix .cc,$(riscv_insn_list))

spike_main/spike.cc

@@ -72,6 +72,7 @@ static void help(int exit_code = 1)
   fprintf(stderr, "  --dm-no-abstract-csr  Debug module won't support abstract to authenticate\n");
   fprintf(stderr, "  --dm-no-halt-groups   Debug module won't support halt groups\n");
   fprintf(stderr, "  --dm-no-impebreak     Debug module won't support implicit ebreak in program buffer\n");
+  fprintf(stderr, "  --blocksz=<size>     Cache block size (B) for CMO operations(powers of 2) [default 16]\n");
 
   exit(exit_code);
 }
@@ -241,6 +242,7 @@ int main(int argc, char** argv)
   uint16_t rbb_port = 0;
   bool use_rbb = false;
   unsigned dmi_rti = 0;
+  reg_t blocksz = 64;
   debug_module_config_t dm_config = {
     .progbufsize = 2,
     .max_sba_data_width = 0,
@@ -376,6 +378,13 @@ int main(int argc, char** argv)
         exit(-1);
      }
   });
+  parser.option(0, "blocksz", 1, [&](const char* s){
+    blocksz = strtoull(s, 0, 0);
+    if (((blocksz & (blocksz - 1))) != 0) {
+      fprintf(stderr, "--blocksz should be power of 2\n");
+      exit(-1);
+    }
+  });
 
   auto argv1 = parser.parse(argv);
   std::vector<std::string> htif_args(argv1, (const char*const*)argv + argc);
@@ -464,6 +473,7 @@ int main(int argc, char** argv)
     if (dc) s.get_core(i)->get_mmu()->register_memtracer(&*dc);
     for (auto e : extensions)
       s.get_core(i)->register_extension(e());
+    s.get_core(i)->get_mmu()->set_cache_blocksz(blocksz);
   }
 
   s.set_debug(debug);