Merge pull request #2266 from riscv-software-src/zama16b

Implement Zama16b extension

disasm/isa_parser.cc

@@ -100,6 +100,8 @@ void isa_parser_t::add_extension(const std::string& ext_str, const char* str)
   } else if (ext_str == "zawrs") {
     extension_table[EXT_ZAWRS] = true;
     extension_table[EXT_ZALRSC] = true;
+  } else if (ext_str == "zama16b") {
+    extension_table[EXT_ZAMA16B] = true;
   } else if (ext_str == "zmmul") {
     extension_table[EXT_ZMMUL] = true;
   } else if (ext_str == "zba") {

riscv/isa_parser.h

@@ -95,6 +95,7 @@ typedef enum {
   EXT_ZACAS,
   EXT_ZABHA,
   EXT_ZAWRS,
+  EXT_ZAMA16B,
   EXT_INTERNAL_ZFH_MOVE,
   EXT_SMCSRIND,
   EXT_SSCSRIND,

riscv/mmu.cc

@@ -370,15 +370,16 @@ void mmu_t::store_slow_path_intrapage(reg_t len, const uint8_t* bytes, mem_acces
 
 void mmu_t::store_slow_path(reg_t original_addr, std::size_t len,
     const std::uint8_t* bytes, xlate_flags_t xlate_flags,
-    bool actually_store, bool UNUSED require_alignment)
+    bool actually_store, bool require_alignment)
 {
   if (likely(!xlate_flags.is_special_access())) {
     // Fast path for simple cases
     auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_store, original_addr, TLB_FLAGS & ~TLB_CHECK_TRIGGERS);
     bool intrapage = (original_addr % PGSIZE) + len <= PGSIZE;
     bool aligned = (original_addr & (len - 1)) == 0;
+    bool misaligned_ok = !require_alignment && intrapage && is_misaligned_enabled();
 
-    if (likely(tlb_hit && (aligned || (intrapage && is_misaligned_enabled())))) {
+    if (likely(tlb_hit && (aligned || misaligned_ok))) {
       if (actually_store)
         perform_intrapage_store(original_addr, host_addr, paddr, len, bytes, xlate_flags);
       return;
@@ -388,9 +389,15 @@ void mmu_t::store_slow_path(reg_t original_addr, std::size_t len,
   auto access_info = generate_access_info(original_addr, STORE, xlate_flags);
   reg_t transformed_addr = access_info.transformed_vaddr;
 
-  if (actually_store && check_triggers_store)
+  if (check_triggers_store) {
+    if (actually_store) {
       check_triggers(triggers::OPERATION_STORE,
         transformed_addr, access_info.effective_virt, len, bytes);
+    } else {
+      check_triggers(triggers::OPERATION_STORE,
+        transformed_addr, access_info.effective_virt, len);
+    }
+  }
 
   if (transformed_addr & (len - 1)) {
     bool gva = access_info.effective_virt;

riscv/mmu.h

@@ -173,7 +173,6 @@ public:
     try { \
       BODY \
     } catch (trap_load_address_misaligned& t) { \
-      /* Misaligned fault will not be triggered by Zicbom */ \
       throw trap_store_address_misaligned(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
     } catch (trap_load_page_fault& t) { \
       throw trap_store_page_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \
@@ -183,11 +182,19 @@ public:
       throw trap_store_guest_page_fault(t.get_tval(), t.get_tval2(), t.get_tinst()); \
     }
 
+  inline bool enforce_amo_alignment(reg_t addr, size_t size)
+  {
+    if (proc->extension_enabled(EXT_ZAMA16B))
+      return (addr / 16) != ((addr + size - 1) / 16);
+
+    return true;
+  }
+
   // template for functions that perform an atomic memory operation
   template<typename T, typename op>
   T amo(reg_t addr, op f) {
     convert_load_traps_to_store_traps({
-      store_slow_path(addr, sizeof(T), nullptr, {}, false, true);
+      store_slow_path(addr, sizeof(T), nullptr, {}, false, enforce_amo_alignment(addr, sizeof(T)));
       auto lhs = load<T>(addr);
       store<T>(addr, f(lhs));
       return lhs;
@@ -197,16 +204,18 @@ public:
   // for shadow stack amoswap
   template<typename T>
   T ssamoswap(reg_t addr, reg_t value) {
+    convert_load_traps_to_store_traps({
       store_slow_path(addr, sizeof(T), nullptr, {.ss_access=true}, false, true);
       auto data = load<T>(addr, {.ss_access=true});
       store<T>(addr, value, {.ss_access=true});
       return data;
+    })
   }
 
   template<typename T>
   T amo_compare_and_swap(reg_t addr, T comp, T swap) {
     convert_load_traps_to_store_traps({
-      store_slow_path(addr, sizeof(T), nullptr, {}, false, true);
+      store_slow_path(addr, sizeof(T), nullptr, {}, false, enforce_amo_alignment(addr, sizeof(T)));
       auto lhs = load<T>(addr);
       if (lhs == comp)
         store<T>(addr, swap);
@@ -253,7 +262,7 @@ public:
       store_slow_path(vaddr, size, nullptr, {}, false, true);
     }
 
-    auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_store, vaddr);
+    auto [tlb_hit, _, paddr] = access_tlb(tlb_store, vaddr);
     if (!tlb_hit)
       paddr = translate(generate_access_info(vaddr, STORE, {}), 1);