diff --git a/disasm/isa_parser.cc b/disasm/isa_parser.cc index 4f48e5f9..3e80fc46 100644 --- a/disasm/isa_parser.cc +++ b/disasm/isa_parser.cc @@ -100,6 +100,8 @@ void isa_parser_t::add_extension(const std::string& ext_str, const char* str) } else if (ext_str == "zawrs") { extension_table[EXT_ZAWRS] = true; extension_table[EXT_ZALRSC] = true; + } else if (ext_str == "zama16b") { + extension_table[EXT_ZAMA16B] = true; } else if (ext_str == "zmmul") { extension_table[EXT_ZMMUL] = true; } else if (ext_str == "zba") { diff --git a/riscv/isa_parser.h b/riscv/isa_parser.h index ada74c76..bf65805e 100644 --- a/riscv/isa_parser.h +++ b/riscv/isa_parser.h @@ -95,6 +95,7 @@ typedef enum { EXT_ZACAS, EXT_ZABHA, EXT_ZAWRS, + EXT_ZAMA16B, EXT_INTERNAL_ZFH_MOVE, EXT_SMCSRIND, EXT_SSCSRIND, diff --git a/riscv/mmu.cc b/riscv/mmu.cc index e2ca0f03..5c57679f 100644 --- a/riscv/mmu.cc +++ b/riscv/mmu.cc @@ -370,15 +370,16 @@ void mmu_t::store_slow_path_intrapage(reg_t len, const uint8_t* bytes, mem_acces void mmu_t::store_slow_path(reg_t original_addr, std::size_t len, const std::uint8_t* bytes, xlate_flags_t xlate_flags, - bool actually_store, bool UNUSED require_alignment) + bool actually_store, bool require_alignment) { if (likely(!xlate_flags.is_special_access())) { // Fast path for simple cases auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_store, original_addr, TLB_FLAGS & ~TLB_CHECK_TRIGGERS); bool intrapage = (original_addr % PGSIZE) + len <= PGSIZE; bool aligned = (original_addr & (len - 1)) == 0; + bool misaligned_ok = !require_alignment && intrapage && is_misaligned_enabled(); - if (likely(tlb_hit && (aligned || (intrapage && is_misaligned_enabled())))) { + if (likely(tlb_hit && (aligned || misaligned_ok))) { if (actually_store) perform_intrapage_store(original_addr, host_addr, paddr, len, bytes, xlate_flags); return; @@ -388,9 +389,15 @@ void mmu_t::store_slow_path(reg_t original_addr, std::size_t len, auto access_info = generate_access_info(original_addr, STORE, xlate_flags); reg_t transformed_addr = access_info.transformed_vaddr; - if (actually_store && check_triggers_store) - check_triggers(triggers::OPERATION_STORE, - transformed_addr, access_info.effective_virt, len, bytes); + if (check_triggers_store) { + if (actually_store) { + check_triggers(triggers::OPERATION_STORE, + transformed_addr, access_info.effective_virt, len, bytes); + } else { + check_triggers(triggers::OPERATION_STORE, + transformed_addr, access_info.effective_virt, len); + } + } if (transformed_addr & (len - 1)) { bool gva = access_info.effective_virt; diff --git a/riscv/mmu.h b/riscv/mmu.h index 5ba0854e..1ab83647 100644 --- a/riscv/mmu.h +++ b/riscv/mmu.h @@ -173,7 +173,6 @@ public: try { \ BODY \ } catch (trap_load_address_misaligned& t) { \ - /* Misaligned fault will not be triggered by Zicbom */ \ throw trap_store_address_misaligned(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \ } catch (trap_load_page_fault& t) { \ throw trap_store_page_fault(t.has_gva(), t.get_tval(), t.get_tval2(), t.get_tinst()); \ @@ -183,11 +182,19 @@ public: throw trap_store_guest_page_fault(t.get_tval(), t.get_tval2(), t.get_tinst()); \ } + inline bool enforce_amo_alignment(reg_t addr, size_t size) + { + if (proc->extension_enabled(EXT_ZAMA16B)) + return (addr / 16) != ((addr + size - 1) / 16); + + return true; + } + // template for functions that perform an atomic memory operation template T amo(reg_t addr, op f) { convert_load_traps_to_store_traps({ - store_slow_path(addr, sizeof(T), nullptr, {}, false, true); + store_slow_path(addr, sizeof(T), nullptr, {}, false, enforce_amo_alignment(addr, sizeof(T))); auto lhs = load(addr); store(addr, f(lhs)); return lhs; @@ -197,16 +204,18 @@ public: // for shadow stack amoswap template T ssamoswap(reg_t addr, reg_t value) { + convert_load_traps_to_store_traps({ store_slow_path(addr, sizeof(T), nullptr, {.ss_access=true}, false, true); auto data = load(addr, {.ss_access=true}); store(addr, value, {.ss_access=true}); return data; + }) } template T amo_compare_and_swap(reg_t addr, T comp, T swap) { convert_load_traps_to_store_traps({ - store_slow_path(addr, sizeof(T), nullptr, {}, false, true); + store_slow_path(addr, sizeof(T), nullptr, {}, false, enforce_amo_alignment(addr, sizeof(T))); auto lhs = load(addr); if (lhs == comp) store(addr, swap); @@ -253,7 +262,7 @@ public: store_slow_path(vaddr, size, nullptr, {}, false, true); } - auto [tlb_hit, host_addr, paddr] = access_tlb(tlb_store, vaddr); + auto [tlb_hit, _, paddr] = access_tlb(tlb_store, vaddr); if (!tlb_hit) paddr = translate(generate_access_info(vaddr, STORE, {}), 1);