Implement Ziccid, maintaining coherence with a Bloom filter

The algorithm is as follows:
- An instruction-side Bloom filter knows which PPNs might be in the I$
- A store-side Bloom filter knows which PPNs might get store TLB hits
- When refilling the store TLB, search the I-Bloom filter; on a hit,
  remove the ITLB entry, rebuild the I-Bloom filter, and flush the I$
- When refilling the ITLB, search the store-Bloom filter; on a hit,
  remove the store TLB entry and rebuild the store-Bloom filter

The effect is that any word that can be stored-to cannot be in any I$.

The old scheme (periodically flush I$) was hacky and didn't correctly
respect the in-order fetch rule (i.e. fetches are in-order wrt.
each other, and so they see ordered stores in order).

riscv/mmu.cc

@@ -102,6 +102,14 @@ mmu_t::insn_parcel_t mmu_t::fetch_slow_path(reg_t vaddr)
     paddr = translate(access_info, sizeof(insn_parcel_t));
     host_addr = (uintptr_t)sim->addr_to_mem(paddr);
 
+    if (proc->extension_enabled(EXT_ZICCID)) {
+      // Maintain exclusion with all store TLBs
+      for (auto [_, p2] : sim->get_harts())
+        p2->mmu->flush_stlb_ppn(paddr >> PGSHIFT);
+
+      tlb_insn_reverse_tags.insert(paddr >> PGSHIFT);
+    }
+
     refill_tlb(vaddr, paddr, (char*)host_addr, FETCH);
   }
 
@@ -316,6 +324,14 @@ void mmu_t::store_slow_path_intrapage(reg_t len, const uint8_t* bytes, mem_acces
     paddr = translate(access_info, len);
     host_addr = (uintptr_t)sim->addr_to_mem(paddr);
 
+    if (proc && proc->extension_enabled(EXT_ZICCID)) {
+      // Maintain exclusion with all instruction TLBs
+      for (auto [_, p2] : sim->get_harts())
+        p2->mmu->flush_itlb_ppn(paddr >> PGSHIFT);
+
+      tlb_store_reverse_tags.insert(paddr >> PGSHIFT);
+    }
+
     if (!access_info.flags.is_special_access())
       refill_tlb(vaddr, paddr, (char*)host_addr, STORE);
   }
@@ -384,6 +400,35 @@ void mmu_t::store_slow_path(reg_t original_addr, reg_t len, const uint8_t* bytes
   }
 }
 
+bool mmu_t::flush_tlb_ppn(reg_t ppn, dtlb_entry_t* tlb, reverse_tags_t& filter)
+{
+  if (!filter.contains(ppn))
+    return false;
+
+  filter.clear();
+
+  for (size_t i = 0; i < TLB_ENTRIES; i++) {
+    auto entry_ppn = tlb[i].data.target_addr >> PGSHIFT;
+    if (entry_ppn == ppn)
+      tlb[i].tag = -1;
+    else if (tlb[i].tag != (reg_t)-1)
+      filter.insert(entry_ppn);
+  }
+
+  return true;
+}
+
+void mmu_t::flush_stlb_ppn(reg_t ppn)
+{
+  flush_tlb_ppn(ppn, tlb_store, tlb_store_reverse_tags);
+}
+
+void mmu_t::flush_itlb_ppn(reg_t ppn)
+{
+  if (flush_tlb_ppn(ppn, tlb_insn, tlb_insn_reverse_tags))
+    flush_icache();
+}
+
 tlb_entry_t mmu_t::refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type)
 {
   reg_t idx = (vaddr >> PGSHIFT) % TLB_ENTRIES;

riscv/mmu.h

@@ -3,6 +3,7 @@
 #ifndef _RISCV_MMU_H
 #define _RISCV_MMU_H
 
+#include "bloom_filter.h"
 #include "decode.h"
 #include "trap.h"
 #include "common.h"
@@ -406,6 +407,14 @@ private:
   dtlb_entry_t tlb_store[TLB_ENTRIES];
   dtlb_entry_t tlb_insn[TLB_ENTRIES];
 
+  typedef bloom_filter_t<reg_t, simple_hash1, simple_hash2, TLB_ENTRIES * 16, 3> reverse_tags_t;
+  reverse_tags_t tlb_store_reverse_tags;
+  reverse_tags_t tlb_insn_reverse_tags;
+
+  bool flush_tlb_ppn(reg_t ppn, dtlb_entry_t* tlb, reverse_tags_t& filter);
+  void flush_itlb_ppn(reg_t ppn);
+  void flush_stlb_ppn(reg_t ppn);
+
   // finish translation on a TLB miss and update the TLB
   tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type);
   const char* fill_from_mmio(reg_t vaddr, reg_t paddr);