Merge pull request #2117 from riscv-software-src/ziccid

Implement Ziccid (I$ coherence)

riscv/bloom_filter.h

@@ -0,0 +1,64 @@
+// See LICENSE for license details.
+
+#ifndef _RISCV_BLOOM_FILTER_H
+#define _RISCV_BLOOM_FILTER_H
+
+#include <bitset>
+#include <cstdint>
+
+struct simple_hash1 {
+  uint64_t operator()(uint64_t x) const
+  {
+    x = (x ^ (x >> 33)) * 0xff51afd7ed558ccd;
+    x = (x ^ (x >> 33)) * 0xc4ceb9fe1a85ec53;
+    return x ^ (x >> 33);
+  }
+};
+
+struct simple_hash2 {
+  uint64_t operator()(uint64_t x) const
+  {
+    x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
+    x = (x ^ (x >> 27)) * 0x94d049b13c66a8ed;
+    return x ^ (x >> 31);
+  }
+};
+
+template <typename T, typename H1, typename H2, size_t M, size_t K> // M: bit array size, K: number of hash functions
+class bloom_filter_t {
+ public:
+  void clear()
+  {
+    bits.reset();
+  }
+
+  void insert(T value)
+  {
+    uint64_t h1 = H1()(value);
+    uint64_t h2 = H2()(value);
+
+    for (size_t i = 0; i < K; i++) {
+      size_t idx = (h1 + i * h2) % M;
+      bits[idx] = true;
+    }
+  }
+
+  bool contains(T value) const
+  {
+    uint64_t h1 = H1()(value);
+    uint64_t h2 = H2()(value);
+
+    for (size_t i = 0; i < K; i++) {
+      size_t idx = (h1 + i * h2) % M;
+      if (!bits[idx])
+        return false;
+    }
+
+    return true;
+  }
+
+ private:
+  std::bitset<M> bits;
+};
+
+#endif

riscv/execute.cc

@@ -223,15 +223,6 @@ void processor_t::step(size_t n)
     }
   }
 
-  if (extension_enabled(EXT_ZICCID)) {
-    // Ziccid requires stores eventually become visible to instruction fetch,
-    // so periodically flush the I$
-    if (ziccid_flush_count-- == 0) {
-      ziccid_flush_count += ZICCID_FLUSH_PERIOD;
-      _mmu->flush_icache();
-    }
-  }
-
   while (n > 0) {
     size_t instret = 0;
     reg_t pc = state.pc;
@@ -314,7 +305,7 @@ void processor_t::step(size_t n)
         for (auto ic_entry = _mmu->access_icache(pc); ; ) {
           auto fetch = ic_entry->data;
           pc = execute_insn_fast(this, pc, fetch);
-          ic_entry = ic_entry->next;
+          ic_entry = &_mmu->icache[_mmu->icache_index(pc)];
           if (unlikely(ic_entry->tag != pc))
             break;
           if (unlikely(instret + 1 == n))

riscv/mmu.cc

@@ -112,6 +112,14 @@ mmu_t::insn_parcel_t mmu_t::fetch_slow_path(reg_t vaddr)
     paddr = translate(access_info, sizeof(insn_parcel_t));
     host_addr = (uintptr_t)sim->addr_to_mem(paddr);
 
+    if (proc->extension_enabled(EXT_ZICCID)) {
+      // Maintain exclusion with all store TLBs
+      for (auto [_, p2] : sim->get_harts())
+        p2->mmu->flush_stlb_ppn(paddr >> PGSHIFT);
+
+      tlb_insn_reverse_tags.insert(paddr >> PGSHIFT);
+    }
+
     refill_tlb(vaddr, paddr, (char*)host_addr, FETCH);
   }
 
@@ -326,6 +334,14 @@ void mmu_t::store_slow_path_intrapage(reg_t len, const uint8_t* bytes, mem_acces
     paddr = translate(access_info, len);
     host_addr = (uintptr_t)sim->addr_to_mem(paddr);
 
+    if (proc && proc->extension_enabled(EXT_ZICCID)) {
+      // Maintain exclusion with all instruction TLBs
+      for (auto [_, p2] : sim->get_harts())
+        p2->mmu->flush_itlb_ppn(paddr >> PGSHIFT);
+
+      tlb_store_reverse_tags.insert(paddr >> PGSHIFT);
+    }
+
     if (!access_info.flags.is_special_access())
       refill_tlb(vaddr, paddr, (char*)host_addr, STORE);
   }
@@ -394,6 +410,35 @@ void mmu_t::store_slow_path(reg_t original_addr, reg_t len, const uint8_t* bytes
   }
 }
 
+bool mmu_t::flush_tlb_ppn(reg_t ppn, dtlb_entry_t* tlb, reverse_tags_t& filter)
+{
+  if (!filter.contains(ppn))
+    return false;
+
+  filter.clear();
+
+  for (size_t i = 0; i < TLB_ENTRIES; i++) {
+    auto entry_ppn = tlb[i].data.target_addr >> PGSHIFT;
+    if (entry_ppn == ppn)
+      tlb[i].tag = -1;
+    else if (tlb[i].tag != (reg_t)-1)
+      filter.insert(entry_ppn);
+  }
+
+  return true;
+}
+
+void mmu_t::flush_stlb_ppn(reg_t ppn)
+{
+  flush_tlb_ppn(ppn, tlb_store, tlb_store_reverse_tags);
+}
+
+void mmu_t::flush_itlb_ppn(reg_t ppn)
+{
+  if (flush_tlb_ppn(ppn, tlb_insn, tlb_insn_reverse_tags))
+    flush_icache();
+}
+
 tlb_entry_t mmu_t::refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type)
 {
   reg_t idx = (vaddr >> PGSHIFT) % TLB_ENTRIES;

riscv/mmu.h

@@ -3,6 +3,7 @@
 #ifndef _RISCV_MMU_H
 #define _RISCV_MMU_H
 
+#include "bloom_filter.h"
 #include "decode.h"
 #include "trap.h"
 #include "common.h"
@@ -42,7 +43,6 @@ struct insn_fetch_t
 
 struct icache_entry_t {
   reg_t tag;
-  struct icache_entry_t* next;
   insn_fetch_t data;
 };
 
@@ -292,7 +292,7 @@ public:
     return have_reservation;
   }
 
-  static const reg_t ICACHE_ENTRIES = 1024;
+  static const reg_t ICACHE_ENTRIES = 4096;
 
   inline size_t icache_index(reg_t addr)
   {
@@ -312,26 +312,15 @@ public:
   inline icache_entry_t* refill_icache(reg_t addr, icache_entry_t* entry)
   {
     insn_bits_t insn = fetch_insn_parcel(addr);
+    unsigned length = insn_length(insn);
 
-    int length = insn_length(insn);
+    for (unsigned pos = sizeof(insn_parcel_t); pos < length; pos += sizeof(insn_parcel_t)) {
-
+      insn |= fetch_insn_parcel(addr + pos) << (8 * pos);
-    if (likely(length == 4)) {
+      length = insn_length(insn);
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 2) << 16;
-    } else if (length == 2) {
-      // entire instruction already fetched
-    } else if (length == 6) {
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 2) << 16;
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 4) << 32;
-    } else {
-      static_assert(sizeof(insn_bits_t) == 8, "insn_bits_t must be uint64_t");
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 2) << 16;
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 4) << 32;
-      insn |= (insn_bits_t)fetch_insn_parcel(addr + 6) << 48;
     }
 
     insn_fetch_t fetch = {proc->decode_insn(insn), insn};
     entry->tag = addr;
-    entry->next = &icache[icache_index(addr + length)];
     entry->data = fetch;
 
     auto [check_tracer, _, paddr] = access_tlb(tlb_insn, addr, TLB_FLAGS, TLB_CHECK_TRACER);
@@ -419,6 +408,14 @@ private:
   dtlb_entry_t tlb_store[TLB_ENTRIES];
   dtlb_entry_t tlb_insn[TLB_ENTRIES];
 
+  typedef bloom_filter_t<reg_t, simple_hash1, simple_hash2, TLB_ENTRIES * 16, 3> reverse_tags_t;
+  reverse_tags_t tlb_store_reverse_tags;
+  reverse_tags_t tlb_insn_reverse_tags;
+
+  bool flush_tlb_ppn(reg_t ppn, dtlb_entry_t* tlb, reverse_tags_t& filter);
+  void flush_itlb_ppn(reg_t ppn);
+  void flush_stlb_ppn(reg_t ppn);
+
   // finish translation on a TLB miss and update the TLB
   tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type);
   const char* fill_from_mmio(reg_t vaddr, reg_t paddr);

riscv/processor.h

@@ -414,9 +414,6 @@ private:
   static const size_t OPCODE_CACHE_SIZE = 4095;
   opcode_cache_entry_t opcode_cache[OPCODE_CACHE_SIZE];
 
-  unsigned ziccid_flush_count = 0;
-  static const unsigned ZICCID_FLUSH_PERIOD = 10;
-
   void take_pending_interrupt() { take_interrupt(state.mip->read() & state.mie->read()); }
   void take_interrupt(reg_t mask); // take first enabled interrupt in mask
   void take_trap(trap_t& t, reg_t epc); // take an exception