Slightly speed up TLB lookups for blink -m mode (#146)

Co-authored-by: tkchia <tkchia-cosmo@gmx.com>
jart · Jul 2, 2023 · caf97b1 · caf97b1
1 parent 671aa0b
commit caf97b1
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 6 deletions.
diff --git a/blink/builtin.h b/blink/builtin.h
@@ -125,6 +125,10 @@
 #endif /* __cplusplus */
 #endif
 
+#ifndef flattencalls
+#define flattencalls __attribute__((__flatten__))
+#endif
+
 #ifndef relegated
 #if __has_attribute(__cold__) || \
     (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 403

diff --git a/blink/machine.h b/blink/machine.h
@@ -806,7 +806,8 @@ void LogCodOp(struct Machine *, const char *);
 #endif
 
 MICRO_OP_SAFE u8 Cpl(struct Machine *m) {
-  return m->mode.genmode != XED_GEN_MODE_REAL ? (m->cs.sel & 3u) : 0u;
+  return !m->metal ? 3u :
+         m->mode.genmode != XED_GEN_MODE_REAL ? (m->cs.sel & 3u) : 0u;
 }
 
 #define BEGIN_NO_PAGE_FAULTS \

diff --git a/blink/memory.c b/blink/memory.c
@@ -184,13 +184,13 @@ u64 FindPageTableEntry(struct Machine *m, u64 page) {
   u64 entry;
   long tlbkey;
   unsigned level, index;
-  if (atomic_load_explicit(&m->invalidated, memory_order_acquire)) {
+  if (UNLIKELY(atomic_load_explicit(&m->invalidated, memory_order_acquire))) {
     ResetTlb(m);
     atomic_store_explicit(&m->invalidated, false, memory_order_relaxed);
   }
   tlbkey = (page >> 12) & (ARRAYLEN(m->tlb) - 1);
-  if (m->tlb[tlbkey].page == page &&
-      ((entry = m->tlb[tlbkey].entry) & PAGE_V)) {
+  if (LIKELY(m->tlb[tlbkey].page == page &&
+             ((entry = m->tlb[tlbkey].entry) & PAGE_V))) {
     STATISTIC(++tlb_hits);
     return entry;
   }
@@ -263,7 +263,8 @@ u64 FindPageTableEntry(struct Machine *m, u64 page) {
 u8 *LookupAddress2(struct Machine *m, i64 virt, u64 mask, u64 need) {
   u8 *host;
   u64 entry;
-  if (m->mode.omode == XED_MODE_LONG ||
+  if (!m->metal ||
+      m->mode.omode == XED_MODE_LONG ||
       (m->mode.genmode != XED_GEN_MODE_REAL && (m->system->cr0 & CR0_PG))) {
     if (!(entry = FindPageTableEntry(m, virt & -4096))) {
       return 0;
@@ -301,7 +302,7 @@ u8 *LookupAddress(struct Machine *m, i64 virt) {
   return LookupAddress2(m, virt, need, need);
 }
 
-u8 *GetAddress(struct Machine *m, i64 v) {
+flattencalls u8 *GetAddress(struct Machine *m, i64 v) {
   if (HasLinearMapping()) return ToHost(v);
   return LookupAddress(m, v);
 }