diff --git a/src/core/bus.cpp b/src/core/bus.cpp
index 5edb10c7c..ac13f931d 100644
--- a/src/core/bus.cpp
+++ b/src/core/bus.cpp
@@ -314,28 +314,31 @@ void UpdateFastmemViews(bool enabled, bool isolate_cache)
   {
     // KUSEG - cached
     MapRAM(0x00000000, !isolate_cache);
-    //MapRAM(0x00200000, !isolate_cache);
-    //MapRAM(0x00400000, !isolate_cache);
-    //MapRAM(0x00600000, !isolate_cache);
+    // MapRAM(0x00200000, !isolate_cache);
+    // MapRAM(0x00400000, !isolate_cache);
+    // MapRAM(0x00600000, !isolate_cache);
 
     // KSEG0 - cached
     MapRAM(0x80000000, !isolate_cache);
-    //MapRAM(0x80200000, !isolate_cache);
-    //MapRAM(0x80400000, !isolate_cache);
-    //MapRAM(0x80600000, !isolate_cache);
+    // MapRAM(0x80200000, !isolate_cache);
+    // MapRAM(0x80400000, !isolate_cache);
+    // MapRAM(0x80600000, !isolate_cache);
   }
 
   // KSEG1 - uncached
   MapRAM(0xA0000000, true);
-  //MapRAM(0xA0200000, true);
-  //MapRAM(0xA0400000, true);
-  //MapRAM(0xA0600000, true);
+  // MapRAM(0xA0200000, true);
+  // MapRAM(0xA0400000, true);
+  // MapRAM(0xA0600000, true);
 }
 
 bool CanUseFastmemForAddress(VirtualMemoryAddress address)
 {
   const PhysicalMemoryAddress paddr = address & CPU::PHYSICAL_MEMORY_ADDRESS_MASK;
-  return IsRAMAddress(paddr);
+
+  // Currently since we don't map the mirrors, don't use fastmem for them.
+  // This is because the swapping of page code bits for SMC is too expensive.
+  return (paddr < RAM_SIZE);
 }
 
 bool IsRAMCodePage(u32 index)
diff --git a/src/core/cpu_recompiler_code_generator_generic.cpp b/src/core/cpu_recompiler_code_generator_generic.cpp
index f03746ca8..ce64e22f7 100644
--- a/src/core/cpu_recompiler_code_generator_generic.cpp
+++ b/src/core/cpu_recompiler_code_generator_generic.cpp
@@ -42,9 +42,15 @@ Value CodeGenerator::EmitLoadGuestMemory(const CodeBlockInstruction& cbi, const
       Value result = m_register_cache.AllocateScratch(size);
 
       if (g_settings.IsUsingFastmem() && Bus::IsRAMAddress(static_cast<u32>(address.constant_value)))
-        EmitLoadGuestRAMFastmem(address, size, result);
+      {
+        // have to mask away the high bits for mirrors, since we don't map them in fastmem
+        EmitLoadGuestRAMFastmem(Value::FromConstantU32(static_cast<u32>(address.constant_value) & Bus::RAM_MASK), size,
+                                result);
+      }
       else
+      {
         EmitLoadGlobal(result.GetHostRegister(), size, ptr);
+      }
 
       m_delayed_cycles_add += read_ticks;
       return result;