diff --git a/src/core/gpu.cpp b/src/core/gpu.cpp
index c4f5e920c..b35d822c2 100644
--- a/src/core/gpu.cpp
+++ b/src/core/gpu.cpp
@@ -112,7 +112,7 @@ void GPU::SoftReset()
   UpdateCommandTickEvent();
 }
 
-bool GPU::DoState(StateWrapper& sw)
+bool GPU::DoState(StateWrapper& sw, bool update_display)
 {
   if (sw.IsReading())
   {
@@ -225,7 +225,9 @@ bool GPU::DoState(StateWrapper& sw)
     m_GPUSTAT.bits = old_GPUSTAT;
 
     UpdateCRTCConfig();
-    UpdateDisplay();
+    if (update_display)
+      UpdateDisplay();
+
     UpdateCRTCTickEvent();
     UpdateCommandTickEvent();
   }
diff --git a/src/core/gpu.h b/src/core/gpu.h
index 8921a42f8..4c8ab4acd 100644
--- a/src/core/gpu.h
+++ b/src/core/gpu.h
@@ -64,7 +64,7 @@ public:
 
   virtual bool Initialize(HostDisplay* host_display);
   virtual void Reset();
-  virtual bool DoState(StateWrapper& sw);
+  virtual bool DoState(StateWrapper& sw, bool update_display);
 
   // Graphics API state reset/restore - call when drawing the UI etc.
   virtual void ResetGraphicsAPIState();
diff --git a/src/core/gpu_hw.cpp b/src/core/gpu_hw.cpp
index 3fd97a530..11d0cd7b0 100644
--- a/src/core/gpu_hw.cpp
+++ b/src/core/gpu_hw.cpp
@@ -95,9 +95,9 @@ void GPU_HW::Reset()
   SetFullVRAMDirtyRectangle();
 }
 
-bool GPU_HW::DoState(StateWrapper& sw)
+bool GPU_HW::DoState(StateWrapper& sw, bool update_display)
 {
-  if (!GPU::DoState(sw))
+  if (!GPU::DoState(sw, update_display))
     return false;
 
   // invalidate the whole VRAM read texture when loading state
diff --git a/src/core/gpu_hw.h b/src/core/gpu_hw.h
index c06f14069..583d2af94 100644
--- a/src/core/gpu_hw.h
+++ b/src/core/gpu_hw.h
@@ -33,7 +33,7 @@ public:
 
   virtual bool Initialize(HostDisplay* host_display) override;
   virtual void Reset() override;
-  virtual bool DoState(StateWrapper& sw) override;
+  virtual bool DoState(StateWrapper& sw, bool update_display) override;
 
   void UpdateResolutionScale() override final;
   std::tuple<u32, u32> GetEffectiveDisplayResolution() override final;
diff --git a/src/core/system.cpp b/src/core/system.cpp
index 810fb0e04..1948c5bee 100644
--- a/src/core/system.cpp
+++ b/src/core/system.cpp
@@ -57,8 +57,8 @@ static bool SetExpansionROM(const char* filename);
 /// Opens CD image, preloading if needed.
 static std::unique_ptr<CDImage> OpenCDImage(const char* path, bool force_preload);
 
-static bool DoLoadState(ByteStream* stream, bool force_software_renderer);
-static bool DoState(StateWrapper& sw);
+static bool DoLoadState(ByteStream* stream, bool force_software_renderer, bool update_display);
+static bool DoState(StateWrapper& sw, bool update_display);
 static bool CreateGPU(GPURenderer renderer);
 
 static bool Initialize(bool force_software_renderer);
@@ -486,14 +486,14 @@ std::optional<DiscRegion> GetRegionForPath(const char* image_path)
   return GetRegionForImage(cdi.get());
 }
 
-bool RecreateGPU(GPURenderer renderer)
+bool RecreateGPU(GPURenderer renderer, bool update_display /* = true*/)
 {
   g_gpu->RestoreGraphicsAPIState();
 
   // save current state
   std::unique_ptr<ByteStream> state_stream = ByteStream_CreateGrowableMemoryStream();
   StateWrapper sw(state_stream.get(), StateWrapper::Mode::Write, SAVE_STATE_VERSION);
-  const bool state_valid = g_gpu->DoState(sw) && TimingEvents::DoState(sw);
+  const bool state_valid = g_gpu->DoState(sw, false) && TimingEvents::DoState(sw);
   if (!state_valid)
     Log_ErrorPrintf("Failed to save old GPU state when switching renderers");
 
@@ -515,7 +515,7 @@ bool RecreateGPU(GPURenderer renderer)
     state_stream->SeekAbsolute(0);
     sw.SetMode(StateWrapper::Mode::Read);
     g_gpu->RestoreGraphicsAPIState();
-    g_gpu->DoState(sw);
+    g_gpu->DoState(sw, update_display);
     TimingEvents::DoState(sw);
     g_gpu->ResetGraphicsAPIState();
   }
@@ -551,7 +551,7 @@ bool Boot(const SystemBootParameters& params)
 
   if (params.state_stream)
   {
-    if (!DoLoadState(params.state_stream.get(), params.force_software_renderer))
+    if (!DoLoadState(params.state_stream.get(), params.force_software_renderer, true))
     {
       Shutdown();
       return false;
@@ -828,7 +828,7 @@ bool CreateGPU(GPURenderer renderer)
   return true;
 }
 
-bool DoState(StateWrapper& sw)
+bool DoState(StateWrapper& sw, bool update_display)
 {
   if (!sw.DoMarker("System"))
     return false;
@@ -853,7 +853,7 @@ bool DoState(StateWrapper& sw)
     return false;
 
   g_gpu->RestoreGraphicsAPIState();
-  const bool gpu_result = sw.DoMarker("GPU") && g_gpu->DoState(sw);
+  const bool gpu_result = sw.DoMarker("GPU") && g_gpu->DoState(sw, update_display);
   g_gpu->ResetGraphicsAPIState();
   if (!gpu_result)
     return false;
@@ -934,15 +934,15 @@ void Reset()
   g_gpu->ResetGraphicsAPIState();
 }
 
-bool LoadState(ByteStream* state)
+bool LoadState(ByteStream* state, bool update_display)
 {
   if (IsShutdown())
     return false;
 
-  return DoLoadState(state, false);
+  return DoLoadState(state, false, update_display);
 }
 
-bool DoLoadState(ByteStream* state, bool force_software_renderer)
+bool DoLoadState(ByteStream* state, bool force_software_renderer, bool update_display)
 {
   SAVE_STATE_HEADER header;
   if (!state->Read2(&header, sizeof(header)))
@@ -1047,7 +1047,7 @@ bool DoLoadState(ByteStream* state, bool force_software_renderer)
     return false;
 
   StateWrapper sw(state, StateWrapper::Mode::Read, header.version);
-  if (!DoState(sw))
+  if (!DoState(sw, update_display))
     return false;
 
   if (s_state == State::Starting)
@@ -1114,7 +1114,7 @@ bool SaveState(ByteStream* state, u32 screenshot_size /* = 128 */)
     g_gpu->RestoreGraphicsAPIState();
 
     StateWrapper sw(state, StateWrapper::Mode::Write, SAVE_STATE_VERSION);
-    const bool result = DoState(sw);
+    const bool result = DoState(sw, false);
 
     g_gpu->ResetGraphicsAPIState();
 
diff --git a/src/core/system.h b/src/core/system.h
index 45aedf46f..85345ebfc 100644
--- a/src/core/system.h
+++ b/src/core/system.h
@@ -138,11 +138,11 @@ bool Boot(const SystemBootParameters& params);
 void Reset();
 void Shutdown();
 
-bool LoadState(ByteStream* state);
+bool LoadState(ByteStream* state, bool update_display = true);
 bool SaveState(ByteStream* state, u32 screenshot_size = 128);
 
 /// Recreates the GPU component, saving/loading the state so it is preserved. Call when the GPU renderer changes.
-bool RecreateGPU(GPURenderer renderer);
+bool RecreateGPU(GPURenderer renderer, bool update_display = true);
 
 void RunFrame();
 
diff --git a/src/duckstation-libretro/libretro_host_interface.cpp b/src/duckstation-libretro/libretro_host_interface.cpp
index 2bc26eb14..890a025c7 100644
--- a/src/duckstation-libretro/libretro_host_interface.cpp
+++ b/src/duckstation-libretro/libretro_host_interface.cpp
@@ -376,7 +376,7 @@ bool LibretroHostInterface::retro_serialize(void* data, size_t size)
 bool LibretroHostInterface::retro_unserialize(const void* data, size_t size)
 {
   std::unique_ptr<ByteStream> stream = ByteStream_CreateReadOnlyMemoryStream(data, static_cast<u32>(size));
-  if (!System::LoadState(stream.get()))
+  if (!System::LoadState(stream.get(), false))
   {
     Log_ErrorPrintf("Failed to load save state from memory stream");
     return false;
@@ -1229,15 +1229,22 @@ void LibretroHostInterface::HardwareRendererContextDestroy()
 void LibretroHostInterface::SwitchToSoftwareRenderer()
 {
   // keep the hw renderer around in case we need it later
+  // but keep it active until we've recreated the GPU so we can save the state
+  std::unique_ptr<HostDisplay> save_display;
   if (m_using_hardware_renderer)
   {
-    m_hw_render_display = std::move(m_display);
-    m_hw_render_display->DestroyResources();
+    save_display = std::move(m_display);
     m_using_hardware_renderer = false;
   }
 
   m_display = std::make_unique<LibretroHostDisplay>();
-  System::RecreateGPU(GPURenderer::Software);
+  System::RecreateGPU(GPURenderer::Software, false);
+
+  if (save_display)
+  {
+    save_display->DestroyResources();
+    m_hw_render_display = std::move(save_display);
+  }
 }
 
 bool LibretroHostInterface::DiskControlSetEjectState(bool ejected)