diff --git a/src/video_core/rasterizer_cache.h b/src/video_core/rasterizer_cache.h
index 4a34491a9f..76743a85ba 100644
--- a/src/video_core/rasterizer_cache.h
+++ b/src/video_core/rasterizer_cache.h
@@ -11,6 +11,7 @@
 
 #include "common/common_types.h"
 #include "core/core.h"
+#include "core/settings.h"
 #include "video_core/rasterizer_interface.h"
 #include "video_core/renderer_base.h"
 
@@ -87,7 +88,12 @@ protected:
     void Unregister(const T& object) {
         auto& rasterizer = Core::System::GetInstance().Renderer().Rasterizer();
         rasterizer.UpdatePagesCachedCount(object->GetAddr(), object->GetSizeInBytes(), -1);
-        object->Flush();
+
+        if (Settings::values.use_accurate_framebuffers) {
+            // Only flush if use_accurate_framebuffers is enabled, as it incurs a performance hit
+            object->Flush();
+        }
+
         object_cache.subtract({GetInterval(object), ObjectSet{object}});
     }
 
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 18db072171..0485dfb7a3 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -637,7 +637,12 @@ void RasterizerOpenGL::FlushAll() {}
 
 void RasterizerOpenGL::FlushRegion(VAddr addr, u64 size) {
     MICROPROFILE_SCOPE(OpenGL_CacheManagement);
-    res_cache.FlushRegion(addr, size);
+
+    if (Settings::values.use_accurate_framebuffers) {
+        // Only flush if use_accurate_framebuffers is enabled, as it incurs a performance hit
+        res_cache.FlushRegion(addr, size);
+    }
+
     shader_cache.FlushRegion(addr, size);
     buffer_cache.FlushRegion(addr, size);
 }