diff --git a/src/video_core/texture_cache/image_base.cpp b/src/video_core/texture_cache/image_base.cpp
index 6052d148ae..e9e725edf7 100644
--- a/src/video_core/texture_cache/image_base.cpp
+++ b/src/video_core/texture_cache/image_base.cpp
@@ -60,9 +60,9 @@ namespace {
 ImageBase::ImageBase(const ImageInfo& info_, GPUVAddr gpu_addr_, VAddr cpu_addr_)
     : info{info_}, guest_size_bytes{CalculateGuestSizeInBytes(info)},
       unswizzled_size_bytes{CalculateUnswizzledSizeBytes(info)},
-      converted_size_bytes{CalculateConvertedSizeBytes(info)}, gpu_addr{gpu_addr_},
-      cpu_addr{cpu_addr_}, cpu_addr_end{cpu_addr + guest_size_bytes},
-      mip_level_offsets{CalculateMipLevelOffsets(info)} {
+      converted_size_bytes{CalculateConvertedSizeBytes(info)}, scale_rating{},
+      scale_tick{}, gpu_addr{gpu_addr_}, cpu_addr{cpu_addr_},
+      cpu_addr_end{cpu_addr + guest_size_bytes}, mip_level_offsets{CalculateMipLevelOffsets(info)} {
     if (info.type == ImageType::e3D) {
         slice_offsets = CalculateSliceOffsets(info);
         slice_subresources = CalculateSliceSubresources(info);
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 10dd52e286..97f107b4dd 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -78,6 +78,8 @@ struct ImageBase {
     u32 guest_size_bytes = 0;
     u32 unswizzled_size_bytes = 0;
     u32 converted_size_bytes = 0;
+    u32 scale_rating = 0;
+    u64 scale_tick = 0;
     ImageFlagBits flags = ImageFlagBits::CpuModified;
 
     GPUVAddr gpu_addr = 0;
diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp
index 022ca90335..7fa8fd4fe0 100644
--- a/src/video_core/texture_cache/image_info.cpp
+++ b/src/video_core/texture_cache/image_info.cpp
@@ -31,6 +31,7 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
             .depth = config.block_depth,
         };
     }
+    rescaleable = false;
     tile_width_spacing = config.tile_width_spacing;
     if (config.texture_type != TextureType::Texture2D &&
         config.texture_type != TextureType::Texture2DNoMipmap) {
@@ -53,12 +54,14 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
     case TextureType::Texture2DNoMipmap:
         ASSERT(config.Depth() == 1);
         type = config.IsPitchLinear() ? ImageType::Linear : ImageType::e2D;
+        rescaleable = !config.IsPitchLinear();
         size.width = config.Width();
         size.height = config.Height();
         resources.layers = config.BaseLayer() + 1;
         break;
     case TextureType::Texture2DArray:
         type = ImageType::e2D;
+        rescaleable = true;
         size.width = config.Width();
         size.height = config.Height();
         resources.layers = config.BaseLayer() + config.Depth();
@@ -98,12 +101,14 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
         // FIXME: Call this without passing *this
         layer_stride = CalculateLayerStride(*this);
         maybe_unaligned_layer_stride = CalculateLayerSize(*this);
+        rescaleable &= (block.depth == 0) && resources.levels == 1;
     }
 }
 
 ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs, size_t index) noexcept {
     const auto& rt = regs.rt[index];
     format = VideoCore::Surface::PixelFormatFromRenderTargetFormat(rt.format);
+    rescaleable = false;
     if (rt.tile_mode.is_pitch_linear) {
         ASSERT(rt.tile_mode.is_3d == 0);
         type = ImageType::Linear;
@@ -129,6 +134,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs, size_t index)
         type = ImageType::e3D;
         size.depth = rt.depth;
     } else {
+        rescaleable = block.depth == 0 && size.height > 256;
         type = ImageType::e2D;
         resources.layers = rt.depth;
     }
@@ -138,6 +144,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs) noexcept {
     format = VideoCore::Surface::PixelFormatFromDepthFormat(regs.zeta.format);
     size.width = regs.zeta_width;
     size.height = regs.zeta_height;
+    rescaleable = false;
     resources.levels = 1;
     layer_stride = regs.zeta.layer_stride * 4;
     maybe_unaligned_layer_stride = layer_stride;
@@ -156,6 +163,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs) noexcept {
         type = ImageType::e3D;
         size.depth = regs.zeta_depth;
     } else {
+        rescaleable = block.depth == 0 && size.height > 256;
         type = ImageType::e2D;
         resources.layers = regs.zeta_depth;
     }
@@ -164,6 +172,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs) noexcept {
 ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept {
     UNIMPLEMENTED_IF_MSG(config.layer != 0, "Surface layer is not zero");
     format = VideoCore::Surface::PixelFormatFromRenderTargetFormat(config.format);
+    rescaleable = false;
     if (config.linear == Tegra::Engines::Fermi2D::MemoryLayout::Pitch) {
         type = ImageType::Linear;
         size = Extent3D{
@@ -174,6 +183,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept {
         pitch = config.pitch;
     } else {
         type = config.block_depth > 0 ? ImageType::e3D : ImageType::e2D;
+
         block = Extent3D{
             .width = config.block_width,
             .height = config.block_height,
@@ -186,6 +196,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept {
             .height = config.height,
             .depth = 1,
         };
+        rescaleable = block.depth == 0 && size.height > 256;
     }
 }
 
diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h
index 16d4cee371..e874d28702 100644
--- a/src/video_core/texture_cache/image_info.h
+++ b/src/video_core/texture_cache/image_info.h
@@ -33,6 +33,7 @@ struct ImageInfo {
     u32 maybe_unaligned_layer_stride = 0;
     u32 num_samples = 1;
     u32 tile_width_spacing = 0;
+    bool rescaleable = false;
 };
 
 } // namespace VideoCommon
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index ce5994d5fe..be40f6b881 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -216,7 +216,10 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
         return;
     }
 
+    u32 scale_rating;
     bool rescaled;
+    std::array<ImageId, NUM_RT> tmp_color_images{};
+    ImageId tmp_depth_image{};
     do {
         flags[Dirty::RenderTargets] = false;
 
@@ -226,10 +229,10 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
         const bool force = flags[Dirty::RenderTargetControl];
         flags[Dirty::RenderTargetControl] = false;
 
+        scale_rating = 0;
+        bool any_rescaled = false;
         bool can_rescale = true;
         bool any_blacklisted = false;
-        std::array<ImageId, NUM_RT> tmp_color_images{};
-        ImageId tmp_depth_image{};
         const auto check_rescale = [&](ImageViewId view_id, ImageId& id_save) {
             if (view_id) {
                 const auto& view = slot_image_views[view_id];
@@ -238,6 +241,10 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
                 auto& image = slot_images[image_id];
                 can_rescale &= ImageCanRescale(image);
                 any_blacklisted |= True(image.flags & ImageFlagBits::Blacklisted);
+                any_rescaled |= True(image.flags & ImageFlagBits::Rescaled);
+                scale_rating = std::max<u32>(scale_rating, image.scale_tick <= frame_tick
+                                                               ? image.scale_rating + 1U
+                                                               : image.scale_rating);
             } else {
                 id_save = CORRUPT_ID;
             }
@@ -257,17 +264,19 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
         check_rescale(render_targets.depth_buffer_id, tmp_depth_image);
 
         if (can_rescale) {
-            rescaled = true;
+            rescaled = any_rescaled || scale_rating >= 2;
             const auto scale_up = [this](ImageId image_id) {
                 if (image_id != CORRUPT_ID) {
                     Image& image = slot_images[image_id];
                     ScaleUp(image);
                 }
             };
-            for (size_t index = 0; index < NUM_RT; ++index) {
-                scale_up(tmp_color_images[index]);
+            if (rescaled) {
+                for (size_t index = 0; index < NUM_RT; ++index) {
+                    scale_up(tmp_color_images[index]);
+                }
+                scale_up(tmp_depth_image);
             }
-            scale_up(tmp_depth_image);
         } else {
             rescaled = false;
             const auto scale_down = [this, any_blacklisted](ImageId image_id) {
@@ -283,10 +292,23 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
                 scale_down(tmp_color_images[index]);
             }
             scale_down(tmp_depth_image);
+            scale_rating = 0;
         }
     } while (has_deleted_images);
     // Rescale End
 
+    const auto set_rating = [this, scale_rating](ImageId image_id) {
+        if (image_id != CORRUPT_ID) {
+            Image& image = slot_images[image_id];
+            image.scale_rating = scale_rating;
+            image.scale_tick = frame_tick + 1;
+        }
+    };
+    for (size_t index = 0; index < NUM_RT; ++index) {
+        set_rating(tmp_color_images[index]);
+    }
+    set_rating(tmp_depth_image);
+
     if (is_rescaling != rescaled) {
         flags[Dirty::RescaleViewports] = true;
         flags[Dirty::RescaleScissors] = true;
@@ -761,10 +783,7 @@ bool TextureCache<P>::ImageCanRescale(Image& image) {
         True(image.flags & ImageFlagBits::RescaleChecked)) {
         return true;
     }
-    const auto& info = image.info;
-    const bool can_this_rescale =
-        (info.type == ImageType::e1D || info.type == ImageType::e2D) && info.block.depth == 0;
-    if (!can_this_rescale) {
+    if (!image.info.rescaleable) {
         image.flags &= ~ImageFlagBits::RescaleChecked;
         return false;
     }
@@ -928,8 +947,7 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     };
     ForEachSparseImageInRegion(gpu_addr, size_bytes, region_check_gpu);
 
-    bool can_rescale =
-        (info.type == ImageType::e1D || info.type == ImageType::e2D) && info.block.depth == 0;
+    bool can_rescale = info.rescaleable;
     bool any_rescaled = false;
     bool any_blacklisted = false;
     for (const ImageId sibling_id : all_siblings) {