diff --git a/src/video_core/renderer_opengl/gl_texture_cache.cpp b/src/video_core/renderer_opengl/gl_texture_cache.cpp
index ec1afd31a4..944a3aa65d 100644
--- a/src/video_core/renderer_opengl/gl_texture_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_texture_cache.cpp
@@ -967,21 +967,24 @@ bool Image::ScaleUp() {
     if (True(flags & ImageFlagBits::Rescaled)) {
         return false;
     }
+    flags |= ImageFlagBits::Rescaled;
     if (!runtime->resolution.active) {
         return false;
     }
     if (gl_format == 0 && gl_type == 0) {
         // compressed textures
+        flags &= ~ImageFlagBits::Rescaled;
         return false;
     }
     if (info.type == ImageType::Linear) {
-        UNIMPLEMENTED();
+        UNREACHABLE();
+        flags &= ~ImageFlagBits::Rescaled;
         return false;
     }
     if (!Scale()) {
+        flags &= ~ImageFlagBits::Rescaled;
         return false;
     }
-    flags |= ImageFlagBits::Rescaled;
     return true;
 }
 
@@ -990,6 +993,9 @@ bool Image::ScaleDown() {
         return false;
     }
     flags &= ~ImageFlagBits::Rescaled;
+    if (!runtime->resolution.active) {
+        return false;
+    }
     current_texture = texture.handle;
     return true;
 }
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 9b90c7d9bf..a4fbbc735f 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -1126,6 +1126,7 @@ bool Image::ScaleUp() {
         return false;
     }
     ASSERT(info.type != ImageType::Linear);
+    flags |= ImageFlagBits::Rescaled;
     const auto& resolution = runtime->resolution;
     if (!resolution.active) {
         return false;
@@ -1188,11 +1189,11 @@ bool Image::ScaleUp() {
                 dst_region, src_region, Tegra::Engines::Fermi2D::Filter::Point, BLIT_OPERATION);
         } else {
             // TODO: Use helper blits where applicable
+            flags &= ~ImageFlagBits::Rescaled;
             LOG_ERROR(Render_Vulkan, "Device does not support scaling format {}", format);
             return false;
         }
     }
-    flags |= ImageFlagBits::Rescaled;
     return true;
 }
 
@@ -1200,8 +1201,12 @@ bool Image::ScaleDown() {
     if (False(flags & ImageFlagBits::Rescaled)) {
         return false;
     }
-    ASSERT(info.type != ImageType::Linear);
     flags &= ~ImageFlagBits::Rescaled;
+    const auto& resolution = runtime->resolution;
+    if (!resolution.active) {
+        return false;
+    }
+    ASSERT(info.type != ImageType::Linear);
     current_image = *original_image;
     return true;
 }
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 261cb6c48a..c06cddae97 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -230,7 +230,8 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
                 auto& image = slot_images[image_id];
                 can_rescale &= ImageCanRescale(image);
                 any_blacklisted |= True(image.flags & ImageFlagBits::Blacklisted);
-                any_rescaled |= True(image.flags & ImageFlagBits::Rescaled);
+                any_rescaled |= True(image.flags & ImageFlagBits::Rescaled) ||
+                                GetFormatType(image.info.format) != SurfaceType::ColorTexture;
                 scale_rating = std::max<u32>(scale_rating, image.scale_tick <= frame_tick
                                                                ? image.scale_rating + 1U
                                                                : image.scale_rating);
@@ -857,7 +858,7 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(Image& image) {
     const f32 add_to_size = Settings::values.resolution_info.up_factor - 1.0f;
     const bool sign = std::signbit(add_to_size);
     const u32 image_size_bytes = std::max(image.guest_size_bytes, image.unswizzled_size_bytes);
-    const u64 tentative_size = image_size_bytes * static_cast<u32>(std::abs(add_to_size));
+    const u64 tentative_size = image_size_bytes * static_cast<u64>(std::abs(add_to_size));
     const u64 fitted_size = Common::AlignUp(tentative_size, 1024);
     return sign ? -fitted_size : fitted_size;
 }
@@ -879,7 +880,7 @@ bool TextureCache<P>::ScaleDown(Image& image) {
     if (!rescaled) {
         return false;
     }
-    total_used_memory += GetScaledImageSizeBytes(image);
+    total_used_memory -= GetScaledImageSizeBytes(image);
     InvalidateScale(image);
     return true;
 }