From 109566fc8feb1d5dadf04aff474b9ec459aa67c8 Mon Sep 17 00:00:00 2001
From: ameerj <52414509+ameerj@users.noreply.github.com>
Date: Mon, 21 Mar 2022 20:21:09 -0400
Subject: [PATCH 1/2] codec: Disable HW_FRAMES method check on Windows

It was reported that this method causes crashes on certain Linux decoding backends, hence the check to avoid it.

This subsequently caused Windows GPU decoders to never be selected and always fall back to CPU decoding, disable the check on Windows for now.
---
 .../command_classes/codecs/codec.cpp          | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/video_core/command_classes/codecs/codec.cpp b/src/video_core/command_classes/codecs/codec.cpp
index 04d0f3a2fe..921f54a41f 100644
--- a/src/video_core/command_classes/codecs/codec.cpp
+++ b/src/video_core/command_classes/codecs/codec.cpp
@@ -57,6 +57,18 @@ AVPixelFormat GetGpuFormat(AVCodecContext* av_codec_ctx, const AVPixelFormat* pi
     av_codec_ctx->pix_fmt = PREFERRED_CPU_FMT;
     return PREFERRED_CPU_FMT;
 }
+
+// List all the currently available hwcontext in ffmpeg
+std::vector<AVHWDeviceType> ListSupportedContexts() {
+    std::vector<AVHWDeviceType> contexts{};
+    AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE;
+    do {
+        current_device_type = av_hwdevice_iterate_types(current_device_type);
+        contexts.push_back(current_device_type);
+    } while (current_device_type != AV_HWDEVICE_TYPE_NONE);
+    return contexts;
+}
+
 } // namespace
 
 void AVFrameDeleter(AVFrame* ptr) {
@@ -77,17 +89,6 @@ Codec::~Codec() {
     av_buffer_unref(&av_gpu_decoder);
 }
 
-// List all the currently available hwcontext in ffmpeg
-static std::vector<AVHWDeviceType> ListSupportedContexts() {
-    std::vector<AVHWDeviceType> contexts{};
-    AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE;
-    do {
-        current_device_type = av_hwdevice_iterate_types(current_device_type);
-        contexts.push_back(current_device_type);
-    } while (current_device_type != AV_HWDEVICE_TYPE_NONE);
-    return contexts;
-}
-
 bool Codec::CreateGpuAvDevice() {
     static constexpr auto HW_CONFIG_METHOD = AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX;
     static const auto supported_contexts = ListSupportedContexts();
@@ -128,15 +129,19 @@ bool Codec::CreateGpuAvDevice() {
                           av_codec->name, av_hwdevice_get_type_name(type));
                 break;
             }
-            if (config->methods & HW_CONFIG_METHOD && config->device_type == type) {
-                av_codec_ctx->pix_fmt = config->pix_fmt;
-                if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) {
+            if ((config->methods & HW_CONFIG_METHOD) != 0 && config->device_type == type) {
+#if defined(__unix__)
+                // Some linux decoding backends are reported to crash with this config method
+                // TODO(ameerj): Properly support this method
+                if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) != 0) {
                     // skip zero-copy decoders, we don't currently support them
                     LOG_DEBUG(Service_NVDRV, "Skipping decoder {} with unsupported capability {}.",
                               av_hwdevice_get_type_name(type), config->methods);
                     continue;
                 }
+#endif
                 LOG_INFO(Service_NVDRV, "Using {} GPU decoder", av_hwdevice_get_type_name(type));
+                av_codec_ctx->pix_fmt = config->pix_fmt;
                 return true;
             }
         }

From 15f9472b150e8881f8e034f9571c9aa578b0821c Mon Sep 17 00:00:00 2001
From: ameerj <52414509+ameerj@users.noreply.github.com>
Date: Mon, 21 Mar 2022 20:22:52 -0400
Subject: [PATCH 2/2] codec: Plug GPU decoder memory leak

---
 src/video_core/command_classes/codecs/codec.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/video_core/command_classes/codecs/codec.cpp b/src/video_core/command_classes/codecs/codec.cpp
index 921f54a41f..c9ab11e538 100644
--- a/src/video_core/command_classes/codecs/codec.cpp
+++ b/src/video_core/command_classes/codecs/codec.cpp
@@ -98,6 +98,8 @@ bool Codec::CreateGpuAvDevice() {
             LOG_DEBUG(Service_NVDRV, "{} explicitly unsupported", av_hwdevice_get_type_name(type));
             continue;
         }
+        // Avoid memory leak from not cleaning up after av_hwdevice_ctx_create
+        av_buffer_unref(&av_gpu_decoder);
         const int hwdevice_res = av_hwdevice_ctx_create(&av_gpu_decoder, type, nullptr, nullptr, 0);
         if (hwdevice_res < 0) {
             LOG_DEBUG(Service_NVDRV, "{} av_hwdevice_ctx_create failed {}",