diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp
index 43c924fa02..61b8a396a8 100644
--- a/src/core/hle/kernel/scheduler.cpp
+++ b/src/core/hle/kernel/scheduler.cpp
@@ -736,15 +736,15 @@ void Scheduler::SwitchContext() {
         previous_thread->context_guard.unlock();
     }
 
-    std::shared_ptr<Common::Fiber> old_context;
+    std::shared_ptr<Common::Fiber>* old_context;
     if (previous_thread != nullptr) {
-        old_context = previous_thread->GetHostContext();
+        old_context = &previous_thread->GetHostContext();
     } else {
-        old_context = idle_thread->GetHostContext();
+        old_context = &idle_thread->GetHostContext();
     }
     guard.unlock();
 
-    Common::Fiber::YieldTo(old_context, switch_fiber);
+    Common::Fiber::YieldTo(*old_context, switch_fiber);
     /// When a thread wakes up, the scheduler may have changed to other in another core.
     auto& next_scheduler = system.Kernel().CurrentScheduler();
     next_scheduler.SwitchContextStep2();
@@ -774,13 +774,13 @@ void Scheduler::SwitchToCurrent() {
                     break;
                 }
             }
-            std::shared_ptr<Common::Fiber> next_context;
+            std::shared_ptr<Common::Fiber>* next_context;
             if (current_thread != nullptr) {
-                next_context = current_thread->GetHostContext();
+                next_context = &current_thread->GetHostContext();
             } else {
-                next_context = idle_thread->GetHostContext();
+                next_context = &idle_thread->GetHostContext();
             }
-            Common::Fiber::YieldTo(switch_fiber, next_context);
+            Common::Fiber::YieldTo(switch_fiber, *next_context);
         }
     }
 }
diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/scheduler.h
index 10dc4b832a..3481071603 100644
--- a/src/core/hle/kernel/scheduler.h
+++ b/src/core/hle/kernel/scheduler.h
@@ -236,7 +236,7 @@ public:
 
     void OnThreadStart();
 
-    std::shared_ptr<Common::Fiber> ControlContext() {
+    std::shared_ptr<Common::Fiber>& ControlContext() {
         return switch_fiber;
     }
 
diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp
index fba2a9c85a..2b10926978 100644
--- a/src/core/hle/kernel/thread.cpp
+++ b/src/core/hle/kernel/thread.cpp
@@ -150,7 +150,7 @@ static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context,
     context.fpcr = 0;
 }
 
-std::shared_ptr<Common::Fiber> Thread::GetHostContext() const {
+std::shared_ptr<Common::Fiber>& Thread::GetHostContext() {
     return host_context;
 }
 
diff --git a/src/core/hle/kernel/thread.h b/src/core/hle/kernel/thread.h
index 3ae0df6ef8..c0342c4625 100644
--- a/src/core/hle/kernel/thread.h
+++ b/src/core/hle/kernel/thread.h
@@ -342,7 +342,7 @@ public:
         was_running = value;
     }
 
-    std::shared_ptr<Common::Fiber> GetHostContext() const;
+    std::shared_ptr<Common::Fiber>& GetHostContext();
 
     ThreadStatus GetStatus() const {
         return status;