BoomMicrophone/suyu-nix-test
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #9135 from liamwhite/service-thread-event

Browse source 
kernel: invert session request handling flow
This commit is contained in:
bunnei 2022-11-03 21:45:56 -07:00 committed by GitHub
commit 37de88040c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 441 additions and 338 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/core/hle/ipc_helpers.h
View file

@ -86,13 +86,13 @@ public:
u32 num_domain_objects{};
const bool always_move_handles{
(static_cast<u32>(flags) & static_cast<u32>(Flags::AlwaysMoveHandles)) != 0};
if (!ctx.Session()->GetSessionRequestManager()->IsDomain() || always_move_handles) {
if (!ctx.GetManager()->IsDomain() || always_move_handles) {
num_handles_to_move = num_objects_to_move;
} else {
num_domain_objects = num_objects_to_move;
}
if (ctx.Session()->GetSessionRequestManager()->IsDomain()) {
if (ctx.GetManager()->IsDomain()) {
raw_data_size +=
static_cast<u32>(sizeof(DomainMessageHeader) / sizeof(u32) + num_domain_objects);
ctx.write_size += num_domain_objects;
@ -125,8 +125,7 @@ public:
if (!ctx.IsTipc()) {
AlignWithPadding();
if (ctx.Session()->GetSessionRequestManager()->IsDomain() &&
ctx.HasDomainMessageHeader()) {
if (ctx.GetManager()->IsDomain() && ctx.HasDomainMessageHeader()) {
IPC::DomainMessageHeader domain_header{};
domain_header.num_objects = num_domain_objects;
PushRaw(domain_header);
@ -146,18 +145,18 @@ public:
template <class T>
void PushIpcInterface(std::shared_ptr<T> iface) {
if (context->Session()->GetSessionRequestManager()->IsDomain()) {
if (context->GetManager()->IsDomain()) {
context->AddDomainObject(std::move(iface));
} else {
kernel.CurrentProcess()->GetResourceLimit()->Reserve(
Kernel::LimitableResource::Sessions, 1);
auto* session = Kernel::KSession::Create(kernel);
session->Initialize(nullptr, iface->GetServiceName(),
session->Initialize(nullptr, iface->GetServiceName());
iface->RegisterSession(&session->GetServerSession(),
std::make_shared<Kernel::SessionRequestManager>(kernel));
context->AddMoveObject(&session->GetClientSession());
iface->ClientConnected(&session->GetServerSession());
}
}
@ -387,7 +386,7 @@ public:
template <class T>
std::weak_ptr<T> PopIpcInterface() {
ASSERT(context->Session()->GetSessionRequestManager()->IsDomain());
ASSERT(context->GetManager()->IsDomain());
ASSERT(context->GetDomainMessageHeader().input_object_count > 0);
return context->GetDomainHandler<T>(Pop<u32>() - 1);
}

55
src/core/hle/kernel/hle_ipc.cpp
View file

@ -16,6 +16,7 @@
#include "core/hle/kernel/k_auto_object.h"
#include "core/hle/kernel/k_handle_table.h"
#include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/k_server_port.h"
#include "core/hle/kernel/k_server_session.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
@ -35,7 +36,21 @@ SessionRequestHandler::SessionRequestHandler(KernelCore& kernel_, const char* se
}
SessionRequestHandler::~SessionRequestHandler() {
kernel.ReleaseServiceThread(service_thread);
kernel.ReleaseServiceThread(service_thread.lock());
}
void SessionRequestHandler::AcceptSession(KServerPort* server_port) {
auto* server_session = server_port->AcceptSession();
ASSERT(server_session != nullptr);
RegisterSession(server_session, std::make_shared<SessionRequestManager>(kernel));
}
void SessionRequestHandler::RegisterSession(KServerSession* server_session,
std::shared_ptr<SessionRequestManager> manager) {
manager->SetSessionHandler(shared_from_this());
service_thread.lock()->RegisterServerSession(server_session, manager);
server_session->Close();
}
SessionRequestManager::SessionRequestManager(KernelCore& kernel_) : kernel{kernel_} {}
@ -92,7 +107,7 @@ Result SessionRequestManager::HandleDomainSyncRequest(KServerSession* server_ses
}
// Set domain handlers in HLE context, used for domain objects (IPC interfaces) as inputs
context.SetSessionRequestManager(server_session->GetSessionRequestManager());
ASSERT(context.GetManager().get() == this);
// If there is a DomainMessageHeader, then this is CommandType "Request"
const auto& domain_message_header = context.GetDomainMessageHeader();
@ -130,31 +145,6 @@ Result SessionRequestManager::HandleDomainSyncRequest(KServerSession* server_ses
return ResultSuccess;
}
Result SessionRequestManager::QueueSyncRequest(KSession* parent,
std::shared_ptr<HLERequestContext>&& context) {
// Ensure we have a session request handler
if (this->HasSessionRequestHandler(*context)) {
if (auto strong_ptr = this->GetServiceThread().lock()) {
strong_ptr->QueueSyncRequest(*parent, std::move(context));
} else {
ASSERT_MSG(false, "strong_ptr is nullptr!");
}
} else {
ASSERT_MSG(false, "handler is invalid!");
}
return ResultSuccess;
}
void SessionRequestHandler::ClientConnected(KServerSession* session) {
session->GetSessionRequestManager()->SetSessionHandler(shared_from_this());
// Ensure our server session is tracked globally.
kernel.RegisterServerObject(session);
}
void SessionRequestHandler::ClientDisconnected(KServerSession* session) {}
HLERequestContext::HLERequestContext(KernelCore& kernel_, Core::Memory::Memory& memory_,
KServerSession* server_session_, KThread* thread_)
: server_session(server_session_), thread(thread_), kernel{kernel_}, memory{memory_} {
@ -214,7 +204,7 @@ void HLERequestContext::ParseCommandBuffer(const KHandleTable& handle_table, u32
// Padding to align to 16 bytes
rp.AlignWithPadding();
if (Session()->GetSessionRequestManager()->IsDomain() &&
if (GetManager()->IsDomain() &&
((command_header->type == IPC::CommandType::Request ||
command_header->type == IPC::CommandType::RequestWithContext) ||
!incoming)) {
@ -223,7 +213,7 @@ void HLERequestContext::ParseCommandBuffer(const KHandleTable& handle_table, u32
if (incoming || domain_message_header) {
domain_message_header = rp.PopRaw<IPC::DomainMessageHeader>();
} else {
if (Session()->GetSessionRequestManager()->IsDomain()) {
if (GetManager()->IsDomain()) {
LOG_WARNING(IPC, "Domain request has no DomainMessageHeader!");
}
}
@ -316,12 +306,11 @@ Result HLERequestContext::WriteToOutgoingCommandBuffer(KThread& requesting_threa
// Write the domain objects to the command buffer, these go after the raw untranslated data.
// TODO(Subv): This completely ignores C buffers.
if (server_session->GetSessionRequestManager()->IsDomain()) {
if (GetManager()->IsDomain()) {
current_offset = domain_offset - static_cast<u32>(outgoing_domain_objects.size());
for (auto& object : outgoing_domain_objects) {
server_session->GetSessionRequestManager()->AppendDomainHandler(std::move(object));
cmd_buf[current_offset++] = static_cast<u32_le>(
server_session->GetSessionRequestManager()->DomainHandlerCount());
GetManager()->AppendDomainHandler(std::move(object));
cmd_buf[current_offset++] = static_cast<u32_le>(GetManager()->DomainHandlerCount());
}
}

29
src/core/hle/kernel/hle_ipc.h
View file

@ -45,11 +45,13 @@ class KAutoObject;
class KernelCore;
class KEvent;
class KHandleTable;
class KServerPort;
class KProcess;
class KServerSession;
class KThread;
class KReadableEvent;
class KSession;
class SessionRequestManager;
class ServiceThread;
enum class ThreadWakeupReason;
@ -76,19 +78,9 @@ public:
virtual Result HandleSyncRequest(Kernel::KServerSession& session,
Kernel::HLERequestContext& context) = 0;
/**
* Signals that a client has just connected to this HLE handler and keeps the
* associated ServerSession alive for the duration of the connection.
* @param server_session Owning pointer to the ServerSession associated with the connection.
*/
void ClientConnected(KServerSession* session);
/**
* Signals that a client has just disconnected from this HLE handler and releases the
* associated ServerSession.
* @param server_session ServerSession associated with the connection.
*/
void ClientDisconnected(KServerSession* session);
void AcceptSession(KServerPort* server_port);
void RegisterSession(KServerSession* server_session,
std::shared_ptr<SessionRequestManager> manager);
std::weak_ptr<ServiceThread> GetServiceThread() const {
return service_thread;
@ -170,7 +162,6 @@ public:
Result HandleDomainSyncRequest(KServerSession* server_session, HLERequestContext& context);
Result CompleteSyncRequest(KServerSession* server_session, HLERequestContext& context);
Result QueueSyncRequest(KSession* parent, std::shared_ptr<HLERequestContext>&& context);
private:
bool convert_to_domain{};
@ -350,11 +341,11 @@ public:
template <typename T>
std::shared_ptr<T> GetDomainHandler(std::size_t index) const {
return std::static_pointer_cast<T>(manager.lock()->DomainHandler(index).lock());
return std::static_pointer_cast<T>(GetManager()->DomainHandler(index).lock());
}
void SetSessionRequestManager(std::weak_ptr<SessionRequestManager> manager_) {
manager = std::move(manager_);
manager = manager_;
}
std::string Description() const;
@ -363,6 +354,10 @@ public:
return *thread;
}
std::shared_ptr<SessionRequestManager> GetManager() const {
return manager.lock();
}
private:
friend class IPC::ResponseBuilder;
@ -396,7 +391,7 @@ private:
u32 handles_offset{};
u32 domain_offset{};
std::weak_ptr<SessionRequestManager> manager;
std::weak_ptr<SessionRequestManager> manager{};
KernelCore& kernel;
Core::Memory::Memory& memory;

5
src/core/hle/kernel/k_client_port.cpp
View file

@ -58,8 +58,7 @@ bool KClientPort::IsSignaled() const {
return num_sessions < max_sessions;
}
Result KClientPort::CreateSession(KClientSession** out,
std::shared_ptr<SessionRequestManager> session_manager) {
Result KClientPort::CreateSession(KClientSession** out) {
// Reserve a new session from the resource limit.
KScopedResourceReservation session_reservation(kernel.CurrentProcess()->GetResourceLimit(),
LimitableResource::Sessions);
@ -104,7 +103,7 @@ Result KClientPort::CreateSession(KClientSession** out,
}
// Initialize the session.
session->Initialize(this, parent->GetName(), session_manager);
session->Initialize(this, parent->GetName());
// Commit the session reservation.
session_reservation.Commit();

3
src/core/hle/kernel/k_client_port.h
View file

@ -52,8 +52,7 @@ public:
void Destroy() override;
bool IsSignaled() const override;
Result CreateSession(KClientSession** out,
std::shared_ptr<SessionRequestManager> session_manager = nullptr);
Result CreateSession(KClientSession** out);
private:
std::atomic<s32> num_sessions{};

6
src/core/hle/kernel/k_port.cpp
View file

@ -57,12 +57,6 @@ Result KPort::EnqueueSession(KServerSession* session) {
server.EnqueueSession(session);
if (auto session_ptr = server.GetSessionRequestHandler().lock()) {
session_ptr->ClientConnected(server.AcceptSession());
} else {
ASSERT(false);
}
return ResultSuccess;
}

6
src/core/hle/kernel/k_server_port.cpp
View file

@ -61,12 +61,6 @@ void KServerPort::Destroy() {
// Close our reference to our parent.
parent->Close();
// Release host emulation members.
session_handler.reset();
// Ensure that the global list tracking server objects does not hold on to a reference.
kernel.UnregisterServerObject(this);
}
bool KServerPort::IsSignaled() const {

19
src/core/hle/kernel/k_server_port.h
View file

@ -27,24 +27,6 @@ public:
void Initialize(KPort* parent_port_, std::string&& name_);
/// Whether or not this server port has an HLE handler available.
bool HasSessionRequestHandler() const {
return !session_handler.expired();
}
/// Gets the HLE handler for this port.
SessionRequestHandlerWeakPtr GetSessionRequestHandler() const {
return session_handler;
}
/**
* Sets the HLE handler template for the port. ServerSessions crated by connecting to this port
* will inherit a reference to this handler.
*/
void SetSessionHandler(SessionRequestHandlerWeakPtr&& handler) {
session_handler = std::move(handler);
}
void EnqueueSession(KServerSession* pending_session);
KServerSession* AcceptSession();
@ -65,7 +47,6 @@ private:
void CleanupSessions();
SessionList session_list;
SessionRequestHandlerWeakPtr session_handler;
KPort* parent{};
};

149
src/core/hle/kernel/k_server_session.cpp
View file

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <tuple>
@ -33,12 +33,10 @@ KServerSession::KServerSession(KernelCore& kernel_)
KServerSession::~KServerSession() = default;
void KServerSession::Initialize(KSession* parent_session_, std::string&& name_,
std::shared_ptr<SessionRequestManager> manager_) {
void KServerSession::Initialize(KSession* parent_session_, std::string&& name_) {
// Set member variables.
parent = parent_session_;
name = std::move(name_);
manager = manager_;
}
void KServerSession::Destroy() {
@ -47,18 +45,99 @@ void KServerSession::Destroy() {
this->CleanupRequests();
parent->Close();
// Release host emulation members.
manager.reset();
// Ensure that the global list tracking server objects does not hold on to a reference.
kernel.UnregisterServerObject(this);
}
void KServerSession::OnClientClosed() {
if (manager && manager->HasSessionHandler()) {
manager->SessionHandler().ClientDisconnected(this);
KScopedLightLock lk{m_lock};
// Handle any pending requests.
KSessionRequest* prev_request = nullptr;
while (true) {
// Declare variables for processing the request.
KSessionRequest* request = nullptr;
KEvent* event = nullptr;
KThread* thread = nullptr;
bool cur_request = false;
bool terminate = false;
// Get the next request.
{
KScopedSchedulerLock sl{kernel};
if (m_current_request != nullptr && m_current_request != prev_request) {
// Set the request, open a reference as we process it.
request = m_current_request;
request->Open();
cur_request = true;
// Get thread and event for the request.
thread = request->GetThread();
event = request->GetEvent();
// If the thread is terminating, handle that.
if (thread->IsTerminationRequested()) {
request->ClearThread();
request->ClearEvent();
terminate = true;
}
prev_request = request;
} else if (!m_request_list.empty()) {
// Pop the request from the front of the list.
request = std::addressof(m_request_list.front());
m_request_list.pop_front();
// Get thread and event for the request.
thread = request->GetThread();
event = request->GetEvent();
}
}
// If there are no requests, we're done.
if (request == nullptr) {
break;
}
// All requests must have threads.
ASSERT(thread != nullptr);
// Ensure that we close the request when done.
SCOPE_EXIT({ request->Close(); });
// If we're terminating, close a reference to the thread and event.
if (terminate) {
thread->Close();
if (event != nullptr) {
event->Close();
}
}
// If we need to, reply.
if (event != nullptr && !cur_request) {
// There must be no mappings.
ASSERT(request->GetSendCount() == 0);
ASSERT(request->GetReceiveCount() == 0);
ASSERT(request->GetExchangeCount() == 0);
// // Get the process and page table.
// KProcess *client_process = thread->GetOwnerProcess();
// auto &client_pt = client_process->GetPageTable();
// // Reply to the request.
// ReplyAsyncError(client_process, request->GetAddress(), request->GetSize(),
// ResultSessionClosed);
// // Unlock the buffer.
// // NOTE: Nintendo does not check the result of this.
// client_pt.UnlockForIpcUserBuffer(request->GetAddress(), request->GetSize());
// Signal the event.
event->Signal();
}
}
// Notify.
this->NotifyAvailable(ResultSessionClosed);
}
bool KServerSession::IsSignaled() const {
@ -73,24 +152,6 @@ bool KServerSession::IsSignaled() const {
return !m_request_list.empty() && m_current_request == nullptr;
}
Result KServerSession::QueueSyncRequest(KThread* thread, Core::Memory::Memory& memory) {
u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(thread->GetTLSAddress()))};
auto context = std::make_shared<HLERequestContext>(kernel, memory, this, thread);
context->PopulateFromIncomingCommandBuffer(kernel.CurrentProcess()->GetHandleTable(), cmd_buf);
return manager->QueueSyncRequest(parent, std::move(context));
}
Result KServerSession::CompleteSyncRequest(HLERequestContext& context) {
Result result = manager->CompleteSyncRequest(this, context);
// The calling thread is waiting for this request to complete, so wake it up.
context.GetThread().EndWait(result);
return result;
}
Result KServerSession::OnRequest(KSessionRequest* request) {
// Create the wait queue.
ThreadQueueImplForKServerSessionRequest wait_queue{kernel};
@ -105,13 +166,6 @@ Result KServerSession::OnRequest(KSessionRequest* request) {
// Check that we're not terminating.
R_UNLESS(!GetCurrentThread(kernel).IsTerminationRequested(), ResultTerminationRequested);
if (manager) {
// HLE request.
auto& memory{kernel.System().Memory()};
this->QueueSyncRequest(GetCurrentThreadPointer(kernel), memory);
} else {
// Non-HLE request.
// Get whether we're empty.
const bool was_empty = m_request_list.empty();
@ -123,7 +177,6 @@ Result KServerSession::OnRequest(KSessionRequest* request) {
if (was_empty) {
this->NotifyAvailable();
}
}
// If we have a request event, this is asynchronous, and we don't need to wait.
R_SUCCEED_IF(request->GetEvent() != nullptr);
@ -136,7 +189,7 @@ Result KServerSession::OnRequest(KSessionRequest* request) {
return GetCurrentThread(kernel).GetWaitResult();
}
Result KServerSession::SendReply() {
Result KServerSession::SendReply(bool is_hle) {
// Lock the session.
KScopedLightLock lk{m_lock};
@ -171,6 +224,10 @@ Result KServerSession::SendReply() {
Result result = ResultSuccess;
if (!closed) {
// If we're not closed, send the reply.
if (is_hle) {
// HLE servers write directly to a pointer to the thread command buffer. Therefore
// the reply has already been written in this case.
} else {
Core::Memory::Memory& memory{kernel.System().Memory()};
KThread* server_thread{GetCurrentThreadPointer(kernel)};
UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess());
@ -178,6 +235,7 @@ Result KServerSession::SendReply() {
auto* src_msg_buffer = memory.GetPointer(server_thread->GetTLSAddress());
auto* dst_msg_buffer = memory.GetPointer(client_message);
std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size);
}
} else {
result = ResultSessionClosed;
}
@ -223,7 +281,8 @@ Result KServerSession::SendReply() {
return result;
}
Result KServerSession::ReceiveRequest() {
Result KServerSession::ReceiveRequest(std::shared_ptr<HLERequestContext>* out_context,
std::weak_ptr<SessionRequestManager> manager) {
// Lock the session.
KScopedLightLock lk{m_lock};
@ -267,12 +326,22 @@ Result KServerSession::ReceiveRequest() {
// Receive the message.
Core::Memory::Memory& memory{kernel.System().Memory()};
if (out_context != nullptr) {
// HLE request.
u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(client_message))};
*out_context = std::make_shared<HLERequestContext>(kernel, memory, this, client_thread);
(*out_context)->SetSessionRequestManager(manager);
(*out_context)
->PopulateFromIncomingCommandBuffer(client_thread->GetOwnerProcess()->GetHandleTable(),
cmd_buf);
} else {
KThread* server_thread{GetCurrentThreadPointer(kernel)};
UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess());
auto* src_msg_buffer = memory.GetPointer(client_message);
auto* dst_msg_buffer = memory.GetPointer(server_thread->GetTLSAddress());
std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size);
}
// We succeeded.
return ResultSuccess;

41
src/core/hle/kernel/k_server_session.h
View file

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
@ -16,21 +16,11 @@
#include "core/hle/kernel/k_synchronization_object.h"
#include "core/hle/result.h"
namespace Core::Memory {
class Memory;
}
namespace Core::Timing {
class CoreTiming;
struct EventType;
} // namespace Core::Timing
namespace Kernel {
class HLERequestContext;
class KernelCore;
class KSession;
class SessionRequestHandler;
class SessionRequestManager;
class KThread;
@ -46,8 +36,7 @@ public:
void Destroy() override;
void Initialize(KSession* parent_session_, std::string&& name_,
std::shared_ptr<SessionRequestManager> manager_);
void Initialize(KSession* parent_session_, std::string&& name_);
KSession* GetParent() {
return parent;
@ -60,32 +49,20 @@ public:
bool IsSignaled() const override;
void OnClientClosed();
/// Gets the session request manager, which forwards requests to the underlying service
std::shared_ptr<SessionRequestManager>& GetSessionRequestManager() {
return manager;
}
/// TODO: flesh these out to match the real kernel
Result OnRequest(KSessionRequest* request);
Result SendReply();
Result ReceiveRequest();
Result SendReply(bool is_hle = false);
Result ReceiveRequest(std::shared_ptr<HLERequestContext>* out_context = nullptr,
std::weak_ptr<SessionRequestManager> manager = {});
Result SendReplyHLE() {
return SendReply(true);
}
private:
/// Frees up waiting client sessions when this server session is about to die
void CleanupRequests();
/// Queues a sync request from the emulated application.
Result QueueSyncRequest(KThread* thread, Core::Memory::Memory& memory);
/// Completes a sync request from the emulated application.
Result CompleteSyncRequest(HLERequestContext& context);
/// This session's HLE request handlers; if nullptr, this is not an HLE server
std::shared_ptr<SessionRequestManager> manager;
/// When set to True, converts the session to a domain at the end of the command
bool convert_to_domain{};
/// KSession that owns this KServerSession
KSession* parent{};

5
src/core/hle/kernel/k_session.cpp
View file

@ -13,8 +13,7 @@ KSession::KSession(KernelCore& kernel_)
: KAutoObjectWithSlabHeapAndContainer{kernel_}, server{kernel_}, client{kernel_} {}
KSession::~KSession() = default;
void KSession::Initialize(KClientPort* port_, const std::string& name_,
std::shared_ptr<SessionRequestManager> manager_) {
void KSession::Initialize(KClientPort* port_, const std::string& name_) {
// Increment reference count.
// Because reference count is one on creation, this will result
// in a reference count of two. Thus, when both server and client are closed
@ -26,7 +25,7 @@ void KSession::Initialize(KClientPort* port_, const std::string& name_,
KAutoObject::Create(std::addressof(client));
// Initialize our sub sessions.
server.Initialize(this, name_ + ":Server", manager_);
server.Initialize(this, name_ + ":Server");
client.Initialize(this, name_ + ":Client");
// Set state and name.

3
src/core/hle/kernel/k_session.h
View file

@ -21,8 +21,7 @@ public:
explicit KSession(KernelCore& kernel_);
~KSession() override;
void Initialize(KClientPort* port_, const std::string& name_,
std::shared_ptr<SessionRequestManager> manager_ = nullptr);
void Initialize(KClientPort* port_, const std::string& name_);
void Finalize() override;

50
src/core/hle/kernel/kernel.cpp
View file

@ -60,7 +60,6 @@ struct KernelCore::Impl {
global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel);
global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel);
global_handle_table->Initialize(KHandleTable::MaxTableSize);
default_service_thread = CreateServiceThread(kernel, "DefaultServiceThread");
is_phantom_mode_for_singlecore = false;
@ -86,6 +85,8 @@ struct KernelCore::Impl {
}
RegisterHostThread();
default_service_thread = CreateServiceThread(kernel, "DefaultServiceThread");
}
void InitializeCores() {
@ -184,17 +185,6 @@ struct KernelCore::Impl {
}
void CloseServices() {
// Close all open server sessions and ports.
std::unordered_set<KAutoObject*> server_objects_;
{
std::scoped_lock lk(server_objects_lock);
server_objects_ = server_objects;
server_objects.clear();
}
for (auto* server_object : server_objects_) {
server_object->Close();
}
// Ensures all service threads gracefully shutdown.
ClearServiceThreads();
}
@ -346,6 +336,8 @@ struct KernelCore::Impl {
return this_id;
}
static inline thread_local bool is_phantom_mode_for_singlecore{false};
bool IsPhantomModeForSingleCore() const {
return is_phantom_mode_for_singlecore;
}
@ -698,24 +690,21 @@ struct KernelCore::Impl {
return {};
}
KClientPort* port = &search->second(system.ServiceManager(), system);
RegisterServerObject(&port->GetParent()->GetServerPort());
return port;
return &search->second(system.ServiceManager(), system);
}
void RegisterServerObject(KAutoObject* server_object) {
std::scoped_lock lk(server_objects_lock);
server_objects.insert(server_object);
void RegisterNamedServiceHandler(std::string name, KServerPort* server_port) {
auto search = service_interface_handlers.find(name);
if (search == service_interface_handlers.end()) {
return;
}
void UnregisterServerObject(KAutoObject* server_object) {
std::scoped_lock lk(server_objects_lock);
server_objects.erase(server_object);
search->second(system.ServiceManager(), server_port);
}
std::weak_ptr<Kernel::ServiceThread> CreateServiceThread(KernelCore& kernel,
const std::string& name) {
auto service_thread = std::make_shared<Kernel::ServiceThread>(kernel, 1, name);
auto service_thread = std::make_shared<Kernel::ServiceThread>(kernel, name);
service_threads_manager.QueueWork(
[this, service_thread]() { service_threads.emplace(service_thread); });
@ -745,7 +734,6 @@ struct KernelCore::Impl {
service_thread_barrier.Sync();
}
std::mutex server_objects_lock;
std::mutex registered_objects_lock;
std::mutex registered_in_use_objects_lock;
@ -774,8 +762,8 @@ struct KernelCore::Impl {
/// Map of named ports managed by the kernel, which can be retrieved using
/// the ConnectToPort SVC.
std::unordered_map<std::string, ServiceInterfaceFactory> service_interface_factory;
std::unordered_map<std::string, ServiceInterfaceHandlerFn> service_interface_handlers;
NamedPortTable named_ports;
std::unordered_set<KAutoObject*> server_objects;
std::unordered_set<KAutoObject*> registered_objects;
std::unordered_set<KAutoObject*> registered_in_use_objects;
@ -814,7 +802,6 @@ struct KernelCore::Impl {
bool is_multicore{};
std::atomic_bool is_shutting_down{};
bool is_phantom_mode_for_singlecore{};
u32 single_core_thread_id{};
std::array<u64, Core::Hardware::NUM_CPU_CORES> svc_ticks{};
@ -981,16 +968,17 @@ void KernelCore::RegisterNamedService(std::string name, ServiceInterfaceFactory&
impl->service_interface_factory.emplace(std::move(name), factory);
}
void KernelCore::RegisterInterfaceForNamedService(std::string name,
ServiceInterfaceHandlerFn&& handler) {
impl->service_interface_handlers.emplace(std::move(name), handler);
}
KClientPort* KernelCore::CreateNamedServicePort(std::string name) {
return impl->CreateNamedServicePort(std::move(name));
}
void KernelCore::RegisterServerObject(KAutoObject* server_object) {
impl->RegisterServerObject(server_object);
}
void KernelCore::UnregisterServerObject(KAutoObject* server_object) {
impl->UnregisterServerObject(server_object);
void KernelCore::RegisterNamedServiceHandler(std::string name, KServerPort* server_port) {
impl->RegisterNamedServiceHandler(std::move(name), server_port);
}
void KernelCore::RegisterKernelObject(KAutoObject* object) {

15
src/core/hle/kernel/kernel.h
View file

@ -45,6 +45,7 @@ class KPort;
class KProcess;
class KResourceLimit;
class KScheduler;
class KServerPort;
class KServerSession;
class KSession;
class KSessionRequest;
@ -63,6 +64,8 @@ class TimeManager;
using ServiceInterfaceFactory =
std::function<KClientPort&(Service::SM::ServiceManager&, Core::System&)>;
using ServiceInterfaceHandlerFn = std::function<void(Service::SM::ServiceManager&, KServerPort*)>;
namespace Init {
struct KSlabResourceCounts;
}
@ -192,16 +195,14 @@ public:
/// Registers a named HLE service, passing a factory used to open a port to that service.
void RegisterNamedService(std::string name, ServiceInterfaceFactory&& factory);
/// Registers a setup function for the named HLE service.
void RegisterInterfaceForNamedService(std::string name, ServiceInterfaceHandlerFn&& handler);
/// Opens a port to a service previously registered with RegisterNamedService.
KClientPort* CreateNamedServicePort(std::string name);
/// Registers a server session or port with the gobal emulation state, to be freed on shutdown.
/// This is necessary because we do not emulate processes for HLE sessions and ports.
void RegisterServerObject(KAutoObject* server_object);
/// Unregisters a server session or port previously registered with RegisterServerSession when
/// it was destroyed during the current emulation session.
void UnregisterServerObject(KAutoObject* server_object);
/// Accepts a session on a port created by CreateNamedServicePort.
void RegisterNamedServiceHandler(std::string name, KServerPort* server_port);
/// Registers all kernel objects with the global emulation state, this is purely for tracking
/// leaks after emulation has been shutdown.

236
src/core/hle/kernel/service_thread.cpp
View file

@ -1,15 +1,18 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <condition_variable>
#include <functional>
#include <map>
#include <mutex>
#include <thread>
#include <vector>
#include <queue>
#include "common/scope_exit.h"
#include "common/thread.h"
#include "core/hle/ipc_helpers.h"
#include "core/hle/kernel/hle_ipc.h"
#include "core/hle/kernel/k_event.h"
#include "core/hle/kernel/k_scoped_resource_reservation.h"
#include "core/hle/kernel/k_session.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
@ -19,101 +22,198 @@ namespace Kernel {
class ServiceThread::Impl final {
public:
explicit Impl(KernelCore& kernel, std::size_t num_threads, const std::string& name);
explicit Impl(KernelCore& kernel, const std::string& service_name);
~Impl();
void QueueSyncRequest(KSession& session, std::shared_ptr<HLERequestContext>&& context);
void WaitAndProcessImpl();
void SessionClosed(KServerSession* server_session,
std::shared_ptr<SessionRequestManager> manager);
void LoopProcess();
void RegisterServerSession(KServerSession* session,
std::shared_ptr<SessionRequestManager> manager);
private:
std::vector<std::jthread> threads;
std::queue<std::function<void()>> requests;
std::mutex queue_mutex;
std::condition_variable_any condition;
const std::string service_name;
KernelCore& kernel;
std::jthread m_thread;
std::mutex m_session_mutex;
std::map<KServerSession*, std::shared_ptr<SessionRequestManager>> m_sessions;
KEvent* m_wakeup_event;
KProcess* m_process;
std::atomic<bool> m_shutdown_requested;
const std::string m_service_name;
};
ServiceThread::Impl::Impl(KernelCore& kernel, std::size_t num_threads, const std::string& name)
: service_name{name} {
for (std::size_t i = 0; i < num_threads; ++i) {
threads.emplace_back([this, &kernel](std::stop_token stop_token) {
Common::SetCurrentThreadName(std::string{service_name}.c_str());
void ServiceThread::Impl::WaitAndProcessImpl() {
// Create local list of waitable sessions.
std::vector<KSynchronizationObject*> objs;
std::vector<std::shared_ptr<SessionRequestManager>> managers;
// Wait for first request before trying to acquire a render context
{
std::unique_lock lock{queue_mutex};
condition.wait(lock, stop_token, [this] { return !requests.empty(); });
// Lock to get the set.
std::scoped_lock lk{m_session_mutex};
// Reserve the needed quantity.
objs.reserve(m_sessions.size() + 1);
managers.reserve(m_sessions.size());
// Copy to our local list.
for (const auto& [session, manager] : m_sessions) {
objs.push_back(session);
managers.push_back(manager);
}
if (stop_token.stop_requested()) {
// Insert the wakeup event at the end.
objs.push_back(&m_wakeup_event->GetReadableEvent());
}
// Wait on the list of sessions.
s32 index{-1};
Result rc = KSynchronizationObject::Wait(kernel, &index, objs.data(),
static_cast<s32>(objs.size()), -1);
ASSERT(!rc.IsFailure());
// If this was the wakeup event, clear it and finish.
if (index >= static_cast<s64>(objs.size() - 1)) {
m_wakeup_event->Clear();
return;
}
// Allocate a dummy guest thread for this host thread.
// This event is from a server session.
auto* server_session = static_cast<KServerSession*>(objs[index]);
auto& manager = managers[index];
// Fetch the HLE request context.
std::shared_ptr<HLERequestContext> context;
rc = server_session->ReceiveRequest(&context, manager);
// If the session was closed, handle that.
if (rc == ResultSessionClosed) {
SessionClosed(server_session, manager);
// Finish.
return;
}
// TODO: handle other cases
ASSERT(rc == ResultSuccess);
// Perform the request.
Result service_rc = manager->CompleteSyncRequest(server_session, *context);
// Reply to the client.
rc = server_session->SendReplyHLE();
if (rc == ResultSessionClosed || service_rc == IPC::ERR_REMOTE_PROCESS_DEAD) {
SessionClosed(server_session, manager);
return;
}
// TODO: handle other cases
ASSERT(rc == ResultSuccess);
ASSERT(service_rc == ResultSuccess);
}
void ServiceThread::Impl::SessionClosed(KServerSession* server_session,
std::shared_ptr<SessionRequestManager> manager) {
{
// Lock to get the set.
std::scoped_lock lk{m_session_mutex};
// Erase the session.
ASSERT(m_sessions.erase(server_session) == 1);
}
// Close our reference to the server session.
server_session->Close();
}
void ServiceThread::Impl::LoopProcess() {
Common::SetCurrentThreadName(m_service_name.c_str());
kernel.RegisterHostThread();
while (true) {
std::function<void()> task;
{
std::unique_lock lock{queue_mutex};
condition.wait(lock, stop_token, [this] { return !requests.empty(); });
if (stop_token.stop_requested()) {
return;
}
if (requests.empty()) {
continue;
}
task = std::move(requests.front());
requests.pop();
}
task();
}
});
while (!m_shutdown_requested.load()) {
WaitAndProcessImpl();
}
}
void ServiceThread::Impl::QueueSyncRequest(KSession& session,
std::shared_ptr<HLERequestContext>&& context) {
{
std::unique_lock lock{queue_mutex};
auto* server_session{&session.GetServerSession()};
// Open a reference to the session to ensure it is not closes while the service request
// completes asynchronously.
void ServiceThread::Impl::RegisterServerSession(KServerSession* server_session,
std::shared_ptr<SessionRequestManager> manager) {
// Open the server session.
server_session->Open();
requests.emplace([server_session, context{std::move(context)}]() {
// Close the reference.
SCOPE_EXIT({ server_session->Close(); });
{
// Lock to get the set.
std::scoped_lock lk{m_session_mutex};
// Complete the service request.
server_session->CompleteSyncRequest(*context);
});
// Insert the session and manager.
m_sessions[server_session] = manager;
}
condition.notify_one();
// Signal the wakeup event.
m_wakeup_event->Signal();
}
ServiceThread::Impl::~Impl() {
condition.notify_all();
for (auto& thread : threads) {
thread.request_stop();
thread.join();
}
// Shut down the processing thread.
m_shutdown_requested.store(true);
m_wakeup_event->Signal();
m_thread.join();
// Lock mutex.
m_session_mutex.lock();
// Close all remaining sessions.
for (const auto& [server_session, manager] : m_sessions) {
server_session->Close();
}
ServiceThread::ServiceThread(KernelCore& kernel, std::size_t num_threads, const std::string& name)
: impl{std::make_unique<Impl>(kernel, num_threads, name)} {}
// Destroy remaining managers.
m_sessions.clear();
// Close event.
m_wakeup_event->GetReadableEvent().Close();
m_wakeup_event->Close();
// Close process.
m_process->Close();
}
ServiceThread::Impl::Impl(KernelCore& kernel_, const std::string& service_name)
: kernel{kernel_}, m_service_name{service_name} {
// Initialize process.
m_process = KProcess::Create(kernel);
KProcess::Initialize(m_process, kernel.System(), service_name,
KProcess::ProcessType::KernelInternal, kernel.GetSystemResourceLimit());
// Reserve a new event from the process resource limit
KScopedResourceReservation event_reservation(m_process, LimitableResource::Events);
ASSERT(event_reservation.Succeeded());
// Initialize event.
m_wakeup_event = KEvent::Create(kernel);
m_wakeup_event->Initialize(m_process);
// Commit the event reservation.
event_reservation.Commit();
// Register the event.
KEvent::Register(kernel, m_wakeup_event);
// Start thread.
m_thread = std::jthread([this] { LoopProcess(); });
}
ServiceThread::ServiceThread(KernelCore& kernel, const std::string& name)
: impl{std::make_unique<Impl>(kernel, name)} {}
ServiceThread::~ServiceThread() = default;
void ServiceThread::QueueSyncRequest(KSession& session,
std::shared_ptr<HLERequestContext>&& context) {
impl->QueueSyncRequest(session, std::move(context));
void ServiceThread::RegisterServerSession(KServerSession* session,
std::shared_ptr<SessionRequestManager> manager) {
impl->RegisterServerSession(session, manager);
}
} // namespace Kernel

6
src/core/hle/kernel/service_thread.h
View file

@ -11,13 +11,15 @@ namespace Kernel {
class HLERequestContext;
class KernelCore;
class KSession;
class SessionRequestManager;
class ServiceThread final {
public:
explicit ServiceThread(KernelCore& kernel, std::size_t num_threads, const std::string& name);
explicit ServiceThread(KernelCore& kernel, const std::string& name);
~ServiceThread();
void QueueSyncRequest(KSession& session, std::shared_ptr<HLERequestContext>&& context);
void RegisterServerSession(KServerSession* session,
std::shared_ptr<SessionRequestManager> manager);
private:
class Impl;

7
src/core/hle/kernel/svc.cpp
View file

@ -24,6 +24,7 @@
#include "core/hle/kernel/k_memory_block.h"
#include "core/hle/kernel/k_memory_layout.h"
#include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_port.h"
#include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/k_readable_event.h"
#include "core/hle/kernel/k_resource_limit.h"
@ -382,9 +383,9 @@ static Result ConnectToNamedPort(Core::System& system, Handle* out, VAddr port_n
// Create a session.
KClientSession* session{};
R_TRY(port->CreateSession(std::addressof(session),
std::make_shared<SessionRequestManager>(kernel)));
port->Close();
R_TRY(port->CreateSession(std::addressof(session)));
kernel.RegisterNamedServiceHandler(port_name, &port->GetParent()->GetServerPort());
// Register the session in the table, close the extra reference.
handle_table.Register(*out, session);

17
src/core/hle/service/service.cpp
View file

@ -99,6 +99,12 @@ ServiceFrameworkBase::ServiceFrameworkBase(Core::System& system_, const char* se
ServiceFrameworkBase::~ServiceFrameworkBase() {
// Wait for other threads to release access before destroying
const auto guard = LockService();
if (named_port != nullptr) {
named_port->GetClientPort().Close();
named_port->GetServerPort().Close();
named_port = nullptr;
}
}
void ServiceFrameworkBase::InstallAsService(SM::ServiceManager& service_manager) {
@ -113,15 +119,16 @@ void ServiceFrameworkBase::InstallAsService(SM::ServiceManager& service_manager)
Kernel::KClientPort& ServiceFrameworkBase::CreatePort() {
const auto guard = LockService();
if (named_port == nullptr) {
ASSERT(!service_registered);
auto* port = Kernel::KPort::Create(kernel);
port->Initialize(max_sessions, false, service_name);
port->GetServerPort().SetSessionHandler(shared_from_this());
named_port = Kernel::KPort::Create(kernel);
named_port->Initialize(max_sessions, false, service_name);
service_registered = true;
}
return port->GetClientPort();
return named_port->GetClientPort();
}
void ServiceFrameworkBase::RegisterHandlersBase(const FunctionInfoBase* functions, std::size_t n) {
@ -199,7 +206,6 @@ Result ServiceFrameworkBase::HandleSyncRequest(Kernel::KServerSession& session,
switch (ctx.GetCommandType()) {
case IPC::CommandType::Close:
case IPC::CommandType::TIPC_Close: {
session.Close();
IPC::ResponseBuilder rb{ctx, 2};
rb.Push(ResultSuccess);
result = IPC::ERR_REMOTE_PROCESS_DEAD;
@ -244,6 +250,7 @@ Services::Services(std::shared_ptr<SM::ServiceManager>& sm, Core::System& system
system.GetFileSystemController().CreateFactories(*system.GetFilesystem(), false);
system.Kernel().RegisterNamedService("sm:", SM::ServiceManager::InterfaceFactory);
system.Kernel().RegisterInterfaceForNamedService("sm:", SM::ServiceManager::SessionHandler);
Account::InstallInterfaces(system);
AM::InstallInterfaces(*sm, *nv_flinger, system);

4
src/core/hle/service/service.h
View file

@ -20,6 +20,7 @@ class System;
namespace Kernel {
class HLERequestContext;
class KClientPort;
class KPort;
class KServerSession;
class ServiceThread;
} // namespace Kernel
@ -98,6 +99,9 @@ protected:
/// Identifier string used to connect to the service.
std::string service_name;
/// Port used by ManageNamedPort.
Kernel::KPort* named_port{};
private:
template <typename T>
friend class ServiceFramework;

44
src/core/hle/service/sm/sm.cpp
View file

@ -23,7 +23,13 @@ constexpr Result ERR_INVALID_NAME(ErrorModule::SM, 6);
constexpr Result ERR_SERVICE_NOT_REGISTERED(ErrorModule::SM, 7);
ServiceManager::ServiceManager(Kernel::KernelCore& kernel_) : kernel{kernel_} {}
ServiceManager::~ServiceManager() = default;
ServiceManager::~ServiceManager() {
for (auto& [name, port] : service_ports) {
port->GetClientPort().Close();
port->GetServerPort().Close();
}
}
void ServiceManager::InvokeControlRequest(Kernel::HLERequestContext& context) {
controller_interface->InvokeRequest(context);
@ -43,6 +49,10 @@ Kernel::KClientPort& ServiceManager::InterfaceFactory(ServiceManager& self, Core
return self.sm_interface->CreatePort();
}
void ServiceManager::SessionHandler(ServiceManager& self, Kernel::KServerPort* server_port) {
self.sm_interface->AcceptSession(server_port);
}
Result ServiceManager::RegisterService(std::string name, u32 max_sessions,
Kernel::SessionRequestHandlerPtr handler) {
@ -53,7 +63,11 @@ Result ServiceManager::RegisterService(std::string name, u32 max_sessions,
return ERR_ALREADY_REGISTERED;
}
registered_services.emplace(std::move(name), handler);
auto* port = Kernel::KPort::Create(kernel);
port->Initialize(ServerSessionCountMax, false, name);
service_ports.emplace(name, port);
registered_services.emplace(name, handler);
return ResultSuccess;
}
@ -68,24 +82,20 @@ Result ServiceManager::UnregisterService(const std::string& name) {
}
registered_services.erase(iter);
service_ports.erase(name);
return ResultSuccess;
}
ResultVal<Kernel::KPort*> ServiceManager::GetServicePort(const std::string& name) {
CASCADE_CODE(ValidateServiceName(name));
auto it = registered_services.find(name);
if (it == registered_services.end()) {
auto it = service_ports.find(name);
if (it == service_ports.end()) {
LOG_ERROR(Service_SM, "Server is not registered! service={}", name);
return ERR_SERVICE_NOT_REGISTERED;
}
auto* port = Kernel::KPort::Create(kernel);
port->Initialize(ServerSessionCountMax, false, name);
auto handler = it->second;
port->GetServerPort().SetSessionHandler(std::move(handler));
return port;
return it->second;
}
/**
@ -144,24 +154,20 @@ ResultVal<Kernel::KClientSession*> SM::GetServiceImpl(Kernel::HLERequestContext&
// Find the named port.
auto port_result = service_manager.GetServicePort(name);
if (port_result.Failed()) {
auto service = service_manager.GetService<Kernel::SessionRequestHandler>(name);
if (port_result.Failed() || !service) {
LOG_ERROR(Service_SM, "called service={} -> error 0x{:08X}", name, port_result.Code().raw);
return port_result.Code();
}
auto& port = port_result.Unwrap();
SCOPE_EXIT({
port->GetClientPort().Close();
port->GetServerPort().Close();
});
// Create a new session.
Kernel::KClientSession* session{};
if (const auto result = port->GetClientPort().CreateSession(
std::addressof(session), std::make_shared<Kernel::SessionRequestManager>(kernel));
result.IsError()) {
if (const auto result = port->GetClientPort().CreateSession(&session); result.IsError()) {
LOG_ERROR(Service_SM, "called service={} -> error 0x{:08X}", name, result.raw);
return result;
}
service->AcceptSession(&port->GetServerPort());
LOG_DEBUG(Service_SM, "called service={} -> session={}", name, session->GetId());

2
src/core/hle/service/sm/sm.h
View file

@ -51,6 +51,7 @@ private:
class ServiceManager {
public:
static Kernel::KClientPort& InterfaceFactory(ServiceManager& self, Core::System& system);
static void SessionHandler(ServiceManager& self, Kernel::KServerPort* server_port);
explicit ServiceManager(Kernel::KernelCore& kernel_);
~ServiceManager();
@ -78,6 +79,7 @@ private:
/// Map of registered services, retrieved using GetServicePort.
std::unordered_map<std::string, Kernel::SessionRequestHandlerPtr> registered_services;
std::unordered_map<std::string, Kernel::KPort*> service_ports;
/// Kernel context
Kernel::KernelCore& kernel;

16
src/core/hle/service/sm/sm_controller.cpp
View file

@ -15,10 +15,9 @@
namespace Service::SM {
void Controller::ConvertCurrentObjectToDomain(Kernel::HLERequestContext& ctx) {
ASSERT_MSG(!ctx.Session()->GetSessionRequestManager()->IsDomain(),
"Session is already a domain");
ASSERT_MSG(!ctx.GetManager()->IsDomain(), "Session is already a domain");
LOG_DEBUG(Service, "called, server_session={}", ctx.Session()->GetId());
ctx.Session()->GetSessionRequestManager()->ConvertToDomainOnRequestEnd();
ctx.GetManager()->ConvertToDomainOnRequestEnd();
IPC::ResponseBuilder rb{ctx, 3};
rb.Push(ResultSuccess);
@ -29,9 +28,7 @@ void Controller::CloneCurrentObject(Kernel::HLERequestContext& ctx) {
LOG_DEBUG(Service, "called");
auto& process = *ctx.GetThread().GetOwnerProcess();
auto& parent_session = *ctx.Session()->GetParent();
auto& session_manager = parent_session.GetServerSession().GetSessionRequestManager();
auto& session_handler = session_manager->SessionHandler();
auto session_manager = ctx.GetManager();
// FIXME: this is duplicated from the SVC, it should just call it instead
// once this is a proper process
@ -46,13 +43,14 @@ void Controller::CloneCurrentObject(Kernel::HLERequestContext& ctx) {
ASSERT(session != nullptr);
// Initialize the session.
session->Initialize(nullptr, parent_session.GetName(), session_manager);
session->Initialize(nullptr, "");
// Commit the session reservation.
session_reservation.Commit();
// Register the session.
session_handler.ClientConnected(&session->GetServerSession());
// Register with manager.
session_manager->SessionHandler().RegisterSession(&session->GetServerSession(),
session_manager);
// We succeeded.
IPC::ResponseBuilder rb{ctx, 2, 0, 1, IPC::ResponseBuilder::Flags::AlwaysMoveHandles};