Nikilite/suyu
1
1
Fork 0
forked from suyu/suyu
Code Pull requests Activity

Merge pull request #6823 from yzct12345/memory-cleanup

Browse source 
memory: Clean up code
This commit is contained in:
bunnei 2021-08-09 17:09:56 -07:00 committed by GitHub
commit 7df790f1ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 166 additions and 494 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

558
src/core/memory.cpp
View file

@ -4,8 +4,6 @@
#include <algorithm> #include <algorithm>
#include <cstring> #include <cstring>
#include <optional>
#include <utility>
#include "common/assert.h" #include "common/assert.h"
#include "common/atomic_ops.h" #include "common/atomic_ops.h"
@ -14,12 +12,10 @@
#include "common/page_table.h" #include "common/page_table.h"
#include "common/settings.h" #include "common/settings.h"
#include "common/swap.h" #include "common/swap.h"
#include "core/arm/arm_interface.h"
#include "core/core.h" #include "core/core.h"
#include "core/device_memory.h" #include "core/device_memory.h"
#include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/physical_memory.h"
#include "core/memory.h" #include "core/memory.h"
#include "video_core/gpu.h" #include "video_core/gpu.h"
@ -62,17 +58,7 @@ struct Memory::Impl {
} }
} }
bool IsValidVirtualAddress(const Kernel::KProcess& process, const VAddr vaddr) const { [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(VAddr vaddr) const {
const auto& page_table = process.PageTable().PageTableImpl();
const auto [pointer, type] = page_table.pointers[vaddr >> PAGE_BITS].PointerType();
return pointer != nullptr || type == Common::PageType::RasterizerCachedMemory;
}
bool IsValidVirtualAddress(VAddr vaddr) const {
return IsValidVirtualAddress(*system.CurrentProcess(), vaddr);
}
u8* GetPointerFromRasterizerCachedMemory(VAddr vaddr) const {
const PAddr paddr{current_page_table->backing_addr[vaddr >> PAGE_BITS]}; const PAddr paddr{current_page_table->backing_addr[vaddr >> PAGE_BITS]};
if (!paddr) { if (!paddr) {
@ -82,18 +68,6 @@ struct Memory::Impl {
return system.DeviceMemory().GetPointer(paddr) + vaddr; return system.DeviceMemory().GetPointer(paddr) + vaddr;
} }
u8* GetPointer(const VAddr vaddr) const {
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
return pointer + vaddr;
}
const auto type = Common::PageTable::PageInfo::ExtractType(raw_pointer);
if (type == Common::PageType::RasterizerCachedMemory) {
return GetPointerFromRasterizerCachedMemory(vaddr);
}
return nullptr;
}
u8 Read8(const VAddr addr) { u8 Read8(const VAddr addr) {
return Read<u8>(addr); return Read<u8>(addr);
} }
@ -179,7 +153,7 @@ struct Memory::Impl {
std::string string; std::string string;
string.reserve(max_length); string.reserve(max_length);
for (std::size_t i = 0; i < max_length; ++i) { for (std::size_t i = 0; i < max_length; ++i) {
const char c = Read8(vaddr); const char c = Read<s8>(vaddr);
if (c == '\0') { if (c == '\0') {
break; break;
} }
@ -190,15 +164,14 @@ struct Memory::Impl {
return string; return string;
} }
void ReadBlock(const Kernel::KProcess& process, const VAddr src_addr, void* dest_buffer, void WalkBlock(const Kernel::KProcess& process, const VAddr addr, const std::size_t size,
const std::size_t size) { auto on_unmapped, auto on_memory, auto on_rasterizer, auto increment) {
const auto& page_table = process.PageTable().PageTableImpl(); const auto& page_table = process.PageTable().PageTableImpl();
std::size_t remaining_size = size; std::size_t remaining_size = size;
std::size_t page_index = src_addr >> PAGE_BITS; std::size_t page_index = addr >> PAGE_BITS;
std::size_t page_offset = src_addr & PAGE_MASK; std::size_t page_offset = addr & PAGE_MASK;
while (remaining_size > 0) { while (remaining_size) {
const std::size_t copy_amount = const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
@ -206,22 +179,18 @@ struct Memory::Impl {
const auto [pointer, type] = page_table.pointers[page_index].PointerType(); const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) { switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, on_unmapped(copy_amount, current_vaddr);
"Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, src_addr, size);
std::memset(dest_buffer, 0, copy_amount);
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(pointer); DEBUG_ASSERT(pointer);
const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS); u8* mem_ptr = pointer + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_buffer, src_ptr, copy_amount); on_memory(copy_amount, mem_ptr);
break; break;
} }
case Common::PageType::RasterizerCachedMemory: { case Common::PageType::RasterizerCachedMemory: {
const u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)}; u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)};
system.GPU().FlushRegion(current_vaddr, copy_amount); on_rasterizer(current_vaddr, copy_amount, host_ptr);
std::memcpy(dest_buffer, host_ptr, copy_amount);
break; break;
} }
default: default:
@ -230,248 +199,122 @@ struct Memory::Impl {
page_index++; page_index++;
page_offset = 0; page_offset = 0;
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; increment(copy_amount);
remaining_size -= copy_amount; remaining_size -= copy_amount;
} }
} }
void ReadBlockUnsafe(const Kernel::KProcess& process, const VAddr src_addr, void* dest_buffer, template <bool UNSAFE>
const std::size_t size) { void ReadBlockImpl(const Kernel::KProcess& process, const VAddr src_addr, void* dest_buffer,
const auto& page_table = process.PageTable().PageTableImpl(); const std::size_t size) {
WalkBlock(
std::size_t remaining_size = size; process, src_addr, size,
std::size_t page_index = src_addr >> PAGE_BITS; [src_addr, size, &dest_buffer](const std::size_t copy_amount,
std::size_t page_offset = src_addr & PAGE_MASK; const VAddr current_vaddr) {
while (remaining_size > 0) {
const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, src_addr, size); current_vaddr, src_addr, size);
std::memset(dest_buffer, 0, copy_amount); std::memset(dest_buffer, 0, copy_amount);
break; },
} [&dest_buffer](const std::size_t copy_amount, const u8* const src_ptr) {
case Common::PageType::Memory: {
DEBUG_ASSERT(pointer);
const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_buffer, src_ptr, copy_amount); std::memcpy(dest_buffer, src_ptr, copy_amount);
break; },
} [&system = system, &dest_buffer](const VAddr current_vaddr,
case Common::PageType::RasterizerCachedMemory: { const std::size_t copy_amount,
const u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)}; const u8* const host_ptr) {
if constexpr (!UNSAFE) {
system.GPU().FlushRegion(current_vaddr, copy_amount);
}
std::memcpy(dest_buffer, host_ptr, copy_amount); std::memcpy(dest_buffer, host_ptr, copy_amount);
break; },
} [&dest_buffer](const std::size_t copy_amount) {
default: dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
UNREACHABLE(); });
}
page_index++;
page_offset = 0;
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
remaining_size -= copy_amount;
}
} }
void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) { void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
ReadBlock(*system.CurrentProcess(), src_addr, dest_buffer, size); ReadBlockImpl<false>(*system.CurrentProcess(), src_addr, dest_buffer, size);
} }
void ReadBlockUnsafe(const VAddr src_addr, void* dest_buffer, const std::size_t size) { void ReadBlockUnsafe(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
ReadBlockUnsafe(*system.CurrentProcess(), src_addr, dest_buffer, size); ReadBlockImpl<true>(*system.CurrentProcess(), src_addr, dest_buffer, size);
} }
void WriteBlock(const Kernel::KProcess& process, const VAddr dest_addr, const void* src_buffer, template <bool UNSAFE>
const std::size_t size) { void WriteBlockImpl(const Kernel::KProcess& process, const VAddr dest_addr,
const auto& page_table = process.PageTable().PageTableImpl(); const void* src_buffer, const std::size_t size) {
std::size_t remaining_size = size; WalkBlock(
std::size_t page_index = dest_addr >> PAGE_BITS; process, dest_addr, size,
std::size_t page_offset = dest_addr & PAGE_MASK; [dest_addr, size](const std::size_t copy_amount, const VAddr current_vaddr) {
while (remaining_size > 0) {
const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, dest_addr, size); current_vaddr, dest_addr, size);
break; },
} [&src_buffer](const std::size_t copy_amount, u8* const dest_ptr) {
case Common::PageType::Memory: {
DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_ptr, src_buffer, copy_amount); std::memcpy(dest_ptr, src_buffer, copy_amount);
break; },
} [&system = system, &src_buffer](const VAddr current_vaddr,
case Common::PageType::RasterizerCachedMemory: { const std::size_t copy_amount, u8* const host_ptr) {
u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)}; if constexpr (!UNSAFE) {
system.GPU().InvalidateRegion(current_vaddr, copy_amount); system.GPU().InvalidateRegion(current_vaddr, copy_amount);
}
std::memcpy(host_ptr, src_buffer, copy_amount); std::memcpy(host_ptr, src_buffer, copy_amount);
break; },
} [&src_buffer](const std::size_t copy_amount) {
default: src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
UNREACHABLE(); });
}
page_index++;
page_offset = 0;
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
remaining_size -= copy_amount;
}
}
void WriteBlockUnsafe(const Kernel::KProcess& process, const VAddr dest_addr,
const void* src_buffer, const std::size_t size) {
const auto& page_table = process.PageTable().PageTableImpl();
std::size_t remaining_size = size;
std::size_t page_index = dest_addr >> PAGE_BITS;
std::size_t page_offset = dest_addr & PAGE_MASK;
while (remaining_size > 0) {
const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory,
"Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, dest_addr, size);
break;
}
case Common::PageType::Memory: {
DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_ptr, src_buffer, copy_amount);
break;
}
case Common::PageType::RasterizerCachedMemory: {
u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)};
std::memcpy(host_ptr, src_buffer, copy_amount);
break;
}
default:
UNREACHABLE();
}
page_index++;
page_offset = 0;
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
remaining_size -= copy_amount;
}
} }
void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
WriteBlock(*system.CurrentProcess(), dest_addr, src_buffer, size); WriteBlockImpl<false>(*system.CurrentProcess(), dest_addr, src_buffer, size);
} }
void WriteBlockUnsafe(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { void WriteBlockUnsafe(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
WriteBlockUnsafe(*system.CurrentProcess(), dest_addr, src_buffer, size); WriteBlockImpl<true>(*system.CurrentProcess(), dest_addr, src_buffer, size);
} }
void ZeroBlock(const Kernel::KProcess& process, const VAddr dest_addr, const std::size_t size) { void ZeroBlock(const Kernel::KProcess& process, const VAddr dest_addr, const std::size_t size) {
const auto& page_table = process.PageTable().PageTableImpl(); WalkBlock(
std::size_t remaining_size = size; process, dest_addr, size,
std::size_t page_index = dest_addr >> PAGE_BITS; [dest_addr, size](const std::size_t copy_amount, const VAddr current_vaddr) {
std::size_t page_offset = dest_addr & PAGE_MASK;
while (remaining_size > 0) {
const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, dest_addr, size); current_vaddr, dest_addr, size);
break; },
} [](const std::size_t copy_amount, u8* const dest_ptr) {
case Common::PageType::Memory: {
DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
std::memset(dest_ptr, 0, copy_amount); std::memset(dest_ptr, 0, copy_amount);
break; },
} [&system = system](const VAddr current_vaddr, const std::size_t copy_amount,
case Common::PageType::RasterizerCachedMemory: { u8* const host_ptr) {
u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)};
system.GPU().InvalidateRegion(current_vaddr, copy_amount); system.GPU().InvalidateRegion(current_vaddr, copy_amount);
std::memset(host_ptr, 0, copy_amount); std::memset(host_ptr, 0, copy_amount);
break; },
} [](const std::size_t copy_amount) {});
default:
UNREACHABLE();
}
page_index++;
page_offset = 0;
remaining_size -= copy_amount;
}
}
void ZeroBlock(const VAddr dest_addr, const std::size_t size) {
ZeroBlock(*system.CurrentProcess(), dest_addr, size);
} }
void CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr, void CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr,
const std::size_t size) { const std::size_t size) {
const auto& page_table = process.PageTable().PageTableImpl(); WalkBlock(
std::size_t remaining_size = size; process, dest_addr, size,
std::size_t page_index = src_addr >> PAGE_BITS; [this, &process, &dest_addr, &src_addr, size](const std::size_t copy_amount,
std::size_t page_offset = src_addr & PAGE_MASK; const VAddr current_vaddr) {
while (remaining_size > 0) {
const std::size_t copy_amount =
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, src_addr, size); current_vaddr, src_addr, size);
ZeroBlock(process, dest_addr, copy_amount); ZeroBlock(process, dest_addr, copy_amount);
break; },
} [this, &process, &dest_addr](const std::size_t copy_amount, const u8* const src_ptr) {
case Common::PageType::Memory: { WriteBlockImpl<false>(process, dest_addr, src_ptr, copy_amount);
DEBUG_ASSERT(pointer); },
const u8* src_ptr = pointer + page_offset + (page_index << PAGE_BITS); [this, &system = system, &process, &dest_addr](
WriteBlock(process, dest_addr, src_ptr, copy_amount); const VAddr current_vaddr, const std::size_t copy_amount, u8* const host_ptr) {
break;
}
case Common::PageType::RasterizerCachedMemory: {
const u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)};
system.GPU().FlushRegion(current_vaddr, copy_amount); system.GPU().FlushRegion(current_vaddr, copy_amount);
WriteBlock(process, dest_addr, host_ptr, copy_amount); WriteBlockImpl<false>(process, dest_addr, host_ptr, copy_amount);
break; },
} [&dest_addr, &src_addr](const std::size_t copy_amount) {
default: dest_addr += static_cast<VAddr>(copy_amount);
UNREACHABLE(); src_addr += static_cast<VAddr>(copy_amount);
} });
page_index++;
page_offset = 0;
dest_addr += static_cast<VAddr>(copy_amount);
src_addr += static_cast<VAddr>(copy_amount);
remaining_size -= copy_amount;
}
}
void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) {
return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size);
} }
void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
@ -514,7 +357,7 @@ struct Memory::Impl {
} else { } else {
// Switch page type to uncached if now uncached // Switch page type to uncached if now uncached
switch (page_type) { switch (page_type) {
case Common::PageType::Unmapped: case Common::PageType::Unmapped: // NOLINT(bugprone-branch-clone)
// It is not necessary for a process to have this region mapped into its address // It is not necessary for a process to have this region mapped into its address
// space, for example, a system module need not have a VRAM mapping. // space, for example, a system module need not have a VRAM mapping.
break; break;
@ -597,6 +440,44 @@ struct Memory::Impl {
} }
} }
[[nodiscard]] u8* GetPointerImpl(VAddr vaddr, auto on_unmapped, auto on_rasterizer) const {
// AARCH64 masks the upper 16 bit of all memory accesses
vaddr &= 0xffffffffffffLL;
if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) {
on_unmapped();
return nullptr;
}
// Avoid adding any extra logic to this fast-path block
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
return &pointer[vaddr];
}
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Unmapped:
on_unmapped();
return nullptr;
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ 0x{:016X}", vaddr);
return nullptr;
case Common::PageType::RasterizerCachedMemory: {
u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
on_rasterizer();
return host_ptr;
}
default:
UNREACHABLE();
}
return nullptr;
}
[[nodiscard]] u8* GetPointer(const VAddr vaddr) const {
return GetPointerImpl(
vaddr, [vaddr]() { LOG_ERROR(HW_Memory, "Unmapped GetPointer @ 0x{:016X}", vaddr); },
[]() {});
}
/** /**
* Reads a particular data type out of memory at the given virtual address. * Reads a particular data type out of memory at the given virtual address.
* *
@ -610,39 +491,17 @@ struct Memory::Impl {
*/ */
template <typename T> template <typename T>
T Read(VAddr vaddr) { T Read(VAddr vaddr) {
// AARCH64 masks the upper 16 bit of all memory accesses T result = 0;
vaddr &= 0xffffffffffffLL; const u8* const ptr = GetPointerImpl(
vaddr,
if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { [vaddr]() {
LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:016X}", sizeof(T) * 8, vaddr);
return 0; },
[&system = system, vaddr]() { system.GPU().FlushRegion(vaddr, sizeof(T)); });
if (ptr) {
std::memcpy(&result, ptr, sizeof(T));
} }
return result;
// Avoid adding any extra logic to this fast-path block
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (const u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
T value;
std::memcpy(&value, &pointer[vaddr], sizeof(T));
return value;
}
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
return 0;
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
break;
case Common::PageType::RasterizerCachedMemory: {
const u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
system.GPU().FlushRegion(vaddr, sizeof(T));
T value;
std::memcpy(&value, host_ptr, sizeof(T));
return value;
}
default:
UNREACHABLE();
}
return {};
} }
/** /**
@ -656,110 +515,46 @@ struct Memory::Impl {
*/ */
template <typename T> template <typename T>
void Write(VAddr vaddr, const T data) { void Write(VAddr vaddr, const T data) {
// AARCH64 masks the upper 16 bit of all memory accesses u8* const ptr = GetPointerImpl(
vaddr &= 0xffffffffffffLL; vaddr,
[vaddr, data]() {
if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { LOG_ERROR(HW_Memory, "Unmapped Write{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8,
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, vaddr, static_cast<u64>(data));
static_cast<u32>(data), vaddr); },
return; [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); });
} if (ptr) {
std::memcpy(ptr, &data, sizeof(T));
// Avoid adding any extra logic to this fast-path block
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
std::memcpy(&pointer[vaddr], &data, sizeof(T));
return;
}
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
static_cast<u32>(data), vaddr);
return;
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
break;
case Common::PageType::RasterizerCachedMemory: {
u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
system.GPU().InvalidateRegion(vaddr, sizeof(T));
std::memcpy(host_ptr, &data, sizeof(T));
break;
}
default:
UNREACHABLE();
} }
} }
template <typename T> template <typename T>
bool WriteExclusive(VAddr vaddr, const T data, const T expected) { bool WriteExclusive(VAddr vaddr, const T data, const T expected) {
// AARCH64 masks the upper 16 bit of all memory accesses u8* const ptr = GetPointerImpl(
vaddr &= 0xffffffffffffLL; vaddr,
[vaddr, data]() {
if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { LOG_ERROR(HW_Memory, "Unmapped WriteExclusive{} @ 0x{:016X} = 0x{:016X}",
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, sizeof(T) * 8, vaddr, static_cast<u64>(data));
static_cast<u32>(data), vaddr); },
return true; [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); });
} if (ptr) {
const auto volatile_pointer = reinterpret_cast<volatile T*>(ptr);
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
// NOTE: Avoid adding any extra logic to this fast-path block
const auto volatile_pointer = reinterpret_cast<volatile T*>(&pointer[vaddr]);
return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
static_cast<u32>(data), vaddr);
return true;
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
break;
case Common::PageType::RasterizerCachedMemory: {
u8* host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
system.GPU().InvalidateRegion(vaddr, sizeof(T));
auto* pointer = reinterpret_cast<volatile T*>(&host_ptr);
return Common::AtomicCompareAndSwap(pointer, data, expected);
}
default:
UNREACHABLE();
}
return true; return true;
} }
bool WriteExclusive128(VAddr vaddr, const u128 data, const u128 expected) { bool WriteExclusive128(VAddr vaddr, const u128 data, const u128 expected) {
// AARCH64 masks the upper 16 bit of all memory accesses u8* const ptr = GetPointerImpl(
vaddr &= 0xffffffffffffLL; vaddr,
[vaddr, data]() {
if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { LOG_ERROR(HW_Memory, "Unmapped WriteExclusive128 @ 0x{:016X} = 0x{:016X}{:016X}",
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, vaddr, static_cast<u64>(data[1]), static_cast<u64>(data[0]));
static_cast<u32>(data[0]), vaddr); },
return true; [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(u128)); });
} if (ptr) {
const auto volatile_pointer = reinterpret_cast<volatile u64*>(ptr);
const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
// NOTE: Avoid adding any extra logic to this fast-path block
const auto volatile_pointer = reinterpret_cast<volatile u64*>(&pointer[vaddr]);
return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8,
static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr);
return true;
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
break;
case Common::PageType::RasterizerCachedMemory: {
u8* host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
system.GPU().InvalidateRegion(vaddr, sizeof(u128));
auto* pointer = reinterpret_cast<volatile u64*>(&host_ptr);
return Common::AtomicCompareAndSwap(pointer, data, expected);
}
default:
UNREACHABLE();
}
return true; return true;
} }
@ -789,12 +584,11 @@ void Memory::UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) {
impl->UnmapRegion(page_table, base, size); impl->UnmapRegion(page_table, base, size);
} }
bool Memory::IsValidVirtualAddress(const Kernel::KProcess& process, const VAddr vaddr) const {
return impl->IsValidVirtualAddress(process, vaddr);
}
bool Memory::IsValidVirtualAddress(const VAddr vaddr) const { bool Memory::IsValidVirtualAddress(const VAddr vaddr) const {
return impl->IsValidVirtualAddress(vaddr); const Kernel::KProcess& process = *system.CurrentProcess();
const auto& page_table = process.PageTable().PageTableImpl();
const auto [pointer, type] = page_table.pointers[vaddr >> PAGE_BITS].PointerType();
return pointer != nullptr || type == Common::PageType::RasterizerCachedMemory;
} }
u8* Memory::GetPointer(VAddr vaddr) { u8* Memory::GetPointer(VAddr vaddr) {
@ -863,64 +657,38 @@ std::string Memory::ReadCString(VAddr vaddr, std::size_t max_length) {
void Memory::ReadBlock(const Kernel::KProcess& process, const VAddr src_addr, void* dest_buffer, void Memory::ReadBlock(const Kernel::KProcess& process, const VAddr src_addr, void* dest_buffer,
const std::size_t size) { const std::size_t size) {
impl->ReadBlock(process, src_addr, dest_buffer, size); impl->ReadBlockImpl<false>(process, src_addr, dest_buffer, size);
} }
void Memory::ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) { void Memory::ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
impl->ReadBlock(src_addr, dest_buffer, size); impl->ReadBlock(src_addr, dest_buffer, size);
} }
void Memory::ReadBlockUnsafe(const Kernel::KProcess& process, const VAddr src_addr,
void* dest_buffer, const std::size_t size) {
impl->ReadBlockUnsafe(process, src_addr, dest_buffer, size);
}
void Memory::ReadBlockUnsafe(const VAddr src_addr, void* dest_buffer, const std::size_t size) { void Memory::ReadBlockUnsafe(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
impl->ReadBlockUnsafe(src_addr, dest_buffer, size); impl->ReadBlockUnsafe(src_addr, dest_buffer, size);
} }
void Memory::WriteBlock(const Kernel::KProcess& process, VAddr dest_addr, const void* src_buffer, void Memory::WriteBlock(const Kernel::KProcess& process, VAddr dest_addr, const void* src_buffer,
std::size_t size) { std::size_t size) {
impl->WriteBlock(process, dest_addr, src_buffer, size); impl->WriteBlockImpl<false>(process, dest_addr, src_buffer, size);
} }
void Memory::WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { void Memory::WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
impl->WriteBlock(dest_addr, src_buffer, size); impl->WriteBlock(dest_addr, src_buffer, size);
} }
void Memory::WriteBlockUnsafe(const Kernel::KProcess& process, VAddr dest_addr,
const void* src_buffer, std::size_t size) {
impl->WriteBlockUnsafe(process, dest_addr, src_buffer, size);
}
void Memory::WriteBlockUnsafe(const VAddr dest_addr, const void* src_buffer, void Memory::WriteBlockUnsafe(const VAddr dest_addr, const void* src_buffer,
const std::size_t size) { const std::size_t size) {
impl->WriteBlockUnsafe(dest_addr, src_buffer, size); impl->WriteBlockUnsafe(dest_addr, src_buffer, size);
} }
void Memory::ZeroBlock(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size) {
impl->ZeroBlock(process, dest_addr, size);
}
void Memory::ZeroBlock(VAddr dest_addr, std::size_t size) {
impl->ZeroBlock(dest_addr, size);
}
void Memory::CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr, void Memory::CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr,
const std::size_t size) { const std::size_t size) {
impl->CopyBlock(process, dest_addr, src_addr, size); impl->CopyBlock(process, dest_addr, src_addr, size);
} }
void Memory::CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) {
impl->CopyBlock(dest_addr, src_addr, size);
}
void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
impl->RasterizerMarkRegionCached(vaddr, size, cached); impl->RasterizerMarkRegionCached(vaddr, size, cached);
} }
bool IsKernelVirtualAddress(const VAddr vaddr) {
return KERNEL_REGION_VADDR <= vaddr && vaddr < KERNEL_REGION_END;
}
} // namespace Core::Memory } // namespace Core::Memory

102
src/core/memory.h
View file

@ -39,11 +39,6 @@ enum : VAddr {
/// Application stack /// Application stack
DEFAULT_STACK_SIZE = 0x100000, DEFAULT_STACK_SIZE = 0x100000,
/// Kernel Virtual Address Range
KERNEL_REGION_VADDR = 0xFFFFFF8000000000,
KERNEL_REGION_SIZE = 0x7FFFE00000,
KERNEL_REGION_END = KERNEL_REGION_VADDR + KERNEL_REGION_SIZE,
}; };
/// Central class that handles all memory operations and state. /// Central class that handles all memory operations and state.
@ -56,7 +51,7 @@ public:
Memory& operator=(const Memory&) = delete; Memory& operator=(const Memory&) = delete;
Memory(Memory&&) = default; Memory(Memory&&) = default;
Memory& operator=(Memory&&) = default; Memory& operator=(Memory&&) = delete;
/** /**
* Resets the state of the Memory system. * Resets the state of the Memory system.
@ -90,17 +85,6 @@ public:
*/ */
void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size); void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size);
/**
* Checks whether or not the supplied address is a valid virtual
* address for the given process.
*
* @param process The emulated process to check the address against.
* @param vaddr The virtual address to check the validity of.
*
* @returns True if the given virtual address is valid, false otherwise.
*/
bool IsValidVirtualAddress(const Kernel::KProcess& process, VAddr vaddr) const;
/** /**
* Checks whether or not the supplied address is a valid virtual * Checks whether or not the supplied address is a valid virtual
* address for the current process. * address for the current process.
@ -109,7 +93,7 @@ public:
* *
* @returns True if the given virtual address is valid, false otherwise. * @returns True if the given virtual address is valid, false otherwise.
*/ */
bool IsValidVirtualAddress(VAddr vaddr) const; [[nodiscard]] bool IsValidVirtualAddress(VAddr vaddr) const;
/** /**
* Gets a pointer to the given address. * Gets a pointer to the given address.
@ -134,7 +118,7 @@ public:
* @returns The pointer to the given address, if the address is valid. * @returns The pointer to the given address, if the address is valid.
* If the address is not valid, nullptr will be returned. * If the address is not valid, nullptr will be returned.
*/ */
const u8* GetPointer(VAddr vaddr) const; [[nodiscard]] const u8* GetPointer(VAddr vaddr) const;
template <typename T> template <typename T>
const T* GetPointer(VAddr vaddr) const { const T* GetPointer(VAddr vaddr) const {
@ -327,27 +311,6 @@ public:
void ReadBlock(const Kernel::KProcess& process, VAddr src_addr, void* dest_buffer, void ReadBlock(const Kernel::KProcess& process, VAddr src_addr, void* dest_buffer,
std::size_t size); std::size_t size);
/**
* Reads a contiguous block of bytes from a specified process' address space.
* This unsafe version does not trigger GPU flushing.
*
* @param process The process to read the data from.
* @param src_addr The virtual address to begin reading from.
* @param dest_buffer The buffer to place the read bytes into.
* @param size The amount of data to read, in bytes.
*
* @note If a size of 0 is specified, then this function reads nothing and
* no attempts to access memory are made at all.
*
* @pre dest_buffer must be at least size bytes in length, otherwise a
* buffer overrun will occur.
*
* @post The range [dest_buffer, size) contains the read bytes from the
* process' address space.
*/
void ReadBlockUnsafe(const Kernel::KProcess& process, VAddr src_addr, void* dest_buffer,
std::size_t size);
/** /**
* Reads a contiguous block of bytes from the current process' address space. * Reads a contiguous block of bytes from the current process' address space.
* *
@ -408,26 +371,6 @@ public:
void WriteBlock(const Kernel::KProcess& process, VAddr dest_addr, const void* src_buffer, void WriteBlock(const Kernel::KProcess& process, VAddr dest_addr, const void* src_buffer,
std::size_t size); std::size_t size);
/**
* Writes a range of bytes into a given process' address space at the specified
* virtual address.
* This unsafe version does not invalidate GPU Memory.
*
* @param process The process to write data into the address space of.
* @param dest_addr The destination virtual address to begin writing the data at.
* @param src_buffer The data to write into the process' address space.
* @param size The size of the data to write, in bytes.
*
* @post The address range [dest_addr, size) in the process' address space
* contains the data that was within src_buffer.
*
* @post If an attempt is made to write into an unmapped region of memory, the writes
* will be ignored and an error will be logged.
*
*/
void WriteBlockUnsafe(const Kernel::KProcess& process, VAddr dest_addr, const void* src_buffer,
std::size_t size);
/** /**
* Writes a range of bytes into the current process' address space at the specified * Writes a range of bytes into the current process' address space at the specified
* virtual address. * virtual address.
@ -467,29 +410,6 @@ public:
*/ */
void WriteBlockUnsafe(VAddr dest_addr, const void* src_buffer, std::size_t size); void WriteBlockUnsafe(VAddr dest_addr, const void* src_buffer, std::size_t size);
/**
* Fills the specified address range within a process' address space with zeroes.
*
* @param process The process that will have a portion of its memory zeroed out.
* @param dest_addr The starting virtual address of the range to zero out.
* @param size The size of the address range to zero out, in bytes.
*
* @post The range [dest_addr, size) within the process' address space is
* filled with zeroes.
*/
void ZeroBlock(const Kernel::KProcess& process, VAddr dest_addr, std::size_t size);
/**
* Fills the specified address range within the current process' address space with zeroes.
*
* @param dest_addr The starting virtual address of the range to zero out.
* @param size The size of the address range to zero out, in bytes.
*
* @post The range [dest_addr, size) within the current process' address space is
* filled with zeroes.
*/
void ZeroBlock(VAddr dest_addr, std::size_t size);
/** /**
* Copies data within a process' address space to another location within the * Copies data within a process' address space to another location within the
* same address space. * same address space.
@ -505,19 +425,6 @@ public:
void CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr, void CopyBlock(const Kernel::KProcess& process, VAddr dest_addr, VAddr src_addr,
std::size_t size); std::size_t size);
/**
* Copies data within the current process' address space to another location within the
* same address space.
*
* @param dest_addr The destination virtual address to begin copying the data into.
* @param src_addr The source virtual address to begin copying the data from.
* @param size The size of the data to copy, in bytes.
*
* @post The range [dest_addr, size) within the current process' address space
* contains the same data within the range [src_addr, size).
*/
void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size);
/** /**
* Marks each page within the specified address range as cached or uncached. * Marks each page within the specified address range as cached or uncached.
* *
@ -535,7 +442,4 @@ private:
std::unique_ptr<Impl> impl; std::unique_ptr<Impl> impl;
}; };
/// Determines if the given VAddr is a kernel address
bool IsKernelVirtualAddress(VAddr vaddr);
} // namespace Core::Memory } // namespace Core::Memory