From fd068ed6b8a8599107bbc5036e055196993a83b9 Mon Sep 17 00:00:00 2001 From: Lynn Date: Thu, 16 Feb 2017 18:18:29 +0000 Subject: [PATCH] Ranged cache invalidation --- include/dynarmic/dynarmic.h | 9 +++- src/CMakeLists.txt | 1 + src/backend_x64/emit_x64.cpp | 49 ++++++++++++++++---- src/backend_x64/emit_x64.h | 17 +++++-- src/backend_x64/interface_x64.cpp | 54 ++++++++++++++-------- src/common/address_range.h | 33 +++++++++++++ src/frontend/ir/basic_block.cpp | 8 ++++ src/frontend/ir/basic_block.h | 9 +++- src/frontend/translate/translate_arm.cpp | 2 + src/frontend/translate/translate_thumb.cpp | 2 + tests/arm/fuzz_arm.cpp | 35 ++++++++++++++ 11 files changed, 188 insertions(+), 31 deletions(-) create mode 100644 src/common/address_range.h diff --git a/include/dynarmic/dynarmic.h b/include/dynarmic/dynarmic.h index c1c73abf..3f172ac6 100644 --- a/include/dynarmic/dynarmic.h +++ b/include/dynarmic/dynarmic.h @@ -8,8 +8,8 @@ #include #include -#include #include +#include #include @@ -38,6 +38,13 @@ public: */ void ClearCache(); + /** + * Invalidate the code cache at a range of addresses. + * @param start_address The starting address of the range to invalidate. + * @param length The length (in bytes) of the range to invalidate. + */ + void InvalidateCacheRange(std::uint32_t start_address, std::size_t length); + /** * Reset CPU state to state at startup. Does not clear code cache. * Cannot be called from a callback. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fcc75d08..1e90fae3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -39,6 +39,7 @@ set(HEADERS ../include/dynarmic/coprocessor_util.h ../include/dynarmic/disassembler.h ../include/dynarmic/dynarmic.h + common/address_range.h common/assert.h common/bit_util.h common/common_types.h diff --git a/src/backend_x64/emit_x64.cpp b/src/backend_x64/emit_x64.cpp index b82618c8..3cce95a6 100644 --- a/src/backend_x64/emit_x64.cpp +++ b/src/backend_x64/emit_x64.cpp @@ -12,8 +12,10 @@ #include "backend_x64/block_of_code.h" #include "backend_x64/emit_x64.h" #include "backend_x64/jitstate.h" +#include "common/address_range.h" #include "common/assert.h" #include "common/bit_util.h" +#include "common/common_types.h" #include "common/variant_util.h" #include "frontend/arm/types.h" #include "frontend/ir/basic_block.h" @@ -74,8 +76,9 @@ EmitX64::EmitX64(BlockOfCode* code, UserCallbacks cb, Jit* jit_interface) EmitX64::BlockDescriptor EmitX64::Emit(IR::Block& block) { code->align(); - const u8* const emitted_code_start_ptr = code->getCurr(); + const u8* const entrypoint = code->getCurr(); + // Start emitting. EmitCondPrelude(block); RegAlloc reg_alloc{code}; @@ -108,11 +111,12 @@ EmitX64::BlockDescriptor EmitX64::Emit(IR::Block& block) { code->int3(); const IR::LocationDescriptor descriptor = block.Location(); - Patch(descriptor, emitted_code_start_ptr); + Patch(descriptor, entrypoint); + + const size_t size = static_cast(code->getCurr() - entrypoint); + EmitX64::BlockDescriptor block_desc{entrypoint, size, block.Location(), block.EndLocation().PC()}; + block_descriptors.emplace(descriptor.UniqueHash(), block_desc); - EmitX64::BlockDescriptor& block_desc = block_descriptors[descriptor.UniqueHash()]; - size_t emitted_code_size = static_cast(code->getCurr() - emitted_code_start_ptr); - block_desc = {emitted_code_start_ptr, emitted_code_size}; return block_desc; } @@ -459,7 +463,7 @@ void EmitX64::EmitPushRSB(RegAlloc& reg_alloc, IR::Block&, IR::Inst* inst) { auto iter = block_descriptors.find(unique_hash_of_target); CodePtr target_code_ptr = iter != block_descriptors.end() - ? iter->second.code_ptr + ? iter->second.entrypoint : code->GetReturnFromRunCodeAddress(); Xbyak::Reg64 code_ptr_reg = reg_alloc.ScratchGpr({HostLoc::RCX}); @@ -3345,7 +3349,7 @@ void EmitX64::EmitTerminal(IR::Term::LinkBlock terminal, IR::LocationDescriptor patch_information[terminal.next.UniqueHash()].jg.emplace_back(code->getCurr()); if (auto next_bb = GetBasicBlock(terminal.next)) { - EmitPatchJg(next_bb->code_ptr); + EmitPatchJg(next_bb->entrypoint); } else { EmitPatchJg(); } @@ -3374,7 +3378,7 @@ void EmitX64::EmitTerminal(IR::Term::LinkBlockFast terminal, IR::LocationDescrip patch_information[terminal.next.UniqueHash()].jmp.emplace_back(code->getCurr()); if (auto next_bb = GetBasicBlock(terminal.next)) { - EmitPatchJmp(terminal.next, next_bb->code_ptr); + EmitPatchJmp(terminal.next, next_bb->entrypoint); } else { EmitPatchJmp(terminal.next); } @@ -3475,5 +3479,34 @@ void EmitX64::ClearCache() { patch_information.clear(); } +void EmitX64::InvalidateCacheRange(const Common::AddressRange& range) { + // Remove cached block descriptors and patch information overlapping with the given range. + + switch (range.which()) { + case 0: // FullAddressRange + ClearCache(); + break; + + case 1: // AddressInterval + auto interval = boost::get(range); + for (auto it = std::begin(block_descriptors); it != std::end(block_descriptors);) { + const IR::LocationDescriptor& descriptor = it->second.start_location; + u32 start = descriptor.PC(); + u32 end = it->second.end_location_pc; + if (interval.Overlaps(start, end)) { + it = block_descriptors.erase(it); + + auto patch_it = patch_information.find(descriptor.UniqueHash()); + if (patch_it != patch_information.end()) { + Unpatch(descriptor); + } + } else { + ++it; + } + } + break; + } +} + } // namespace BackendX64 } // namespace Dynarmic diff --git a/src/backend_x64/emit_x64.h b/src/backend_x64/emit_x64.h index f2aaabd2..5eb8af51 100644 --- a/src/backend_x64/emit_x64.h +++ b/src/backend_x64/emit_x64.h @@ -14,6 +14,7 @@ #include #include "backend_x64/reg_alloc.h" +#include "common/address_range.h" #include "dynarmic/callbacks.h" #include "frontend/ir/location_descriptor.h" #include "frontend/ir/terminal.h" @@ -34,8 +35,11 @@ class BlockOfCode; class EmitX64 final { public: struct BlockDescriptor { - CodePtr code_ptr; ///< Entrypoint of emitted code - size_t size; ///< Length in bytes of emitted code + CodePtr entrypoint; // Entrypoint of emitted code + size_t size; // Length in bytes of emitted code + + IR::LocationDescriptor start_location; + u32 end_location_pc; }; EmitX64(BlockOfCode* code, UserCallbacks cb, Jit* jit_interface); @@ -49,9 +53,16 @@ public: /// Looks up an emitted host block in the cache. boost::optional GetBasicBlock(IR::LocationDescriptor descriptor) const; - /// Empties the cache. + /// Empties the entire cache. void ClearCache(); + /** + * Invalidate the cache at a range of addresses. + * @param start_address The starting address of the range to invalidate. + * @param length The length (in bytes) of the range to invalidate. + */ + void InvalidateCacheRange(const Common::AddressRange& range); + private: // Microinstruction emitters #define OPCODE(name, type, ...) void Emit##name(RegAlloc& reg_alloc, IR::Block& block, IR::Inst* inst); diff --git a/src/backend_x64/interface_x64.cpp b/src/backend_x64/interface_x64.cpp index 63d524d2..632cee3c 100644 --- a/src/backend_x64/interface_x64.cpp +++ b/src/backend_x64/interface_x64.cpp @@ -5,6 +5,7 @@ */ #include +#include #include @@ -35,6 +36,7 @@ struct Jit::Impl { , jit_state() , emitter(&block_of_code, callbacks, jit) , callbacks(callbacks) + , jit_interface(jit) {} BlockOfCode block_of_code; @@ -42,20 +44,21 @@ struct Jit::Impl { EmitX64 emitter; const UserCallbacks callbacks; - bool clear_cache_required = false; + // Requests made during execution to invalidate the cache are queued up here. + std::queue invalid_cache_ranges; size_t Execute(size_t cycle_count) { u32 pc = jit_state.Reg[15]; IR::LocationDescriptor descriptor{pc, Arm::PSR{jit_state.Cpsr}, Arm::FPSCR{jit_state.FPSCR_mode}}; - CodePtr code_ptr = GetBasicBlock(descriptor).code_ptr; - return block_of_code.RunCode(&jit_state, code_ptr, cycle_count); + CodePtr entrypoint = GetBasicBlock(descriptor).entrypoint; + return block_of_code.RunCode(&jit_state, entrypoint, cycle_count); } std::string Disassemble(const IR::LocationDescriptor& descriptor) { auto block = GetBasicBlock(descriptor); - std::string result = fmt::format("address: {}\nsize: {} bytes\n", block.code_ptr, block.size); + std::string result = fmt::format("address: {}\nsize: {} bytes\n", block.entrypoint, block.size); #ifdef DYNARMIC_USE_LLVM LLVMInitializeX86TargetInfo(); @@ -64,7 +67,7 @@ struct Jit::Impl { LLVMDisasmContextRef llvm_ctx = LLVMCreateDisasm("x86_64", nullptr, 0, nullptr, nullptr); LLVMSetDisasmOptions(llvm_ctx, LLVMDisassembler_Option_AsmPrinterVariant); - const u8* pos = static_cast(block.code_ptr); + const u8* pos = static_cast(block.entrypoint); const u8* end = pos + block.size; size_t remaining = block.size; @@ -91,14 +94,31 @@ struct Jit::Impl { return result; } - void ClearCache() { - block_of_code.ClearCache(); - emitter.ClearCache(); + void PerformCacheInvalidation() { + if (invalid_cache_ranges.empty()) { + return; + } + jit_state.ResetRSB(); - clear_cache_required = false; + block_of_code.ClearCache(); + while (!invalid_cache_ranges.empty()) { + emitter.InvalidateCacheRange(invalid_cache_ranges.front()); + invalid_cache_ranges.pop(); + } + } + + void HandleNewCacheRange() { + if (jit_interface->is_executing) { + jit_state.halt_requested = true; + return; + } + + PerformCacheInvalidation(); } private: + Jit* jit_interface; + EmitX64::BlockDescriptor GetBasicBlock(IR::LocationDescriptor descriptor) { auto block = emitter.GetBasicBlock(descriptor); if (block) @@ -130,21 +150,19 @@ size_t Jit::Run(size_t cycle_count) { cycles_executed += impl->Execute(cycle_count - cycles_executed); } - if (impl->clear_cache_required) { - impl->ClearCache(); - } + impl->PerformCacheInvalidation(); return cycles_executed; } void Jit::ClearCache() { - if (is_executing) { - impl->jit_state.halt_requested = true; - impl->clear_cache_required = true; - return; - } + impl->invalid_cache_ranges.push(Common::FullAddressRange{}); + impl->HandleNewCacheRange(); +} - impl->ClearCache(); +void Jit::InvalidateCacheRange(std::uint32_t start_address, std::size_t length) { + impl->invalid_cache_ranges.push(Common::AddressInterval{start_address, length}); + impl->HandleNewCacheRange(); } void Jit::Reset() { diff --git a/src/common/address_range.h b/src/common/address_range.h new file mode 100644 index 00000000..b8d7875b --- /dev/null +++ b/src/common/address_range.h @@ -0,0 +1,33 @@ +/* This file is part of the dynarmic project. + * Copyright (c) 2016 MerryMage + * This software may be used and distributed according to the terms of the GNU + * General Public License version 2 or any later version. + */ + +#pragma once + +#include + +#include + +#include "common/common_types.h" + +namespace Dynarmic { +namespace Common { + +struct FullAddressRange {}; + +struct AddressInterval { + u32 start_address; + std::size_t length; + + // Does this interval overlap with [from, to)? + bool Overlaps(u32 from, u32 to) const { + return start_address <= to && from <= start_address + length; + } +}; + +using AddressRange = boost::variant; + +} // namespace Common +} // namespace Dynarmic diff --git a/src/frontend/ir/basic_block.cpp b/src/frontend/ir/basic_block.cpp index 0a358e44..2219262b 100644 --- a/src/frontend/ir/basic_block.cpp +++ b/src/frontend/ir/basic_block.cpp @@ -35,6 +35,14 @@ LocationDescriptor Block::Location() const { return location; } +LocationDescriptor Block::EndLocation() const { + return end_location; +} + +void Block::SetEndLocation(const LocationDescriptor& descriptor) { + end_location = descriptor; +} + Arm::Cond Block::GetCondition() const { return cond; } diff --git a/src/frontend/ir/basic_block.h b/src/frontend/ir/basic_block.h index 31310adf..b41e81f9 100644 --- a/src/frontend/ir/basic_block.h +++ b/src/frontend/ir/basic_block.h @@ -40,7 +40,8 @@ public: using reverse_iterator = InstructionList::reverse_iterator; using const_reverse_iterator = InstructionList::const_reverse_iterator; - explicit Block(const LocationDescriptor& location) : location(location) {} + explicit Block(const LocationDescriptor& location) + : location(location), end_location(location) {} bool empty() const { return instructions.empty(); } size_type size() const { return instructions.size(); } @@ -78,6 +79,10 @@ public: /// Gets the starting location for this basic block. LocationDescriptor Location() const; + /// Gets the end location for this basic block. + LocationDescriptor EndLocation() const; + /// Sets the end location for this basic block. + void SetEndLocation(const LocationDescriptor& descriptor); /// Gets the condition required to pass in order to execute this block. Arm::Cond GetCondition() const; @@ -116,6 +121,8 @@ public: private: /// Description of the starting location of this block LocationDescriptor location; + /// Description of the end location of this block + LocationDescriptor end_location; /// Conditional to pass in order to execute this block Arm::Cond cond = Arm::Cond::AL; /// Block to execute next if `cond` did not pass. diff --git a/src/frontend/translate/translate_arm.cpp b/src/frontend/translate/translate_arm.cpp index 77774dea..1902c954 100644 --- a/src/frontend/translate/translate_arm.cpp +++ b/src/frontend/translate/translate_arm.cpp @@ -60,6 +60,8 @@ IR::Block TranslateArm(IR::LocationDescriptor descriptor, MemoryReadCodeFuncType ASSERT_MSG(visitor.ir.block.HasTerminal(), "Terminal has not been set"); + visitor.ir.block.SetEndLocation(visitor.ir.current_location); + return std::move(visitor.ir.block); } diff --git a/src/frontend/translate/translate_thumb.cpp b/src/frontend/translate/translate_thumb.cpp index 367c7b08..6a173274 100644 --- a/src/frontend/translate/translate_thumb.cpp +++ b/src/frontend/translate/translate_thumb.cpp @@ -909,6 +909,8 @@ IR::Block TranslateThumb(IR::LocationDescriptor descriptor, MemoryReadCodeFuncTy visitor.ir.block.CycleCount()++; } + visitor.ir.block.SetEndLocation(visitor.ir.current_location); + return std::move(visitor.ir.block); } diff --git a/tests/arm/fuzz_arm.cpp b/tests/arm/fuzz_arm.cpp index 52014cab..84f6de77 100644 --- a/tests/arm/fuzz_arm.cpp +++ b/tests/arm/fuzz_arm.cpp @@ -1166,3 +1166,38 @@ TEST_CASE("Fuzz ARM packing instructions", "[JitX64]") { }); } } + +TEST_CASE("arm: Test InvalidateCacheRange", "[arm]") { + Dynarmic::Jit jit{GetUserCallbacks()}; + code_mem.fill({}); + code_mem[0] = 0xe3a00005; // mov r0, #5 + code_mem[1] = 0xe3a0100D; // mov r1, #13 + code_mem[2] = 0xe0812000; // add r2, r1, r0 + code_mem[3] = 0xeafffffe; // b +#0 (infinite loop) + + jit.Regs() = {}; + jit.Cpsr() = 0x000001d0; // User-mode + + jit.Run(4); + + REQUIRE(jit.Regs()[0] == 5); + REQUIRE(jit.Regs()[1] == 13); + REQUIRE(jit.Regs()[2] == 18); + REQUIRE(jit.Regs()[15] == 0x0000000c); + REQUIRE(jit.Cpsr() == 0x000001d0); + + // Change the code + code_mem[1] = 0xe3a01007; // mov r1, #7 + jit.InvalidateCacheRange(/*start_memory_location = */ 4, /* length_in_bytes = */ 4); + + // Reset position of PC + jit.Regs()[15] = 0; + + jit.Run(4); + + REQUIRE(jit.Regs()[0] == 5); + REQUIRE(jit.Regs()[1] == 7); + REQUIRE(jit.Regs()[2] == 12); + REQUIRE(jit.Regs()[15] == 0x0000000c); + REQUIRE(jit.Cpsr() == 0x000001d0); +}