From f78ba9ac557798ce451283f50a0dc19c10363fd0 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 22 Oct 2022 14:04:00 +0200 Subject: [PATCH 1/3] JitArm64: Never check downcount on block entry Jumping between linked blocks currently works as follows: First, at the end of the first block, we check if the downcount is greater than zero. If it is, we jump to the `normalEntry` of the block. So far so good. But if the downcount wasn't greater than zero, we jump to the `checkedEntry` of the block, which checks the downcount *again* and then jumps to `do_timing` if it's less than zero (which seems like an off by one error - Jit64 doesn't do anything like this). This second check is rather redundant. Let's jump to `do_timing` where we previously jumped to `checkedEntry`. Jit64 doesn't check the downcount on block entry. See 5236dc3. --- Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 31 ++++---- .../Core/PowerPC/JitArm64/JitArm64Cache.cpp | 79 ++++++++++++------- .../Core/PowerPC/JitArm64/JitArm64Cache.h | 3 + 3 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index ef7737d69c..f8b2a99f72 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -361,11 +361,14 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return LK &= m_enable_blr_optimization; + const u8* host_address_after_return; if (LK) { - // Push {ARM_PC+20; PPC_PC} on the stack + // Push {ARM_PC; PPC_PC} on the stack MOVI2R(ARM64Reg::X1, exit_address_after_return); - ADR(ARM64Reg::X0, 20); + constexpr s32 adr_offset = JitArm64BlockCache::BLOCK_LINK_SIZE + sizeof(u32) * 2; + host_address_after_return = GetCodePtr() + adr_offset; + ADR(ARM64Reg::X0, adr_offset); STP(IndexType::Pre, ARM64Reg::X0, ARM64Reg::X1, ARM64Reg::SP, -16); } @@ -381,6 +384,8 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return if (LK) { + DEBUG_ASSERT(GetCodePtr() == host_address_after_return || HasWriteFailed()); + // Write the regular exit node after the return. linkData.exitAddress = exit_address_after_return; linkData.exitPtrs = GetWritableCodePtr(); @@ -411,10 +416,13 @@ void JitArm64::WriteExit(Arm64Gen::ARM64Reg dest, bool LK, u32 exit_address_afte { // Push {ARM_PC, PPC_PC} on the stack MOVI2R(ARM64Reg::X1, exit_address_after_return); - ADR(ARM64Reg::X0, 12); + constexpr s32 adr_offset = sizeof(u32) * 3; + const u8* host_address_after_return = GetCodePtr() + adr_offset; + ADR(ARM64Reg::X0, adr_offset); STP(IndexType::Pre, ARM64Reg::X0, ARM64Reg::X1, ARM64Reg::SP, -16); BL(dispatcher); + DEBUG_ASSERT(GetCodePtr() == host_address_after_return || HasWriteFailed()); // Write the regular exit node after the return. JitBlock* b = js.curBlock; @@ -440,11 +448,14 @@ void JitArm64::FakeLKExit(u32 exit_address_after_return) ARM64Reg after_reg = gpr.GetReg(); ARM64Reg code_reg = gpr.GetReg(); MOVI2R(after_reg, exit_address_after_return); - ADR(EncodeRegTo64(code_reg), 12); + constexpr s32 adr_offset = sizeof(u32) * 3; + const u8* host_address_after_return = GetCodePtr() + adr_offset; + ADR(EncodeRegTo64(code_reg), adr_offset); STP(IndexType::Pre, EncodeRegTo64(code_reg), EncodeRegTo64(after_reg), ARM64Reg::SP, -16); gpr.Unlock(after_reg, code_reg); FixupBranch skip_exit = BL(); + DEBUG_ASSERT(GetCodePtr() == host_address_after_return || HasWriteFailed()); gpr.Unlock(ARM64Reg::W30); // Write the regular exit node after the return. @@ -839,17 +850,7 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) u8* const start = GetWritableCodePtr(); b->checkedEntry = start; - - // Downcount flag check, Only valid for linked blocks - { - FixupBranch bail = B(CC_PL); - MOVI2R(DISPATCHER_PC, js.blockStart); - B(do_timing); - SetJumpTarget(bail); - } - - // Normal entry doesn't need to check for downcount. - b->normalEntry = GetWritableCodePtr(); + b->normalEntry = start; // Conditionally add profiling code. if (jo.profile_blocks) diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp index 6551ed2d45..f0e6922df8 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp @@ -22,52 +22,73 @@ void JitArm64BlockCache::Init() void JitArm64BlockCache::WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, const JitBlock::LinkData& source, const JitBlock* dest) { + const u8* start = emit.GetCodePtr(); + if (!dest) { - // Use a fixed amount of instructions, so we can assume to use 3 instructions on patching. - emit.MOVZ(DISPATCHER_PC, source.exitAddress & 0xFFFF, ShiftAmount::Shift0); - emit.MOVK(DISPATCHER_PC, source.exitAddress >> 16, ShiftAmount::Shift16); - + emit.MOVI2R(DISPATCHER_PC, source.exitAddress); if (source.call) + { + while (emit.GetCodePtr() < start + BLOCK_LINK_FAST_BL_OFFSET && !emit.HasWriteFailed()) + emit.NOP(); emit.BL(m_jit.GetAsmRoutines()->dispatcher); + } else + { emit.B(m_jit.GetAsmRoutines()->dispatcher); - return; + } + } + else + { + if (source.call) + { + // The "fast" BL should be the last instruction, so that the return address matches the + // address that was pushed onto the stack by the function that called WriteLinkBlock + FixupBranch fast = emit.B(CC_GT); + emit.MOVI2R(DISPATCHER_PC, source.exitAddress); + emit.BL(m_jit.GetAsmRoutines()->do_timing); + while (emit.GetCodePtr() < start + BLOCK_LINK_FAST_BL_OFFSET && !emit.HasWriteFailed()) + emit.BRK(101); + emit.SetJumpTarget(fast); + emit.BL(dest->normalEntry); + } + else + { + // Are we able to jump directly to the block? + s64 block_distance = ((s64)dest->normalEntry - (s64)emit.GetCodePtr()) >> 2; + if (block_distance >= -0x40000 && block_distance <= 0x3FFFF) + { + emit.B(CC_GT, dest->normalEntry); + emit.MOVI2R(DISPATCHER_PC, source.exitAddress); + emit.B(m_jit.GetAsmRoutines()->do_timing); + } + else + { + FixupBranch slow = emit.B(CC_LE); + emit.B(dest->normalEntry); + emit.SetJumpTarget(slow); + emit.MOVI2R(DISPATCHER_PC, source.exitAddress); + emit.B(m_jit.GetAsmRoutines()->do_timing); + } + } } - if (source.call) + // Use a fixed number of instructions so we have enough room for any patching needed later. + const u8* end = start + BLOCK_LINK_SIZE; + while (emit.GetCodePtr() < end) { - // The "fast" BL must be the third instruction. So just use the former two to inline the - // downcount check here. It's better to do this near jump before the long jump to the other - // block. - FixupBranch fast_link = emit.B(CC_GT); - emit.BL(dest->checkedEntry); - emit.SetJumpTarget(fast_link); - emit.BL(dest->normalEntry); - return; - } - - // Are we able to jump directly to the normal entry? - s64 distance = ((s64)dest->normalEntry - (s64)emit.GetCodePtr()) >> 2; - if (distance >= -0x40000 && distance <= 0x3FFFF) - { - emit.B(CC_GT, dest->normalEntry); - emit.B(dest->checkedEntry); emit.BRK(101); - return; + if (emit.HasWriteFailed()) + return; } - - FixupBranch fast_link = emit.B(CC_GT); - emit.B(dest->checkedEntry); - emit.SetJumpTarget(fast_link); - emit.B(dest->normalEntry); + ASSERT(emit.GetCodePtr() == end); } void JitArm64BlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) { const Common::ScopedJITPageWriteAndNoExecute enable_jit_page_writes; u8* location = source.exitPtrs; - ARM64XEmitter emit(location, location + 12); + ARM64XEmitter emit(location, location + BLOCK_LINK_SIZE); WriteLinkBlock(emit, source, dest); emit.FlushIcache(); diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h index 0ba537df90..dc068054d7 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h @@ -29,6 +29,9 @@ public: void WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, const JitBlock::LinkData& source, const JitBlock* dest = nullptr); + static constexpr size_t BLOCK_LINK_SIZE = 5 * sizeof(u32); + static constexpr size_t BLOCK_LINK_FAST_BL_OFFSET = BLOCK_LINK_SIZE - sizeof(u32); + private: void WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) override; void WriteDestroyBlock(const JitBlock& block) override; From 1813f0fdb598542f3771769d388bbab76b8fc6b4 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 22 Oct 2022 14:09:03 +0200 Subject: [PATCH 2/3] Jit: Remove checkedEntry It's now always identical to normalEntry. --- .../Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp | 3 +-- Source/Core/Core/PowerPC/Jit64/Jit.cpp | 6 ++---- Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp | 9 +++------ Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 6 ++---- Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp | 7 +++---- Source/Core/Core/PowerPC/JitCommon/JitCache.cpp | 4 ++-- Source/Core/Core/PowerPC/JitCommon/JitCache.h | 3 --- Source/Core/Core/PowerPC/JitInterface.cpp | 2 +- 8 files changed, 14 insertions(+), 26 deletions(-) diff --git a/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp b/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp index 34f7d51593..eca1afd32d 100644 --- a/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp +++ b/Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp @@ -311,7 +311,6 @@ void CachedInterpreter::Jit(u32 address) js.numFloatingPointInst = 0; js.curBlock = b; - b->checkedEntry = GetCodePtr(); b->normalEntry = GetCodePtr(); for (u32 i = 0; i < code_block.m_num_instructions; i++) @@ -374,7 +373,7 @@ void CachedInterpreter::Jit(u32 address) } m_code.emplace_back(); - b->codeSize = (u32)(GetCodePtr() - b->checkedEntry); + b->codeSize = static_cast(GetCodePtr() - b->normalEntry); b->originalSize = code_block.m_num_instructions; m_block_cache.FinalizeBlock(*b, jo.enableBlocklink, code_block.m_physical_addresses); diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp index 313b840132..1f4e86ffbf 100644 --- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp +++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp @@ -830,9 +830,7 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) js.numFloatingPointInst = 0; // TODO: Test if this or AlignCode16 make a difference from GetCodePtr - u8* const start = AlignCode4(); - b->checkedEntry = start; - b->normalEntry = start; + b->normalEntry = AlignCode4(); // Used to get a trace of the last few blocks before a crash, sometimes VERY useful if (m_im_here_debug) @@ -1140,7 +1138,7 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) return false; } - b->codeSize = (u32)(GetCodePtr() - start); + b->codeSize = static_cast(GetCodePtr() - b->normalEntry); b->originalSize = code_block.m_num_instructions; #ifdef JIT_LOG_GENERATED_CODE diff --git a/Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp b/Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp index 3b0b7e5471..351c6e1c09 100644 --- a/Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp +++ b/Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp @@ -14,8 +14,7 @@ JitBlockCache::JitBlockCache(JitBase& jit) : JitBaseBlockCache{jit} void JitBlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) { u8* location = source.exitPtrs; - const u8* address = - dest ? dest->checkedEntry : m_jit.GetAsmRoutines()->dispatcher_no_timing_check; + const u8* address = dest ? dest->normalEntry : m_jit.GetAsmRoutines()->dispatcher_no_timing_check; if (source.call) { Gen::XEmitter emit(location, location + 5); @@ -42,11 +41,9 @@ void JitBlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBl void JitBlockCache::WriteDestroyBlock(const JitBlock& block) { - // Only clear the entry points as we might still be within this block. - Gen::XEmitter emit(block.checkedEntry, block.checkedEntry + 1); + // Only clear the entry point as we might still be within this block. + Gen::XEmitter emit(block.normalEntry, block.normalEntry + 1); emit.INT3(); - Gen::XEmitter emit2(block.normalEntry, block.normalEntry + 1); - emit2.INT3(); } void JitBlockCache::Init() diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index f8b2a99f72..8bf5178616 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -848,9 +848,7 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) js.numLoadStoreInst = 0; js.numFloatingPointInst = 0; - u8* const start = GetWritableCodePtr(); - b->checkedEntry = start; - b->normalEntry = start; + b->normalEntry = GetWritableCodePtr(); // Conditionally add profiling code. if (jo.profile_blocks) @@ -1117,7 +1115,7 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC) return false; } - b->codeSize = (u32)(GetCodePtr() - start); + b->codeSize = static_cast(GetCodePtr() - b->normalEntry); b->originalSize = code_block.m_num_instructions; FlushIcache(); diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp index f0e6922df8..2fcefa019c 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp @@ -96,11 +96,10 @@ void JitArm64BlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const void JitArm64BlockCache::WriteDestroyBlock(const JitBlock& block) { - // Only clear the entry points as we might still be within this block. - ARM64XEmitter emit(block.checkedEntry, block.normalEntry + 4); + // Only clear the entry point as we might still be within this block. + ARM64XEmitter emit(block.normalEntry, block.normalEntry + 4); const Common::ScopedJITPageWriteAndNoExecute enable_jit_page_writes; - while (emit.GetWritableCodePtr() <= block.normalEntry) - emit.BRK(0x123); + emit.BRK(0x123); emit.FlushIcache(); } diff --git a/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp b/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp index 6b746d94d1..a41ebb71b2 100644 --- a/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp +++ b/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp @@ -163,12 +163,12 @@ void JitBaseBlockCache::FinalizeBlock(JitBlock& block, bool block_link, if (Common::JitRegister::IsEnabled() && (symbol = g_symbolDB.GetSymbolFromAddr(block.effectiveAddress)) != nullptr) { - Common::JitRegister::Register(block.checkedEntry, block.codeSize, "JIT_PPC_{}_{:08x}", + Common::JitRegister::Register(block.normalEntry, block.codeSize, "JIT_PPC_{}_{:08x}", symbol->function_name.c_str(), block.physicalAddress); } else { - Common::JitRegister::Register(block.checkedEntry, block.codeSize, "JIT_PPC_{:08x}", + Common::JitRegister::Register(block.normalEntry, block.codeSize, "JIT_PPC_{:08x}", block.physicalAddress); } } diff --git a/Source/Core/Core/PowerPC/JitCommon/JitCache.h b/Source/Core/Core/PowerPC/JitCommon/JitCache.h index 61b5966a14..fb72cbe07e 100644 --- a/Source/Core/Core/PowerPC/JitCommon/JitCache.h +++ b/Source/Core/Core/PowerPC/JitCommon/JitCache.h @@ -30,9 +30,6 @@ struct JitBlockData u8* far_begin; u8* far_end; - // A special entry point for block linking; usually used to check the - // downcount. - u8* checkedEntry; // The normal entry point for the block, returned by Dispatch(). u8* normalEntry; diff --git a/Source/Core/Core/PowerPC/JitInterface.cpp b/Source/Core/Core/PowerPC/JitInterface.cpp index 5e369067f6..0aed99b225 100644 --- a/Source/Core/Core/PowerPC/JitInterface.cpp +++ b/Source/Core/Core/PowerPC/JitInterface.cpp @@ -187,7 +187,7 @@ JitInterface::GetHostCode(u32 address) const } GetHostCodeResult result; - result.code = block->checkedEntry; + result.code = block->normalEntry; result.code_size = block->codeSize; result.entry_address = block->effectiveAddress; return result; From d2c5d79614c728735cf637d5c53eb6591828dc79 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 22 Oct 2022 16:30:12 +0200 Subject: [PATCH 3/3] JitArm64: Use farcode in WriteLinkBlock Now block link nearcode is back to a length of three instructions. Unfortunately, the code I'm adding to Jit.cpp ends up being a bit messy because we need to handle the case of already being in farcode... --- Source/Core/Core/PowerPC/JitArm64/Jit.cpp | 75 +++++++++++++++++++ .../Core/PowerPC/JitArm64/JitArm64Cache.cpp | 15 ++-- .../Core/PowerPC/JitArm64/JitArm64Cache.h | 2 +- Source/Core/Core/PowerPC/JitCommon/JitCache.h | 3 + 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp index 8bf5178616..7f93d359af 100644 --- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp @@ -372,12 +372,29 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return STP(IndexType::Pre, ARM64Reg::X0, ARM64Reg::X1, ARM64Reg::SP, -16); } + constexpr size_t primary_farcode_size = 3 * sizeof(u32); + const bool switch_to_far_code = !IsInFarCode(); + const u8* primary_farcode_addr; + if (switch_to_far_code) + { + SwitchToFarCode(); + primary_farcode_addr = GetCodePtr(); + SwitchToNearCode(); + } + else + { + primary_farcode_addr = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE + + (LK ? JitArm64BlockCache::BLOCK_LINK_SIZE : 0); + } + const u8* return_farcode_addr = primary_farcode_addr + primary_farcode_size; + JitBlock* b = js.curBlock; JitBlock::LinkData linkData; linkData.exitAddress = destination; linkData.exitPtrs = GetWritableCodePtr(); linkData.linkStatus = false; linkData.call = LK; + linkData.exitFarcode = primary_farcode_addr; b->linkData.push_back(linkData); blocks.WriteLinkBlock(*this, linkData); @@ -391,10 +408,32 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return linkData.exitPtrs = GetWritableCodePtr(); linkData.linkStatus = false; linkData.call = false; + linkData.exitFarcode = return_farcode_addr; b->linkData.push_back(linkData); blocks.WriteLinkBlock(*this, linkData); } + + if (switch_to_far_code) + SwitchToFarCode(); + DEBUG_ASSERT(GetCodePtr() == primary_farcode_addr || HasWriteFailed()); + MOVI2R(DISPATCHER_PC, destination); + if (LK) + BL(GetAsmRoutines()->do_timing); + else + B(GetAsmRoutines()->do_timing); + + if (LK) + { + if (GetCodePtr() == return_farcode_addr - sizeof(u32)) + BRK(101); + DEBUG_ASSERT(GetCodePtr() == return_farcode_addr || HasWriteFailed()); + MOVI2R(DISPATCHER_PC, exit_address_after_return); + B(GetAsmRoutines()->do_timing); + } + + if (switch_to_far_code) + SwitchToNearCode(); } void JitArm64::WriteExit(Arm64Gen::ARM64Reg dest, bool LK, u32 exit_address_after_return) @@ -431,9 +470,27 @@ void JitArm64::WriteExit(Arm64Gen::ARM64Reg dest, bool LK, u32 exit_address_afte linkData.exitPtrs = GetWritableCodePtr(); linkData.linkStatus = false; linkData.call = false; + const bool switch_to_far_code = !IsInFarCode(); + if (switch_to_far_code) + { + SwitchToFarCode(); + linkData.exitFarcode = GetCodePtr(); + SwitchToNearCode(); + } + else + { + linkData.exitFarcode = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE; + } b->linkData.push_back(linkData); blocks.WriteLinkBlock(*this, linkData); + + if (switch_to_far_code) + SwitchToFarCode(); + MOVI2R(DISPATCHER_PC, exit_address_after_return); + B(GetAsmRoutines()->do_timing); + if (switch_to_far_code) + SwitchToNearCode(); } } @@ -465,10 +522,28 @@ void JitArm64::FakeLKExit(u32 exit_address_after_return) linkData.exitPtrs = GetWritableCodePtr(); linkData.linkStatus = false; linkData.call = false; + const bool switch_to_far_code = !IsInFarCode(); + if (switch_to_far_code) + { + SwitchToFarCode(); + linkData.exitFarcode = GetCodePtr(); + SwitchToNearCode(); + } + else + { + linkData.exitFarcode = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE; + } b->linkData.push_back(linkData); blocks.WriteLinkBlock(*this, linkData); + if (switch_to_far_code) + SwitchToFarCode(); + MOVI2R(DISPATCHER_PC, exit_address_after_return); + B(GetAsmRoutines()->do_timing); + if (switch_to_far_code) + SwitchToNearCode(); + SetJumpTarget(skip_exit); } diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp index 2fcefa019c..e991fe04c9 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp @@ -29,8 +29,9 @@ void JitArm64BlockCache::WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, emit.MOVI2R(DISPATCHER_PC, source.exitAddress); if (source.call) { - while (emit.GetCodePtr() < start + BLOCK_LINK_FAST_BL_OFFSET && !emit.HasWriteFailed()) + if (emit.GetCodePtr() == start + BLOCK_LINK_FAST_BL_OFFSET - sizeof(u32)) emit.NOP(); + DEBUG_ASSERT(emit.GetCodePtr() == start + BLOCK_LINK_FAST_BL_OFFSET || emit.HasWriteFailed()); emit.BL(m_jit.GetAsmRoutines()->dispatcher); } else @@ -45,10 +46,8 @@ void JitArm64BlockCache::WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, // The "fast" BL should be the last instruction, so that the return address matches the // address that was pushed onto the stack by the function that called WriteLinkBlock FixupBranch fast = emit.B(CC_GT); - emit.MOVI2R(DISPATCHER_PC, source.exitAddress); - emit.BL(m_jit.GetAsmRoutines()->do_timing); - while (emit.GetCodePtr() < start + BLOCK_LINK_FAST_BL_OFFSET && !emit.HasWriteFailed()) - emit.BRK(101); + emit.B(source.exitFarcode); + DEBUG_ASSERT(emit.GetCodePtr() == start + BLOCK_LINK_FAST_BL_OFFSET || emit.HasWriteFailed()); emit.SetJumpTarget(fast); emit.BL(dest->normalEntry); } @@ -59,16 +58,14 @@ void JitArm64BlockCache::WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, if (block_distance >= -0x40000 && block_distance <= 0x3FFFF) { emit.B(CC_GT, dest->normalEntry); - emit.MOVI2R(DISPATCHER_PC, source.exitAddress); - emit.B(m_jit.GetAsmRoutines()->do_timing); + emit.B(source.exitFarcode); } else { FixupBranch slow = emit.B(CC_LE); emit.B(dest->normalEntry); emit.SetJumpTarget(slow); - emit.MOVI2R(DISPATCHER_PC, source.exitAddress); - emit.B(m_jit.GetAsmRoutines()->do_timing); + emit.B(source.exitFarcode); } } } diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h index dc068054d7..1214e8c308 100644 --- a/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h +++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h @@ -29,7 +29,7 @@ public: void WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, const JitBlock::LinkData& source, const JitBlock* dest = nullptr); - static constexpr size_t BLOCK_LINK_SIZE = 5 * sizeof(u32); + static constexpr size_t BLOCK_LINK_SIZE = 3 * sizeof(u32); static constexpr size_t BLOCK_LINK_FAST_BL_OFFSET = BLOCK_LINK_SIZE - sizeof(u32); private: diff --git a/Source/Core/Core/PowerPC/JitCommon/JitCache.h b/Source/Core/Core/PowerPC/JitCommon/JitCache.h index fb72cbe07e..d3f353e815 100644 --- a/Source/Core/Core/PowerPC/JitCommon/JitCache.h +++ b/Source/Core/Core/PowerPC/JitCommon/JitCache.h @@ -70,6 +70,9 @@ struct JitBlock : public JitBlockData struct LinkData { u8* exitPtrs; // to be able to rewrite the exit jump +#ifdef _M_ARM_64 + const u8* exitFarcode; +#endif u32 exitAddress; bool linkStatus; // is it already linked? bool call;