From b2e4c16ef8552019ea158dcc5d28d8635c851b15 Mon Sep 17 00:00:00 2001 From: MerryMage Date: Mon, 23 Jul 2018 22:58:52 +0100 Subject: [PATCH] A64: Implement FRSQRTS (vector), single/double variant --- .../emit_x64_vector_floating_point.cpp | 61 +++++++++++++++++ src/frontend/A64/decoder/a64.inc | 2 +- src/frontend/A64/translate/impl/impl.h | 66 +++++++++---------- .../A64/translate/impl/simd_three_same.cpp | 14 ++++ src/frontend/ir/ir_emitter.cpp | 11 ++++ src/frontend/ir/ir_emitter.h | 1 + src/frontend/ir/opcodes.inc | 2 + 7 files changed, 123 insertions(+), 34 deletions(-) diff --git a/src/backend_x64/emit_x64_vector_floating_point.cpp b/src/backend_x64/emit_x64_vector_floating_point.cpp index 20e516ae..01687b16 100644 --- a/src/backend_x64/emit_x64_vector_floating_point.cpp +++ b/src/backend_x64/emit_x64_vector_floating_point.cpp @@ -250,6 +250,50 @@ inline void EmitOneArgumentFallback(BlockOfCode& code, EmitContext& ctx, IR::Ins ctx.reg_alloc.DefineValue(inst, xmm0); } +template +inline void EmitTwoArgumentFallback(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Lambda lambda) { + const auto fn = static_cast*>(lambda); + + auto args = ctx.reg_alloc.GetArgumentInfo(inst); + const Xbyak::Xmm arg1 = ctx.reg_alloc.UseXmm(args[0]); + const Xbyak::Xmm arg2 = ctx.reg_alloc.UseXmm(args[1]); + ctx.reg_alloc.EndOfAllocScope(); + ctx.reg_alloc.HostCall(nullptr); + +#ifdef _WIN32 + constexpr u32 stack_space = 4 * 16; + code.sub(rsp, stack_space + ABI_SHADOW_SPACE); + code.lea(code.ABI_PARAM1, ptr[rsp + ABI_SHADOW_SPACE + 1 * 16]); + code.lea(code.ABI_PARAM2, ptr[rsp + ABI_SHADOW_SPACE + 2 * 16]); + code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE + 3 * 16]); + code.mov(code.ABI_PARAM4.cvt32(), ctx.FPCR()); + code.lea(rax, code.ptr[code.r15 + code.GetJitStateInfo().offsetof_fpsr_exc]); + code.mov(qword[rsp + ABI_SHADOW_SPACE + 0 * 16], rax); +#else + constexpr u32 stack_space = 3 * 16; + code.sub(rsp, stack_space + ABI_SHADOW_SPACE); + code.lea(code.ABI_PARAM1, ptr[rsp + ABI_SHADOW_SPACE + 0 * 16]); + code.lea(code.ABI_PARAM2, ptr[rsp + ABI_SHADOW_SPACE + 1 * 16]); + code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE + 2 * 16]); + code.mov(code.ABI_PARAM4.cvt32(), ctx.FPCR()); + code.lea(code.ABI_PARAM5, code.ptr[code.r15 + code.GetJitStateInfo().offsetof_fpsr_exc]); +#endif + + code.movaps(xword[code.ABI_PARAM2], arg1); + code.movaps(xword[code.ABI_PARAM3], arg2); + code.CallFunction(fn); + +#ifdef _WIN32 + code.movaps(xmm0, xword[rsp + ABI_SHADOW_SPACE + 1 * 16]); +#else + code.movaps(xmm0, xword[rsp + ABI_SHADOW_SPACE + 0 * 16]); +#endif + + code.add(rsp, stack_space + ABI_SHADOW_SPACE); + + ctx.reg_alloc.DefineValue(inst, xmm0); +} + void EmitX64::EmitFPVectorAbs16(EmitContext& ctx, IR::Inst* inst) { auto args = ctx.reg_alloc.GetArgumentInfo(inst); @@ -410,6 +454,23 @@ void EmitX64::EmitFPVectorRSqrtEstimate64(EmitContext& ctx, IR::Inst* inst) { EmitRSqrtEstimate(code, ctx, inst); } +template +static void EmitRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) { + EmitTwoArgumentFallback(code, ctx, inst, [](VectorArray& result, const VectorArray& op1, const VectorArray& op2, FP::FPCR fpcr, FP::FPSR& fpsr) { + for (size_t i = 0; i < result.size(); i++) { + result[i] = FP::FPRSqrtStepFused(op1[i], op2[i], fpcr, fpsr); + } + }); +} + +void EmitX64::EmitFPVectorRSqrtStepFused32(EmitContext& ctx, IR::Inst* inst) { + EmitRSqrtStepFused(code, ctx, inst); +} + +void EmitX64::EmitFPVectorRSqrtStepFused64(EmitContext& ctx, IR::Inst* inst) { + EmitRSqrtStepFused(code, ctx, inst); +} + void EmitX64::EmitFPVectorS32ToSingle(EmitContext& ctx, IR::Inst* inst) { auto args = ctx.reg_alloc.GetArgumentInfo(inst); const Xbyak::Xmm xmm = ctx.reg_alloc.UseScratchXmm(args[0]); diff --git a/src/frontend/A64/decoder/a64.inc b/src/frontend/A64/decoder/a64.inc index ab15234e..1401bb60 100644 --- a/src/frontend/A64/decoder/a64.inc +++ b/src/frontend/A64/decoder/a64.inc @@ -739,7 +739,7 @@ INST(BIC_asimd_reg, "BIC (vector, register)", "0Q001 INST(FSUB_2, "FSUB (vector)", "0Q0011101z1mmmmm110101nnnnnddddd") //INST(FMLSL_vec_1, "FMLSL, FMLSL2 (vector)", "0Q0011101z1mmmmm111011nnnnnddddd") //INST(FMIN_2, "FMIN (vector)", "0Q0011101z1mmmmm111101nnnnnddddd") -//INST(FRSQRTS_4, "FRSQRTS", "0Q0011101z1mmmmm111111nnnnnddddd") +INST(FRSQRTS_4, "FRSQRTS", "0Q0011101z1mmmmm111111nnnnnddddd") INST(ORR_asimd_reg, "ORR (vector, register)", "0Q001110101mmmmm000111nnnnnddddd") INST(ORN_asimd, "ORN (vector)", "0Q001110111mmmmm000111nnnnnddddd") INST(UHADD, "UHADD", "0Q101110zz1mmmmm000001nnnnnddddd") diff --git a/src/frontend/A64/translate/impl/impl.h b/src/frontend/A64/translate/impl/impl.h index 0da93c88..7dde70d7 100644 --- a/src/frontend/A64/translate/impl/impl.h +++ b/src/frontend/A64/translate/impl/impl.h @@ -404,40 +404,22 @@ struct TranslatorVisitor final { // Data Processing - FP and SIMD - Scalar three bool FMULX_vec_1(Vec Vm, Vec Vn, Vec Vd); bool FMULX_vec_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FMULX_vec_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMULX_vec_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FCMEQ_reg_1(Vec Vm, Vec Vn, Vec Vd); bool FCMEQ_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FCMEQ_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FCMEQ_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FRECPS_1(Vec Vm, Vec Vn, Vec Vd); bool FRECPS_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FRECPS_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FRSQRTS_1(Vec Vm, Vec Vn, Vec Vd); bool FRSQRTS_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FCMGE_reg_1(Vec Vm, Vec Vn, Vec Vd); bool FCMGE_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FCMGE_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FCMGE_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FACGE_1(Vec Vm, Vec Vn, Vec Vd); bool FACGE_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FACGE_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FACGE_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FABD_1(Vec Vm, Vec Vn, Vec Vd); bool FABD_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FABD_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FABD_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FCMGT_reg_1(Vec Vm, Vec Vn, Vec Vd); bool FCMGT_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FCMGT_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FCMGT_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FACGT_1(Vec Vm, Vec Vn, Vec Vd); bool FACGT_2(bool sz, Vec Vm, Vec Vn, Vec Vd); - bool FACGT_3(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FACGT_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); // Data Processing - FP and SIMD - Two register misc FP16 bool FCVTNS_1(Vec Vn, Vec Vd); @@ -697,36 +679,30 @@ struct TranslatorVisitor final { bool INS_elt(Imm<5> imm5, Imm<4> imm4, Vec Vn, Vec Vd); // Data Processing - FP and SIMD - SIMD Three same + bool FMULX_vec_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FCMEQ_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FRECPS_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FCMGE_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FACGE_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FABD_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FCMGT_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FACGT_3(bool Q, Vec Vm, Vec Vn, Vec Vd); bool FMAXNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMAXNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMLA_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMLA_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FADD_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FADD_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMAX_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMAX_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMINNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMINNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMLS_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMLS_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FSUB_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMIN_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMIN_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMAXNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMAXNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FADDP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FADDP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMUL_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMUL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMAXP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMAXP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FDIV_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FDIV_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMINNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMINNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); bool FMINP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd); - bool FMINP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); // Data Processing - FP and SIMD - SIMD Two-register misc bool FRINTN_1(bool Q, Vec Vn, Vec Vd); @@ -854,6 +830,30 @@ struct TranslatorVisitor final { bool BSL(bool Q, Vec Vm, Vec Vn, Vec Vd); bool BIT(bool Q, Vec Vm, Vec Vn, Vec Vd); bool BIF(bool Q, Vec Vm, Vec Vn, Vec Vd); + bool FMAXNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMLA_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FADD_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMAX_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMINNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMLS_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMIN_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMAXNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FADDP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMUL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMAXP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FDIV_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMINNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMINP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FMULX_vec_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FCMEQ_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FCMGE_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FACGE_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FABD_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FCMGT_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); + bool FACGT_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd); // Data Processing - FP and SIMD - SIMD modified immediate bool MOVI(bool Q, bool op, Imm<1> a, Imm<1> b, Imm<1> c, Imm<4> cmode, Imm<1> d, Imm<1> e, Imm<1> f, Imm<1> g, Imm<1> h, Vec Vd); diff --git a/src/frontend/A64/translate/impl/simd_three_same.cpp b/src/frontend/A64/translate/impl/simd_three_same.cpp index 2cf94673..366a444c 100644 --- a/src/frontend/A64/translate/impl/simd_three_same.cpp +++ b/src/frontend/A64/translate/impl/simd_three_same.cpp @@ -592,6 +592,20 @@ bool TranslatorVisitor::FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { return true; } +bool TranslatorVisitor::FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { + if (sz && !Q) { + return ReservedValue(); + } + const size_t esize = sz ? 64 : 32; + const size_t datasize = Q ? 128 : 64; + + const IR::U128 operand1 = V(datasize, Vn); + const IR::U128 operand2 = V(datasize, Vm); + const IR::U128 result = ir.FPVectorRSqrtStepFused(esize, operand1, operand2); + V(datasize, Vd, result); + return true; +} + bool TranslatorVisitor::ORR_asimd_reg(bool Q, Vec Vm, Vec Vn, Vec Vd) { const size_t datasize = Q ? 128 : 64; diff --git a/src/frontend/ir/ir_emitter.cpp b/src/frontend/ir/ir_emitter.cpp index 863aaecf..84d462d2 100644 --- a/src/frontend/ir/ir_emitter.cpp +++ b/src/frontend/ir/ir_emitter.cpp @@ -1690,6 +1690,17 @@ U128 IREmitter::FPVectorRSqrtEstimate(size_t esize, const U128& a) { return {}; } +U128 IREmitter::FPVectorRSqrtStepFused(size_t esize, const U128& a, const U128& b) { + switch (esize) { + case 32: + return Inst(Opcode::FPVectorRSqrtStepFused32, a, b); + case 64: + return Inst(Opcode::FPVectorRSqrtStepFused64, a, b); + } + UNREACHABLE(); + return {}; +} + U128 IREmitter::FPVectorSub(size_t esize, const U128& a, const U128& b) { switch (esize) { case 32: diff --git a/src/frontend/ir/ir_emitter.h b/src/frontend/ir/ir_emitter.h index 18724816..6820b0a0 100644 --- a/src/frontend/ir/ir_emitter.h +++ b/src/frontend/ir/ir_emitter.h @@ -300,6 +300,7 @@ public: U128 FPVectorPairedAdd(size_t esize, const U128& a, const U128& b); U128 FPVectorPairedAddLower(size_t esize, const U128& a, const U128& b); U128 FPVectorRSqrtEstimate(size_t esize, const U128& a); + U128 FPVectorRSqrtStepFused(size_t esize, const U128& a, const U128& b); U128 FPVectorSub(size_t esize, const U128& a, const U128& b); U128 FPVectorS32ToSingle(const U128& a); U128 FPVectorS64ToDouble(const U128& a); diff --git a/src/frontend/ir/opcodes.inc b/src/frontend/ir/opcodes.inc index e294dc86..2e7ec331 100644 --- a/src/frontend/ir/opcodes.inc +++ b/src/frontend/ir/opcodes.inc @@ -437,6 +437,8 @@ OPCODE(FPVectorPairedAdd32, T::U128, T::U128, T::U OPCODE(FPVectorPairedAdd64, T::U128, T::U128, T::U128 ) OPCODE(FPVectorRSqrtEstimate32, T::U128, T::U128 ) OPCODE(FPVectorRSqrtEstimate64, T::U128, T::U128 ) +OPCODE(FPVectorRSqrtStepFused32, T::U128, T::U128, T::U128 ) +OPCODE(FPVectorRSqrtStepFused64, T::U128, T::U128, T::U128 ) OPCODE(FPVectorS32ToSingle, T::U128, T::U128 ) OPCODE(FPVectorS64ToDouble, T::U128, T::U128 ) OPCODE(FPVectorSub32, T::U128, T::U128, T::U128 )