A64: Implement FRSQRTS (vector), single/double variant

src/backend_x64/emit_x64_vector_floating_point.cpp

@@ -250,6 +250,50 @@ inline void EmitOneArgumentFallback(BlockOfCode& code, EmitContext& ctx, IR::Ins
     ctx.reg_alloc.DefineValue(inst, xmm0);
 }
 
+template <typename Lambda>
+inline void EmitTwoArgumentFallback(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst, Lambda lambda) {
+    const auto fn = static_cast<mp::equivalent_function_type_t<Lambda>*>(lambda);
+
+    auto args = ctx.reg_alloc.GetArgumentInfo(inst);
+    const Xbyak::Xmm arg1 = ctx.reg_alloc.UseXmm(args[0]);
+    const Xbyak::Xmm arg2 = ctx.reg_alloc.UseXmm(args[1]);
+    ctx.reg_alloc.EndOfAllocScope();
+    ctx.reg_alloc.HostCall(nullptr);
+
+#ifdef _WIN32
+    constexpr u32 stack_space = 4 * 16;
+    code.sub(rsp, stack_space + ABI_SHADOW_SPACE);
+    code.lea(code.ABI_PARAM1, ptr[rsp + ABI_SHADOW_SPACE + 1 * 16]);
+    code.lea(code.ABI_PARAM2, ptr[rsp + ABI_SHADOW_SPACE + 2 * 16]);
+    code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE + 3 * 16]);
+    code.mov(code.ABI_PARAM4.cvt32(), ctx.FPCR());
+    code.lea(rax, code.ptr[code.r15 + code.GetJitStateInfo().offsetof_fpsr_exc]);
+    code.mov(qword[rsp + ABI_SHADOW_SPACE + 0 * 16], rax);
+#else
+    constexpr u32 stack_space = 3 * 16;
+    code.sub(rsp, stack_space + ABI_SHADOW_SPACE);
+    code.lea(code.ABI_PARAM1, ptr[rsp + ABI_SHADOW_SPACE + 0 * 16]);
+    code.lea(code.ABI_PARAM2, ptr[rsp + ABI_SHADOW_SPACE + 1 * 16]);
+    code.lea(code.ABI_PARAM3, ptr[rsp + ABI_SHADOW_SPACE + 2 * 16]);
+    code.mov(code.ABI_PARAM4.cvt32(), ctx.FPCR());
+    code.lea(code.ABI_PARAM5, code.ptr[code.r15 + code.GetJitStateInfo().offsetof_fpsr_exc]);
+#endif
+
+    code.movaps(xword[code.ABI_PARAM2], arg1);
+    code.movaps(xword[code.ABI_PARAM3], arg2);
+    code.CallFunction(fn);
+
+#ifdef _WIN32
+    code.movaps(xmm0, xword[rsp + ABI_SHADOW_SPACE + 1 * 16]);
+#else
+    code.movaps(xmm0, xword[rsp + ABI_SHADOW_SPACE + 0 * 16]);
+#endif
+
+    code.add(rsp, stack_space + ABI_SHADOW_SPACE);
+
+    ctx.reg_alloc.DefineValue(inst, xmm0);
+}
+
 void EmitX64::EmitFPVectorAbs16(EmitContext& ctx, IR::Inst* inst) {
     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
 
@@ -410,6 +454,23 @@ void EmitX64::EmitFPVectorRSqrtEstimate64(EmitContext& ctx, IR::Inst* inst) {
     EmitRSqrtEstimate<u64>(code, ctx, inst);
 }
 
+template<typename FPT>
+static void EmitRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
+    EmitTwoArgumentFallback(code, ctx, inst, [](VectorArray<FPT>& result, const VectorArray<FPT>& op1, const VectorArray<FPT>& op2, FP::FPCR fpcr, FP::FPSR& fpsr) {
+        for (size_t i = 0; i < result.size(); i++) {
+            result[i] = FP::FPRSqrtStepFused<FPT>(op1[i], op2[i], fpcr, fpsr);
+        }
+    });
+}
+
+void EmitX64::EmitFPVectorRSqrtStepFused32(EmitContext& ctx, IR::Inst* inst) {
+    EmitRSqrtStepFused<u32>(code, ctx, inst);
+}
+
+void EmitX64::EmitFPVectorRSqrtStepFused64(EmitContext& ctx, IR::Inst* inst) {
+    EmitRSqrtStepFused<u64>(code, ctx, inst);
+}
+
 void EmitX64::EmitFPVectorS32ToSingle(EmitContext& ctx, IR::Inst* inst) {
     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
     const Xbyak::Xmm xmm = ctx.reg_alloc.UseScratchXmm(args[0]);

src/frontend/A64/decoder/a64.inc

@@ -739,7 +739,7 @@ INST(BIC_asimd_reg,          "BIC (vector, register)",                    "0Q001
 INST(FSUB_2,                 "FSUB (vector)",                             "0Q0011101z1mmmmm110101nnnnnddddd")
 //INST(FMLSL_vec_1,            "FMLSL, FMLSL2 (vector)",                    "0Q0011101z1mmmmm111011nnnnnddddd")
 //INST(FMIN_2,                 "FMIN (vector)",                             "0Q0011101z1mmmmm111101nnnnnddddd")
-//INST(FRSQRTS_4,              "FRSQRTS",                                   "0Q0011101z1mmmmm111111nnnnnddddd")
+INST(FRSQRTS_4,              "FRSQRTS",                                   "0Q0011101z1mmmmm111111nnnnnddddd")
 INST(ORR_asimd_reg,          "ORR (vector, register)",                    "0Q001110101mmmmm000111nnnnnddddd")
 INST(ORN_asimd,              "ORN (vector)",                              "0Q001110111mmmmm000111nnnnnddddd")
 INST(UHADD,                  "UHADD",                                     "0Q101110zz1mmmmm000001nnnnnddddd")

src/frontend/A64/translate/impl/impl.h

@@ -404,40 +404,22 @@ struct TranslatorVisitor final {
     // Data Processing - FP and SIMD - Scalar three
     bool FMULX_vec_1(Vec Vm, Vec Vn, Vec Vd);
     bool FMULX_vec_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FMULX_vec_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMULX_vec_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FCMEQ_reg_1(Vec Vm, Vec Vn, Vec Vd);
     bool FCMEQ_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMEQ_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMEQ_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FRECPS_1(Vec Vm, Vec Vn, Vec Vd);
     bool FRECPS_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FRECPS_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FRSQRTS_1(Vec Vm, Vec Vn, Vec Vd);
     bool FRSQRTS_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FCMGE_reg_1(Vec Vm, Vec Vn, Vec Vd);
     bool FCMGE_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMGE_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMGE_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FACGE_1(Vec Vm, Vec Vn, Vec Vd);
     bool FACGE_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FACGE_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FACGE_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FABD_1(Vec Vm, Vec Vn, Vec Vd);
     bool FABD_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FABD_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FABD_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FCMGT_reg_1(Vec Vm, Vec Vn, Vec Vd);
     bool FCMGT_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMGT_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FCMGT_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FACGT_1(Vec Vm, Vec Vn, Vec Vd);
     bool FACGT_2(bool sz, Vec Vm, Vec Vn, Vec Vd);
-    bool FACGT_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FACGT_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
 
     // Data Processing - FP and SIMD - Two register misc FP16
     bool FCVTNS_1(Vec Vn, Vec Vd);
@@ -697,36 +679,30 @@ struct TranslatorVisitor final {
     bool INS_elt(Imm<5> imm5, Imm<4> imm4, Vec Vn, Vec Vd);
 
     // Data Processing - FP and SIMD - SIMD Three same
+    bool FMULX_vec_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMEQ_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FRECPS_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMGE_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FACGE_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FABD_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMGT_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FACGT_3(bool Q, Vec Vm, Vec Vn, Vec Vd);
     bool FMAXNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMAXNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMLA_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMLA_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FADD_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FADD_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMAX_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMAX_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMINNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMINNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMLS_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMLS_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FSUB_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMIN_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMIN_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMAXNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMAXNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FADDP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FADDP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMUL_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMUL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMAXP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMAXP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FDIV_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FDIV_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMINNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMINNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
     bool FMINP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd);
-    bool FMINP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
 
     // Data Processing - FP and SIMD - SIMD Two-register misc
     bool FRINTN_1(bool Q, Vec Vn, Vec Vd);
@@ -854,6 +830,30 @@ struct TranslatorVisitor final {
     bool BSL(bool Q, Vec Vm, Vec Vn, Vec Vd);
     bool BIT(bool Q, Vec Vm, Vec Vn, Vec Vd);
     bool BIF(bool Q, Vec Vm, Vec Vn, Vec Vd);
+    bool FMAXNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMLA_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FADD_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMAX_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMINNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMLS_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMIN_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMAXNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FADDP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMUL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMAXP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FDIV_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMINNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMINP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FMULX_vec_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMEQ_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMGE_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FACGE_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FABD_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FCMGT_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
+    bool FACGT_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd);
 
     // Data Processing - FP and SIMD - SIMD modified immediate
     bool MOVI(bool Q, bool op, Imm<1> a, Imm<1> b, Imm<1> c, Imm<4> cmode, Imm<1> d, Imm<1> e, Imm<1> f, Imm<1> g, Imm<1> h, Vec Vd);

src/frontend/A64/translate/impl/simd_three_same.cpp

@@ -592,6 +592,20 @@ bool TranslatorVisitor::FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) {
     return true;
 }
 
+bool TranslatorVisitor::FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) {
+    if (sz && !Q) {
+        return ReservedValue();
+    }
+    const size_t esize = sz ? 64 : 32;
+    const size_t datasize = Q ? 128 : 64;
+
+    const IR::U128 operand1 = V(datasize, Vn);
+    const IR::U128 operand2 = V(datasize, Vm);
+    const IR::U128 result = ir.FPVectorRSqrtStepFused(esize, operand1, operand2);
+    V(datasize, Vd, result);
+    return true;
+}
+
 bool TranslatorVisitor::ORR_asimd_reg(bool Q, Vec Vm, Vec Vn, Vec Vd) {
     const size_t datasize = Q ? 128 : 64;
 

src/frontend/ir/ir_emitter.cpp

@@ -1690,6 +1690,17 @@ U128 IREmitter::FPVectorRSqrtEstimate(size_t esize, const U128& a) {
     return {};
 }
 
+U128 IREmitter::FPVectorRSqrtStepFused(size_t esize, const U128& a, const U128& b) {
+    switch (esize) {
+    case 32:
+        return Inst<U128>(Opcode::FPVectorRSqrtStepFused32, a, b);
+    case 64:
+        return Inst<U128>(Opcode::FPVectorRSqrtStepFused64, a, b);
+    }
+    UNREACHABLE();
+    return {};
+}
+
 U128 IREmitter::FPVectorSub(size_t esize, const U128& a, const U128& b) {
     switch (esize) {
     case 32:

src/frontend/ir/ir_emitter.h

@@ -300,6 +300,7 @@ public:
     U128 FPVectorPairedAdd(size_t esize, const U128& a, const U128& b);
     U128 FPVectorPairedAddLower(size_t esize, const U128& a, const U128& b);
     U128 FPVectorRSqrtEstimate(size_t esize, const U128& a);
+    U128 FPVectorRSqrtStepFused(size_t esize, const U128& a, const U128& b);
     U128 FPVectorSub(size_t esize, const U128& a, const U128& b);
     U128 FPVectorS32ToSingle(const U128& a);
     U128 FPVectorS64ToDouble(const U128& a);

src/frontend/ir/opcodes.inc

@@ -437,6 +437,8 @@ OPCODE(FPVectorPairedAdd32,                 T::U128,        T::U128,        T::U
 OPCODE(FPVectorPairedAdd64,                 T::U128,        T::U128,        T::U128                         )
 OPCODE(FPVectorRSqrtEstimate32,             T::U128,        T::U128                                         )
 OPCODE(FPVectorRSqrtEstimate64,             T::U128,        T::U128                                         )
+OPCODE(FPVectorRSqrtStepFused32,            T::U128,        T::U128,        T::U128                         )
+OPCODE(FPVectorRSqrtStepFused64,            T::U128,        T::U128,        T::U128                         )
 OPCODE(FPVectorS32ToSingle,                 T::U128,        T::U128                                         )
 OPCODE(FPVectorS64ToDouble,                 T::U128,        T::U128                                         )
 OPCODE(FPVectorSub32,                       T::U128,        T::U128,        T::U128                         )