diff --git a/src/backend_x64/emit_x64.cpp b/src/backend_x64/emit_x64.cpp
index b433391c..6fb70618 100644
--- a/src/backend_x64/emit_x64.cpp
+++ b/src/backend_x64/emit_x64.cpp
@@ -567,10 +567,6 @@ void EmitX64::EmitLogicalShiftLeft(RegAlloc& reg_alloc, IR::Block& block, IR::In
     // TODO: Consider using BMI2 instructions like SHLX when arm-in-host flags is implemented.
 
     if (!carry_inst) {
-        if (!carry_arg.IsImmediate()) {
-            inst->GetArg(2).GetInst()->DecrementRemainingUses();
-        }
-
         if (shift_arg.IsImmediate()) {
             Xbyak::Reg32 result = reg_alloc.UseScratchGpr(operand_arg).cvt32();
             u8 shift = shift_arg.GetImmediateU8();
@@ -669,10 +665,6 @@ void EmitX64::EmitLogicalShiftRight(RegAlloc& reg_alloc, IR::Block& block, IR::I
     auto& carry_arg = args[2];
 
     if (!carry_inst) {
-        if (!carry_arg.IsImmediate()) {
-            inst->GetArg(2).GetInst()->DecrementRemainingUses();
-        }
-
         if (shift_arg.IsImmediate()) {
             Xbyak::Reg32 result = reg_alloc.UseScratchGpr(operand_arg).cvt32();
             u8 shift = shift_arg.GetImmediateU8();
@@ -788,10 +780,6 @@ void EmitX64::EmitArithmeticShiftRight(RegAlloc& reg_alloc, IR::Block& block, IR
     auto& carry_arg = args[2];
 
     if (!carry_inst) {
-        if (!carry_arg.IsImmediate()) {
-            inst->GetArg(2).GetInst()->DecrementRemainingUses();
-        }
-
         if (shift_arg.IsImmediate()) {
             u8 shift = shift_arg.GetImmediateU8();
             Xbyak::Reg32 result = reg_alloc.UseScratchGpr(operand_arg).cvt32();
@@ -880,10 +868,6 @@ void EmitX64::EmitRotateRight(RegAlloc& reg_alloc, IR::Block& block, IR::Inst* i
     auto& carry_arg = args[2];
 
     if (!carry_inst) {
-        if (!carry_arg.IsImmediate()) {
-            inst->GetArg(2).GetInst()->DecrementRemainingUses();
-        }
-
         if (shift_arg.IsImmediate()) {
             u8 shift = shift_arg.GetImmediateU8();
             Xbyak::Reg32 result = reg_alloc.UseScratchGpr(operand_arg).cvt32();
diff --git a/src/backend_x64/reg_alloc.cpp b/src/backend_x64/reg_alloc.cpp
index 011803de..a79d843b 100644
--- a/src/backend_x64/reg_alloc.cpp
+++ b/src/backend_x64/reg_alloc.cpp
@@ -114,6 +114,9 @@ std::array<Argument, 3> RegAlloc::GetArgumentInfo(IR::Inst* inst) {
     for (size_t i = 0; i < inst->NumArgs(); i++) {
         IR::Value arg = inst->GetArg(i);
         ret[i].value = arg;
+        if (!arg.IsImmediate()) {
+            arg.GetInst()->DecrementRemainingUses();
+        }
     }
     return ret;
 }
@@ -326,7 +329,6 @@ void RegAlloc::DefineValueImpl(IR::Inst* def_inst, const IR::Value& use_inst) {
         return;
     }
 
-    use_inst.GetInst()->DecrementRemainingUses();
     DEBUG_ASSERT_MSG(ValueLocation(use_inst.GetInst()), "use_inst must already be defined");
     HostLoc location = *ValueLocation(use_inst.GetInst());
     DefineValueImpl(def_inst, location);