Revamp the generation of runtime division checks on ARM64

Fixes #64795

This patch wraps GT_DIV/GT_UDIV nodes on integral types with GT_QMARK
trees that contain the necessary conformance checks (overflow/divide-by-zero)
when compiling the code with FullOpts enabled. Currently these checks are added
during the Emit phase, this is still the case for MinOpts.

The aim is to allow the compiler to make decisions on code position
and instruction selection for these checks. For example on ARM64 this
enables certain scenarios to choose the cbz instruction over cmp/beq,
can lead to more compact code. It also allows some of the comparisons
in the checks to be hoisted out of loops.

src/coreclr/jit/compiler.h

@@ -4681,6 +4681,8 @@ class Compiler
 
     GenTree* impThrowIfNull(GenTreeCall* call);
 
+    void impImportDivision(bool isSigned);
+
 #ifdef DEBUG
     var_types impImportJitTestLabelMark(int numArgs);
 #endif // DEBUG

src/coreclr/jit/fgbasic.cpp

@@ -4643,6 +4643,8 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr)
 BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt)
 {
     assert(!curr->IsLIR()); // No statements in LIR, so you can't use this function.
+    assert(curr->bbStmtList != nullptr);
+    assert(fgBlockContainsStatementBounded(curr, stmt));
 
     BasicBlock* newBlock = fgSplitBlockAtEnd(curr);
 
@@ -4651,7 +4653,7 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* st
         newBlock->bbStmtList = stmt->GetNextStmt();
         if (newBlock->bbStmtList != nullptr)
         {
-            newBlock->bbStmtList->SetPrevStmt(curr->bbStmtList->GetPrevStmt());
+            newBlock->bbStmtList->SetPrevStmt(curr->lastStmt());
         }
         curr->bbStmtList->SetPrevStmt(stmt);
         stmt->SetNextStmt(nullptr);

src/coreclr/jit/importer.cpp

@@ -7292,12 +7292,9 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                 // Other binary math operations
 
             case CEE_DIV:
-                oper = GT_DIV;
-                goto MATH_MAYBE_CALL_NO_OVF;
-
             case CEE_DIV_UN:
-                oper = GT_UDIV;
-                goto MATH_MAYBE_CALL_NO_OVF;
+                impImportDivision(opcode == CEE_DIV);
+                break;
 
             case CEE_REM:
                 oper = GT_MOD;
@@ -13823,3 +13820,137 @@ methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED
     memory->m_tokenConstraint = tokenConstrained;
     return memory;
 }
+
+void Compiler::impImportDivision(bool isSigned)
+{
+    typeInfo tiRetVal = typeInfo();
+
+    genTreeOps oper = isSigned ? GT_DIV : GT_UDIV;
+
+    GenTree* divisor  = impPopStack().val;
+    GenTree* dividend = impPopStack().val;
+
+    // Can't do arithmetic with references
+    assert(genActualType(dividend) != TYP_REF && genActualType(divisor) != TYP_REF);
+
+    // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only
+    // if it is in the stack)
+    impBashVarAddrsToI(dividend, divisor);
+
+    var_types resultType = impGetByRefResultType(oper, !isSigned, &dividend, &divisor);
+
+    // Cannot perform div.un with floating point.
+    assert(!(!isSigned && varTypeIsFloating(resultType)));
+
+    // Special case: "int/1"
+    if (divisor->IsIntegralConst(1))
+    {
+        impPushOnStack(dividend, tiRetVal);
+        return;
+    }
+
+    // These operators can later be transformed into 'GT_CALL'
+
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]);
+#ifndef TARGET_ARM
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]);
+#endif
+    // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying
+    // that we'll need to transform into a general large node, but rather specifically
+    // to a call: by doing it this way, things keep working if there are multiple sizes,
+    // and a CALL is no longer the largest.
+    // That said, as of now it *is* a large node, so we'll do this with an assert rather
+    // than an "if".
+    assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE);
+    GenTree* divNode = new (this, GT_CALL) GenTreeOp(oper, resultType, dividend, divisor DEBUGARG(/*largeNode*/ true));
+
+    // Special case: integer/long division may throw an exception
+
+    divNode         = gtFoldExpr(divNode);
+    GenTree* result = divNode;
+
+    // Is the result still a division after folding?
+    const bool isDivisionAfterFold = result->OperIs(GT_DIV, GT_UDIV);
+
+#if defined(TARGET_ARM64)
+    const bool isSuitableOptimizationArch = true;
+#else
+    const bool isSuitableOptimizationArch = false;
+#endif // defined(TARGET_ARM64)
+
+    if (opts.OptimizationEnabled() && isSuitableOptimizationArch && !varTypeIsFloating(resultType) &&
+        isDivisionAfterFold)
+    {
+        // Spill the divisor, as (divisor == 0) is always checked.
+        GenTree* divisorCopy = nullptr;
+        impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("divisor used in runtime checks"));
+
+        // Update the original division to use this temp as the divisor.
+        divNode->AsOp()->gtOp2 = gtClone(divisorCopy, true);
+        assert(divNode->AsOp()->gtOp2 != nullptr);
+
+        result =
+            gtNewQmarkNode(resultType,
+                           // (divisor == 0)
+                           gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(0, genActualType(divisorCopy))),
+                           gtNewColonNode(resultType, gtNewHelperCallNode(CORINFO_HELP_THROWDIVZERO, resultType),
+                                          result));
+
+        // No need to generate check in Emitter.
+        divNode->gtFlags |= GTF_DIV_MOD_NO_BY_ZERO;
+
+        // Expand division into QMark containing runtime checks.
+        // We can skip this when both the dividend and divisor are unsigned.
+        if (isSigned && !(varTypeIsUnsigned(dividend) && varTypeIsUnsigned(divisor)))
+        {
+            // Spill the dividend for the check if it's complex.
+            GenTree* dividendCopy = nullptr;
+            impCloneExpr(dividend, &dividendCopy, CHECK_SPILL_NONE,
+                         nullptr DEBUGARG("dividend used in runtime checks"));
+
+            // Update the original division to use this temp as the dividend.
+            divNode->AsOp()->gtOp1 = gtClone(dividendCopy, true);
+            assert(divNode->AsOp()->gtOp1 != nullptr);
+
+            // Clone of the divisor should be easy, it was either simple enough to clone or spilled already.
+            divisorCopy = gtClone(divisorCopy, true);
+            assert(divisorCopy != nullptr);
+
+            const ssize_t minValue = genActualType(dividendCopy) == TYP_LONG ? INT64_MIN : INT32_MIN;
+
+            // (dividend == MinValue && divisor == -1)
+            GenTreeOp* const divisorIsMinusOne =
+                gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(-1, genActualType(divisorCopy)));
+            GenTreeOp* const dividendIsMinValue =
+                gtNewOperNode(GT_EQ, TYP_INT, dividendCopy, gtNewIconNode(minValue, genActualType(dividendCopy)));
+            GenTreeOp* const combinedTest = gtNewOperNode(GT_AND, TYP_INT, divisorIsMinusOne, dividendIsMinValue);
+            GenTree*         condition    = gtNewOperNode(GT_EQ, TYP_INT, combinedTest, gtNewTrue());
+
+            result = gtNewQmarkNode(resultType, condition,
+                                    gtNewColonNode(resultType, gtNewHelperCallNode(CORINFO_HELP_OVERFLOW, resultType),
+                                                   result));
+
+            // No need to generate check in Emitter.
+            divNode->gtFlags |= GTF_DIV_MOD_NO_OVERFLOW;
+        }
+
+        // Spilling the overall Qmark helps with later passes.
+        unsigned tmp            = lvaGrabTemp(true DEBUGARG("spilling to hold checked division tree"));
+        lvaGetDesc(tmp)->lvType = resultType;
+        impStoreToTemp(tmp, result, CHECK_SPILL_NONE);
+
+        result = gtNewLclVarNode(tmp, resultType);
+    }
+    else
+    {
+        // The division could still throw if it was not folded away.
+        if (isDivisionAfterFold)
+            divNode->gtFlags |= GTF_EXCEPT;
+    }
+
+    impPushOnStack(result, tiRetVal);
+    return;
+}