Revamp the generation of runtime division checks on ARM64

src/coreclr/jit/compiler.h

@@ -4681,6 +4681,8 @@ class Compiler
 
     GenTree* impThrowIfNull(GenTreeCall* call);
 
+    void impImportDivision(bool isSigned);
+
 #ifdef DEBUG
     var_types impImportJitTestLabelMark(int numArgs);
 #endif // DEBUG

src/coreclr/jit/fgbasic.cpp

@@ -4643,6 +4643,8 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr)
 BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt)
 {
     assert(!curr->IsLIR()); // No statements in LIR, so you can't use this function.
+    assert(curr->bbStmtList != nullptr);
+    assert(fgBlockContainsStatementBounded(curr, stmt));
 
     BasicBlock* newBlock = fgSplitBlockAtEnd(curr);
 
@@ -4651,7 +4653,7 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* st
         newBlock->bbStmtList = stmt->GetNextStmt();
         if (newBlock->bbStmtList != nullptr)
         {
-            newBlock->bbStmtList->SetPrevStmt(curr->bbStmtList->GetPrevStmt());
+            newBlock->bbStmtList->SetPrevStmt(curr->lastStmt());
         }
         curr->bbStmtList->SetPrevStmt(stmt);
         stmt->SetNextStmt(nullptr);

src/coreclr/jit/importer.cpp

@@ -7292,12 +7292,9 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                 // Other binary math operations
 
             case CEE_DIV:
-                oper = GT_DIV;
-                goto MATH_MAYBE_CALL_NO_OVF;
-
             case CEE_DIV_UN:
-                oper = GT_UDIV;
-                goto MATH_MAYBE_CALL_NO_OVF;
+                impImportDivision(opcode == CEE_DIV);
+                break;
 
             case CEE_REM:
                 oper = GT_MOD;
@@ -13823,3 +13820,137 @@ methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED
     memory->m_tokenConstraint = tokenConstrained;
     return memory;
 }
+
+void Compiler::impImportDivision(bool isSigned)
+{
+    typeInfo tiRetVal = typeInfo();
+
+    genTreeOps oper = isSigned ? GT_DIV : GT_UDIV;
+
+    GenTree* divisor  = impPopStack().val;
+    GenTree* dividend = impPopStack().val;
+
+    // Can't do arithmetic with references
+    assert(genActualType(dividend) != TYP_REF && genActualType(divisor) != TYP_REF);
+
+    // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only
+    // if it is in the stack)
+    impBashVarAddrsToI(dividend, divisor);
+
+    var_types resultType = impGetByRefResultType(oper, !isSigned, &dividend, &divisor);
+
+    // Cannot perform div.un with floating point.
+    assert(!(!isSigned && varTypeIsFloating(resultType)));
+
+    // Special case: "int/1"
+    if (divisor->IsIntegralConst(1))
+    {
+        impPushOnStack(dividend, tiRetVal);
+        return;
+    }
+
+    // These operators can later be transformed into 'GT_CALL'
+
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]);
+#ifndef TARGET_ARM
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]);
+    assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]);
+#endif
+    // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying
+    // that we'll need to transform into a general large node, but rather specifically
+    // to a call: by doing it this way, things keep working if there are multiple sizes,
+    // and a CALL is no longer the largest.
+    // That said, as of now it *is* a large node, so we'll do this with an assert rather
+    // than an "if".
+    assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE);
+    GenTree* divNode = new (this, GT_CALL) GenTreeOp(oper, resultType, dividend, divisor DEBUGARG(/*largeNode*/ true));
+
+    // Special case: integer/long division may throw an exception
+
+    divNode         = gtFoldExpr(divNode);
+    GenTree* result = divNode;
+
+    // Is the result still a division after folding?
+    const bool isDivisionAfterFold = result->OperIs(GT_DIV, GT_UDIV);
+
+#if defined(TARGET_ARM64)
+    const bool isSuitableOptimizationArch = true;
+#else
+    const bool isSuitableOptimizationArch = false;
+#endif // defined(TARGET_ARM64)
+
+    if (opts.OptimizationEnabled() && isSuitableOptimizationArch && !varTypeIsFloating(resultType) &&
+        isDivisionAfterFold)
+    {
+        // Spill the divisor, as (divisor == 0) is always checked.
+        GenTree* divisorCopy = nullptr;
+        impCloneExpr(divisor, &divisorCopy, CHECK_SPILL_NONE, nullptr DEBUGARG("divisor used in runtime checks"));
+
+        // Update the original division to use this temp as the divisor.
+        divNode->AsOp()->gtOp2 = gtClone(divisorCopy, true);
+        assert(divNode->AsOp()->gtOp2 != nullptr);
+
+        result =
+            gtNewQmarkNode(resultType,
+                           // (divisor == 0)
+                           gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(0, genActualType(divisorCopy))),
+                           gtNewColonNode(resultType, gtNewHelperCallNode(CORINFO_HELP_THROWDIVZERO, resultType),
+                                          result));
+
+        // No need to generate check in Emitter.
+        divNode->gtFlags |= GTF_DIV_MOD_NO_BY_ZERO;
+
+        // Expand division into QMark containing runtime checks.
+        // We can skip this when both the dividend and divisor are unsigned.
+        if (isSigned && !(varTypeIsUnsigned(dividend) && varTypeIsUnsigned(divisor)))
+        {
+            // Spill the dividend for the check if it's complex.
+            GenTree* dividendCopy = nullptr;
+            impCloneExpr(dividend, &dividendCopy, CHECK_SPILL_NONE,
+                         nullptr DEBUGARG("dividend used in runtime checks"));
+
+            // Update the original division to use this temp as the dividend.
+            divNode->AsOp()->gtOp1 = gtClone(dividendCopy, true);
+            assert(divNode->AsOp()->gtOp1 != nullptr);
+
+            // Clone of the divisor should be easy, it was either simple enough to clone or spilled already.
+            divisorCopy = gtClone(divisorCopy, true);
+            assert(divisorCopy != nullptr);
+
+            const ssize_t minValue = genActualType(dividendCopy) == TYP_LONG ? INT64_MIN : INT32_MIN;
+
+            // (dividend == MinValue && divisor == -1)
+            GenTreeOp* const divisorIsMinusOne =
+                gtNewOperNode(GT_EQ, TYP_INT, divisorCopy, gtNewIconNode(-1, genActualType(divisorCopy)));
+            GenTreeOp* const dividendIsMinValue =
+                gtNewOperNode(GT_EQ, TYP_INT, dividendCopy, gtNewIconNode(minValue, genActualType(dividendCopy)));
+            GenTreeOp* const combinedTest = gtNewOperNode(GT_AND, TYP_INT, divisorIsMinusOne, dividendIsMinValue);
+            GenTree*         condition    = gtNewOperNode(GT_EQ, TYP_INT, combinedTest, gtNewTrue());
+
+            result = gtNewQmarkNode(resultType, condition,
+                                    gtNewColonNode(resultType, gtNewHelperCallNode(CORINFO_HELP_OVERFLOW, resultType),
+                                                   result));
+
+            // No need to generate check in Emitter.
+            divNode->gtFlags |= GTF_DIV_MOD_NO_OVERFLOW;
+        }
+
+        // Spilling the overall Qmark helps with later passes.
+        unsigned tmp            = lvaGrabTemp(true DEBUGARG("spilling to hold checked division tree"));
+        lvaGetDesc(tmp)->lvType = resultType;
+        impStoreToTemp(tmp, result, CHECK_SPILL_NONE);
+
+        result = gtNewLclVarNode(tmp, resultType);
+    }
+    else
+    {
+        // The division could still throw if it was not folded away.
+        if (isDivisionAfterFold)
+            divNode->gtFlags |= GTF_EXCEPT;
+    }
+
+    impPushOnStack(result, tiRetVal);
+    return;
+}