[JSC] PutCellButterflySlot forbids the double format for the variable it stores

JSTests/stress/array-sort-inline-int32-double-comparator.js

@@ -0,0 +1,49 @@
+//@ defaultRun
+//@ runNoCJIT("--validateGraphAtEachPhase=true")
+
+// The inlined sort keeps the pivot of its insertion sort in a DFG variable, stores it back into the scratch
+// as is, and reads the scratch of an Int32 array back with no type check. A comparator that does double
+// arithmetic on its first parameter must not make that variable a double. The sort would then store boxed
+// doubles into an ArrayWithInt32, compare other numbers than the elements, and return them out of order.
+
+function shouldBe(actual, expected) {
+    if (actual !== expected)
+        throw new Error("bad value: " + actual + " expected: " + expected);
+}
+
+var float64 = new Float64Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]);
+
+function make(name, comparator) {
+    return new Function("float64", `return function ${name}(i) { return [17, 8, i % 9 + 1, i * 7 % 9 + 1].sort(${comparator}); };`)(float64);
+}
+
+function test(optimized, expected, expectedIndexOf8) {
+    for (var i = 0; i < testLoopCount; ++i) {
+        var actual = optimized(i);
+        // join() and indexOf() read the elements of an ArrayWithInt32 as Int32s.
+        shouldBe(actual.join(), expected[i % 9]);
+        shouldBe(actual.indexOf(8), expectedIndexOf8[i % 9]);
+    }
+}
+noInline(test);
+
+var comparators = [
+    "(a, b) => (a + 0.5) - (b + 0.5)",
+    "(a, b) => a - float64[b]",
+];
+
+for (var comparator of comparators) {
+    var optimized = make("optimized", comparator);
+    var reference = make("reference", comparator);
+    noInline(optimized);
+    noDFG(reference);
+
+    var expected = [];
+    var expectedIndexOf8 = [];
+    for (var i = 0; i < 9; ++i) {
+        var sorted = reference(i);
+        expected.push(sorted.join());
+        expectedIndexOf8.push(sorted.indexOf(8));
+    }
+    test(optimized, expected, expectedIndexOf8);
+}

Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp

@@ -932,6 +932,20 @@ class PredictionPropagationPhase : public Phase {
             break;
         }
 
+        case PutCellButterflySlot: {
+            // The inlined sort stores this value as is, and ArraySortCommit copies the scratch as is into
+            // the array. For an ArrayWithInt32 the sort also reads the scratch slots back as Int32s with
+            // no check. A variable in the double format gives a boxed double here, also for an element
+            // that came out of the array as an Int32. The sort would then compare other numbers than the
+            // elements and leave boxed doubles in an Int32 array. If this node gets an array mode, only
+            // Array::Int32 needs this.
+            m_graph.voteChildren(node, VoteValue, weight);
+            Node* value = node->child3().node();
+            if (value->op() == GetLocal)
+                value->variableAccessData()->mergeDoubleFormatState(CantUseDoubleFormat);
+            break;
+        }
+
         case DataViewSet: {
             DataViewData data = node->dataViewData();
             if (data.isFloatingPoint)