Fix bigint modulo operation. Fixes bug 6056

tests/corelib/big_integer_vm_test.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 // Testing Bigints.
 // TODO(srdjan): Make sure the numbers are Bigint and not Mint or Smi.
 
 #library("BigIntegerTest.dart");
 
 class BigIntegerTest {
 
@@ skipping 88 matching lines @@
     var a = 12345670000000000000;
     var b = 10000000000000000;
     Expect.equals(1234.567, a / b);
     // Bigint and double.
     a = 200.0;
     b = 100000000000;
     Expect.equals(0.000000002, a / b);
     Expect.equals(500000000.0, b / a);
   }
 
-  static testBigintRemainder() {
+  static testBigintModulo() {
     // Bigint and Smi.
     var a = 1000000000005;
     var b = 10;
     Expect.equals(5, a % b);
     Expect.equals(10, b % a);
     // Bigint & Bigint
     a = 10000000000000000001;
     b = 10000000000000000000;
     Expect.equals(1, a % b);
     Expect.equals(10000000000000000000, b % a);
     // Bigint & double.
     a = 100000000001.0;
     b = 100000000000;
     Expect.equals(1.0, a % b);
     Expect.equals(100000000000.0, b % a);
+    // Transitioning from Mint to Bigint.
+    var iStart = 4611686018427387900;
+    var prevX = -23 % iStart;
+    for (int i = iStart + 1; i < iStart + 10; i++) {
+      var x = -23 % i;
+      Expect.equals(1, x - prevX);
+      Expect.isTrue(x > 0);
+      prevX = x;
+    }
   }
 
   static testBigintNegate() {
     var a = 0xF000000000F;
     var b = ~a;  // negate.
     Expect.equals(-0xF0000000010, b);
     Expect.equals(0, a & b);
     Expect.equals(-1, a | b);
   }
 
   static testShiftAmount() {
     Expect.equals(0, 12 >> 111111111111111111111111111111);
     Expect.equals(-1, -12 >> 111111111111111111111111111111);
     bool exceptionCaught = false;
     try {
       var a = 1 << 1111111111111111111111111111;
     } on OutOfMemoryError catch (e) {
       exceptionCaught = true;
     }
     Expect.equals(true, exceptionCaught);
   }
 
   static testMain() {
     Expect.equals(1234567890123456789, foo());
     testSmiOverflow();
     testBigintAdd();
     testBigintSub();
     testBigintMul();
-    testBigintRemainder();
+    testBigintModulo();
     testBigintTruncDiv();
     testBigintDiv();
     testBigintNegate();
     testShiftAmount();
     Expect.equals(1234567890123456, (1234567890123456).abs());
     Expect.equals(1234567890123456, (-1234567890123456).abs());
     var a = 10000000000000000000;
     var b = 10000000000000000001;
     Expect.equals(false, a.hashCode() == b.hashCode());
     Expect.equals(true, a.hashCode() == (b - 1).hashCode());
   }
 }
 
 main() {
   BigIntegerTest.testMain();
 }