Avoid bigints entering non-bigint gcd implementation.

runtime/lib/integers.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 abstract class _IntegerImplementation {
   // The Dart class _Bigint extending _IntegerImplementation requires a
   // default constructor.
 
   num operator +(num other) {
     var result = other._addFromInteger(this);
@@ skipping 376 matching lines @@
   // Returns gcd of abs(this) and abs(other).
   int gcd(int other) {
     if (other is! int) {
       throw new ArgumentError.value(other, "other", "not an integer");
     }
     int x = this.abs();
     int y = other.abs();
     if (x == 0) return y;
     if (y == 0) return x;
     if ((x == 1) || (y == 1)) return 1;
-    if (other is _Bigint) {
-      return _toBigint().gcd(other);
+    if (y is _Bigint) {
+      return x._toBigint().gcd(y);

[Lasse Reichstein Nielsen, 2016/12/06 09:05:23]

If other is -0x8000000000000000 (aka min-mint) then y == -other is a bigint, but
other is not. Similar for `this` and `x`. Might as well go to the bigint
implementation (and avoid doing the negation again).

[regis, 2016/12/06 17:08:15]

Agreed.

     }
     return _binaryGcd(x, y, false);
   }
 }
 
 class _Smi extends _IntegerImplementation implements int {
   factory _Smi._uninstantiable() {
     throw new UnsupportedError(
         "_Smi can only be allocated by the VM");
   }
@@ skipping 207 matching lines @@
   // Shift by mint exceeds range that can be handled by the VM.
   int _shrFromInt(int other) {
     if (other < 0) {
       return -1;
     } else {
       return 0;
     }
   }
   int _shlFromInt(int other) native "Mint_shlFromInt";
 }