| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. | 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. |
| 6 #if defined(TARGET_ARCH_X64) | 6 #if defined(TARGET_ARCH_X64) |
| 7 | 7 |
| 8 #include "vm/intrinsifier.h" | 8 #include "vm/intrinsifier.h" |
| 9 | 9 |
| 10 #include "vm/assembler.h" | 10 #include "vm/assembler.h" |
| (...skipping 911 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 922 __ j(NOT_ZERO, &carry_loop, Assembler::kNearJump); | 922 __ j(NOT_ZERO, &carry_loop, Assembler::kNearJump); |
| 923 | 923 |
| 924 __ Bind(&done); | 924 __ Bind(&done); |
| 925 // Returning Object::null() is not required, since this method is private. | 925 // Returning Object::null() is not required, since this method is private. |
| 926 __ ret(); | 926 __ ret(); |
| 927 } | 927 } |
| 928 | 928 |
| 929 | 929 |
| 930 void Intrinsifier::Bigint_mulAdd(Assembler* assembler) { | 930 void Intrinsifier::Bigint_mulAdd(Assembler* assembler) { |
| 931 // Pseudo code: | 931 // Pseudo code: |
| 932 // static void _mulAdd(Uint32List x_digits, int xi, | 932 // static int _mulAdd(Uint32List x_digits, int xi, |
| 933 // Uint32List m_digits, int i, | 933 // Uint32List m_digits, int i, |
| 934 // Uint32List a_digits, int j, int n) { | 934 // Uint32List a_digits, int j, int n) { |
| 935 // uint32_t x = x_digits[xi >> 1]; // xi is Smi. | 935 // uint32_t x = x_digits[xi >> 1]; // xi is Smi. |
| 936 // if (x == 0 || n == 0) { | 936 // if (x == 0 || n == 0) { |
| 937 // return; | 937 // return 1; |
| 938 // } | 938 // } |
| 939 // uint32_t* mip = &m_digits[i >> 1]; // i is Smi. | 939 // uint32_t* mip = &m_digits[i >> 1]; // i is Smi. |
| 940 // uint32_t* ajp = &a_digits[j >> 1]; // j is Smi. | 940 // uint32_t* ajp = &a_digits[j >> 1]; // j is Smi. |
| 941 // uint32_t c = 0; | 941 // uint32_t c = 0; |
| 942 // SmiUntag(n); | 942 // SmiUntag(n); |
| 943 // do { | 943 // do { |
| 944 // uint32_t mi = *mip++; | 944 // uint32_t mi = *mip++; |
| 945 // uint32_t aj = *ajp; | 945 // uint32_t aj = *ajp; |
| 946 // uint64_t t = x*mi + aj + c; // 32-bit * 32-bit -> 64-bit. | 946 // uint64_t t = x*mi + aj + c; // 32-bit * 32-bit -> 64-bit. |
| 947 // *ajp++ = low32(t); | 947 // *ajp++ = low32(t); |
| 948 // c = high32(t); | 948 // c = high32(t); |
| 949 // } while (--n > 0); | 949 // } while (--n > 0); |
| 950 // while (c != 0) { | 950 // while (c != 0) { |
| 951 // uint64_t t = *ajp + c; | 951 // uint64_t t = *ajp + c; |
| 952 // *ajp++ = low32(t); | 952 // *ajp++ = low32(t); |
| 953 // c = high32(t); // c == 0 or 1. | 953 // c = high32(t); // c == 0 or 1. |
| 954 // } | 954 // } |
| 955 // return 1; |
| 955 // } | 956 // } |
| 956 | 957 |
| 957 Label done; | 958 Label done; |
| 958 // RBX = x, done if x == 0 | 959 // RBX = x, done if x == 0 |
| 959 __ movq(RCX, Address(RSP, 7 * kWordSize)); // x_digits | 960 __ movq(RCX, Address(RSP, 7 * kWordSize)); // x_digits |
| 960 __ movq(RAX, Address(RSP, 6 * kWordSize)); // xi is Smi | 961 __ movq(RAX, Address(RSP, 6 * kWordSize)); // xi is Smi |
| 961 __ movl(RBX, FieldAddress(RCX, RAX, TIMES_2, TypedData::data_offset())); | 962 __ movl(RBX, FieldAddress(RCX, RAX, TIMES_2, TypedData::data_offset())); |
| 962 __ testl(RBX, RBX); | 963 __ testl(RBX, RBX); |
| 963 __ j(ZERO, &done, Assembler::kNearJump); | 964 __ j(ZERO, &done, Assembler::kNearJump); |
| 964 | 965 |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1020 __ addl(Address(RSI, 0), RCX); | 1021 __ addl(Address(RSI, 0), RCX); |
| 1021 __ j(NOT_CARRY, &done, Assembler::kNearJump); | 1022 __ j(NOT_CARRY, &done, Assembler::kNearJump); |
| 1022 | 1023 |
| 1023 Label propagate_carry_loop; | 1024 Label propagate_carry_loop; |
| 1024 __ Bind(&propagate_carry_loop); | 1025 __ Bind(&propagate_carry_loop); |
| 1025 __ addq(RSI, Immediate(Bigint::kBytesPerDigit)); | 1026 __ addq(RSI, Immediate(Bigint::kBytesPerDigit)); |
| 1026 __ incl(Address(RSI, 0)); // c == 0 or 1 | 1027 __ incl(Address(RSI, 0)); // c == 0 or 1 |
| 1027 __ j(CARRY, &propagate_carry_loop, Assembler::kNearJump); | 1028 __ j(CARRY, &propagate_carry_loop, Assembler::kNearJump); |
| 1028 | 1029 |
| 1029 __ Bind(&done); | 1030 __ Bind(&done); |
| 1030 // Returning Object::null() is not required, since this method is private. | 1031 __ movq(RAX, Immediate(Smi::RawValue(1))); // One digit processed. |
| 1031 __ ret(); | 1032 __ ret(); |
| 1032 } | 1033 } |
| 1033 | 1034 |
| 1034 | 1035 |
| 1035 void Intrinsifier::Bigint_sqrAdd(Assembler* assembler) { | 1036 void Intrinsifier::Bigint_sqrAdd(Assembler* assembler) { |
| 1036 // Pseudo code: | 1037 // Pseudo code: |
| 1037 // static void _sqrAdd(Uint32List x_digits, int i, | 1038 // static int _sqrAdd(Uint32List x_digits, int i, |
| 1038 // Uint32List a_digits, int used) { | 1039 // Uint32List a_digits, int used) { |
| 1039 // uint32_t* xip = &x_digits[i >> 1]; // i is Smi. | 1040 // uint32_t* xip = &x_digits[i >> 1]; // i is Smi. |
| 1040 // uint32_t x = *xip++; | 1041 // uint32_t x = *xip++; |
| 1041 // if (x == 0) return; | 1042 // if (x == 0) return 1; |
| 1042 // uint32_t* ajp = &a_digits[i]; // j == 2*i, i is Smi. | 1043 // uint32_t* ajp = &a_digits[i]; // j == 2*i, i is Smi. |
| 1043 // uint32_t aj = *ajp; | 1044 // uint32_t aj = *ajp; |
| 1044 // uint64_t t = x*x + aj; | 1045 // uint64_t t = x*x + aj; |
| 1045 // *ajp++ = low32(t); | 1046 // *ajp++ = low32(t); |
| 1046 // uint64_t c = high32(t); | 1047 // uint64_t c = high32(t); |
| 1047 // int n = ((used - i) >> 1) - 1; // used and i are Smi. | 1048 // int n = ((used - i) >> 1) - 1; // used and i are Smi. |
| 1048 // while (--n >= 0) { | 1049 // while (--n >= 0) { |
| 1049 // uint32_t xi = *xip++; | 1050 // uint32_t xi = *xip++; |
| 1050 // uint32_t aj = *ajp; | 1051 // uint32_t aj = *ajp; |
| 1051 // uint96_t t = 2*x*xi + aj + c; // 2-bit * 32-bit * 32-bit -> 65-bit. | 1052 // uint96_t t = 2*x*xi + aj + c; // 2-bit * 32-bit * 32-bit -> 65-bit. |
| 1052 // *ajp++ = low32(t); | 1053 // *ajp++ = low32(t); |
| 1053 // c = high64(t); // 33-bit. | 1054 // c = high64(t); // 33-bit. |
| 1054 // } | 1055 // } |
| 1055 // uint32_t aj = *ajp; | 1056 // uint32_t aj = *ajp; |
| 1056 // uint64_t t = aj + c; // 32-bit + 33-bit -> 34-bit. | 1057 // uint64_t t = aj + c; // 32-bit + 33-bit -> 34-bit. |
| 1057 // *ajp++ = low32(t); | 1058 // *ajp++ = low32(t); |
| 1058 // *ajp = high32(t); | 1059 // *ajp = high32(t); |
| 1060 // return 1; |
| 1059 // } | 1061 // } |
| 1060 | 1062 |
| 1061 // RDI = xip = &x_digits[i >> 1] | 1063 // RDI = xip = &x_digits[i >> 1] |
| 1062 __ movq(RDI, Address(RSP, 4 * kWordSize)); // x_digits | 1064 __ movq(RDI, Address(RSP, 4 * kWordSize)); // x_digits |
| 1063 __ movq(RAX, Address(RSP, 3 * kWordSize)); // i is Smi | 1065 __ movq(RAX, Address(RSP, 3 * kWordSize)); // i is Smi |
| 1064 __ leaq(RDI, FieldAddress(RDI, RAX, TIMES_2, TypedData::data_offset())); | 1066 __ leaq(RDI, FieldAddress(RDI, RAX, TIMES_2, TypedData::data_offset())); |
| 1065 | 1067 |
| 1066 // RBX = x = *xip++, return if x == 0 | 1068 // RBX = x = *xip++, return if x == 0 |
| 1067 Label x_zero; | 1069 Label x_zero; |
| 1068 __ movl(RBX, Address(RDI, 0)); | 1070 __ movl(RBX, Address(RDI, 0)); |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1138 // uint64_t t = aj + c | 1140 // uint64_t t = aj + c |
| 1139 __ addl(R12, Address(RSI, 0)); // t = c, t += *ajp | 1141 __ addl(R12, Address(RSI, 0)); // t = c, t += *ajp |
| 1140 __ adcl(R13, Immediate(0)); | 1142 __ adcl(R13, Immediate(0)); |
| 1141 | 1143 |
| 1142 // *ajp++ = low32(t) | 1144 // *ajp++ = low32(t) |
| 1143 // *ajp = high32(t) | 1145 // *ajp = high32(t) |
| 1144 __ movl(Address(RSI, 0), R12); | 1146 __ movl(Address(RSI, 0), R12); |
| 1145 __ movl(Address(RSI, Bigint::kBytesPerDigit), R13); | 1147 __ movl(Address(RSI, Bigint::kBytesPerDigit), R13); |
| 1146 | 1148 |
| 1147 __ Bind(&x_zero); | 1149 __ Bind(&x_zero); |
| 1148 // Returning Object::null() is not required, since this method is private. | 1150 __ movq(RAX, Immediate(Smi::RawValue(1))); // One digit processed. |
| 1149 __ ret(); | 1151 __ ret(); |
| 1150 } | 1152 } |
| 1151 | 1153 |
| 1152 | 1154 |
| 1153 void Intrinsifier::Bigint_estQuotientDigit(Assembler* assembler) { | 1155 void Intrinsifier::Bigint_estQuotientDigit(Assembler* assembler) { |
| 1154 // Pseudo code: | 1156 // Pseudo code: |
| 1155 // static void _estQuotientDigit(Uint32List args, Uint32List digits, int i) { | 1157 // static int _estQuotientDigit(Uint32List args, Uint32List digits, int i) { |
| 1156 // uint32_t yt = args[_YT]; // _YT == 0. | 1158 // uint32_t yt = args[_YT]; // _YT == 1. |
| 1157 // uint32_t* dp = &digits[i >> 1]; // i is Smi. | 1159 // uint32_t* dp = &digits[i >> 1]; // i is Smi. |
| 1158 // uint32_t dh = dp[0]; // dh == digits[i >> 1]. | 1160 // uint32_t dh = dp[0]; // dh == digits[i >> 1]. |
| 1159 // uint32_t qd; | 1161 // uint32_t qd; |
| 1160 // if (dh == yt) { | 1162 // if (dh == yt) { |
| 1161 // qd = DIGIT_MASK; | 1163 // qd = DIGIT_MASK; |
| 1162 // } else { | 1164 // } else { |
| 1163 // dl = dp[-1]; // dl == digits[(i - 1) >> 1]. | 1165 // dl = dp[-1]; // dl == digits[(i - 1) >> 1]. |
| 1164 // qd = dh:dl / yt; // No overflow possible, because dh < yt. | 1166 // qd = dh:dl / yt; // No overflow possible, because dh < yt. |
| 1165 // } | 1167 // } |
| 1166 // args[_QD] = qd; // _QD == 1; | 1168 // args[_QD] = qd; // _QD == 2. |
| 1169 // return 1; |
| 1167 // } | 1170 // } |
| 1168 | 1171 |
| 1169 // RDI = args | 1172 // RDI = args |
| 1170 __ movq(RDI, Address(RSP, 3 * kWordSize)); // args | 1173 __ movq(RDI, Address(RSP, 3 * kWordSize)); // args |
| 1171 | 1174 |
| 1172 // RCX = yt = args[0] | 1175 // RCX = yt = args[1] |
| 1173 __ movl(RCX, FieldAddress(RDI, TypedData::data_offset())); | 1176 __ movl(RCX, |
| 1177 FieldAddress(RDI, TypedData::data_offset() + Bigint::kBytesPerDigit)); |
| 1174 | 1178 |
| 1175 // RBX = dp = &digits[i >> 1] | 1179 // RBX = dp = &digits[i >> 1] |
| 1176 __ movq(RBX, Address(RSP, 2 * kWordSize)); // digits | 1180 __ movq(RBX, Address(RSP, 2 * kWordSize)); // digits |
| 1177 __ movq(RAX, Address(RSP, 1 * kWordSize)); // i is Smi | 1181 __ movq(RAX, Address(RSP, 1 * kWordSize)); // i is Smi |
| 1178 __ leaq(RBX, FieldAddress(RBX, RAX, TIMES_2, TypedData::data_offset())); | 1182 __ leaq(RBX, FieldAddress(RBX, RAX, TIMES_2, TypedData::data_offset())); |
| 1179 | 1183 |
| 1180 // RDX = dh = dp[0] | 1184 // RDX = dh = dp[0] |
| 1181 __ movl(RDX, Address(RBX, 0)); | 1185 __ movl(RDX, Address(RBX, 0)); |
| 1182 | 1186 |
| 1183 // RAX = qd = DIGIT_MASK = -1 | 1187 // RAX = qd = DIGIT_MASK = -1 |
| 1184 __ movl(RAX, Immediate(-1)); | 1188 __ movl(RAX, Immediate(-1)); |
| 1185 | 1189 |
| 1186 // Return qd if dh == yt | 1190 // Return qd if dh == yt |
| 1187 Label return_qd; | 1191 Label return_qd; |
| 1188 __ cmpl(RDX, RCX); | 1192 __ cmpl(RDX, RCX); |
| 1189 __ j(EQUAL, &return_qd, Assembler::kNearJump); | 1193 __ j(EQUAL, &return_qd, Assembler::kNearJump); |
| 1190 | 1194 |
| 1191 // RAX = dl = dp[-1] | 1195 // RAX = dl = dp[-1] |
| 1192 __ movl(RAX, Address(RBX, -Bigint::kBytesPerDigit)); | 1196 __ movl(RAX, Address(RBX, -Bigint::kBytesPerDigit)); |
| 1193 | 1197 |
| 1194 // RAX = qd = dh:dl / yt = RDX:RAX / RCX | 1198 // RAX = qd = dh:dl / yt = RDX:RAX / RCX |
| 1195 __ divl(RCX); | 1199 __ divl(RCX); |
| 1196 | 1200 |
| 1197 __ Bind(&return_qd); | 1201 __ Bind(&return_qd); |
| 1198 // args[1] = qd | 1202 // args[2] = qd |
| 1199 __ movl(FieldAddress(RDI, TypedData::data_offset() + Bigint::kBytesPerDigit), | 1203 __ movl(FieldAddress(RDI, |
| 1204 TypedData::data_offset() + 2*Bigint::kBytesPerDigit), |
| 1200 RAX); | 1205 RAX); |
| 1201 | 1206 |
| 1202 // Returning Object::null() is not required, since this method is private. | 1207 __ movq(RAX, Immediate(Smi::RawValue(1))); // One digit processed. |
| 1203 __ ret(); | 1208 __ ret(); |
| 1204 } | 1209 } |
| 1205 | 1210 |
| 1206 | 1211 |
| 1207 void Intrinsifier::Montgomery_mulMod(Assembler* assembler) { | 1212 void Intrinsifier::Montgomery_mulMod(Assembler* assembler) { |
| 1208 // Pseudo code: | 1213 // Pseudo code: |
| 1209 // static void _mulMod(Uint32List args, Uint32List digits, int i) { | 1214 // static int _mulMod(Uint32List args, Uint32List digits, int i) { |
| 1210 // uint32_t rho = args[_RHO]; // _RHO == 0. | 1215 // uint32_t rho = args[_RHO]; // _RHO == 2. |
| 1211 // uint32_t d = digits[i >> 1]; // i is Smi. | 1216 // uint32_t d = digits[i >> 1]; // i is Smi. |
| 1212 // uint64_t t = rho*d; | 1217 // uint64_t t = rho*d; |
| 1213 // args[_MU] = t mod DIGIT_BASE; // _MU == 1. | 1218 // args[_MU] = t mod DIGIT_BASE; // _MU == 4. |
| 1219 // return 1; |
| 1214 // } | 1220 // } |
| 1215 | 1221 |
| 1216 // RDI = args | 1222 // RDI = args |
| 1217 __ movq(RDI, Address(RSP, 3 * kWordSize)); // args | 1223 __ movq(RDI, Address(RSP, 3 * kWordSize)); // args |
| 1218 | 1224 |
| 1219 // RCX = rho = args[0] | 1225 // RCX = rho = args[2] |
| 1220 __ movl(RCX, FieldAddress(RDI, TypedData::data_offset())); | 1226 __ movl(RCX, |
| 1227 FieldAddress(RDI, |
| 1228 TypedData::data_offset() + 2*Bigint::kBytesPerDigit)); |
| 1221 | 1229 |
| 1222 // RAX = digits[i >> 1] | 1230 // RAX = digits[i >> 1] |
| 1223 __ movq(RBX, Address(RSP, 2 * kWordSize)); // digits | 1231 __ movq(RBX, Address(RSP, 2 * kWordSize)); // digits |
| 1224 __ movq(RAX, Address(RSP, 1 * kWordSize)); // i is Smi | 1232 __ movq(RAX, Address(RSP, 1 * kWordSize)); // i is Smi |
| 1225 __ movl(RAX, FieldAddress(RBX, RAX, TIMES_2, TypedData::data_offset())); | 1233 __ movl(RAX, FieldAddress(RBX, RAX, TIMES_2, TypedData::data_offset())); |
| 1226 | 1234 |
| 1227 // RDX:RAX = t = rho*d | 1235 // RDX:RAX = t = rho*d |
| 1228 __ mull(RCX); | 1236 __ mull(RCX); |
| 1229 | 1237 |
| 1230 // args[1] = t mod DIGIT_BASE = low32(t) | 1238 // args[4] = t mod DIGIT_BASE = low32(t) |
| 1231 __ movl(FieldAddress(RDI, TypedData::data_offset() + Bigint::kBytesPerDigit), | 1239 __ movl(FieldAddress(RDI, |
| 1240 TypedData::data_offset() + 4*Bigint::kBytesPerDigit), |
| 1232 RAX); | 1241 RAX); |
| 1233 | 1242 |
| 1234 // Returning Object::null() is not required, since this method is private. | 1243 __ movq(RAX, Immediate(Smi::RawValue(1))); // One digit processed. |
| 1235 __ ret(); | 1244 __ ret(); |
| 1236 } | 1245 } |
| 1237 | 1246 |
| 1238 | 1247 |
| 1239 // Check if the last argument is a double, jump to label 'is_smi' if smi | 1248 // Check if the last argument is a double, jump to label 'is_smi' if smi |
| 1240 // (easy to convert to double), otherwise jump to label 'not_double_smi', | 1249 // (easy to convert to double), otherwise jump to label 'not_double_smi', |
| 1241 // Returns the last argument in RAX. | 1250 // Returns the last argument in RAX. |
| 1242 static void TestLastArgumentIsDouble(Assembler* assembler, | 1251 static void TestLastArgumentIsDouble(Assembler* assembler, |
| 1243 Label* is_smi, | 1252 Label* is_smi, |
| 1244 Label* not_double_smi) { | 1253 Label* not_double_smi) { |
| (...skipping 752 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1997 // Set return value to Isolate::current_tag_. | 2006 // Set return value to Isolate::current_tag_. |
| 1998 __ movq(RAX, Address(RBX, Isolate::current_tag_offset())); | 2007 __ movq(RAX, Address(RBX, Isolate::current_tag_offset())); |
| 1999 __ ret(); | 2008 __ ret(); |
| 2000 } | 2009 } |
| 2001 | 2010 |
| 2002 #undef __ | 2011 #undef __ |
| 2003 | 2012 |
| 2004 } // namespace dart | 2013 } // namespace dart |
| 2005 | 2014 |
| 2006 #endif // defined TARGET_ARCH_X64 | 2015 #endif // defined TARGET_ARCH_X64 |
| OLD | NEW |