| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. | 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. |
| 6 #if defined(TARGET_ARCH_X64) | 6 #if defined(TARGET_ARCH_X64) |
| 7 | 7 |
| 8 #include "vm/intrinsifier.h" | 8 #include "vm/intrinsifier.h" |
| 9 | 9 |
| 10 #include "vm/assembler.h" | 10 #include "vm/assembler.h" |
| 11 #include "vm/flow_graph_compiler.h" | 11 #include "vm/flow_graph_compiler.h" |
| 12 #include "vm/instructions.h" | 12 #include "vm/instructions.h" |
| 13 #include "vm/object_store.h" | 13 #include "vm/object_store.h" |
| 14 #include "vm/symbols.h" | 14 #include "vm/symbols.h" |
| 15 | 15 |
| 16 namespace dart { | 16 namespace dart { |
| 17 | 17 |
| 18 DECLARE_FLAG(bool, enable_type_checks); | 18 DECLARE_FLAG(bool, enable_type_checks); |
| 19 | 19 |
| 20 // When entering intrinsics code: | 20 // When entering intrinsics code: |
| 21 // RBX: IC Data | 21 // RBX: IC Data |
| 22 // R10: Arguments descriptor | 22 // R10: Arguments descriptor |
| 23 // TOS: Return address | 23 // TOS: Return address |
| 24 // The RBX, R10 registers can be destroyed only if there is no slow-path (i.e., | 24 // The RBX, R10 registers can be destroyed only if there is no slow-path (i.e., |
| 25 // the methods returns true). | 25 // the methods returns true). |
| 26 | 26 |
| 27 #define __ assembler-> | 27 #define __ assembler-> |
| 28 | 28 |
| 29 | 29 |
| 30 void Intrinsifier::ObjectArrayLength(Assembler* assembler) { | 30 intptr_t Intrinsifier::ParameterSlotFromSp() { return 0; } |
| 31 __ movq(RAX, Address(RSP, + 1 * kWordSize)); | |
| 32 __ movq(RAX, FieldAddress(RAX, Array::length_offset())); | |
| 33 __ ret(); | |
| 34 } | |
| 35 | |
| 36 | |
| 37 void Intrinsifier::ImmutableArrayLength(Assembler* assembler) { | |
| 38 ObjectArrayLength(assembler); | |
| 39 } | |
| 40 | |
| 41 | |
| 42 void Intrinsifier::ObjectArrayGetIndexed(Assembler* assembler) { | |
| 43 Label fall_through; | |
| 44 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | |
| 45 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // Array. | |
| 46 __ testq(RCX, Immediate(kSmiTagMask)); | |
| 47 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | |
| 48 // Range check. | |
| 49 __ cmpq(RCX, FieldAddress(RAX, Array::length_offset())); | |
| 50 // Runtime throws exception. | |
| 51 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | |
| 52 // Note that RBX is Smi, i.e, times 2. | |
| 53 ASSERT(kSmiTagShift == 1); | |
| 54 __ movq(RAX, FieldAddress(RAX, RCX, TIMES_4, Array::data_offset())); | |
| 55 __ ret(); | |
| 56 __ Bind(&fall_through); | |
| 57 } | |
| 58 | |
| 59 | |
| 60 void Intrinsifier::ImmutableArrayGetIndexed(Assembler* assembler) { | |
| 61 ObjectArrayGetIndexed(assembler); | |
| 62 } | |
| 63 | 31 |
| 64 | 32 |
| 65 void Intrinsifier::ObjectArraySetIndexed(Assembler* assembler) { | 33 void Intrinsifier::ObjectArraySetIndexed(Assembler* assembler) { |
| 66 if (FLAG_enable_type_checks) { | 34 if (FLAG_enable_type_checks) { |
| 67 return; | 35 return; |
| 68 } | 36 } |
| 69 __ movq(RDX, Address(RSP, + 1 * kWordSize)); // Value. | 37 __ movq(RDX, Address(RSP, + 1 * kWordSize)); // Value. |
| 70 __ movq(RCX, Address(RSP, + 2 * kWordSize)); // Index. | 38 __ movq(RCX, Address(RSP, + 2 * kWordSize)); // Index. |
| 71 __ movq(RAX, Address(RSP, + 3 * kWordSize)); // Array. | 39 __ movq(RAX, Address(RSP, + 3 * kWordSize)); // Array. |
| 72 Label fall_through; | 40 Label fall_through; |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 121 | 89 |
| 122 // Set the length field in the growable array object to 0. | 90 // Set the length field in the growable array object to 0. |
| 123 __ movq(FieldAddress(RAX, GrowableObjectArray::length_offset()), | 91 __ movq(FieldAddress(RAX, GrowableObjectArray::length_offset()), |
| 124 Immediate(0)); | 92 Immediate(0)); |
| 125 __ ret(); // returns the newly allocated object in RAX. | 93 __ ret(); // returns the newly allocated object in RAX. |
| 126 | 94 |
| 127 __ Bind(&fall_through); | 95 __ Bind(&fall_through); |
| 128 } | 96 } |
| 129 | 97 |
| 130 | 98 |
| 131 // Get length of growable object array. | |
| 132 // On stack: growable array (+1), return-address (+0). | |
| 133 void Intrinsifier::GrowableArrayLength(Assembler* assembler) { | |
| 134 __ movq(RAX, Address(RSP, + 1 * kWordSize)); | |
| 135 __ movq(RAX, FieldAddress(RAX, GrowableObjectArray::length_offset())); | |
| 136 __ ret(); | |
| 137 } | |
| 138 | |
| 139 | |
| 140 void Intrinsifier::GrowableArrayCapacity(Assembler* assembler) { | |
| 141 __ movq(RAX, Address(RSP, + 1 * kWordSize)); | |
| 142 __ movq(RAX, FieldAddress(RAX, GrowableObjectArray::data_offset())); | |
| 143 __ movq(RAX, FieldAddress(RAX, Array::length_offset())); | |
| 144 __ ret(); | |
| 145 } | |
| 146 | |
| 147 | |
| 148 // Access growable object array at specified index. | 99 // Access growable object array at specified index. |
| 149 // On stack: growable array (+2), index (+1), return-address (+0). | 100 // On stack: growable array (+2), index (+1), return-address (+0). |
| 150 void Intrinsifier::GrowableArrayGetIndexed(Assembler* assembler) { | 101 void Intrinsifier::GrowableArrayGetIndexed(Assembler* assembler) { |
| 151 Label fall_through; | 102 Label fall_through; |
| 152 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | 103 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. |
| 153 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // GrowableArray. | 104 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // GrowableArray. |
| 154 __ testq(RCX, Immediate(kSmiTagMask)); | 105 __ testq(RCX, Immediate(kSmiTagMask)); |
| 155 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | 106 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. |
| 156 // Range check using _length field. | 107 // Range check using _length field. |
| 157 __ cmpq(RCX, FieldAddress(RAX, GrowableObjectArray::length_offset())); | 108 __ cmpq(RCX, FieldAddress(RAX, GrowableObjectArray::length_offset())); |
| (...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 351 __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); \ | 302 __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); \ |
| 352 __ movq(Address(RDI, 0), RBX); \ | 303 __ movq(Address(RDI, 0), RBX); \ |
| 353 __ addq(RDI, Immediate(kWordSize)); \ | 304 __ addq(RDI, Immediate(kWordSize)); \ |
| 354 __ jmp(&init_loop, Assembler::kNearJump); \ | 305 __ jmp(&init_loop, Assembler::kNearJump); \ |
| 355 __ Bind(&done); \ | 306 __ Bind(&done); \ |
| 356 \ | 307 \ |
| 357 __ ret(); \ | 308 __ ret(); \ |
| 358 __ Bind(&fall_through); \ | 309 __ Bind(&fall_through); \ |
| 359 | 310 |
| 360 | 311 |
| 361 // Gets the length of a TypedData. | |
| 362 void Intrinsifier::TypedDataLength(Assembler* assembler) { | |
| 363 __ movq(RAX, Address(RSP, + 1 * kWordSize)); | |
| 364 __ movq(RAX, FieldAddress(RAX, TypedData::length_offset())); | |
| 365 __ ret(); | |
| 366 } | |
| 367 | |
| 368 | |
| 369 void Intrinsifier::Uint8ArrayGetIndexed(Assembler* assembler) { | |
| 370 Label fall_through; | |
| 371 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | |
| 372 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // Array. | |
| 373 __ testq(RCX, Immediate(kSmiTagMask)); | |
| 374 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | |
| 375 // Range check. | |
| 376 __ cmpq(RCX, FieldAddress(RAX, TypedData::length_offset())); | |
| 377 // Runtime throws exception. | |
| 378 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | |
| 379 | |
| 380 __ SmiUntag(RCX); | |
| 381 __ movzxb(RAX, FieldAddress(RAX, RCX, TIMES_1, TypedData::data_offset())); | |
| 382 __ SmiTag(RAX); | |
| 383 __ ret(); | |
| 384 __ Bind(&fall_through); | |
| 385 } | |
| 386 | |
| 387 | |
| 388 void Intrinsifier::ExternalUint8ArrayGetIndexed(Assembler* assembler) { | |
| 389 Label fall_through; | |
| 390 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | |
| 391 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // Array. | |
| 392 __ testq(RCX, Immediate(kSmiTagMask)); | |
| 393 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | |
| 394 // Range check. | |
| 395 __ cmpq(RCX, FieldAddress(RAX, TypedData::length_offset())); | |
| 396 // Runtime throws exception. | |
| 397 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | |
| 398 | |
| 399 __ movq(RAX, FieldAddress(RAX, ExternalTypedData::data_offset())); | |
| 400 __ SmiUntag(RCX); | |
| 401 __ movzxb(RAX, Address(RAX, RCX, TIMES_1, 0)); | |
| 402 __ SmiTag(RAX); | |
| 403 __ ret(); | |
| 404 __ Bind(&fall_through); | |
| 405 } | |
| 406 | |
| 407 | |
| 408 void Intrinsifier::Float64ArrayGetIndexed(Assembler* assembler) { | |
| 409 Label fall_through; | |
| 410 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | |
| 411 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // Array. | |
| 412 __ testq(RCX, Immediate(kSmiTagMask)); | |
| 413 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | |
| 414 // Range check. | |
| 415 __ cmpq(RCX, FieldAddress(RAX, TypedData::length_offset())); | |
| 416 // Runtime throws exception. | |
| 417 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | |
| 418 | |
| 419 Address element_address = | |
| 420 Assembler::ElementAddressForRegIndex(false, // Not external. | |
| 421 kTypedDataFloat64ArrayCid, | |
| 422 8, // Index scale. | |
| 423 RAX, // Array. | |
| 424 RCX); // Index. | |
| 425 | |
| 426 __ movsd(XMM0, element_address); | |
| 427 | |
| 428 const Class& double_class = Class::Handle( | |
| 429 Isolate::Current()->object_store()->double_class()); | |
| 430 __ TryAllocate(double_class, | |
| 431 &fall_through, | |
| 432 Assembler::kNearJump, | |
| 433 RAX, // Result register. | |
| 434 kNoRegister); | |
| 435 __ movsd(FieldAddress(RAX, Double::value_offset()), XMM0); | |
| 436 __ ret(); | |
| 437 __ Bind(&fall_through); | |
| 438 } | |
| 439 | |
| 440 | |
| 441 void Intrinsifier::Float64ArraySetIndexed(Assembler* assembler) { | |
| 442 Label fall_through; | |
| 443 __ movq(RCX, Address(RSP, + 2 * kWordSize)); // Index. | |
| 444 __ movq(RAX, Address(RSP, + 3 * kWordSize)); // Array. | |
| 445 __ testq(RCX, Immediate(kSmiTagMask)); | |
| 446 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | |
| 447 // Range check. | |
| 448 __ cmpq(RCX, FieldAddress(RAX, TypedData::length_offset())); | |
| 449 // Runtime throws exception. | |
| 450 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | |
| 451 | |
| 452 __ movq(RDX, Address(RSP, + 1 * kWordSize)); // Value | |
| 453 __ testq(RDX, Immediate(kSmiTagMask)); | |
| 454 __ j(ZERO, &fall_through, Assembler::kNearJump); // Value is Smi. | |
| 455 | |
| 456 __ LoadClassId(RDI, RDX); | |
| 457 __ cmpq(RDI, Immediate(kTypedDataFloat64ArrayCid)); | |
| 458 __ j(NOT_EQUAL, &fall_through, Assembler::kNearJump); | |
| 459 | |
| 460 __ movsd(XMM0, FieldAddress(RDX, Double::value_offset())); | |
| 461 | |
| 462 Address element_address = | |
| 463 Assembler::ElementAddressForRegIndex(false, // Not external. | |
| 464 kTypedDataFloat64ArrayCid, | |
| 465 8, // Index scale. | |
| 466 RAX, // Array. | |
| 467 RCX); // Index. | |
| 468 | |
| 469 __ movsd(element_address, XMM0); | |
| 470 __ ret(); | |
| 471 __ Bind(&fall_through); | |
| 472 } | |
| 473 | |
| 474 | |
| 475 static ScaleFactor GetScaleFactor(intptr_t size) { | 312 static ScaleFactor GetScaleFactor(intptr_t size) { |
| 476 switch (size) { | 313 switch (size) { |
| 477 case 1: return TIMES_1; | 314 case 1: return TIMES_1; |
| 478 case 2: return TIMES_2; | 315 case 2: return TIMES_2; |
| 479 case 4: return TIMES_4; | 316 case 4: return TIMES_4; |
| 480 case 8: return TIMES_8; | 317 case 8: return TIMES_8; |
| 481 case 16: return TIMES_16; | 318 case 16: return TIMES_16; |
| 482 } | 319 } |
| 483 UNREACHABLE(); | 320 UNREACHABLE(); |
| 484 return static_cast<ScaleFactor>(0); | 321 return static_cast<ScaleFactor>(0); |
| (...skipping 777 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1262 __ movq(RAX, Address(RSP, + 1 * kWordSize)); // String object. | 1099 __ movq(RAX, Address(RSP, + 1 * kWordSize)); // String object. |
| 1263 __ movq(RAX, FieldAddress(RAX, String::hash_offset())); | 1100 __ movq(RAX, FieldAddress(RAX, String::hash_offset())); |
| 1264 __ cmpq(RAX, Immediate(0)); | 1101 __ cmpq(RAX, Immediate(0)); |
| 1265 __ j(EQUAL, &fall_through, Assembler::kNearJump); | 1102 __ j(EQUAL, &fall_through, Assembler::kNearJump); |
| 1266 __ ret(); | 1103 __ ret(); |
| 1267 __ Bind(&fall_through); | 1104 __ Bind(&fall_through); |
| 1268 // Hash not yet computed. | 1105 // Hash not yet computed. |
| 1269 } | 1106 } |
| 1270 | 1107 |
| 1271 | 1108 |
| 1272 void Intrinsifier::StringBaseLength(Assembler* assembler) { | |
| 1273 __ movq(RAX, Address(RSP, + 1 * kWordSize)); // String object. | |
| 1274 __ movq(RAX, FieldAddress(RAX, String::length_offset())); | |
| 1275 __ ret(); | |
| 1276 } | |
| 1277 | |
| 1278 | |
| 1279 void Intrinsifier::StringBaseCodeUnitAt(Assembler* assembler) { | 1109 void Intrinsifier::StringBaseCodeUnitAt(Assembler* assembler) { |
| 1280 Label fall_through, try_two_byte_string; | 1110 Label fall_through, try_two_byte_string; |
| 1281 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. | 1111 __ movq(RCX, Address(RSP, + 1 * kWordSize)); // Index. |
| 1282 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // String. | 1112 __ movq(RAX, Address(RSP, + 2 * kWordSize)); // String. |
| 1283 __ testq(RCX, Immediate(kSmiTagMask)); | 1113 __ testq(RCX, Immediate(kSmiTagMask)); |
| 1284 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. | 1114 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Non-smi index. |
| 1285 // Range check. | 1115 // Range check. |
| 1286 __ cmpq(RCX, FieldAddress(RAX, String::length_offset())); | 1116 __ cmpq(RCX, FieldAddress(RAX, String::length_offset())); |
| 1287 // Runtime throws exception. | 1117 // Runtime throws exception. |
| 1288 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); | 1118 __ j(ABOVE_EQUAL, &fall_through, Assembler::kNearJump); |
| (...skipping 405 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1694 // Set return value to Isolate::current_tag_. | 1524 // Set return value to Isolate::current_tag_. |
| 1695 __ movq(RAX, Address(RBX, Isolate::current_tag_offset())); | 1525 __ movq(RAX, Address(RBX, Isolate::current_tag_offset())); |
| 1696 __ ret(); | 1526 __ ret(); |
| 1697 } | 1527 } |
| 1698 | 1528 |
| 1699 #undef __ | 1529 #undef __ |
| 1700 | 1530 |
| 1701 } // namespace dart | 1531 } // namespace dart |
| 1702 | 1532 |
| 1703 #endif // defined TARGET_ARCH_X64 | 1533 #endif // defined TARGET_ARCH_X64 |
| OLD | NEW |