| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/globals.h" // NOLINT | 5 #include "vm/globals.h" // NOLINT |
| 6 #if defined(TARGET_ARCH_IA32) | 6 #if defined(TARGET_ARCH_IA32) |
| 7 | 7 |
| 8 #include "vm/assembler.h" | 8 #include "vm/assembler.h" |
| 9 #include "vm/code_generator.h" | 9 #include "vm/code_generator.h" |
| 10 #include "vm/cpu.h" | 10 #include "vm/cpu.h" |
| 11 #include "vm/heap.h" | 11 #include "vm/heap.h" |
| 12 #include "vm/instructions.h" | 12 #include "vm/instructions.h" |
| 13 #include "vm/memory_region.h" | 13 #include "vm/memory_region.h" |
| 14 #include "vm/runtime_entry.h" | 14 #include "vm/runtime_entry.h" |
| 15 #include "vm/stack_frame.h" | 15 #include "vm/stack_frame.h" |
| 16 #include "vm/stub_code.h" | 16 #include "vm/stub_code.h" |
| 17 #include "vm/verified_memory.h" | |
| 18 | 17 |
| 19 namespace dart { | 18 namespace dart { |
| 20 | 19 |
| 21 DECLARE_FLAG(bool, inline_alloc); | 20 DECLARE_FLAG(bool, inline_alloc); |
| 22 | 21 |
| 23 | 22 |
| 24 class DirectCallRelocation : public AssemblerFixup { | 23 class DirectCallRelocation : public AssemblerFixup { |
| 25 public: | 24 public: |
| 26 void Process(const MemoryRegion& region, intptr_t position) { | 25 void Process(const MemoryRegion& region, intptr_t position) { |
| 27 // Direct calls are relative to the following instruction on x86. | 26 // Direct calls are relative to the following instruction on x86. |
| (...skipping 1963 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1991 | 1990 |
| 1992 | 1991 |
| 1993 void Assembler::hlt() { | 1992 void Assembler::hlt() { |
| 1994 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 1993 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
| 1995 EmitUint8(0xF4); | 1994 EmitUint8(0xF4); |
| 1996 } | 1995 } |
| 1997 | 1996 |
| 1998 | 1997 |
| 1999 void Assembler::j(Condition condition, Label* label, bool near) { | 1998 void Assembler::j(Condition condition, Label* label, bool near) { |
| 2000 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 1999 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
| 2001 if (VerifiedMemory::enabled()) { | |
| 2002 near = Assembler::kFarJump; | |
| 2003 } | |
| 2004 if (label->IsBound()) { | 2000 if (label->IsBound()) { |
| 2005 static const int kShortSize = 2; | 2001 static const int kShortSize = 2; |
| 2006 static const int kLongSize = 6; | 2002 static const int kLongSize = 6; |
| 2007 intptr_t offset = label->Position() - buffer_.Size(); | 2003 intptr_t offset = label->Position() - buffer_.Size(); |
| 2008 ASSERT(offset <= 0); | 2004 ASSERT(offset <= 0); |
| 2009 if (Utils::IsInt(8, offset - kShortSize)) { | 2005 if (Utils::IsInt(8, offset - kShortSize)) { |
| 2010 EmitUint8(0x70 + condition); | 2006 EmitUint8(0x70 + condition); |
| 2011 EmitUint8((offset - kShortSize) & 0xFF); | 2007 EmitUint8((offset - kShortSize) & 0xFF); |
| 2012 } else { | 2008 } else { |
| 2013 EmitUint8(0x0F); | 2009 EmitUint8(0x0F); |
| (...skipping 22 matching lines...) Expand all Loading... |
| 2036 | 2032 |
| 2037 void Assembler::jmp(Register reg) { | 2033 void Assembler::jmp(Register reg) { |
| 2038 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
| 2039 EmitUint8(0xFF); | 2035 EmitUint8(0xFF); |
| 2040 EmitRegisterOperand(4, reg); | 2036 EmitRegisterOperand(4, reg); |
| 2041 } | 2037 } |
| 2042 | 2038 |
| 2043 | 2039 |
| 2044 void Assembler::jmp(Label* label, bool near) { | 2040 void Assembler::jmp(Label* label, bool near) { |
| 2045 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2041 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
| 2046 if (VerifiedMemory::enabled()) { | |
| 2047 near = Assembler::kFarJump; | |
| 2048 } | |
| 2049 if (label->IsBound()) { | 2042 if (label->IsBound()) { |
| 2050 static const int kShortSize = 2; | 2043 static const int kShortSize = 2; |
| 2051 static const int kLongSize = 5; | 2044 static const int kLongSize = 5; |
| 2052 intptr_t offset = label->Position() - buffer_.Size(); | 2045 intptr_t offset = label->Position() - buffer_.Size(); |
| 2053 ASSERT(offset <= 0); | 2046 ASSERT(offset <= 0); |
| 2054 if (Utils::IsInt(8, offset - kShortSize)) { | 2047 if (Utils::IsInt(8, offset - kShortSize)) { |
| 2055 EmitUint8(0xEB); | 2048 EmitUint8(0xEB); |
| 2056 EmitUint8((offset - kShortSize) & 0xFF); | 2049 EmitUint8((offset - kShortSize) & 0xFF); |
| 2057 } else { | 2050 } else { |
| 2058 EmitUint8(0xE9); | 2051 EmitUint8(0xE9); |
| (...skipping 200 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2259 // Mask out higher, uninteresting bits which were polluted by dest. | 2252 // Mask out higher, uninteresting bits which were polluted by dest. |
| 2260 andl(value, Immediate(kObjectAlignment - 1)); | 2253 andl(value, Immediate(kObjectAlignment - 1)); |
| 2261 // Compare with the expected bit pattern. | 2254 // Compare with the expected bit pattern. |
| 2262 cmpl(value, Immediate( | 2255 cmpl(value, Immediate( |
| 2263 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + | 2256 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + |
| 2264 kOldObjectAlignmentOffset + kHeapObjectTag)); | 2257 kOldObjectAlignmentOffset + kHeapObjectTag)); |
| 2265 j(NOT_ZERO, no_update, Assembler::kNearJump); | 2258 j(NOT_ZERO, no_update, Assembler::kNearJump); |
| 2266 } | 2259 } |
| 2267 | 2260 |
| 2268 | 2261 |
| 2269 void Assembler::VerifyHeapWord(const Address& address, | |
| 2270 FieldContent old_content) { | |
| 2271 #if defined(DEBUG) | |
| 2272 switch (old_content) { | |
| 2273 case kEmptyOrSmiOrNull: | |
| 2274 VerifyUninitialized(address); | |
| 2275 break; | |
| 2276 case kHeapObjectOrSmi: | |
| 2277 VerifyObjectOrSmi(address); | |
| 2278 break; | |
| 2279 case kOnlySmi: | |
| 2280 VerifySmi(address); | |
| 2281 break; | |
| 2282 } | |
| 2283 #endif // DEBUG | |
| 2284 if (VerifiedMemory::enabled()) { | |
| 2285 Register addr_reg = EDX; | |
| 2286 Register value = EBX; | |
| 2287 // Preserve registers. | |
| 2288 pushl(addr_reg); | |
| 2289 pushl(value); | |
| 2290 leal(addr_reg, address); | |
| 2291 // ASSERT(*address == *(address + offset)) | |
| 2292 movl(value, Address(addr_reg, 0)); | |
| 2293 cmpl(value, Address(addr_reg, VerifiedMemory::offset())); | |
| 2294 Label ok; | |
| 2295 j(EQUAL, &ok, Assembler::kNearJump); | |
| 2296 Stop("Write barrier verification failed"); | |
| 2297 Bind(&ok); | |
| 2298 popl(value); | |
| 2299 popl(addr_reg); | |
| 2300 } | |
| 2301 } | |
| 2302 | |
| 2303 | |
| 2304 void Assembler::VerifiedWrite(const Address& dest, | |
| 2305 Register value, | |
| 2306 FieldContent old_content) { | |
| 2307 VerifyHeapWord(dest, old_content); | |
| 2308 movl(dest, value); | |
| 2309 if (VerifiedMemory::enabled()) { | |
| 2310 Register temp = (value == EDX) ? ECX : EDX; | |
| 2311 pushl(temp); | |
| 2312 leal(temp, dest); | |
| 2313 movl(Address(temp, VerifiedMemory::offset()), value); | |
| 2314 popl(temp); | |
| 2315 } | |
| 2316 } | |
| 2317 | |
| 2318 | |
| 2319 #if defined(DEBUG) | |
| 2320 void Assembler::VerifyObjectOrSmi(const Address& dest) { | |
| 2321 Label ok; | |
| 2322 testb(dest, Immediate(kHeapObjectTag)); | |
| 2323 j(ZERO, &ok, Assembler::kNearJump); | |
| 2324 // Non-smi case: Verify object pointer is word-aligned when untagged. | |
| 2325 COMPILE_ASSERT(kHeapObjectTag == 1); | |
| 2326 testb(dest, Immediate((kWordSize - 1) - kHeapObjectTag)); | |
| 2327 j(ZERO, &ok, Assembler::kNearJump); | |
| 2328 Stop("Expected heap object or Smi"); | |
| 2329 Bind(&ok); | |
| 2330 } | |
| 2331 | |
| 2332 | |
| 2333 void Assembler::VerifyUninitialized(const Address& dest) { | |
| 2334 Label ok; | |
| 2335 testb(dest, Immediate(kHeapObjectTag)); | |
| 2336 j(ZERO, &ok, Assembler::kNearJump); | |
| 2337 // Non-smi case: Check for the special zap word or null. | |
| 2338 #if defined(DEBUG) | |
| 2339 cmpl(dest, Immediate(Heap::kZap32Bits)); | |
| 2340 j(EQUAL, &ok, Assembler::kNearJump); | |
| 2341 #else | |
| 2342 #error Only supported in DEBUG mode | |
| 2343 #endif | |
| 2344 cmpl(dest, Immediate(reinterpret_cast<uint32_t>(Object::null()))); | |
| 2345 j(EQUAL, &ok, Assembler::kNearJump); | |
| 2346 Stop("Expected zapped, Smi or null"); | |
| 2347 Bind(&ok); | |
| 2348 } | |
| 2349 | |
| 2350 | |
| 2351 void Assembler::VerifySmi(const Address& dest, const char* stop_msg) { | |
| 2352 Label done; | |
| 2353 testb(dest, Immediate(kHeapObjectTag)); | |
| 2354 j(ZERO, &done, Assembler::kNearJump); | |
| 2355 Stop(stop_msg); | |
| 2356 Bind(&done); | |
| 2357 } | |
| 2358 #endif // defined(DEBUG) | |
| 2359 | |
| 2360 | |
| 2361 // Destroys the value register. | 2262 // Destroys the value register. |
| 2362 void Assembler::StoreIntoObject(Register object, | 2263 void Assembler::StoreIntoObject(Register object, |
| 2363 const Address& dest, | 2264 const Address& dest, |
| 2364 Register value, | 2265 Register value, |
| 2365 bool can_value_be_smi) { | 2266 bool can_value_be_smi) { |
| 2366 ASSERT(object != value); | 2267 ASSERT(object != value); |
| 2367 VerifiedWrite(dest, value, kHeapObjectOrSmi); | 2268 movl(dest, value); |
| 2368 Label done; | 2269 Label done; |
| 2369 if (can_value_be_smi) { | 2270 if (can_value_be_smi) { |
| 2370 StoreIntoObjectFilter(object, value, &done); | 2271 StoreIntoObjectFilter(object, value, &done); |
| 2371 } else { | 2272 } else { |
| 2372 StoreIntoObjectFilterNoSmi(object, value, &done); | 2273 StoreIntoObjectFilterNoSmi(object, value, &done); |
| 2373 } | 2274 } |
| 2374 // A store buffer update is required. | 2275 // A store buffer update is required. |
| 2375 if (value != EDX) { | 2276 if (value != EDX) { |
| 2376 pushl(EDX); // Preserve EDX. | 2277 pushl(EDX); // Preserve EDX. |
| 2377 } | 2278 } |
| 2378 if (object != EDX) { | 2279 if (object != EDX) { |
| 2379 movl(EDX, object); | 2280 movl(EDX, object); |
| 2380 } | 2281 } |
| 2381 call(Address(THR, Thread::update_store_buffer_entry_point_offset())); | 2282 call(Address(THR, Thread::update_store_buffer_entry_point_offset())); |
| 2382 if (value != EDX) { | 2283 if (value != EDX) { |
| 2383 popl(EDX); // Restore EDX. | 2284 popl(EDX); // Restore EDX. |
| 2384 } | 2285 } |
| 2385 Bind(&done); | 2286 Bind(&done); |
| 2386 } | 2287 } |
| 2387 | 2288 |
| 2388 | 2289 |
| 2389 void Assembler::StoreIntoObjectNoBarrier(Register object, | 2290 void Assembler::StoreIntoObjectNoBarrier(Register object, |
| 2390 const Address& dest, | 2291 const Address& dest, |
| 2391 Register value, | 2292 Register value) { |
| 2392 FieldContent old_content) { | 2293 movl(dest, value); |
| 2393 VerifiedWrite(dest, value, old_content); | |
| 2394 #if defined(DEBUG) | 2294 #if defined(DEBUG) |
| 2395 Label done; | 2295 Label done; |
| 2396 pushl(value); | 2296 pushl(value); |
| 2397 StoreIntoObjectFilter(object, value, &done); | 2297 StoreIntoObjectFilter(object, value, &done); |
| 2398 Stop("Store buffer update is required"); | 2298 Stop("Store buffer update is required"); |
| 2399 Bind(&done); | 2299 Bind(&done); |
| 2400 popl(value); | 2300 popl(value); |
| 2401 #endif // defined(DEBUG) | 2301 #endif // defined(DEBUG) |
| 2402 // No store buffer update. | 2302 // No store buffer update. |
| 2403 } | 2303 } |
| 2404 | 2304 |
| 2405 | 2305 |
| 2406 void Assembler::UnverifiedStoreOldObject(const Address& dest, | 2306 void Assembler::UnverifiedStoreOldObject(const Address& dest, |
| 2407 const Object& value) { | 2307 const Object& value) { |
| 2408 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); | 2308 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); |
| 2409 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); | 2309 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); |
| 2410 ASSERT(value.IsOld()); | 2310 ASSERT(value.IsOld()); |
| 2411 ASSERT(!value.InVMHeap()); | 2311 ASSERT(!value.InVMHeap()); |
| 2412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
| 2413 EmitUint8(0xC7); | 2313 EmitUint8(0xC7); |
| 2414 EmitOperand(0, dest); | 2314 EmitOperand(0, dest); |
| 2415 buffer_.EmitObject(value); | 2315 buffer_.EmitObject(value); |
| 2416 } | 2316 } |
| 2417 | 2317 |
| 2418 | 2318 |
| 2419 void Assembler::StoreIntoObjectNoBarrier(Register object, | 2319 void Assembler::StoreIntoObjectNoBarrier(Register object, |
| 2420 const Address& dest, | 2320 const Address& dest, |
| 2421 const Object& value, | 2321 const Object& value) { |
| 2422 FieldContent old_content) { | |
| 2423 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); | 2322 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); |
| 2424 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); | 2323 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); |
| 2425 VerifyHeapWord(dest, old_content); | |
| 2426 if (value.IsSmi() || value.InVMHeap()) { | 2324 if (value.IsSmi() || value.InVMHeap()) { |
| 2427 Immediate imm_value(reinterpret_cast<int32_t>(value.raw())); | 2325 Immediate imm_value(reinterpret_cast<int32_t>(value.raw())); |
| 2428 movl(dest, imm_value); | 2326 movl(dest, imm_value); |
| 2429 if (VerifiedMemory::enabled()) { | |
| 2430 Register temp = ECX; | |
| 2431 pushl(temp); | |
| 2432 leal(temp, dest); | |
| 2433 movl(Address(temp, VerifiedMemory::offset()), imm_value); | |
| 2434 popl(temp); | |
| 2435 } | |
| 2436 } else { | 2327 } else { |
| 2437 UnverifiedStoreOldObject(dest, value); | 2328 UnverifiedStoreOldObject(dest, value); |
| 2438 if (VerifiedMemory::enabled()) { | |
| 2439 Register temp = EDX; | |
| 2440 pushl(temp); | |
| 2441 leal(temp, dest); | |
| 2442 UnverifiedStoreOldObject(Address(temp, VerifiedMemory::offset()), value); | |
| 2443 popl(temp); | |
| 2444 } | |
| 2445 } | 2329 } |
| 2446 // No store buffer update. | 2330 // No store buffer update. |
| 2447 } | 2331 } |
| 2448 | 2332 |
| 2449 | 2333 |
| 2450 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { | 2334 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { |
| 2451 #if defined(DEBUG) | 2335 #if defined(DEBUG) |
| 2452 Label done; | 2336 Label done; |
| 2453 testl(value, Immediate(kHeapObjectTag)); | 2337 testl(value, Immediate(kHeapObjectTag)); |
| 2454 j(ZERO, &done); | 2338 j(ZERO, &done); |
| 2455 Stop("New value must be Smi."); | 2339 Stop("New value must be Smi."); |
| 2456 Bind(&done); | 2340 Bind(&done); |
| 2457 #endif // defined(DEBUG) | 2341 #endif // defined(DEBUG) |
| 2458 VerifiedWrite(dest, value, kOnlySmi); | 2342 movl(dest, value); |
| 2459 } | 2343 } |
| 2460 | 2344 |
| 2461 | 2345 |
| 2462 void Assembler::ZeroInitSmiField(const Address& dest) { | 2346 void Assembler::ZeroInitSmiField(const Address& dest) { |
| 2463 VerifyHeapWord(dest, kEmptyOrSmiOrNull); | |
| 2464 Immediate zero(Smi::RawValue(0)); | 2347 Immediate zero(Smi::RawValue(0)); |
| 2465 movl(dest, zero); | 2348 movl(dest, zero); |
| 2466 if (VerifiedMemory::enabled()) { | |
| 2467 Register temp = ECX; | |
| 2468 pushl(temp); | |
| 2469 leal(temp, dest); | |
| 2470 movl(Address(temp, VerifiedMemory::offset()), zero); | |
| 2471 popl(temp); | |
| 2472 } | |
| 2473 } | 2349 } |
| 2474 | 2350 |
| 2475 | 2351 |
| 2476 void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { | 2352 void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { |
| 2477 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on | 2353 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on |
| 2478 // the length of this instruction sequence. | 2354 // the length of this instruction sequence. |
| 2479 VerifyHeapWord(dest, kOnlySmi); | |
| 2480 Immediate inc_imm(Smi::RawValue(increment)); | 2355 Immediate inc_imm(Smi::RawValue(increment)); |
| 2481 addl(dest, inc_imm); | 2356 addl(dest, inc_imm); |
| 2482 if (VerifiedMemory::enabled()) { | |
| 2483 Register temp = ECX; | |
| 2484 pushl(temp); | |
| 2485 leal(temp, dest); | |
| 2486 addl(Address(temp, VerifiedMemory::offset()), inc_imm); | |
| 2487 popl(temp); | |
| 2488 } | |
| 2489 } | 2357 } |
| 2490 | 2358 |
| 2491 | 2359 |
| 2492 void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { | 2360 void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { |
| 2493 // TODO(5410843): Need to have a code constants table. | 2361 // TODO(5410843): Need to have a code constants table. |
| 2494 int64_t constant = bit_cast<int64_t, double>(value); | 2362 int64_t constant = bit_cast<int64_t, double>(value); |
| 2495 pushl(Immediate(Utils::High32Bits(constant))); | 2363 pushl(Immediate(Utils::High32Bits(constant))); |
| 2496 pushl(Immediate(Utils::Low32Bits(constant))); | 2364 pushl(Immediate(Utils::Low32Bits(constant))); |
| 2497 movsd(dst, Address(ESP, 0)); | 2365 movsd(dst, Address(ESP, 0)); |
| 2498 addl(ESP, Immediate(2 * kWordSize)); | 2366 addl(ESP, Immediate(2 * kWordSize)); |
| (...skipping 681 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3180 | 3048 |
| 3181 const char* Assembler::FpuRegisterName(FpuRegister reg) { | 3049 const char* Assembler::FpuRegisterName(FpuRegister reg) { |
| 3182 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); | 3050 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); |
| 3183 return xmm_reg_names[reg]; | 3051 return xmm_reg_names[reg]; |
| 3184 } | 3052 } |
| 3185 | 3053 |
| 3186 | 3054 |
| 3187 } // namespace dart | 3055 } // namespace dart |
| 3188 | 3056 |
| 3189 #endif // defined TARGET_ARCH_IA32 | 3057 #endif // defined TARGET_ARCH_IA32 |
| OLD | NEW |