OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/globals.h" // NOLINT | 5 #include "vm/globals.h" // NOLINT |
6 #if defined(TARGET_ARCH_IA32) | 6 #if defined(TARGET_ARCH_IA32) |
7 | 7 |
8 #include "vm/assembler.h" | 8 #include "vm/assembler.h" |
9 #include "vm/code_generator.h" | 9 #include "vm/code_generator.h" |
10 #include "vm/cpu.h" | 10 #include "vm/cpu.h" |
11 #include "vm/heap.h" | 11 #include "vm/heap.h" |
12 #include "vm/instructions.h" | 12 #include "vm/instructions.h" |
13 #include "vm/memory_region.h" | 13 #include "vm/memory_region.h" |
14 #include "vm/runtime_entry.h" | 14 #include "vm/runtime_entry.h" |
15 #include "vm/stack_frame.h" | 15 #include "vm/stack_frame.h" |
16 #include "vm/stub_code.h" | 16 #include "vm/stub_code.h" |
17 #include "vm/verified_memory.h" | |
18 | 17 |
19 namespace dart { | 18 namespace dart { |
20 | 19 |
21 DECLARE_FLAG(bool, inline_alloc); | 20 DECLARE_FLAG(bool, inline_alloc); |
22 | 21 |
23 | 22 |
24 class DirectCallRelocation : public AssemblerFixup { | 23 class DirectCallRelocation : public AssemblerFixup { |
25 public: | 24 public: |
26 void Process(const MemoryRegion& region, intptr_t position) { | 25 void Process(const MemoryRegion& region, intptr_t position) { |
27 // Direct calls are relative to the following instruction on x86. | 26 // Direct calls are relative to the following instruction on x86. |
(...skipping 1963 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1991 | 1990 |
1992 | 1991 |
1993 void Assembler::hlt() { | 1992 void Assembler::hlt() { |
1994 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 1993 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
1995 EmitUint8(0xF4); | 1994 EmitUint8(0xF4); |
1996 } | 1995 } |
1997 | 1996 |
1998 | 1997 |
1999 void Assembler::j(Condition condition, Label* label, bool near) { | 1998 void Assembler::j(Condition condition, Label* label, bool near) { |
2000 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 1999 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
2001 if (VerifiedMemory::enabled()) { | |
2002 near = Assembler::kFarJump; | |
2003 } | |
2004 if (label->IsBound()) { | 2000 if (label->IsBound()) { |
2005 static const int kShortSize = 2; | 2001 static const int kShortSize = 2; |
2006 static const int kLongSize = 6; | 2002 static const int kLongSize = 6; |
2007 intptr_t offset = label->Position() - buffer_.Size(); | 2003 intptr_t offset = label->Position() - buffer_.Size(); |
2008 ASSERT(offset <= 0); | 2004 ASSERT(offset <= 0); |
2009 if (Utils::IsInt(8, offset - kShortSize)) { | 2005 if (Utils::IsInt(8, offset - kShortSize)) { |
2010 EmitUint8(0x70 + condition); | 2006 EmitUint8(0x70 + condition); |
2011 EmitUint8((offset - kShortSize) & 0xFF); | 2007 EmitUint8((offset - kShortSize) & 0xFF); |
2012 } else { | 2008 } else { |
2013 EmitUint8(0x0F); | 2009 EmitUint8(0x0F); |
(...skipping 22 matching lines...) Expand all Loading... |
2036 | 2032 |
2037 void Assembler::jmp(Register reg) { | 2033 void Assembler::jmp(Register reg) { |
2038 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
2039 EmitUint8(0xFF); | 2035 EmitUint8(0xFF); |
2040 EmitRegisterOperand(4, reg); | 2036 EmitRegisterOperand(4, reg); |
2041 } | 2037 } |
2042 | 2038 |
2043 | 2039 |
2044 void Assembler::jmp(Label* label, bool near) { | 2040 void Assembler::jmp(Label* label, bool near) { |
2045 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2041 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
2046 if (VerifiedMemory::enabled()) { | |
2047 near = Assembler::kFarJump; | |
2048 } | |
2049 if (label->IsBound()) { | 2042 if (label->IsBound()) { |
2050 static const int kShortSize = 2; | 2043 static const int kShortSize = 2; |
2051 static const int kLongSize = 5; | 2044 static const int kLongSize = 5; |
2052 intptr_t offset = label->Position() - buffer_.Size(); | 2045 intptr_t offset = label->Position() - buffer_.Size(); |
2053 ASSERT(offset <= 0); | 2046 ASSERT(offset <= 0); |
2054 if (Utils::IsInt(8, offset - kShortSize)) { | 2047 if (Utils::IsInt(8, offset - kShortSize)) { |
2055 EmitUint8(0xEB); | 2048 EmitUint8(0xEB); |
2056 EmitUint8((offset - kShortSize) & 0xFF); | 2049 EmitUint8((offset - kShortSize) & 0xFF); |
2057 } else { | 2050 } else { |
2058 EmitUint8(0xE9); | 2051 EmitUint8(0xE9); |
(...skipping 200 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2259 // Mask out higher, uninteresting bits which were polluted by dest. | 2252 // Mask out higher, uninteresting bits which were polluted by dest. |
2260 andl(value, Immediate(kObjectAlignment - 1)); | 2253 andl(value, Immediate(kObjectAlignment - 1)); |
2261 // Compare with the expected bit pattern. | 2254 // Compare with the expected bit pattern. |
2262 cmpl(value, Immediate( | 2255 cmpl(value, Immediate( |
2263 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + | 2256 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + |
2264 kOldObjectAlignmentOffset + kHeapObjectTag)); | 2257 kOldObjectAlignmentOffset + kHeapObjectTag)); |
2265 j(NOT_ZERO, no_update, Assembler::kNearJump); | 2258 j(NOT_ZERO, no_update, Assembler::kNearJump); |
2266 } | 2259 } |
2267 | 2260 |
2268 | 2261 |
2269 void Assembler::VerifyHeapWord(const Address& address, | |
2270 FieldContent old_content) { | |
2271 #if defined(DEBUG) | |
2272 switch (old_content) { | |
2273 case kEmptyOrSmiOrNull: | |
2274 VerifyUninitialized(address); | |
2275 break; | |
2276 case kHeapObjectOrSmi: | |
2277 VerifyObjectOrSmi(address); | |
2278 break; | |
2279 case kOnlySmi: | |
2280 VerifySmi(address); | |
2281 break; | |
2282 } | |
2283 #endif // DEBUG | |
2284 if (VerifiedMemory::enabled()) { | |
2285 Register addr_reg = EDX; | |
2286 Register value = EBX; | |
2287 // Preserve registers. | |
2288 pushl(addr_reg); | |
2289 pushl(value); | |
2290 leal(addr_reg, address); | |
2291 // ASSERT(*address == *(address + offset)) | |
2292 movl(value, Address(addr_reg, 0)); | |
2293 cmpl(value, Address(addr_reg, VerifiedMemory::offset())); | |
2294 Label ok; | |
2295 j(EQUAL, &ok, Assembler::kNearJump); | |
2296 Stop("Write barrier verification failed"); | |
2297 Bind(&ok); | |
2298 popl(value); | |
2299 popl(addr_reg); | |
2300 } | |
2301 } | |
2302 | |
2303 | |
2304 void Assembler::VerifiedWrite(const Address& dest, | |
2305 Register value, | |
2306 FieldContent old_content) { | |
2307 VerifyHeapWord(dest, old_content); | |
2308 movl(dest, value); | |
2309 if (VerifiedMemory::enabled()) { | |
2310 Register temp = (value == EDX) ? ECX : EDX; | |
2311 pushl(temp); | |
2312 leal(temp, dest); | |
2313 movl(Address(temp, VerifiedMemory::offset()), value); | |
2314 popl(temp); | |
2315 } | |
2316 } | |
2317 | |
2318 | |
2319 #if defined(DEBUG) | |
2320 void Assembler::VerifyObjectOrSmi(const Address& dest) { | |
2321 Label ok; | |
2322 testb(dest, Immediate(kHeapObjectTag)); | |
2323 j(ZERO, &ok, Assembler::kNearJump); | |
2324 // Non-smi case: Verify object pointer is word-aligned when untagged. | |
2325 COMPILE_ASSERT(kHeapObjectTag == 1); | |
2326 testb(dest, Immediate((kWordSize - 1) - kHeapObjectTag)); | |
2327 j(ZERO, &ok, Assembler::kNearJump); | |
2328 Stop("Expected heap object or Smi"); | |
2329 Bind(&ok); | |
2330 } | |
2331 | |
2332 | |
2333 void Assembler::VerifyUninitialized(const Address& dest) { | |
2334 Label ok; | |
2335 testb(dest, Immediate(kHeapObjectTag)); | |
2336 j(ZERO, &ok, Assembler::kNearJump); | |
2337 // Non-smi case: Check for the special zap word or null. | |
2338 #if defined(DEBUG) | |
2339 cmpl(dest, Immediate(Heap::kZap32Bits)); | |
2340 j(EQUAL, &ok, Assembler::kNearJump); | |
2341 #else | |
2342 #error Only supported in DEBUG mode | |
2343 #endif | |
2344 cmpl(dest, Immediate(reinterpret_cast<uint32_t>(Object::null()))); | |
2345 j(EQUAL, &ok, Assembler::kNearJump); | |
2346 Stop("Expected zapped, Smi or null"); | |
2347 Bind(&ok); | |
2348 } | |
2349 | |
2350 | |
2351 void Assembler::VerifySmi(const Address& dest, const char* stop_msg) { | |
2352 Label done; | |
2353 testb(dest, Immediate(kHeapObjectTag)); | |
2354 j(ZERO, &done, Assembler::kNearJump); | |
2355 Stop(stop_msg); | |
2356 Bind(&done); | |
2357 } | |
2358 #endif // defined(DEBUG) | |
2359 | |
2360 | |
2361 // Destroys the value register. | 2262 // Destroys the value register. |
2362 void Assembler::StoreIntoObject(Register object, | 2263 void Assembler::StoreIntoObject(Register object, |
2363 const Address& dest, | 2264 const Address& dest, |
2364 Register value, | 2265 Register value, |
2365 bool can_value_be_smi) { | 2266 bool can_value_be_smi) { |
2366 ASSERT(object != value); | 2267 ASSERT(object != value); |
2367 VerifiedWrite(dest, value, kHeapObjectOrSmi); | 2268 movl(dest, value); |
2368 Label done; | 2269 Label done; |
2369 if (can_value_be_smi) { | 2270 if (can_value_be_smi) { |
2370 StoreIntoObjectFilter(object, value, &done); | 2271 StoreIntoObjectFilter(object, value, &done); |
2371 } else { | 2272 } else { |
2372 StoreIntoObjectFilterNoSmi(object, value, &done); | 2273 StoreIntoObjectFilterNoSmi(object, value, &done); |
2373 } | 2274 } |
2374 // A store buffer update is required. | 2275 // A store buffer update is required. |
2375 if (value != EDX) { | 2276 if (value != EDX) { |
2376 pushl(EDX); // Preserve EDX. | 2277 pushl(EDX); // Preserve EDX. |
2377 } | 2278 } |
2378 if (object != EDX) { | 2279 if (object != EDX) { |
2379 movl(EDX, object); | 2280 movl(EDX, object); |
2380 } | 2281 } |
2381 call(Address(THR, Thread::update_store_buffer_entry_point_offset())); | 2282 call(Address(THR, Thread::update_store_buffer_entry_point_offset())); |
2382 if (value != EDX) { | 2283 if (value != EDX) { |
2383 popl(EDX); // Restore EDX. | 2284 popl(EDX); // Restore EDX. |
2384 } | 2285 } |
2385 Bind(&done); | 2286 Bind(&done); |
2386 } | 2287 } |
2387 | 2288 |
2388 | 2289 |
2389 void Assembler::StoreIntoObjectNoBarrier(Register object, | 2290 void Assembler::StoreIntoObjectNoBarrier(Register object, |
2390 const Address& dest, | 2291 const Address& dest, |
2391 Register value, | 2292 Register value) { |
2392 FieldContent old_content) { | 2293 movl(dest, value); |
2393 VerifiedWrite(dest, value, old_content); | |
2394 #if defined(DEBUG) | 2294 #if defined(DEBUG) |
2395 Label done; | 2295 Label done; |
2396 pushl(value); | 2296 pushl(value); |
2397 StoreIntoObjectFilter(object, value, &done); | 2297 StoreIntoObjectFilter(object, value, &done); |
2398 Stop("Store buffer update is required"); | 2298 Stop("Store buffer update is required"); |
2399 Bind(&done); | 2299 Bind(&done); |
2400 popl(value); | 2300 popl(value); |
2401 #endif // defined(DEBUG) | 2301 #endif // defined(DEBUG) |
2402 // No store buffer update. | 2302 // No store buffer update. |
2403 } | 2303 } |
2404 | 2304 |
2405 | 2305 |
2406 void Assembler::UnverifiedStoreOldObject(const Address& dest, | 2306 void Assembler::UnverifiedStoreOldObject(const Address& dest, |
2407 const Object& value) { | 2307 const Object& value) { |
2408 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); | 2308 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); |
2409 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); | 2309 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); |
2410 ASSERT(value.IsOld()); | 2310 ASSERT(value.IsOld()); |
2411 ASSERT(!value.InVMHeap()); | 2311 ASSERT(!value.InVMHeap()); |
2412 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 2312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
2413 EmitUint8(0xC7); | 2313 EmitUint8(0xC7); |
2414 EmitOperand(0, dest); | 2314 EmitOperand(0, dest); |
2415 buffer_.EmitObject(value); | 2315 buffer_.EmitObject(value); |
2416 } | 2316 } |
2417 | 2317 |
2418 | 2318 |
2419 void Assembler::StoreIntoObjectNoBarrier(Register object, | 2319 void Assembler::StoreIntoObjectNoBarrier(Register object, |
2420 const Address& dest, | 2320 const Address& dest, |
2421 const Object& value, | 2321 const Object& value) { |
2422 FieldContent old_content) { | |
2423 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); | 2322 ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal()); |
2424 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); | 2323 ASSERT(!value.IsField() || Field::Cast(value).IsOriginal()); |
2425 VerifyHeapWord(dest, old_content); | |
2426 if (value.IsSmi() || value.InVMHeap()) { | 2324 if (value.IsSmi() || value.InVMHeap()) { |
2427 Immediate imm_value(reinterpret_cast<int32_t>(value.raw())); | 2325 Immediate imm_value(reinterpret_cast<int32_t>(value.raw())); |
2428 movl(dest, imm_value); | 2326 movl(dest, imm_value); |
2429 if (VerifiedMemory::enabled()) { | |
2430 Register temp = ECX; | |
2431 pushl(temp); | |
2432 leal(temp, dest); | |
2433 movl(Address(temp, VerifiedMemory::offset()), imm_value); | |
2434 popl(temp); | |
2435 } | |
2436 } else { | 2327 } else { |
2437 UnverifiedStoreOldObject(dest, value); | 2328 UnverifiedStoreOldObject(dest, value); |
2438 if (VerifiedMemory::enabled()) { | |
2439 Register temp = EDX; | |
2440 pushl(temp); | |
2441 leal(temp, dest); | |
2442 UnverifiedStoreOldObject(Address(temp, VerifiedMemory::offset()), value); | |
2443 popl(temp); | |
2444 } | |
2445 } | 2329 } |
2446 // No store buffer update. | 2330 // No store buffer update. |
2447 } | 2331 } |
2448 | 2332 |
2449 | 2333 |
2450 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { | 2334 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { |
2451 #if defined(DEBUG) | 2335 #if defined(DEBUG) |
2452 Label done; | 2336 Label done; |
2453 testl(value, Immediate(kHeapObjectTag)); | 2337 testl(value, Immediate(kHeapObjectTag)); |
2454 j(ZERO, &done); | 2338 j(ZERO, &done); |
2455 Stop("New value must be Smi."); | 2339 Stop("New value must be Smi."); |
2456 Bind(&done); | 2340 Bind(&done); |
2457 #endif // defined(DEBUG) | 2341 #endif // defined(DEBUG) |
2458 VerifiedWrite(dest, value, kOnlySmi); | 2342 movl(dest, value); |
2459 } | 2343 } |
2460 | 2344 |
2461 | 2345 |
2462 void Assembler::ZeroInitSmiField(const Address& dest) { | 2346 void Assembler::ZeroInitSmiField(const Address& dest) { |
2463 VerifyHeapWord(dest, kEmptyOrSmiOrNull); | |
2464 Immediate zero(Smi::RawValue(0)); | 2347 Immediate zero(Smi::RawValue(0)); |
2465 movl(dest, zero); | 2348 movl(dest, zero); |
2466 if (VerifiedMemory::enabled()) { | |
2467 Register temp = ECX; | |
2468 pushl(temp); | |
2469 leal(temp, dest); | |
2470 movl(Address(temp, VerifiedMemory::offset()), zero); | |
2471 popl(temp); | |
2472 } | |
2473 } | 2349 } |
2474 | 2350 |
2475 | 2351 |
2476 void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { | 2352 void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { |
2477 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on | 2353 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on |
2478 // the length of this instruction sequence. | 2354 // the length of this instruction sequence. |
2479 VerifyHeapWord(dest, kOnlySmi); | |
2480 Immediate inc_imm(Smi::RawValue(increment)); | 2355 Immediate inc_imm(Smi::RawValue(increment)); |
2481 addl(dest, inc_imm); | 2356 addl(dest, inc_imm); |
2482 if (VerifiedMemory::enabled()) { | |
2483 Register temp = ECX; | |
2484 pushl(temp); | |
2485 leal(temp, dest); | |
2486 addl(Address(temp, VerifiedMemory::offset()), inc_imm); | |
2487 popl(temp); | |
2488 } | |
2489 } | 2357 } |
2490 | 2358 |
2491 | 2359 |
2492 void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { | 2360 void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { |
2493 // TODO(5410843): Need to have a code constants table. | 2361 // TODO(5410843): Need to have a code constants table. |
2494 int64_t constant = bit_cast<int64_t, double>(value); | 2362 int64_t constant = bit_cast<int64_t, double>(value); |
2495 pushl(Immediate(Utils::High32Bits(constant))); | 2363 pushl(Immediate(Utils::High32Bits(constant))); |
2496 pushl(Immediate(Utils::Low32Bits(constant))); | 2364 pushl(Immediate(Utils::Low32Bits(constant))); |
2497 movsd(dst, Address(ESP, 0)); | 2365 movsd(dst, Address(ESP, 0)); |
2498 addl(ESP, Immediate(2 * kWordSize)); | 2366 addl(ESP, Immediate(2 * kWordSize)); |
(...skipping 681 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3180 | 3048 |
3181 const char* Assembler::FpuRegisterName(FpuRegister reg) { | 3049 const char* Assembler::FpuRegisterName(FpuRegister reg) { |
3182 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); | 3050 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); |
3183 return xmm_reg_names[reg]; | 3051 return xmm_reg_names[reg]; |
3184 } | 3052 } |
3185 | 3053 |
3186 | 3054 |
3187 } // namespace dart | 3055 } // namespace dart |
3188 | 3056 |
3189 #endif // defined TARGET_ARCH_IA32 | 3057 #endif // defined TARGET_ARCH_IA32 |
OLD | NEW |