OLD | NEW |
---|---|
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/globals.h" // NOLINT | 5 #include "vm/globals.h" // NOLINT |
6 #if defined(TARGET_ARCH_X64) | 6 #if defined(TARGET_ARCH_X64) |
7 | 7 |
8 #include "vm/assembler.h" | 8 #include "vm/assembler.h" |
9 #include "vm/cpu.h" | 9 #include "vm/cpu.h" |
10 #include "vm/heap.h" | 10 #include "vm/heap.h" |
(...skipping 1583 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1594 } else { | 1594 } else { |
1595 EmitRegisterREX(reg, REX_NONE); | 1595 EmitRegisterREX(reg, REX_NONE); |
1596 EmitUint8(0xF7); | 1596 EmitUint8(0xF7); |
1597 EmitUint8(0xC0 | (reg & 7)); | 1597 EmitUint8(0xC0 | (reg & 7)); |
1598 } | 1598 } |
1599 EmitImmediate(imm); | 1599 EmitImmediate(imm); |
1600 } | 1600 } |
1601 } | 1601 } |
1602 | 1602 |
1603 | 1603 |
1604 void Assembler::testb(const Address& address, const Immediate& imm) { | |
1605 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | |
1606 EmitOperandREX(0, address, REX_NONE); | |
1607 EmitUint8(0xF6); | |
1608 EmitOperand(0, address); | |
1609 ASSERT(imm.is_int8()); | |
1610 EmitUint8(imm.value() & 0xFF); | |
1611 } | |
1612 | |
1613 | |
1604 void Assembler::testq(Register reg1, Register reg2) { | 1614 void Assembler::testq(Register reg1, Register reg2) { |
1605 AssemblerBuffer::EnsureCapacity ensured(&buffer_); | 1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); |
1606 Operand operand(reg2); | 1616 Operand operand(reg2); |
1607 EmitOperandREX(reg1, operand, REX_W); | 1617 EmitOperandREX(reg1, operand, REX_W); |
1608 EmitUint8(0x85); | 1618 EmitUint8(0x85); |
1609 EmitOperand(reg1 & 7, operand); | 1619 EmitOperand(reg1 & 7, operand); |
1610 } | 1620 } |
1611 | 1621 |
1612 | 1622 |
1613 void Assembler::testq(Register reg, const Immediate& imm) { | 1623 void Assembler::testq(Register reg, const Immediate& imm) { |
(...skipping 1368 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2982 // Mask out higher, uninteresting bits which were polluted by dest. | 2992 // Mask out higher, uninteresting bits which were polluted by dest. |
2983 andl(value, Immediate(kObjectAlignment - 1)); | 2993 andl(value, Immediate(kObjectAlignment - 1)); |
2984 // Compare with the expected bit pattern. | 2994 // Compare with the expected bit pattern. |
2985 cmpl(value, Immediate( | 2995 cmpl(value, Immediate( |
2986 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + | 2996 (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag + |
2987 kOldObjectAlignmentOffset + kHeapObjectTag)); | 2997 kOldObjectAlignmentOffset + kHeapObjectTag)); |
2988 j(NOT_ZERO, no_update, Assembler::kNearJump); | 2998 j(NOT_ZERO, no_update, Assembler::kNearJump); |
2989 } | 2999 } |
2990 | 3000 |
2991 | 3001 |
2992 void Assembler::VerifyHeapWord(const Address& address) { | 3002 void Assembler::VerifyHeapWord(const Address& address, |
3003 FieldContent old_content) { | |
3004 #if defined(DEBUG) | |
3005 switch (old_content) { | |
3006 case kEmptyOrSmiOrNull: | |
3007 VerifyUninitialized(address); | |
3008 break; | |
3009 case kHeapObjectOrSmi: | |
3010 VerifyObjectOrSmi(address); | |
3011 break; | |
3012 case kOnlySmi: | |
3013 VerifySmi(address); | |
3014 break; | |
3015 } | |
3016 #endif // DEBUG | |
2993 if (VerifiedMemory::enabled()) { | 3017 if (VerifiedMemory::enabled()) { |
2994 Register addr_reg = RDX; | 3018 Register addr_reg = RDX; |
2995 Register value = RBX; | 3019 Register value = RBX; |
2996 // Preserve registers. | 3020 // Preserve registers. |
2997 pushq(addr_reg); | 3021 pushq(addr_reg); |
2998 pushq(value); | 3022 pushq(value); |
2999 leaq(addr_reg, address); | 3023 leaq(addr_reg, address); |
3000 // ASSERT(*address == *(address + offset)) | 3024 // ASSERT(*address == *(address + offset)) |
3001 movq(value, Address(addr_reg, 0)); | 3025 movq(value, Address(addr_reg, 0)); |
3002 cmpq(value, Address(addr_reg, VerifiedMemory::offset())); | 3026 cmpq(value, Address(addr_reg, VerifiedMemory::offset())); |
3003 Label ok; | 3027 Label ok; |
3004 j(EQUAL, &ok); | 3028 j(EQUAL, &ok); |
3005 static const bool kFixedLengthEncoding = true; | 3029 static const bool kFixedLengthEncoding = true; |
3006 Stop("Write barrier verification failed", kFixedLengthEncoding); | 3030 Stop("Write barrier verification failed", kFixedLengthEncoding); |
3007 Bind(&ok); | 3031 Bind(&ok); |
3008 popq(value); | 3032 popq(value); |
3009 popq(addr_reg); | 3033 popq(addr_reg); |
3010 } | 3034 } |
3011 } | 3035 } |
3012 | 3036 |
3013 | 3037 |
3014 void Assembler::VerifiedWrite(const Address& dest, Register value) { | 3038 void Assembler::VerifiedWrite(const Address& dest, |
3015 VerifyHeapWord(dest); | 3039 Register value, |
3040 FieldContent old_content) { | |
3041 VerifyHeapWord(dest, old_content); | |
3016 movq(dest, value); | 3042 movq(dest, value); |
3017 if (VerifiedMemory::enabled()) { | 3043 if (VerifiedMemory::enabled()) { |
3018 Register temp = (value == RDX) ? RCX : RDX; | 3044 Register temp = (value == RDX) ? RCX : RDX; |
3019 pushq(temp); | 3045 pushq(temp); |
3020 leaq(temp, dest); | 3046 leaq(temp, dest); |
3021 movq(Address(temp, VerifiedMemory::offset()), value); | 3047 movq(Address(temp, VerifiedMemory::offset()), value); |
3022 popq(temp); | 3048 popq(temp); |
3023 } | 3049 } |
3024 } | 3050 } |
3025 | 3051 |
3026 | 3052 |
3053 #if defined(DEBUG) | |
3054 void Assembler::VerifyObjectOrSmi(const Address& dest) { | |
3055 Label ok; | |
3056 testb(dest, Immediate(kHeapObjectTag)); | |
3057 j(ZERO, &ok, Assembler::kNearJump); | |
3058 // Non-smi case: Verify object pointer is word-aligned when untagged. | |
3059 COMPILE_ASSERT(kHeapObjectTag == 1); | |
3060 testb(dest, Immediate((kWordSize - 1) - kHeapObjectTag)); | |
3061 j(ZERO, &ok, Assembler::kNearJump); | |
3062 Stop("Expected heap object or Smi"); | |
3063 Bind(&ok); | |
3064 } | |
3065 | |
3066 | |
3067 void Assembler::VerifyUninitialized(const Address& dest) { | |
Ivan Posva
2015/01/09 00:16:56
Missed this in the original review:
#if defined(D
koda
2015/01/09 02:02:34
Done.
| |
3068 Label ok; | |
3069 testb(dest, Immediate(kHeapObjectTag)); | |
3070 j(ZERO, &ok, Assembler::kNearJump); | |
3071 // Non-smi case: Check for the special zap word or null. | |
3072 cmpq(dest, Immediate(Heap::kZap64Bits)); | |
3073 j(EQUAL, &ok, Assembler::kNearJump); | |
3074 cmpq(dest, Immediate(reinterpret_cast<uint64_t>(Object::null()))); | |
3075 j(EQUAL, &ok, Assembler::kNearJump); | |
3076 Stop("Expected zapped, Smi or null"); | |
3077 Bind(&ok); | |
3078 } | |
3079 | |
3080 | |
3081 void Assembler::VerifySmi(const Address& dest, const char* stop_msg) { | |
3082 Label done; | |
3083 testb(dest, Immediate(kHeapObjectTag)); | |
3084 j(ZERO, &done, Assembler::kNearJump); | |
3085 Stop(stop_msg); | |
3086 Bind(&done); | |
3087 } | |
3088 #endif // defined(DEBUG) | |
3089 | |
3090 | |
3027 void Assembler::StoreIntoObject(Register object, | 3091 void Assembler::StoreIntoObject(Register object, |
3028 const Address& dest, | 3092 const Address& dest, |
3029 Register value, | 3093 Register value, |
3030 bool can_value_be_smi) { | 3094 bool can_value_be_smi) { |
3031 ASSERT(object != value); | 3095 ASSERT(object != value); |
3032 VerifiedWrite(dest, value); | 3096 VerifiedWrite(dest, value, kHeapObjectOrSmi); |
3033 Label done; | 3097 Label done; |
3034 if (can_value_be_smi) { | 3098 if (can_value_be_smi) { |
3035 StoreIntoObjectFilter(object, value, &done); | 3099 StoreIntoObjectFilter(object, value, &done); |
3036 } else { | 3100 } else { |
3037 StoreIntoObjectFilterNoSmi(object, value, &done); | 3101 StoreIntoObjectFilterNoSmi(object, value, &done); |
3038 } | 3102 } |
3039 // A store buffer update is required. | 3103 // A store buffer update is required. |
3040 if (value != RDX) pushq(RDX); | 3104 if (value != RDX) pushq(RDX); |
3041 if (object != RDX) { | 3105 if (object != RDX) { |
3042 movq(RDX, object); | 3106 movq(RDX, object); |
3043 } | 3107 } |
3044 StubCode* stub_code = Isolate::Current()->stub_code(); | 3108 StubCode* stub_code = Isolate::Current()->stub_code(); |
3045 Call(&stub_code->UpdateStoreBufferLabel(), PP); | 3109 Call(&stub_code->UpdateStoreBufferLabel(), PP); |
3046 if (value != RDX) popq(RDX); | 3110 if (value != RDX) popq(RDX); |
3047 Bind(&done); | 3111 Bind(&done); |
3048 } | 3112 } |
3049 | 3113 |
3050 | 3114 |
3051 void Assembler::StoreIntoObjectNoBarrier(Register object, | 3115 void Assembler::StoreIntoObjectNoBarrier(Register object, |
3052 const Address& dest, | 3116 const Address& dest, |
3053 Register value) { | 3117 Register value, |
3054 VerifiedWrite(dest, value); | 3118 FieldContent old_content) { |
3119 VerifiedWrite(dest, value, old_content); | |
3055 #if defined(DEBUG) | 3120 #if defined(DEBUG) |
3056 Label done; | 3121 Label done; |
3057 pushq(value); | 3122 pushq(value); |
3058 StoreIntoObjectFilter(object, value, &done); | 3123 StoreIntoObjectFilter(object, value, &done); |
3059 Stop("Store buffer update is required"); | 3124 Stop("Store buffer update is required"); |
3060 Bind(&done); | 3125 Bind(&done); |
3061 popq(value); | 3126 popq(value); |
3062 #endif // defined(DEBUG) | 3127 #endif // defined(DEBUG) |
3063 // No store buffer update. | 3128 // No store buffer update. |
3064 } | 3129 } |
3065 | 3130 |
3066 | 3131 |
3067 void Assembler::StoreIntoObjectNoBarrier(Register object, | 3132 void Assembler::StoreIntoObjectNoBarrier(Register object, |
3068 const Address& dest, | 3133 const Address& dest, |
3069 const Object& value, | 3134 const Object& value, |
3070 Register pp) { | 3135 Register pp, |
3136 FieldContent old_content) { | |
3137 VerifyHeapWord(dest, old_content); | |
3071 if (VerifiedMemory::enabled()) { | 3138 if (VerifiedMemory::enabled()) { |
3072 VerifyHeapWord(dest); | |
3073 Register temp = (pp == RCX) ? RDX : RCX; | 3139 Register temp = (pp == RCX) ? RDX : RCX; |
3074 pushq(temp); | 3140 pushq(temp); |
3075 leaq(temp, dest); | 3141 leaq(temp, dest); |
3076 StoreObject(Address(temp, 0), value, pp); | 3142 StoreObject(Address(temp, 0), value, pp); |
3077 StoreObject(Address(temp, VerifiedMemory::offset()), value, pp); | 3143 StoreObject(Address(temp, VerifiedMemory::offset()), value, pp); |
3078 popq(temp); | 3144 popq(temp); |
3079 } else { | 3145 } else { |
3080 StoreObject(dest, value, pp); | 3146 StoreObject(dest, value, pp); |
3081 } | 3147 } |
3082 // TODO(koda): Use 'object', verify that generational barrier's not needed. | 3148 // TODO(koda): Use 'object', verify that generational barrier's not needed. |
3083 } | 3149 } |
3084 | 3150 |
3085 | 3151 |
3086 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { | 3152 void Assembler::StoreIntoSmiField(const Address& dest, Register value) { |
3087 VerifiedWrite(dest, value); | |
3088 #if defined(DEBUG) | 3153 #if defined(DEBUG) |
3089 Label done; | 3154 Label done; |
3090 testq(value, Immediate(kHeapObjectTag)); | 3155 testq(value, Immediate(kHeapObjectTag)); |
3091 j(ZERO, &done); | 3156 j(ZERO, &done); |
3092 Stop("Smi expected"); | 3157 Stop("New value must be Smi."); |
3093 Bind(&done); | 3158 Bind(&done); |
3094 #endif // defined(DEBUG) | 3159 #endif // defined(DEBUG) |
3160 VerifiedWrite(dest, value, kOnlySmi); | |
3095 } | 3161 } |
3096 | 3162 |
3097 | 3163 |
3098 void Assembler::ZeroInitSmiField(const Address& dest) { | 3164 void Assembler::ZeroInitSmiField(const Address& dest) { |
3099 // TODO(koda): Add VerifySmi once we distinguish initalization. | 3165 // TODO(koda): Add VerifySmi once we distinguish initalization. |
3100 VerifyHeapWord(dest); | 3166 VerifyHeapWord(dest, kEmptyOrSmiOrNull); |
3101 Immediate zero(Smi::RawValue(0)); | 3167 Immediate zero(Smi::RawValue(0)); |
3102 movq(dest, zero); | 3168 movq(dest, zero); |
3103 if (VerifiedMemory::enabled()) { | 3169 if (VerifiedMemory::enabled()) { |
3104 Register temp = RCX; | 3170 Register temp = RCX; |
3105 pushq(temp); | 3171 pushq(temp); |
3106 leaq(temp, dest); | 3172 leaq(temp, dest); |
3107 movq(Address(temp, VerifiedMemory::offset()), zero); | 3173 movq(Address(temp, VerifiedMemory::offset()), zero); |
3108 popq(temp); | 3174 popq(temp); |
3109 } | 3175 } |
3110 } | 3176 } |
3111 | 3177 |
3112 | 3178 |
3113 void Assembler::IncrementSmiField(const Address& dest, int64_t increment) { | 3179 void Assembler::IncrementSmiField(const Address& dest, int64_t increment) { |
3114 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on | 3180 // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on |
3115 // the length of this instruction sequence. | 3181 // the length of this instruction sequence. |
3116 // TODO(koda): Add VerifySmi once we distinguish initalization. | 3182 // TODO(koda): Add VerifySmi once we distinguish initalization. |
3117 VerifyHeapWord(dest); | 3183 VerifyHeapWord(dest, kOnlySmi); |
3118 Immediate inc_imm(Smi::RawValue(increment)); | 3184 Immediate inc_imm(Smi::RawValue(increment)); |
3119 addq(dest, inc_imm); | 3185 addq(dest, inc_imm); |
3120 if (VerifiedMemory::enabled()) { | 3186 if (VerifiedMemory::enabled()) { |
3121 Register temp = RCX; | 3187 Register temp = RCX; |
3122 pushq(temp); | 3188 pushq(temp); |
3123 leaq(temp, dest); | 3189 leaq(temp, dest); |
3124 addq(Address(temp, VerifiedMemory::offset()), inc_imm); | 3190 addq(Address(temp, VerifiedMemory::offset()), inc_imm); |
3125 popq(temp); | 3191 popq(temp); |
3126 } | 3192 } |
3127 } | 3193 } |
(...skipping 801 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
3929 | 3995 |
3930 | 3996 |
3931 const char* Assembler::FpuRegisterName(FpuRegister reg) { | 3997 const char* Assembler::FpuRegisterName(FpuRegister reg) { |
3932 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); | 3998 ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters)); |
3933 return xmm_reg_names[reg]; | 3999 return xmm_reg_names[reg]; |
3934 } | 4000 } |
3935 | 4001 |
3936 } // namespace dart | 4002 } // namespace dart |
3937 | 4003 |
3938 #endif // defined TARGET_ARCH_X64 | 4004 #endif // defined TARGET_ARCH_X64 |
OLD | NEW |