OLD | NEW |
1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "v8.h" | 5 #include "v8.h" |
6 | 6 |
7 #if V8_TARGET_ARCH_IA32 | 7 #if V8_TARGET_ARCH_IA32 |
8 | 8 |
9 #include "codegen.h" | 9 #include "codegen.h" |
10 #include "heap.h" | 10 #include "heap.h" |
(...skipping 17 matching lines...) Expand all Loading... |
28 masm->LeaveFrame(StackFrame::INTERNAL); | 28 masm->LeaveFrame(StackFrame::INTERNAL); |
29 ASSERT(masm->has_frame()); | 29 ASSERT(masm->has_frame()); |
30 masm->set_has_frame(false); | 30 masm->set_has_frame(false); |
31 } | 31 } |
32 | 32 |
33 | 33 |
34 #define __ masm. | 34 #define __ masm. |
35 | 35 |
36 | 36 |
37 UnaryMathFunction CreateExpFunction() { | 37 UnaryMathFunction CreateExpFunction() { |
38 if (!CpuFeatures::IsSupported(SSE2)) return &std::exp; | |
39 if (!FLAG_fast_math) return &std::exp; | 38 if (!FLAG_fast_math) return &std::exp; |
40 size_t actual_size; | 39 size_t actual_size; |
41 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true)); | 40 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true)); |
42 if (buffer == NULL) return &std::exp; | 41 if (buffer == NULL) return &std::exp; |
43 ExternalReference::InitializeMathExpData(); | 42 ExternalReference::InitializeMathExpData(); |
44 | 43 |
45 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); | 44 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); |
46 // esp[1 * kPointerSize]: raw double input | 45 // esp[1 * kPointerSize]: raw double input |
47 // esp[0 * kPointerSize]: return address | 46 // esp[0 * kPointerSize]: return address |
48 { | 47 { |
49 CpuFeatureScope use_sse2(&masm, SSE2); | |
50 XMMRegister input = xmm1; | 48 XMMRegister input = xmm1; |
51 XMMRegister result = xmm2; | 49 XMMRegister result = xmm2; |
52 __ movsd(input, Operand(esp, 1 * kPointerSize)); | 50 __ movsd(input, Operand(esp, 1 * kPointerSize)); |
53 __ push(eax); | 51 __ push(eax); |
54 __ push(ebx); | 52 __ push(ebx); |
55 | 53 |
56 MathExpGenerator::EmitMathExp(&masm, input, result, xmm0, eax, ebx); | 54 MathExpGenerator::EmitMathExp(&masm, input, result, xmm0, eax, ebx); |
57 | 55 |
58 __ pop(ebx); | 56 __ pop(ebx); |
59 __ pop(eax); | 57 __ pop(eax); |
(...skipping 11 matching lines...) Expand all Loading... |
71 return FUNCTION_CAST<UnaryMathFunction>(buffer); | 69 return FUNCTION_CAST<UnaryMathFunction>(buffer); |
72 } | 70 } |
73 | 71 |
74 | 72 |
75 UnaryMathFunction CreateSqrtFunction() { | 73 UnaryMathFunction CreateSqrtFunction() { |
76 size_t actual_size; | 74 size_t actual_size; |
77 // Allocate buffer in executable space. | 75 // Allocate buffer in executable space. |
78 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, | 76 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, |
79 &actual_size, | 77 &actual_size, |
80 true)); | 78 true)); |
81 // If SSE2 is not available, we can use libc's implementation to ensure | 79 if (buffer == NULL) return &std::sqrt; |
82 // consistency since code by fullcodegen's calls into runtime in that case. | |
83 if (buffer == NULL || !CpuFeatures::IsSupported(SSE2)) return &std::sqrt; | |
84 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); | 80 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); |
85 // esp[1 * kPointerSize]: raw double input | 81 // esp[1 * kPointerSize]: raw double input |
86 // esp[0 * kPointerSize]: return address | 82 // esp[0 * kPointerSize]: return address |
87 // Move double input into registers. | 83 // Move double input into registers. |
88 { | 84 { |
89 CpuFeatureScope use_sse2(&masm, SSE2); | |
90 __ movsd(xmm0, Operand(esp, 1 * kPointerSize)); | 85 __ movsd(xmm0, Operand(esp, 1 * kPointerSize)); |
91 __ sqrtsd(xmm0, xmm0); | 86 __ sqrtsd(xmm0, xmm0); |
92 __ movsd(Operand(esp, 1 * kPointerSize), xmm0); | 87 __ movsd(Operand(esp, 1 * kPointerSize), xmm0); |
93 // Load result into floating point register as return value. | 88 // Load result into floating point register as return value. |
94 __ fld_d(Operand(esp, 1 * kPointerSize)); | 89 __ fld_d(Operand(esp, 1 * kPointerSize)); |
95 __ Ret(); | 90 __ Ret(); |
96 } | 91 } |
97 | 92 |
98 CodeDesc desc; | 93 CodeDesc desc; |
99 masm.GetCode(&desc); | 94 masm.GetCode(&desc); |
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
236 Register src = esi; | 231 Register src = esi; |
237 Register count = ecx; | 232 Register count = ecx; |
238 Register loop_count = edx; | 233 Register loop_count = edx; |
239 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); | 234 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); |
240 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); | 235 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); |
241 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); | 236 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); |
242 | 237 |
243 __ cmp(dst, src); | 238 __ cmp(dst, src); |
244 __ j(equal, &pop_and_return); | 239 __ j(equal, &pop_and_return); |
245 | 240 |
246 if (CpuFeatures::IsSupported(SSE2)) { | 241 __ prefetch(Operand(src, 0), 1); |
247 CpuFeatureScope sse2_scope(&masm, SSE2); | 242 __ cmp(count, kSmallCopySize); |
248 __ prefetch(Operand(src, 0), 1); | 243 __ j(below_equal, &small_size); |
| 244 __ cmp(count, kMediumCopySize); |
| 245 __ j(below_equal, &medium_size); |
| 246 __ cmp(dst, src); |
| 247 __ j(above, &backward); |
| 248 |
| 249 { |
| 250 // |dst| is a lower address than |src|. Copy front-to-back. |
| 251 Label unaligned_source, move_last_15, skip_last_move; |
| 252 __ mov(eax, src); |
| 253 __ sub(eax, dst); |
| 254 __ cmp(eax, kMinMoveDistance); |
| 255 __ j(below, &forward_much_overlap); |
| 256 // Copy first 16 bytes. |
| 257 __ movdqu(xmm0, Operand(src, 0)); |
| 258 __ movdqu(Operand(dst, 0), xmm0); |
| 259 // Determine distance to alignment: 16 - (dst & 0xF). |
| 260 __ mov(edx, dst); |
| 261 __ and_(edx, 0xF); |
| 262 __ neg(edx); |
| 263 __ add(edx, Immediate(16)); |
| 264 __ add(dst, edx); |
| 265 __ add(src, edx); |
| 266 __ sub(count, edx); |
| 267 // dst is now aligned. Main copy loop. |
| 268 __ mov(loop_count, count); |
| 269 __ shr(loop_count, 6); |
| 270 // Check if src is also aligned. |
| 271 __ test(src, Immediate(0xF)); |
| 272 __ j(not_zero, &unaligned_source); |
| 273 // Copy loop for aligned source and destination. |
| 274 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED); |
| 275 // At most 15 bytes to copy. Copy 16 bytes at end of string. |
| 276 __ bind(&move_last_15); |
| 277 __ and_(count, 0xF); |
| 278 __ j(zero, &skip_last_move, Label::kNear); |
| 279 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); |
| 280 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); |
| 281 __ bind(&skip_last_move); |
| 282 MemMoveEmitPopAndReturn(&masm); |
| 283 |
| 284 // Copy loop for unaligned source and aligned destination. |
| 285 __ bind(&unaligned_source); |
| 286 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED); |
| 287 __ jmp(&move_last_15); |
| 288 |
| 289 // Less than kMinMoveDistance offset between dst and src. |
| 290 Label loop_until_aligned, last_15_much_overlap; |
| 291 __ bind(&loop_until_aligned); |
| 292 __ mov_b(eax, Operand(src, 0)); |
| 293 __ inc(src); |
| 294 __ mov_b(Operand(dst, 0), eax); |
| 295 __ inc(dst); |
| 296 __ dec(count); |
| 297 __ bind(&forward_much_overlap); // Entry point into this block. |
| 298 __ test(dst, Immediate(0xF)); |
| 299 __ j(not_zero, &loop_until_aligned); |
| 300 // dst is now aligned, src can't be. Main copy loop. |
| 301 __ mov(loop_count, count); |
| 302 __ shr(loop_count, 6); |
| 303 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, |
| 304 FORWARD, MOVE_UNALIGNED); |
| 305 __ bind(&last_15_much_overlap); |
| 306 __ and_(count, 0xF); |
| 307 __ j(zero, &pop_and_return); |
249 __ cmp(count, kSmallCopySize); | 308 __ cmp(count, kSmallCopySize); |
250 __ j(below_equal, &small_size); | 309 __ j(below_equal, &small_size); |
251 __ cmp(count, kMediumCopySize); | 310 __ jmp(&medium_size); |
252 __ j(below_equal, &medium_size); | 311 } |
253 __ cmp(dst, src); | 312 |
254 __ j(above, &backward); | 313 { |
255 | 314 // |dst| is a higher address than |src|. Copy backwards. |
256 { | 315 Label unaligned_source, move_first_15, skip_last_move; |
257 // |dst| is a lower address than |src|. Copy front-to-back. | 316 __ bind(&backward); |
258 Label unaligned_source, move_last_15, skip_last_move; | 317 // |dst| and |src| always point to the end of what's left to copy. |
259 __ mov(eax, src); | 318 __ add(dst, count); |
260 __ sub(eax, dst); | 319 __ add(src, count); |
261 __ cmp(eax, kMinMoveDistance); | 320 __ mov(eax, dst); |
262 __ j(below, &forward_much_overlap); | 321 __ sub(eax, src); |
263 // Copy first 16 bytes. | 322 __ cmp(eax, kMinMoveDistance); |
264 __ movdqu(xmm0, Operand(src, 0)); | 323 __ j(below, &backward_much_overlap); |
265 __ movdqu(Operand(dst, 0), xmm0); | 324 // Copy last 16 bytes. |
266 // Determine distance to alignment: 16 - (dst & 0xF). | 325 __ movdqu(xmm0, Operand(src, -0x10)); |
267 __ mov(edx, dst); | 326 __ movdqu(Operand(dst, -0x10), xmm0); |
268 __ and_(edx, 0xF); | 327 // Find distance to alignment: dst & 0xF |
269 __ neg(edx); | 328 __ mov(edx, dst); |
270 __ add(edx, Immediate(16)); | 329 __ and_(edx, 0xF); |
271 __ add(dst, edx); | 330 __ sub(dst, edx); |
272 __ add(src, edx); | 331 __ sub(src, edx); |
273 __ sub(count, edx); | 332 __ sub(count, edx); |
274 // dst is now aligned. Main copy loop. | 333 // dst is now aligned. Main copy loop. |
275 __ mov(loop_count, count); | 334 __ mov(loop_count, count); |
276 __ shr(loop_count, 6); | 335 __ shr(loop_count, 6); |
277 // Check if src is also aligned. | 336 // Check if src is also aligned. |
278 __ test(src, Immediate(0xF)); | 337 __ test(src, Immediate(0xF)); |
279 __ j(not_zero, &unaligned_source); | 338 __ j(not_zero, &unaligned_source); |
280 // Copy loop for aligned source and destination. | 339 // Copy loop for aligned source and destination. |
281 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED); | 340 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED); |
282 // At most 15 bytes to copy. Copy 16 bytes at end of string. | 341 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. |
283 __ bind(&move_last_15); | 342 __ bind(&move_first_15); |
284 __ and_(count, 0xF); | 343 __ and_(count, 0xF); |
285 __ j(zero, &skip_last_move, Label::kNear); | 344 __ j(zero, &skip_last_move, Label::kNear); |
286 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); | 345 __ sub(src, count); |
287 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); | 346 __ sub(dst, count); |
288 __ bind(&skip_last_move); | 347 __ movdqu(xmm0, Operand(src, 0)); |
289 MemMoveEmitPopAndReturn(&masm); | 348 __ movdqu(Operand(dst, 0), xmm0); |
290 | 349 __ bind(&skip_last_move); |
291 // Copy loop for unaligned source and aligned destination. | 350 MemMoveEmitPopAndReturn(&masm); |
292 __ bind(&unaligned_source); | 351 |
293 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED); | 352 // Copy loop for unaligned source and aligned destination. |
294 __ jmp(&move_last_15); | 353 __ bind(&unaligned_source); |
295 | 354 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED); |
296 // Less than kMinMoveDistance offset between dst and src. | 355 __ jmp(&move_first_15); |
297 Label loop_until_aligned, last_15_much_overlap; | 356 |
298 __ bind(&loop_until_aligned); | 357 // Less than kMinMoveDistance offset between dst and src. |
299 __ mov_b(eax, Operand(src, 0)); | 358 Label loop_until_aligned, first_15_much_overlap; |
300 __ inc(src); | 359 __ bind(&loop_until_aligned); |
301 __ mov_b(Operand(dst, 0), eax); | 360 __ dec(src); |
302 __ inc(dst); | 361 __ dec(dst); |
303 __ dec(count); | 362 __ mov_b(eax, Operand(src, 0)); |
304 __ bind(&forward_much_overlap); // Entry point into this block. | 363 __ mov_b(Operand(dst, 0), eax); |
305 __ test(dst, Immediate(0xF)); | 364 __ dec(count); |
306 __ j(not_zero, &loop_until_aligned); | 365 __ bind(&backward_much_overlap); // Entry point into this block. |
307 // dst is now aligned, src can't be. Main copy loop. | 366 __ test(dst, Immediate(0xF)); |
308 __ mov(loop_count, count); | 367 __ j(not_zero, &loop_until_aligned); |
309 __ shr(loop_count, 6); | 368 // dst is now aligned, src can't be. Main copy loop. |
310 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, | 369 __ mov(loop_count, count); |
311 FORWARD, MOVE_UNALIGNED); | 370 __ shr(loop_count, 6); |
312 __ bind(&last_15_much_overlap); | 371 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, |
313 __ and_(count, 0xF); | 372 BACKWARD, MOVE_UNALIGNED); |
314 __ j(zero, &pop_and_return); | 373 __ bind(&first_15_much_overlap); |
315 __ cmp(count, kSmallCopySize); | 374 __ and_(count, 0xF); |
316 __ j(below_equal, &small_size); | 375 __ j(zero, &pop_and_return); |
317 __ jmp(&medium_size); | 376 // Small/medium handlers expect dst/src to point to the beginning. |
| 377 __ sub(dst, count); |
| 378 __ sub(src, count); |
| 379 __ cmp(count, kSmallCopySize); |
| 380 __ j(below_equal, &small_size); |
| 381 __ jmp(&medium_size); |
| 382 } |
| 383 { |
| 384 // Special handlers for 9 <= copy_size < 64. No assumptions about |
| 385 // alignment or move distance, so all reads must be unaligned and |
| 386 // must happen before any writes. |
| 387 Label medium_handlers, f9_16, f17_32, f33_48, f49_63; |
| 388 |
| 389 __ bind(&f9_16); |
| 390 __ movsd(xmm0, Operand(src, 0)); |
| 391 __ movsd(xmm1, Operand(src, count, times_1, -8)); |
| 392 __ movsd(Operand(dst, 0), xmm0); |
| 393 __ movsd(Operand(dst, count, times_1, -8), xmm1); |
| 394 MemMoveEmitPopAndReturn(&masm); |
| 395 |
| 396 __ bind(&f17_32); |
| 397 __ movdqu(xmm0, Operand(src, 0)); |
| 398 __ movdqu(xmm1, Operand(src, count, times_1, -0x10)); |
| 399 __ movdqu(Operand(dst, 0x00), xmm0); |
| 400 __ movdqu(Operand(dst, count, times_1, -0x10), xmm1); |
| 401 MemMoveEmitPopAndReturn(&masm); |
| 402 |
| 403 __ bind(&f33_48); |
| 404 __ movdqu(xmm0, Operand(src, 0x00)); |
| 405 __ movdqu(xmm1, Operand(src, 0x10)); |
| 406 __ movdqu(xmm2, Operand(src, count, times_1, -0x10)); |
| 407 __ movdqu(Operand(dst, 0x00), xmm0); |
| 408 __ movdqu(Operand(dst, 0x10), xmm1); |
| 409 __ movdqu(Operand(dst, count, times_1, -0x10), xmm2); |
| 410 MemMoveEmitPopAndReturn(&masm); |
| 411 |
| 412 __ bind(&f49_63); |
| 413 __ movdqu(xmm0, Operand(src, 0x00)); |
| 414 __ movdqu(xmm1, Operand(src, 0x10)); |
| 415 __ movdqu(xmm2, Operand(src, 0x20)); |
| 416 __ movdqu(xmm3, Operand(src, count, times_1, -0x10)); |
| 417 __ movdqu(Operand(dst, 0x00), xmm0); |
| 418 __ movdqu(Operand(dst, 0x10), xmm1); |
| 419 __ movdqu(Operand(dst, 0x20), xmm2); |
| 420 __ movdqu(Operand(dst, count, times_1, -0x10), xmm3); |
| 421 MemMoveEmitPopAndReturn(&masm); |
| 422 |
| 423 __ bind(&medium_handlers); |
| 424 __ dd(conv.address(&f9_16)); |
| 425 __ dd(conv.address(&f17_32)); |
| 426 __ dd(conv.address(&f33_48)); |
| 427 __ dd(conv.address(&f49_63)); |
| 428 |
| 429 __ bind(&medium_size); // Entry point into this block. |
| 430 __ mov(eax, count); |
| 431 __ dec(eax); |
| 432 __ shr(eax, 4); |
| 433 if (FLAG_debug_code) { |
| 434 Label ok; |
| 435 __ cmp(eax, 3); |
| 436 __ j(below_equal, &ok); |
| 437 __ int3(); |
| 438 __ bind(&ok); |
318 } | 439 } |
319 | 440 __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers))); |
320 { | 441 __ jmp(eax); |
321 // |dst| is a higher address than |src|. Copy backwards. | 442 } |
322 Label unaligned_source, move_first_15, skip_last_move; | 443 { |
323 __ bind(&backward); | 444 // Specialized copiers for copy_size <= 8 bytes. |
324 // |dst| and |src| always point to the end of what's left to copy. | 445 Label small_handlers, f0, f1, f2, f3, f4, f5_8; |
325 __ add(dst, count); | 446 __ bind(&f0); |
326 __ add(src, count); | 447 MemMoveEmitPopAndReturn(&masm); |
327 __ mov(eax, dst); | 448 |
328 __ sub(eax, src); | 449 __ bind(&f1); |
329 __ cmp(eax, kMinMoveDistance); | 450 __ mov_b(eax, Operand(src, 0)); |
330 __ j(below, &backward_much_overlap); | 451 __ mov_b(Operand(dst, 0), eax); |
331 // Copy last 16 bytes. | 452 MemMoveEmitPopAndReturn(&masm); |
332 __ movdqu(xmm0, Operand(src, -0x10)); | 453 |
333 __ movdqu(Operand(dst, -0x10), xmm0); | 454 __ bind(&f2); |
334 // Find distance to alignment: dst & 0xF | 455 __ mov_w(eax, Operand(src, 0)); |
335 __ mov(edx, dst); | 456 __ mov_w(Operand(dst, 0), eax); |
336 __ and_(edx, 0xF); | 457 MemMoveEmitPopAndReturn(&masm); |
337 __ sub(dst, edx); | 458 |
338 __ sub(src, edx); | 459 __ bind(&f3); |
339 __ sub(count, edx); | 460 __ mov_w(eax, Operand(src, 0)); |
340 // dst is now aligned. Main copy loop. | 461 __ mov_b(edx, Operand(src, 2)); |
341 __ mov(loop_count, count); | 462 __ mov_w(Operand(dst, 0), eax); |
342 __ shr(loop_count, 6); | 463 __ mov_b(Operand(dst, 2), edx); |
343 // Check if src is also aligned. | 464 MemMoveEmitPopAndReturn(&masm); |
344 __ test(src, Immediate(0xF)); | 465 |
345 __ j(not_zero, &unaligned_source); | 466 __ bind(&f4); |
346 // Copy loop for aligned source and destination. | 467 __ mov(eax, Operand(src, 0)); |
347 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED); | 468 __ mov(Operand(dst, 0), eax); |
348 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. | 469 MemMoveEmitPopAndReturn(&masm); |
349 __ bind(&move_first_15); | 470 |
350 __ and_(count, 0xF); | 471 __ bind(&f5_8); |
351 __ j(zero, &skip_last_move, Label::kNear); | 472 __ mov(eax, Operand(src, 0)); |
352 __ sub(src, count); | 473 __ mov(edx, Operand(src, count, times_1, -4)); |
353 __ sub(dst, count); | 474 __ mov(Operand(dst, 0), eax); |
354 __ movdqu(xmm0, Operand(src, 0)); | 475 __ mov(Operand(dst, count, times_1, -4), edx); |
355 __ movdqu(Operand(dst, 0), xmm0); | 476 MemMoveEmitPopAndReturn(&masm); |
356 __ bind(&skip_last_move); | 477 |
357 MemMoveEmitPopAndReturn(&masm); | 478 __ bind(&small_handlers); |
358 | 479 __ dd(conv.address(&f0)); |
359 // Copy loop for unaligned source and aligned destination. | 480 __ dd(conv.address(&f1)); |
360 __ bind(&unaligned_source); | 481 __ dd(conv.address(&f2)); |
361 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED); | 482 __ dd(conv.address(&f3)); |
362 __ jmp(&move_first_15); | 483 __ dd(conv.address(&f4)); |
363 | 484 __ dd(conv.address(&f5_8)); |
364 // Less than kMinMoveDistance offset between dst and src. | 485 __ dd(conv.address(&f5_8)); |
365 Label loop_until_aligned, first_15_much_overlap; | 486 __ dd(conv.address(&f5_8)); |
366 __ bind(&loop_until_aligned); | 487 __ dd(conv.address(&f5_8)); |
367 __ dec(src); | 488 |
368 __ dec(dst); | 489 __ bind(&small_size); // Entry point into this block. |
369 __ mov_b(eax, Operand(src, 0)); | 490 if (FLAG_debug_code) { |
370 __ mov_b(Operand(dst, 0), eax); | 491 Label ok; |
371 __ dec(count); | 492 __ cmp(count, 8); |
372 __ bind(&backward_much_overlap); // Entry point into this block. | 493 __ j(below_equal, &ok); |
373 __ test(dst, Immediate(0xF)); | 494 __ int3(); |
374 __ j(not_zero, &loop_until_aligned); | 495 __ bind(&ok); |
375 // dst is now aligned, src can't be. Main copy loop. | |
376 __ mov(loop_count, count); | |
377 __ shr(loop_count, 6); | |
378 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, | |
379 BACKWARD, MOVE_UNALIGNED); | |
380 __ bind(&first_15_much_overlap); | |
381 __ and_(count, 0xF); | |
382 __ j(zero, &pop_and_return); | |
383 // Small/medium handlers expect dst/src to point to the beginning. | |
384 __ sub(dst, count); | |
385 __ sub(src, count); | |
386 __ cmp(count, kSmallCopySize); | |
387 __ j(below_equal, &small_size); | |
388 __ jmp(&medium_size); | |
389 } | 496 } |
390 { | 497 __ mov(eax, Operand(count, times_4, conv.address(&small_handlers))); |
391 // Special handlers for 9 <= copy_size < 64. No assumptions about | 498 __ jmp(eax); |
392 // alignment or move distance, so all reads must be unaligned and | |
393 // must happen before any writes. | |
394 Label medium_handlers, f9_16, f17_32, f33_48, f49_63; | |
395 | |
396 __ bind(&f9_16); | |
397 __ movsd(xmm0, Operand(src, 0)); | |
398 __ movsd(xmm1, Operand(src, count, times_1, -8)); | |
399 __ movsd(Operand(dst, 0), xmm0); | |
400 __ movsd(Operand(dst, count, times_1, -8), xmm1); | |
401 MemMoveEmitPopAndReturn(&masm); | |
402 | |
403 __ bind(&f17_32); | |
404 __ movdqu(xmm0, Operand(src, 0)); | |
405 __ movdqu(xmm1, Operand(src, count, times_1, -0x10)); | |
406 __ movdqu(Operand(dst, 0x00), xmm0); | |
407 __ movdqu(Operand(dst, count, times_1, -0x10), xmm1); | |
408 MemMoveEmitPopAndReturn(&masm); | |
409 | |
410 __ bind(&f33_48); | |
411 __ movdqu(xmm0, Operand(src, 0x00)); | |
412 __ movdqu(xmm1, Operand(src, 0x10)); | |
413 __ movdqu(xmm2, Operand(src, count, times_1, -0x10)); | |
414 __ movdqu(Operand(dst, 0x00), xmm0); | |
415 __ movdqu(Operand(dst, 0x10), xmm1); | |
416 __ movdqu(Operand(dst, count, times_1, -0x10), xmm2); | |
417 MemMoveEmitPopAndReturn(&masm); | |
418 | |
419 __ bind(&f49_63); | |
420 __ movdqu(xmm0, Operand(src, 0x00)); | |
421 __ movdqu(xmm1, Operand(src, 0x10)); | |
422 __ movdqu(xmm2, Operand(src, 0x20)); | |
423 __ movdqu(xmm3, Operand(src, count, times_1, -0x10)); | |
424 __ movdqu(Operand(dst, 0x00), xmm0); | |
425 __ movdqu(Operand(dst, 0x10), xmm1); | |
426 __ movdqu(Operand(dst, 0x20), xmm2); | |
427 __ movdqu(Operand(dst, count, times_1, -0x10), xmm3); | |
428 MemMoveEmitPopAndReturn(&masm); | |
429 | |
430 __ bind(&medium_handlers); | |
431 __ dd(conv.address(&f9_16)); | |
432 __ dd(conv.address(&f17_32)); | |
433 __ dd(conv.address(&f33_48)); | |
434 __ dd(conv.address(&f49_63)); | |
435 | |
436 __ bind(&medium_size); // Entry point into this block. | |
437 __ mov(eax, count); | |
438 __ dec(eax); | |
439 __ shr(eax, 4); | |
440 if (FLAG_debug_code) { | |
441 Label ok; | |
442 __ cmp(eax, 3); | |
443 __ j(below_equal, &ok); | |
444 __ int3(); | |
445 __ bind(&ok); | |
446 } | |
447 __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers))); | |
448 __ jmp(eax); | |
449 } | |
450 { | |
451 // Specialized copiers for copy_size <= 8 bytes. | |
452 Label small_handlers, f0, f1, f2, f3, f4, f5_8; | |
453 __ bind(&f0); | |
454 MemMoveEmitPopAndReturn(&masm); | |
455 | |
456 __ bind(&f1); | |
457 __ mov_b(eax, Operand(src, 0)); | |
458 __ mov_b(Operand(dst, 0), eax); | |
459 MemMoveEmitPopAndReturn(&masm); | |
460 | |
461 __ bind(&f2); | |
462 __ mov_w(eax, Operand(src, 0)); | |
463 __ mov_w(Operand(dst, 0), eax); | |
464 MemMoveEmitPopAndReturn(&masm); | |
465 | |
466 __ bind(&f3); | |
467 __ mov_w(eax, Operand(src, 0)); | |
468 __ mov_b(edx, Operand(src, 2)); | |
469 __ mov_w(Operand(dst, 0), eax); | |
470 __ mov_b(Operand(dst, 2), edx); | |
471 MemMoveEmitPopAndReturn(&masm); | |
472 | |
473 __ bind(&f4); | |
474 __ mov(eax, Operand(src, 0)); | |
475 __ mov(Operand(dst, 0), eax); | |
476 MemMoveEmitPopAndReturn(&masm); | |
477 | |
478 __ bind(&f5_8); | |
479 __ mov(eax, Operand(src, 0)); | |
480 __ mov(edx, Operand(src, count, times_1, -4)); | |
481 __ mov(Operand(dst, 0), eax); | |
482 __ mov(Operand(dst, count, times_1, -4), edx); | |
483 MemMoveEmitPopAndReturn(&masm); | |
484 | |
485 __ bind(&small_handlers); | |
486 __ dd(conv.address(&f0)); | |
487 __ dd(conv.address(&f1)); | |
488 __ dd(conv.address(&f2)); | |
489 __ dd(conv.address(&f3)); | |
490 __ dd(conv.address(&f4)); | |
491 __ dd(conv.address(&f5_8)); | |
492 __ dd(conv.address(&f5_8)); | |
493 __ dd(conv.address(&f5_8)); | |
494 __ dd(conv.address(&f5_8)); | |
495 | |
496 __ bind(&small_size); // Entry point into this block. | |
497 if (FLAG_debug_code) { | |
498 Label ok; | |
499 __ cmp(count, 8); | |
500 __ j(below_equal, &ok); | |
501 __ int3(); | |
502 __ bind(&ok); | |
503 } | |
504 __ mov(eax, Operand(count, times_4, conv.address(&small_handlers))); | |
505 __ jmp(eax); | |
506 } | |
507 } else { | |
508 // No SSE2. | |
509 Label forward; | |
510 __ cmp(count, 0); | |
511 __ j(equal, &pop_and_return); | |
512 __ cmp(dst, src); | |
513 __ j(above, &backward); | |
514 __ jmp(&forward); | |
515 { | |
516 // Simple forward copier. | |
517 Label forward_loop_1byte, forward_loop_4byte; | |
518 __ bind(&forward_loop_4byte); | |
519 __ mov(eax, Operand(src, 0)); | |
520 __ sub(count, Immediate(4)); | |
521 __ add(src, Immediate(4)); | |
522 __ mov(Operand(dst, 0), eax); | |
523 __ add(dst, Immediate(4)); | |
524 __ bind(&forward); // Entry point. | |
525 __ cmp(count, 3); | |
526 __ j(above, &forward_loop_4byte); | |
527 __ bind(&forward_loop_1byte); | |
528 __ cmp(count, 0); | |
529 __ j(below_equal, &pop_and_return); | |
530 __ mov_b(eax, Operand(src, 0)); | |
531 __ dec(count); | |
532 __ inc(src); | |
533 __ mov_b(Operand(dst, 0), eax); | |
534 __ inc(dst); | |
535 __ jmp(&forward_loop_1byte); | |
536 } | |
537 { | |
538 // Simple backward copier. | |
539 Label backward_loop_1byte, backward_loop_4byte, entry_shortcut; | |
540 __ bind(&backward); | |
541 __ add(src, count); | |
542 __ add(dst, count); | |
543 __ cmp(count, 3); | |
544 __ j(below_equal, &entry_shortcut); | |
545 | |
546 __ bind(&backward_loop_4byte); | |
547 __ sub(src, Immediate(4)); | |
548 __ sub(count, Immediate(4)); | |
549 __ mov(eax, Operand(src, 0)); | |
550 __ sub(dst, Immediate(4)); | |
551 __ mov(Operand(dst, 0), eax); | |
552 __ cmp(count, 3); | |
553 __ j(above, &backward_loop_4byte); | |
554 __ bind(&backward_loop_1byte); | |
555 __ cmp(count, 0); | |
556 __ j(below_equal, &pop_and_return); | |
557 __ bind(&entry_shortcut); | |
558 __ dec(src); | |
559 __ dec(count); | |
560 __ mov_b(eax, Operand(src, 0)); | |
561 __ dec(dst); | |
562 __ mov_b(Operand(dst, 0), eax); | |
563 __ jmp(&backward_loop_1byte); | |
564 } | |
565 } | 499 } |
566 | 500 |
567 __ bind(&pop_and_return); | 501 __ bind(&pop_and_return); |
568 MemMoveEmitPopAndReturn(&masm); | 502 MemMoveEmitPopAndReturn(&masm); |
569 | 503 |
570 CodeDesc desc; | 504 CodeDesc desc; |
571 masm.GetCode(&desc); | 505 masm.GetCode(&desc); |
572 ASSERT(!RelocInfo::RequiresRelocation(desc)); | 506 ASSERT(!RelocInfo::RequiresRelocation(desc)); |
573 CPU::FlushICache(buffer, actual_size); | 507 CPU::FlushICache(buffer, actual_size); |
574 OS::ProtectCode(buffer, actual_size); | 508 OS::ProtectCode(buffer, actual_size); |
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
664 kDontSaveFPRegs, | 598 kDontSaveFPRegs, |
665 EMIT_REMEMBERED_SET, | 599 EMIT_REMEMBERED_SET, |
666 OMIT_SMI_CHECK); | 600 OMIT_SMI_CHECK); |
667 | 601 |
668 __ mov(edi, FieldOperand(esi, FixedArray::kLengthOffset)); | 602 __ mov(edi, FieldOperand(esi, FixedArray::kLengthOffset)); |
669 | 603 |
670 // Prepare for conversion loop. | 604 // Prepare for conversion loop. |
671 ExternalReference canonical_the_hole_nan_reference = | 605 ExternalReference canonical_the_hole_nan_reference = |
672 ExternalReference::address_of_the_hole_nan(); | 606 ExternalReference::address_of_the_hole_nan(); |
673 XMMRegister the_hole_nan = xmm1; | 607 XMMRegister the_hole_nan = xmm1; |
674 if (CpuFeatures::IsSupported(SSE2)) { | 608 __ movsd(the_hole_nan, |
675 CpuFeatureScope use_sse2(masm, SSE2); | 609 Operand::StaticVariable(canonical_the_hole_nan_reference)); |
676 __ movsd(the_hole_nan, | |
677 Operand::StaticVariable(canonical_the_hole_nan_reference)); | |
678 } | |
679 __ jmp(&entry); | 610 __ jmp(&entry); |
680 | 611 |
681 // Call into runtime if GC is required. | 612 // Call into runtime if GC is required. |
682 __ bind(&gc_required); | 613 __ bind(&gc_required); |
683 // Restore registers before jumping into runtime. | 614 // Restore registers before jumping into runtime. |
684 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); | 615 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); |
685 __ pop(ebx); | 616 __ pop(ebx); |
686 __ pop(eax); | 617 __ pop(eax); |
687 __ jmp(fail); | 618 __ jmp(fail); |
688 | 619 |
689 // Convert and copy elements | 620 // Convert and copy elements |
690 // esi: source FixedArray | 621 // esi: source FixedArray |
691 __ bind(&loop); | 622 __ bind(&loop); |
692 __ mov(ebx, FieldOperand(esi, edi, times_2, FixedArray::kHeaderSize)); | 623 __ mov(ebx, FieldOperand(esi, edi, times_2, FixedArray::kHeaderSize)); |
693 // ebx: current element from source | 624 // ebx: current element from source |
694 // edi: index of current element | 625 // edi: index of current element |
695 __ JumpIfNotSmi(ebx, &convert_hole); | 626 __ JumpIfNotSmi(ebx, &convert_hole); |
696 | 627 |
697 // Normal smi, convert it to double and store. | 628 // Normal smi, convert it to double and store. |
698 __ SmiUntag(ebx); | 629 __ SmiUntag(ebx); |
699 if (CpuFeatures::IsSupported(SSE2)) { | 630 __ Cvtsi2sd(xmm0, ebx); |
700 CpuFeatureScope fscope(masm, SSE2); | 631 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), |
701 __ Cvtsi2sd(xmm0, ebx); | 632 xmm0); |
702 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), | |
703 xmm0); | |
704 } else { | |
705 __ push(ebx); | |
706 __ fild_s(Operand(esp, 0)); | |
707 __ pop(ebx); | |
708 __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize)); | |
709 } | |
710 __ jmp(&entry); | 633 __ jmp(&entry); |
711 | 634 |
712 // Found hole, store hole_nan_as_double instead. | 635 // Found hole, store hole_nan_as_double instead. |
713 __ bind(&convert_hole); | 636 __ bind(&convert_hole); |
714 | 637 |
715 if (FLAG_debug_code) { | 638 if (FLAG_debug_code) { |
716 __ cmp(ebx, masm->isolate()->factory()->the_hole_value()); | 639 __ cmp(ebx, masm->isolate()->factory()->the_hole_value()); |
717 __ Assert(equal, kObjectFoundInSmiOnlyArray); | 640 __ Assert(equal, kObjectFoundInSmiOnlyArray); |
718 } | 641 } |
719 | 642 |
720 if (CpuFeatures::IsSupported(SSE2)) { | 643 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), |
721 CpuFeatureScope use_sse2(masm, SSE2); | 644 the_hole_nan); |
722 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), | |
723 the_hole_nan); | |
724 } else { | |
725 __ fld_d(Operand::StaticVariable(canonical_the_hole_nan_reference)); | |
726 __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize)); | |
727 } | |
728 | 645 |
729 __ bind(&entry); | 646 __ bind(&entry); |
730 __ sub(edi, Immediate(Smi::FromInt(1))); | 647 __ sub(edi, Immediate(Smi::FromInt(1))); |
731 __ j(not_sign, &loop); | 648 __ j(not_sign, &loop); |
732 | 649 |
733 __ pop(ebx); | 650 __ pop(ebx); |
734 __ pop(eax); | 651 __ pop(eax); |
735 | 652 |
736 // Restore esi. | 653 // Restore esi. |
737 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); | 654 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
819 // eax: destination FixedArray | 736 // eax: destination FixedArray |
820 __ bind(&loop); | 737 __ bind(&loop); |
821 // ebx: index of current element (smi-tagged) | 738 // ebx: index of current element (smi-tagged) |
822 uint32_t offset = FixedDoubleArray::kHeaderSize + sizeof(kHoleNanLower32); | 739 uint32_t offset = FixedDoubleArray::kHeaderSize + sizeof(kHoleNanLower32); |
823 __ cmp(FieldOperand(edi, ebx, times_4, offset), Immediate(kHoleNanUpper32)); | 740 __ cmp(FieldOperand(edi, ebx, times_4, offset), Immediate(kHoleNanUpper32)); |
824 __ j(equal, &convert_hole); | 741 __ j(equal, &convert_hole); |
825 | 742 |
826 // Non-hole double, copy value into a heap number. | 743 // Non-hole double, copy value into a heap number. |
827 __ AllocateHeapNumber(edx, esi, no_reg, &gc_required); | 744 __ AllocateHeapNumber(edx, esi, no_reg, &gc_required); |
828 // edx: new heap number | 745 // edx: new heap number |
829 if (CpuFeatures::IsSupported(SSE2)) { | 746 __ movsd(xmm0, |
830 CpuFeatureScope fscope(masm, SSE2); | 747 FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); |
831 __ movsd(xmm0, | 748 __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0); |
832 FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); | |
833 __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0); | |
834 } else { | |
835 __ mov(esi, FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); | |
836 __ mov(FieldOperand(edx, HeapNumber::kValueOffset), esi); | |
837 __ mov(esi, FieldOperand(edi, ebx, times_4, offset)); | |
838 __ mov(FieldOperand(edx, HeapNumber::kValueOffset + kPointerSize), esi); | |
839 } | |
840 __ mov(FieldOperand(eax, ebx, times_2, FixedArray::kHeaderSize), edx); | 749 __ mov(FieldOperand(eax, ebx, times_2, FixedArray::kHeaderSize), edx); |
841 __ mov(esi, ebx); | 750 __ mov(esi, ebx); |
842 __ RecordWriteArray(eax, | 751 __ RecordWriteArray(eax, |
843 edx, | 752 edx, |
844 esi, | 753 esi, |
845 kDontSaveFPRegs, | 754 kDontSaveFPRegs, |
846 EMIT_REMEMBERED_SET, | 755 EMIT_REMEMBERED_SET, |
847 OMIT_SMI_CHECK); | 756 OMIT_SMI_CHECK); |
848 __ jmp(&entry, Label::kNear); | 757 __ jmp(&entry, Label::kNear); |
849 | 758 |
(...skipping 251 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1101 Code* stub = GetCodeAgeStub(isolate, age, parity); | 1010 Code* stub = GetCodeAgeStub(isolate, age, parity); |
1102 CodePatcher patcher(sequence, young_length); | 1011 CodePatcher patcher(sequence, young_length); |
1103 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); | 1012 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); |
1104 } | 1013 } |
1105 } | 1014 } |
1106 | 1015 |
1107 | 1016 |
1108 } } // namespace v8::internal | 1017 } } // namespace v8::internal |
1109 | 1018 |
1110 #endif // V8_TARGET_ARCH_IA32 | 1019 #endif // V8_TARGET_ARCH_IA32 |
OLD | NEW |