| OLD | NEW |
| 1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "v8.h" | 5 #include "v8.h" |
| 6 | 6 |
| 7 #if V8_TARGET_ARCH_IA32 | 7 #if V8_TARGET_ARCH_IA32 |
| 8 | 8 |
| 9 #include "codegen.h" | 9 #include "codegen.h" |
| 10 #include "heap.h" | 10 #include "heap.h" |
| (...skipping 17 matching lines...) Expand all Loading... |
| 28 masm->LeaveFrame(StackFrame::INTERNAL); | 28 masm->LeaveFrame(StackFrame::INTERNAL); |
| 29 ASSERT(masm->has_frame()); | 29 ASSERT(masm->has_frame()); |
| 30 masm->set_has_frame(false); | 30 masm->set_has_frame(false); |
| 31 } | 31 } |
| 32 | 32 |
| 33 | 33 |
| 34 #define __ masm. | 34 #define __ masm. |
| 35 | 35 |
| 36 | 36 |
| 37 UnaryMathFunction CreateExpFunction() { | 37 UnaryMathFunction CreateExpFunction() { |
| 38 if (!CpuFeatures::IsSupported(SSE2)) return &std::exp; | |
| 39 if (!FLAG_fast_math) return &std::exp; | 38 if (!FLAG_fast_math) return &std::exp; |
| 40 size_t actual_size; | 39 size_t actual_size; |
| 41 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true)); | 40 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true)); |
| 42 if (buffer == NULL) return &std::exp; | 41 if (buffer == NULL) return &std::exp; |
| 43 ExternalReference::InitializeMathExpData(); | 42 ExternalReference::InitializeMathExpData(); |
| 44 | 43 |
| 45 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); | 44 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); |
| 46 // esp[1 * kPointerSize]: raw double input | 45 // esp[1 * kPointerSize]: raw double input |
| 47 // esp[0 * kPointerSize]: return address | 46 // esp[0 * kPointerSize]: return address |
| 48 { | 47 { |
| 49 CpuFeatureScope use_sse2(&masm, SSE2); | |
| 50 XMMRegister input = xmm1; | 48 XMMRegister input = xmm1; |
| 51 XMMRegister result = xmm2; | 49 XMMRegister result = xmm2; |
| 52 __ movsd(input, Operand(esp, 1 * kPointerSize)); | 50 __ movsd(input, Operand(esp, 1 * kPointerSize)); |
| 53 __ push(eax); | 51 __ push(eax); |
| 54 __ push(ebx); | 52 __ push(ebx); |
| 55 | 53 |
| 56 MathExpGenerator::EmitMathExp(&masm, input, result, xmm0, eax, ebx); | 54 MathExpGenerator::EmitMathExp(&masm, input, result, xmm0, eax, ebx); |
| 57 | 55 |
| 58 __ pop(ebx); | 56 __ pop(ebx); |
| 59 __ pop(eax); | 57 __ pop(eax); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 71 return FUNCTION_CAST<UnaryMathFunction>(buffer); | 69 return FUNCTION_CAST<UnaryMathFunction>(buffer); |
| 72 } | 70 } |
| 73 | 71 |
| 74 | 72 |
| 75 UnaryMathFunction CreateSqrtFunction() { | 73 UnaryMathFunction CreateSqrtFunction() { |
| 76 size_t actual_size; | 74 size_t actual_size; |
| 77 // Allocate buffer in executable space. | 75 // Allocate buffer in executable space. |
| 78 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, | 76 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, |
| 79 &actual_size, | 77 &actual_size, |
| 80 true)); | 78 true)); |
| 81 // If SSE2 is not available, we can use libc's implementation to ensure | 79 if (buffer == NULL) return &std::sqrt; |
| 82 // consistency since code by fullcodegen's calls into runtime in that case. | |
| 83 if (buffer == NULL || !CpuFeatures::IsSupported(SSE2)) return &std::sqrt; | |
| 84 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); | 80 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); |
| 85 // esp[1 * kPointerSize]: raw double input | 81 // esp[1 * kPointerSize]: raw double input |
| 86 // esp[0 * kPointerSize]: return address | 82 // esp[0 * kPointerSize]: return address |
| 87 // Move double input into registers. | 83 // Move double input into registers. |
| 88 { | 84 { |
| 89 CpuFeatureScope use_sse2(&masm, SSE2); | |
| 90 __ movsd(xmm0, Operand(esp, 1 * kPointerSize)); | 85 __ movsd(xmm0, Operand(esp, 1 * kPointerSize)); |
| 91 __ sqrtsd(xmm0, xmm0); | 86 __ sqrtsd(xmm0, xmm0); |
| 92 __ movsd(Operand(esp, 1 * kPointerSize), xmm0); | 87 __ movsd(Operand(esp, 1 * kPointerSize), xmm0); |
| 93 // Load result into floating point register as return value. | 88 // Load result into floating point register as return value. |
| 94 __ fld_d(Operand(esp, 1 * kPointerSize)); | 89 __ fld_d(Operand(esp, 1 * kPointerSize)); |
| 95 __ Ret(); | 90 __ Ret(); |
| 96 } | 91 } |
| 97 | 92 |
| 98 CodeDesc desc; | 93 CodeDesc desc; |
| 99 masm.GetCode(&desc); | 94 masm.GetCode(&desc); |
| (...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 236 Register src = esi; | 231 Register src = esi; |
| 237 Register count = ecx; | 232 Register count = ecx; |
| 238 Register loop_count = edx; | 233 Register loop_count = edx; |
| 239 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); | 234 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); |
| 240 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); | 235 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); |
| 241 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); | 236 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); |
| 242 | 237 |
| 243 __ cmp(dst, src); | 238 __ cmp(dst, src); |
| 244 __ j(equal, &pop_and_return); | 239 __ j(equal, &pop_and_return); |
| 245 | 240 |
| 246 if (CpuFeatures::IsSupported(SSE2)) { | 241 __ prefetch(Operand(src, 0), 1); |
| 247 CpuFeatureScope sse2_scope(&masm, SSE2); | 242 __ cmp(count, kSmallCopySize); |
| 248 __ prefetch(Operand(src, 0), 1); | 243 __ j(below_equal, &small_size); |
| 244 __ cmp(count, kMediumCopySize); |
| 245 __ j(below_equal, &medium_size); |
| 246 __ cmp(dst, src); |
| 247 __ j(above, &backward); |
| 248 |
| 249 { |
| 250 // |dst| is a lower address than |src|. Copy front-to-back. |
| 251 Label unaligned_source, move_last_15, skip_last_move; |
| 252 __ mov(eax, src); |
| 253 __ sub(eax, dst); |
| 254 __ cmp(eax, kMinMoveDistance); |
| 255 __ j(below, &forward_much_overlap); |
| 256 // Copy first 16 bytes. |
| 257 __ movdqu(xmm0, Operand(src, 0)); |
| 258 __ movdqu(Operand(dst, 0), xmm0); |
| 259 // Determine distance to alignment: 16 - (dst & 0xF). |
| 260 __ mov(edx, dst); |
| 261 __ and_(edx, 0xF); |
| 262 __ neg(edx); |
| 263 __ add(edx, Immediate(16)); |
| 264 __ add(dst, edx); |
| 265 __ add(src, edx); |
| 266 __ sub(count, edx); |
| 267 // dst is now aligned. Main copy loop. |
| 268 __ mov(loop_count, count); |
| 269 __ shr(loop_count, 6); |
| 270 // Check if src is also aligned. |
| 271 __ test(src, Immediate(0xF)); |
| 272 __ j(not_zero, &unaligned_source); |
| 273 // Copy loop for aligned source and destination. |
| 274 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED); |
| 275 // At most 15 bytes to copy. Copy 16 bytes at end of string. |
| 276 __ bind(&move_last_15); |
| 277 __ and_(count, 0xF); |
| 278 __ j(zero, &skip_last_move, Label::kNear); |
| 279 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); |
| 280 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); |
| 281 __ bind(&skip_last_move); |
| 282 MemMoveEmitPopAndReturn(&masm); |
| 283 |
| 284 // Copy loop for unaligned source and aligned destination. |
| 285 __ bind(&unaligned_source); |
| 286 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED); |
| 287 __ jmp(&move_last_15); |
| 288 |
| 289 // Less than kMinMoveDistance offset between dst and src. |
| 290 Label loop_until_aligned, last_15_much_overlap; |
| 291 __ bind(&loop_until_aligned); |
| 292 __ mov_b(eax, Operand(src, 0)); |
| 293 __ inc(src); |
| 294 __ mov_b(Operand(dst, 0), eax); |
| 295 __ inc(dst); |
| 296 __ dec(count); |
| 297 __ bind(&forward_much_overlap); // Entry point into this block. |
| 298 __ test(dst, Immediate(0xF)); |
| 299 __ j(not_zero, &loop_until_aligned); |
| 300 // dst is now aligned, src can't be. Main copy loop. |
| 301 __ mov(loop_count, count); |
| 302 __ shr(loop_count, 6); |
| 303 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, |
| 304 FORWARD, MOVE_UNALIGNED); |
| 305 __ bind(&last_15_much_overlap); |
| 306 __ and_(count, 0xF); |
| 307 __ j(zero, &pop_and_return); |
| 249 __ cmp(count, kSmallCopySize); | 308 __ cmp(count, kSmallCopySize); |
| 250 __ j(below_equal, &small_size); | 309 __ j(below_equal, &small_size); |
| 251 __ cmp(count, kMediumCopySize); | 310 __ jmp(&medium_size); |
| 252 __ j(below_equal, &medium_size); | 311 } |
| 253 __ cmp(dst, src); | 312 |
| 254 __ j(above, &backward); | 313 { |
| 255 | 314 // |dst| is a higher address than |src|. Copy backwards. |
| 256 { | 315 Label unaligned_source, move_first_15, skip_last_move; |
| 257 // |dst| is a lower address than |src|. Copy front-to-back. | 316 __ bind(&backward); |
| 258 Label unaligned_source, move_last_15, skip_last_move; | 317 // |dst| and |src| always point to the end of what's left to copy. |
| 259 __ mov(eax, src); | 318 __ add(dst, count); |
| 260 __ sub(eax, dst); | 319 __ add(src, count); |
| 261 __ cmp(eax, kMinMoveDistance); | 320 __ mov(eax, dst); |
| 262 __ j(below, &forward_much_overlap); | 321 __ sub(eax, src); |
| 263 // Copy first 16 bytes. | 322 __ cmp(eax, kMinMoveDistance); |
| 264 __ movdqu(xmm0, Operand(src, 0)); | 323 __ j(below, &backward_much_overlap); |
| 265 __ movdqu(Operand(dst, 0), xmm0); | 324 // Copy last 16 bytes. |
| 266 // Determine distance to alignment: 16 - (dst & 0xF). | 325 __ movdqu(xmm0, Operand(src, -0x10)); |
| 267 __ mov(edx, dst); | 326 __ movdqu(Operand(dst, -0x10), xmm0); |
| 268 __ and_(edx, 0xF); | 327 // Find distance to alignment: dst & 0xF |
| 269 __ neg(edx); | 328 __ mov(edx, dst); |
| 270 __ add(edx, Immediate(16)); | 329 __ and_(edx, 0xF); |
| 271 __ add(dst, edx); | 330 __ sub(dst, edx); |
| 272 __ add(src, edx); | 331 __ sub(src, edx); |
| 273 __ sub(count, edx); | 332 __ sub(count, edx); |
| 274 // dst is now aligned. Main copy loop. | 333 // dst is now aligned. Main copy loop. |
| 275 __ mov(loop_count, count); | 334 __ mov(loop_count, count); |
| 276 __ shr(loop_count, 6); | 335 __ shr(loop_count, 6); |
| 277 // Check if src is also aligned. | 336 // Check if src is also aligned. |
| 278 __ test(src, Immediate(0xF)); | 337 __ test(src, Immediate(0xF)); |
| 279 __ j(not_zero, &unaligned_source); | 338 __ j(not_zero, &unaligned_source); |
| 280 // Copy loop for aligned source and destination. | 339 // Copy loop for aligned source and destination. |
| 281 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED); | 340 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED); |
| 282 // At most 15 bytes to copy. Copy 16 bytes at end of string. | 341 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. |
| 283 __ bind(&move_last_15); | 342 __ bind(&move_first_15); |
| 284 __ and_(count, 0xF); | 343 __ and_(count, 0xF); |
| 285 __ j(zero, &skip_last_move, Label::kNear); | 344 __ j(zero, &skip_last_move, Label::kNear); |
| 286 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); | 345 __ sub(src, count); |
| 287 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); | 346 __ sub(dst, count); |
| 288 __ bind(&skip_last_move); | 347 __ movdqu(xmm0, Operand(src, 0)); |
| 289 MemMoveEmitPopAndReturn(&masm); | 348 __ movdqu(Operand(dst, 0), xmm0); |
| 290 | 349 __ bind(&skip_last_move); |
| 291 // Copy loop for unaligned source and aligned destination. | 350 MemMoveEmitPopAndReturn(&masm); |
| 292 __ bind(&unaligned_source); | 351 |
| 293 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED); | 352 // Copy loop for unaligned source and aligned destination. |
| 294 __ jmp(&move_last_15); | 353 __ bind(&unaligned_source); |
| 295 | 354 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED); |
| 296 // Less than kMinMoveDistance offset between dst and src. | 355 __ jmp(&move_first_15); |
| 297 Label loop_until_aligned, last_15_much_overlap; | 356 |
| 298 __ bind(&loop_until_aligned); | 357 // Less than kMinMoveDistance offset between dst and src. |
| 299 __ mov_b(eax, Operand(src, 0)); | 358 Label loop_until_aligned, first_15_much_overlap; |
| 300 __ inc(src); | 359 __ bind(&loop_until_aligned); |
| 301 __ mov_b(Operand(dst, 0), eax); | 360 __ dec(src); |
| 302 __ inc(dst); | 361 __ dec(dst); |
| 303 __ dec(count); | 362 __ mov_b(eax, Operand(src, 0)); |
| 304 __ bind(&forward_much_overlap); // Entry point into this block. | 363 __ mov_b(Operand(dst, 0), eax); |
| 305 __ test(dst, Immediate(0xF)); | 364 __ dec(count); |
| 306 __ j(not_zero, &loop_until_aligned); | 365 __ bind(&backward_much_overlap); // Entry point into this block. |
| 307 // dst is now aligned, src can't be. Main copy loop. | 366 __ test(dst, Immediate(0xF)); |
| 308 __ mov(loop_count, count); | 367 __ j(not_zero, &loop_until_aligned); |
| 309 __ shr(loop_count, 6); | 368 // dst is now aligned, src can't be. Main copy loop. |
| 310 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, | 369 __ mov(loop_count, count); |
| 311 FORWARD, MOVE_UNALIGNED); | 370 __ shr(loop_count, 6); |
| 312 __ bind(&last_15_much_overlap); | 371 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, |
| 313 __ and_(count, 0xF); | 372 BACKWARD, MOVE_UNALIGNED); |
| 314 __ j(zero, &pop_and_return); | 373 __ bind(&first_15_much_overlap); |
| 315 __ cmp(count, kSmallCopySize); | 374 __ and_(count, 0xF); |
| 316 __ j(below_equal, &small_size); | 375 __ j(zero, &pop_and_return); |
| 317 __ jmp(&medium_size); | 376 // Small/medium handlers expect dst/src to point to the beginning. |
| 377 __ sub(dst, count); |
| 378 __ sub(src, count); |
| 379 __ cmp(count, kSmallCopySize); |
| 380 __ j(below_equal, &small_size); |
| 381 __ jmp(&medium_size); |
| 382 } |
| 383 { |
| 384 // Special handlers for 9 <= copy_size < 64. No assumptions about |
| 385 // alignment or move distance, so all reads must be unaligned and |
| 386 // must happen before any writes. |
| 387 Label medium_handlers, f9_16, f17_32, f33_48, f49_63; |
| 388 |
| 389 __ bind(&f9_16); |
| 390 __ movsd(xmm0, Operand(src, 0)); |
| 391 __ movsd(xmm1, Operand(src, count, times_1, -8)); |
| 392 __ movsd(Operand(dst, 0), xmm0); |
| 393 __ movsd(Operand(dst, count, times_1, -8), xmm1); |
| 394 MemMoveEmitPopAndReturn(&masm); |
| 395 |
| 396 __ bind(&f17_32); |
| 397 __ movdqu(xmm0, Operand(src, 0)); |
| 398 __ movdqu(xmm1, Operand(src, count, times_1, -0x10)); |
| 399 __ movdqu(Operand(dst, 0x00), xmm0); |
| 400 __ movdqu(Operand(dst, count, times_1, -0x10), xmm1); |
| 401 MemMoveEmitPopAndReturn(&masm); |
| 402 |
| 403 __ bind(&f33_48); |
| 404 __ movdqu(xmm0, Operand(src, 0x00)); |
| 405 __ movdqu(xmm1, Operand(src, 0x10)); |
| 406 __ movdqu(xmm2, Operand(src, count, times_1, -0x10)); |
| 407 __ movdqu(Operand(dst, 0x00), xmm0); |
| 408 __ movdqu(Operand(dst, 0x10), xmm1); |
| 409 __ movdqu(Operand(dst, count, times_1, -0x10), xmm2); |
| 410 MemMoveEmitPopAndReturn(&masm); |
| 411 |
| 412 __ bind(&f49_63); |
| 413 __ movdqu(xmm0, Operand(src, 0x00)); |
| 414 __ movdqu(xmm1, Operand(src, 0x10)); |
| 415 __ movdqu(xmm2, Operand(src, 0x20)); |
| 416 __ movdqu(xmm3, Operand(src, count, times_1, -0x10)); |
| 417 __ movdqu(Operand(dst, 0x00), xmm0); |
| 418 __ movdqu(Operand(dst, 0x10), xmm1); |
| 419 __ movdqu(Operand(dst, 0x20), xmm2); |
| 420 __ movdqu(Operand(dst, count, times_1, -0x10), xmm3); |
| 421 MemMoveEmitPopAndReturn(&masm); |
| 422 |
| 423 __ bind(&medium_handlers); |
| 424 __ dd(conv.address(&f9_16)); |
| 425 __ dd(conv.address(&f17_32)); |
| 426 __ dd(conv.address(&f33_48)); |
| 427 __ dd(conv.address(&f49_63)); |
| 428 |
| 429 __ bind(&medium_size); // Entry point into this block. |
| 430 __ mov(eax, count); |
| 431 __ dec(eax); |
| 432 __ shr(eax, 4); |
| 433 if (FLAG_debug_code) { |
| 434 Label ok; |
| 435 __ cmp(eax, 3); |
| 436 __ j(below_equal, &ok); |
| 437 __ int3(); |
| 438 __ bind(&ok); |
| 318 } | 439 } |
| 319 | 440 __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers))); |
| 320 { | 441 __ jmp(eax); |
| 321 // |dst| is a higher address than |src|. Copy backwards. | 442 } |
| 322 Label unaligned_source, move_first_15, skip_last_move; | 443 { |
| 323 __ bind(&backward); | 444 // Specialized copiers for copy_size <= 8 bytes. |
| 324 // |dst| and |src| always point to the end of what's left to copy. | 445 Label small_handlers, f0, f1, f2, f3, f4, f5_8; |
| 325 __ add(dst, count); | 446 __ bind(&f0); |
| 326 __ add(src, count); | 447 MemMoveEmitPopAndReturn(&masm); |
| 327 __ mov(eax, dst); | 448 |
| 328 __ sub(eax, src); | 449 __ bind(&f1); |
| 329 __ cmp(eax, kMinMoveDistance); | 450 __ mov_b(eax, Operand(src, 0)); |
| 330 __ j(below, &backward_much_overlap); | 451 __ mov_b(Operand(dst, 0), eax); |
| 331 // Copy last 16 bytes. | 452 MemMoveEmitPopAndReturn(&masm); |
| 332 __ movdqu(xmm0, Operand(src, -0x10)); | 453 |
| 333 __ movdqu(Operand(dst, -0x10), xmm0); | 454 __ bind(&f2); |
| 334 // Find distance to alignment: dst & 0xF | 455 __ mov_w(eax, Operand(src, 0)); |
| 335 __ mov(edx, dst); | 456 __ mov_w(Operand(dst, 0), eax); |
| 336 __ and_(edx, 0xF); | 457 MemMoveEmitPopAndReturn(&masm); |
| 337 __ sub(dst, edx); | 458 |
| 338 __ sub(src, edx); | 459 __ bind(&f3); |
| 339 __ sub(count, edx); | 460 __ mov_w(eax, Operand(src, 0)); |
| 340 // dst is now aligned. Main copy loop. | 461 __ mov_b(edx, Operand(src, 2)); |
| 341 __ mov(loop_count, count); | 462 __ mov_w(Operand(dst, 0), eax); |
| 342 __ shr(loop_count, 6); | 463 __ mov_b(Operand(dst, 2), edx); |
| 343 // Check if src is also aligned. | 464 MemMoveEmitPopAndReturn(&masm); |
| 344 __ test(src, Immediate(0xF)); | 465 |
| 345 __ j(not_zero, &unaligned_source); | 466 __ bind(&f4); |
| 346 // Copy loop for aligned source and destination. | 467 __ mov(eax, Operand(src, 0)); |
| 347 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED); | 468 __ mov(Operand(dst, 0), eax); |
| 348 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. | 469 MemMoveEmitPopAndReturn(&masm); |
| 349 __ bind(&move_first_15); | 470 |
| 350 __ and_(count, 0xF); | 471 __ bind(&f5_8); |
| 351 __ j(zero, &skip_last_move, Label::kNear); | 472 __ mov(eax, Operand(src, 0)); |
| 352 __ sub(src, count); | 473 __ mov(edx, Operand(src, count, times_1, -4)); |
| 353 __ sub(dst, count); | 474 __ mov(Operand(dst, 0), eax); |
| 354 __ movdqu(xmm0, Operand(src, 0)); | 475 __ mov(Operand(dst, count, times_1, -4), edx); |
| 355 __ movdqu(Operand(dst, 0), xmm0); | 476 MemMoveEmitPopAndReturn(&masm); |
| 356 __ bind(&skip_last_move); | 477 |
| 357 MemMoveEmitPopAndReturn(&masm); | 478 __ bind(&small_handlers); |
| 358 | 479 __ dd(conv.address(&f0)); |
| 359 // Copy loop for unaligned source and aligned destination. | 480 __ dd(conv.address(&f1)); |
| 360 __ bind(&unaligned_source); | 481 __ dd(conv.address(&f2)); |
| 361 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED); | 482 __ dd(conv.address(&f3)); |
| 362 __ jmp(&move_first_15); | 483 __ dd(conv.address(&f4)); |
| 363 | 484 __ dd(conv.address(&f5_8)); |
| 364 // Less than kMinMoveDistance offset between dst and src. | 485 __ dd(conv.address(&f5_8)); |
| 365 Label loop_until_aligned, first_15_much_overlap; | 486 __ dd(conv.address(&f5_8)); |
| 366 __ bind(&loop_until_aligned); | 487 __ dd(conv.address(&f5_8)); |
| 367 __ dec(src); | 488 |
| 368 __ dec(dst); | 489 __ bind(&small_size); // Entry point into this block. |
| 369 __ mov_b(eax, Operand(src, 0)); | 490 if (FLAG_debug_code) { |
| 370 __ mov_b(Operand(dst, 0), eax); | 491 Label ok; |
| 371 __ dec(count); | 492 __ cmp(count, 8); |
| 372 __ bind(&backward_much_overlap); // Entry point into this block. | 493 __ j(below_equal, &ok); |
| 373 __ test(dst, Immediate(0xF)); | 494 __ int3(); |
| 374 __ j(not_zero, &loop_until_aligned); | 495 __ bind(&ok); |
| 375 // dst is now aligned, src can't be. Main copy loop. | |
| 376 __ mov(loop_count, count); | |
| 377 __ shr(loop_count, 6); | |
| 378 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, | |
| 379 BACKWARD, MOVE_UNALIGNED); | |
| 380 __ bind(&first_15_much_overlap); | |
| 381 __ and_(count, 0xF); | |
| 382 __ j(zero, &pop_and_return); | |
| 383 // Small/medium handlers expect dst/src to point to the beginning. | |
| 384 __ sub(dst, count); | |
| 385 __ sub(src, count); | |
| 386 __ cmp(count, kSmallCopySize); | |
| 387 __ j(below_equal, &small_size); | |
| 388 __ jmp(&medium_size); | |
| 389 } | 496 } |
| 390 { | 497 __ mov(eax, Operand(count, times_4, conv.address(&small_handlers))); |
| 391 // Special handlers for 9 <= copy_size < 64. No assumptions about | 498 __ jmp(eax); |
| 392 // alignment or move distance, so all reads must be unaligned and | |
| 393 // must happen before any writes. | |
| 394 Label medium_handlers, f9_16, f17_32, f33_48, f49_63; | |
| 395 | |
| 396 __ bind(&f9_16); | |
| 397 __ movsd(xmm0, Operand(src, 0)); | |
| 398 __ movsd(xmm1, Operand(src, count, times_1, -8)); | |
| 399 __ movsd(Operand(dst, 0), xmm0); | |
| 400 __ movsd(Operand(dst, count, times_1, -8), xmm1); | |
| 401 MemMoveEmitPopAndReturn(&masm); | |
| 402 | |
| 403 __ bind(&f17_32); | |
| 404 __ movdqu(xmm0, Operand(src, 0)); | |
| 405 __ movdqu(xmm1, Operand(src, count, times_1, -0x10)); | |
| 406 __ movdqu(Operand(dst, 0x00), xmm0); | |
| 407 __ movdqu(Operand(dst, count, times_1, -0x10), xmm1); | |
| 408 MemMoveEmitPopAndReturn(&masm); | |
| 409 | |
| 410 __ bind(&f33_48); | |
| 411 __ movdqu(xmm0, Operand(src, 0x00)); | |
| 412 __ movdqu(xmm1, Operand(src, 0x10)); | |
| 413 __ movdqu(xmm2, Operand(src, count, times_1, -0x10)); | |
| 414 __ movdqu(Operand(dst, 0x00), xmm0); | |
| 415 __ movdqu(Operand(dst, 0x10), xmm1); | |
| 416 __ movdqu(Operand(dst, count, times_1, -0x10), xmm2); | |
| 417 MemMoveEmitPopAndReturn(&masm); | |
| 418 | |
| 419 __ bind(&f49_63); | |
| 420 __ movdqu(xmm0, Operand(src, 0x00)); | |
| 421 __ movdqu(xmm1, Operand(src, 0x10)); | |
| 422 __ movdqu(xmm2, Operand(src, 0x20)); | |
| 423 __ movdqu(xmm3, Operand(src, count, times_1, -0x10)); | |
| 424 __ movdqu(Operand(dst, 0x00), xmm0); | |
| 425 __ movdqu(Operand(dst, 0x10), xmm1); | |
| 426 __ movdqu(Operand(dst, 0x20), xmm2); | |
| 427 __ movdqu(Operand(dst, count, times_1, -0x10), xmm3); | |
| 428 MemMoveEmitPopAndReturn(&masm); | |
| 429 | |
| 430 __ bind(&medium_handlers); | |
| 431 __ dd(conv.address(&f9_16)); | |
| 432 __ dd(conv.address(&f17_32)); | |
| 433 __ dd(conv.address(&f33_48)); | |
| 434 __ dd(conv.address(&f49_63)); | |
| 435 | |
| 436 __ bind(&medium_size); // Entry point into this block. | |
| 437 __ mov(eax, count); | |
| 438 __ dec(eax); | |
| 439 __ shr(eax, 4); | |
| 440 if (FLAG_debug_code) { | |
| 441 Label ok; | |
| 442 __ cmp(eax, 3); | |
| 443 __ j(below_equal, &ok); | |
| 444 __ int3(); | |
| 445 __ bind(&ok); | |
| 446 } | |
| 447 __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers))); | |
| 448 __ jmp(eax); | |
| 449 } | |
| 450 { | |
| 451 // Specialized copiers for copy_size <= 8 bytes. | |
| 452 Label small_handlers, f0, f1, f2, f3, f4, f5_8; | |
| 453 __ bind(&f0); | |
| 454 MemMoveEmitPopAndReturn(&masm); | |
| 455 | |
| 456 __ bind(&f1); | |
| 457 __ mov_b(eax, Operand(src, 0)); | |
| 458 __ mov_b(Operand(dst, 0), eax); | |
| 459 MemMoveEmitPopAndReturn(&masm); | |
| 460 | |
| 461 __ bind(&f2); | |
| 462 __ mov_w(eax, Operand(src, 0)); | |
| 463 __ mov_w(Operand(dst, 0), eax); | |
| 464 MemMoveEmitPopAndReturn(&masm); | |
| 465 | |
| 466 __ bind(&f3); | |
| 467 __ mov_w(eax, Operand(src, 0)); | |
| 468 __ mov_b(edx, Operand(src, 2)); | |
| 469 __ mov_w(Operand(dst, 0), eax); | |
| 470 __ mov_b(Operand(dst, 2), edx); | |
| 471 MemMoveEmitPopAndReturn(&masm); | |
| 472 | |
| 473 __ bind(&f4); | |
| 474 __ mov(eax, Operand(src, 0)); | |
| 475 __ mov(Operand(dst, 0), eax); | |
| 476 MemMoveEmitPopAndReturn(&masm); | |
| 477 | |
| 478 __ bind(&f5_8); | |
| 479 __ mov(eax, Operand(src, 0)); | |
| 480 __ mov(edx, Operand(src, count, times_1, -4)); | |
| 481 __ mov(Operand(dst, 0), eax); | |
| 482 __ mov(Operand(dst, count, times_1, -4), edx); | |
| 483 MemMoveEmitPopAndReturn(&masm); | |
| 484 | |
| 485 __ bind(&small_handlers); | |
| 486 __ dd(conv.address(&f0)); | |
| 487 __ dd(conv.address(&f1)); | |
| 488 __ dd(conv.address(&f2)); | |
| 489 __ dd(conv.address(&f3)); | |
| 490 __ dd(conv.address(&f4)); | |
| 491 __ dd(conv.address(&f5_8)); | |
| 492 __ dd(conv.address(&f5_8)); | |
| 493 __ dd(conv.address(&f5_8)); | |
| 494 __ dd(conv.address(&f5_8)); | |
| 495 | |
| 496 __ bind(&small_size); // Entry point into this block. | |
| 497 if (FLAG_debug_code) { | |
| 498 Label ok; | |
| 499 __ cmp(count, 8); | |
| 500 __ j(below_equal, &ok); | |
| 501 __ int3(); | |
| 502 __ bind(&ok); | |
| 503 } | |
| 504 __ mov(eax, Operand(count, times_4, conv.address(&small_handlers))); | |
| 505 __ jmp(eax); | |
| 506 } | |
| 507 } else { | |
| 508 // No SSE2. | |
| 509 Label forward; | |
| 510 __ cmp(count, 0); | |
| 511 __ j(equal, &pop_and_return); | |
| 512 __ cmp(dst, src); | |
| 513 __ j(above, &backward); | |
| 514 __ jmp(&forward); | |
| 515 { | |
| 516 // Simple forward copier. | |
| 517 Label forward_loop_1byte, forward_loop_4byte; | |
| 518 __ bind(&forward_loop_4byte); | |
| 519 __ mov(eax, Operand(src, 0)); | |
| 520 __ sub(count, Immediate(4)); | |
| 521 __ add(src, Immediate(4)); | |
| 522 __ mov(Operand(dst, 0), eax); | |
| 523 __ add(dst, Immediate(4)); | |
| 524 __ bind(&forward); // Entry point. | |
| 525 __ cmp(count, 3); | |
| 526 __ j(above, &forward_loop_4byte); | |
| 527 __ bind(&forward_loop_1byte); | |
| 528 __ cmp(count, 0); | |
| 529 __ j(below_equal, &pop_and_return); | |
| 530 __ mov_b(eax, Operand(src, 0)); | |
| 531 __ dec(count); | |
| 532 __ inc(src); | |
| 533 __ mov_b(Operand(dst, 0), eax); | |
| 534 __ inc(dst); | |
| 535 __ jmp(&forward_loop_1byte); | |
| 536 } | |
| 537 { | |
| 538 // Simple backward copier. | |
| 539 Label backward_loop_1byte, backward_loop_4byte, entry_shortcut; | |
| 540 __ bind(&backward); | |
| 541 __ add(src, count); | |
| 542 __ add(dst, count); | |
| 543 __ cmp(count, 3); | |
| 544 __ j(below_equal, &entry_shortcut); | |
| 545 | |
| 546 __ bind(&backward_loop_4byte); | |
| 547 __ sub(src, Immediate(4)); | |
| 548 __ sub(count, Immediate(4)); | |
| 549 __ mov(eax, Operand(src, 0)); | |
| 550 __ sub(dst, Immediate(4)); | |
| 551 __ mov(Operand(dst, 0), eax); | |
| 552 __ cmp(count, 3); | |
| 553 __ j(above, &backward_loop_4byte); | |
| 554 __ bind(&backward_loop_1byte); | |
| 555 __ cmp(count, 0); | |
| 556 __ j(below_equal, &pop_and_return); | |
| 557 __ bind(&entry_shortcut); | |
| 558 __ dec(src); | |
| 559 __ dec(count); | |
| 560 __ mov_b(eax, Operand(src, 0)); | |
| 561 __ dec(dst); | |
| 562 __ mov_b(Operand(dst, 0), eax); | |
| 563 __ jmp(&backward_loop_1byte); | |
| 564 } | |
| 565 } | 499 } |
| 566 | 500 |
| 567 __ bind(&pop_and_return); | 501 __ bind(&pop_and_return); |
| 568 MemMoveEmitPopAndReturn(&masm); | 502 MemMoveEmitPopAndReturn(&masm); |
| 569 | 503 |
| 570 CodeDesc desc; | 504 CodeDesc desc; |
| 571 masm.GetCode(&desc); | 505 masm.GetCode(&desc); |
| 572 ASSERT(!RelocInfo::RequiresRelocation(desc)); | 506 ASSERT(!RelocInfo::RequiresRelocation(desc)); |
| 573 CPU::FlushICache(buffer, actual_size); | 507 CPU::FlushICache(buffer, actual_size); |
| 574 OS::ProtectCode(buffer, actual_size); | 508 OS::ProtectCode(buffer, actual_size); |
| (...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 664 kDontSaveFPRegs, | 598 kDontSaveFPRegs, |
| 665 EMIT_REMEMBERED_SET, | 599 EMIT_REMEMBERED_SET, |
| 666 OMIT_SMI_CHECK); | 600 OMIT_SMI_CHECK); |
| 667 | 601 |
| 668 __ mov(edi, FieldOperand(esi, FixedArray::kLengthOffset)); | 602 __ mov(edi, FieldOperand(esi, FixedArray::kLengthOffset)); |
| 669 | 603 |
| 670 // Prepare for conversion loop. | 604 // Prepare for conversion loop. |
| 671 ExternalReference canonical_the_hole_nan_reference = | 605 ExternalReference canonical_the_hole_nan_reference = |
| 672 ExternalReference::address_of_the_hole_nan(); | 606 ExternalReference::address_of_the_hole_nan(); |
| 673 XMMRegister the_hole_nan = xmm1; | 607 XMMRegister the_hole_nan = xmm1; |
| 674 if (CpuFeatures::IsSupported(SSE2)) { | 608 __ movsd(the_hole_nan, |
| 675 CpuFeatureScope use_sse2(masm, SSE2); | 609 Operand::StaticVariable(canonical_the_hole_nan_reference)); |
| 676 __ movsd(the_hole_nan, | |
| 677 Operand::StaticVariable(canonical_the_hole_nan_reference)); | |
| 678 } | |
| 679 __ jmp(&entry); | 610 __ jmp(&entry); |
| 680 | 611 |
| 681 // Call into runtime if GC is required. | 612 // Call into runtime if GC is required. |
| 682 __ bind(&gc_required); | 613 __ bind(&gc_required); |
| 683 // Restore registers before jumping into runtime. | 614 // Restore registers before jumping into runtime. |
| 684 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); | 615 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); |
| 685 __ pop(ebx); | 616 __ pop(ebx); |
| 686 __ pop(eax); | 617 __ pop(eax); |
| 687 __ jmp(fail); | 618 __ jmp(fail); |
| 688 | 619 |
| 689 // Convert and copy elements | 620 // Convert and copy elements |
| 690 // esi: source FixedArray | 621 // esi: source FixedArray |
| 691 __ bind(&loop); | 622 __ bind(&loop); |
| 692 __ mov(ebx, FieldOperand(esi, edi, times_2, FixedArray::kHeaderSize)); | 623 __ mov(ebx, FieldOperand(esi, edi, times_2, FixedArray::kHeaderSize)); |
| 693 // ebx: current element from source | 624 // ebx: current element from source |
| 694 // edi: index of current element | 625 // edi: index of current element |
| 695 __ JumpIfNotSmi(ebx, &convert_hole); | 626 __ JumpIfNotSmi(ebx, &convert_hole); |
| 696 | 627 |
| 697 // Normal smi, convert it to double and store. | 628 // Normal smi, convert it to double and store. |
| 698 __ SmiUntag(ebx); | 629 __ SmiUntag(ebx); |
| 699 if (CpuFeatures::IsSupported(SSE2)) { | 630 __ Cvtsi2sd(xmm0, ebx); |
| 700 CpuFeatureScope fscope(masm, SSE2); | 631 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), |
| 701 __ Cvtsi2sd(xmm0, ebx); | 632 xmm0); |
| 702 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), | |
| 703 xmm0); | |
| 704 } else { | |
| 705 __ push(ebx); | |
| 706 __ fild_s(Operand(esp, 0)); | |
| 707 __ pop(ebx); | |
| 708 __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize)); | |
| 709 } | |
| 710 __ jmp(&entry); | 633 __ jmp(&entry); |
| 711 | 634 |
| 712 // Found hole, store hole_nan_as_double instead. | 635 // Found hole, store hole_nan_as_double instead. |
| 713 __ bind(&convert_hole); | 636 __ bind(&convert_hole); |
| 714 | 637 |
| 715 if (FLAG_debug_code) { | 638 if (FLAG_debug_code) { |
| 716 __ cmp(ebx, masm->isolate()->factory()->the_hole_value()); | 639 __ cmp(ebx, masm->isolate()->factory()->the_hole_value()); |
| 717 __ Assert(equal, kObjectFoundInSmiOnlyArray); | 640 __ Assert(equal, kObjectFoundInSmiOnlyArray); |
| 718 } | 641 } |
| 719 | 642 |
| 720 if (CpuFeatures::IsSupported(SSE2)) { | 643 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), |
| 721 CpuFeatureScope use_sse2(masm, SSE2); | 644 the_hole_nan); |
| 722 __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize), | |
| 723 the_hole_nan); | |
| 724 } else { | |
| 725 __ fld_d(Operand::StaticVariable(canonical_the_hole_nan_reference)); | |
| 726 __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize)); | |
| 727 } | |
| 728 | 645 |
| 729 __ bind(&entry); | 646 __ bind(&entry); |
| 730 __ sub(edi, Immediate(Smi::FromInt(1))); | 647 __ sub(edi, Immediate(Smi::FromInt(1))); |
| 731 __ j(not_sign, &loop); | 648 __ j(not_sign, &loop); |
| 732 | 649 |
| 733 __ pop(ebx); | 650 __ pop(ebx); |
| 734 __ pop(eax); | 651 __ pop(eax); |
| 735 | 652 |
| 736 // Restore esi. | 653 // Restore esi. |
| 737 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); | 654 __ mov(esi, Operand(ebp, StandardFrameConstants::kContextOffset)); |
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 819 // eax: destination FixedArray | 736 // eax: destination FixedArray |
| 820 __ bind(&loop); | 737 __ bind(&loop); |
| 821 // ebx: index of current element (smi-tagged) | 738 // ebx: index of current element (smi-tagged) |
| 822 uint32_t offset = FixedDoubleArray::kHeaderSize + sizeof(kHoleNanLower32); | 739 uint32_t offset = FixedDoubleArray::kHeaderSize + sizeof(kHoleNanLower32); |
| 823 __ cmp(FieldOperand(edi, ebx, times_4, offset), Immediate(kHoleNanUpper32)); | 740 __ cmp(FieldOperand(edi, ebx, times_4, offset), Immediate(kHoleNanUpper32)); |
| 824 __ j(equal, &convert_hole); | 741 __ j(equal, &convert_hole); |
| 825 | 742 |
| 826 // Non-hole double, copy value into a heap number. | 743 // Non-hole double, copy value into a heap number. |
| 827 __ AllocateHeapNumber(edx, esi, no_reg, &gc_required); | 744 __ AllocateHeapNumber(edx, esi, no_reg, &gc_required); |
| 828 // edx: new heap number | 745 // edx: new heap number |
| 829 if (CpuFeatures::IsSupported(SSE2)) { | 746 __ movsd(xmm0, |
| 830 CpuFeatureScope fscope(masm, SSE2); | 747 FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); |
| 831 __ movsd(xmm0, | 748 __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0); |
| 832 FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); | |
| 833 __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0); | |
| 834 } else { | |
| 835 __ mov(esi, FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize)); | |
| 836 __ mov(FieldOperand(edx, HeapNumber::kValueOffset), esi); | |
| 837 __ mov(esi, FieldOperand(edi, ebx, times_4, offset)); | |
| 838 __ mov(FieldOperand(edx, HeapNumber::kValueOffset + kPointerSize), esi); | |
| 839 } | |
| 840 __ mov(FieldOperand(eax, ebx, times_2, FixedArray::kHeaderSize), edx); | 749 __ mov(FieldOperand(eax, ebx, times_2, FixedArray::kHeaderSize), edx); |
| 841 __ mov(esi, ebx); | 750 __ mov(esi, ebx); |
| 842 __ RecordWriteArray(eax, | 751 __ RecordWriteArray(eax, |
| 843 edx, | 752 edx, |
| 844 esi, | 753 esi, |
| 845 kDontSaveFPRegs, | 754 kDontSaveFPRegs, |
| 846 EMIT_REMEMBERED_SET, | 755 EMIT_REMEMBERED_SET, |
| 847 OMIT_SMI_CHECK); | 756 OMIT_SMI_CHECK); |
| 848 __ jmp(&entry, Label::kNear); | 757 __ jmp(&entry, Label::kNear); |
| 849 | 758 |
| (...skipping 251 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1101 Code* stub = GetCodeAgeStub(isolate, age, parity); | 1010 Code* stub = GetCodeAgeStub(isolate, age, parity); |
| 1102 CodePatcher patcher(sequence, young_length); | 1011 CodePatcher patcher(sequence, young_length); |
| 1103 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); | 1012 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); |
| 1104 } | 1013 } |
| 1105 } | 1014 } |
| 1106 | 1015 |
| 1107 | 1016 |
| 1108 } } // namespace v8::internal | 1017 } } // namespace v8::internal |
| 1109 | 1018 |
| 1110 #endif // V8_TARGET_ARCH_IA32 | 1019 #endif // V8_TARGET_ARCH_IA32 |
| OLD | NEW |