Require SSE2 support for the ia32 port.

src/ia32/codegen-ia32.cc

Index: src/ia32/codegen-ia32.cc
diff --git a/src/ia32/codegen-ia32.cc b/src/ia32/codegen-ia32.cc
index 19b66aee3b59216c4c03709703aeff4f092e2d30..04d00648cb944516de086e15c19a5c87d3b7dd5c 100644
--- a/src/ia32/codegen-ia32.cc
+++ b/src/ia32/codegen-ia32.cc
@@ -35,7 +35,6 @@ void StubRuntimeCallHelper::AfterCall(MacroAssembler* masm) const {
 
 
 UnaryMathFunction CreateExpFunction() {
-  if (!CpuFeatures::IsSupported(SSE2)) return &std::exp;
   if (!FLAG_fast_math) return &std::exp;
   size_t actual_size;
   byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true));
@@ -46,7 +45,6 @@ UnaryMathFunction CreateExpFunction() {
   // esp[1 * kPointerSize]: raw double input
   // esp[0 * kPointerSize]: return address
   {
-    CpuFeatureScope use_sse2(&masm, SSE2);
     XMMRegister input = xmm1;
     XMMRegister result = xmm2;
     __ movsd(input, Operand(esp, 1 * kPointerSize));
@@ -78,15 +76,12 @@ UnaryMathFunction CreateSqrtFunction() {
   byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB,
                                                  &actual_size,
                                                  true));
-  // If SSE2 is not available, we can use libc's implementation to ensure
-  // consistency since code by fullcodegen's calls into runtime in that case.
-  if (buffer == NULL || !CpuFeatures::IsSupported(SSE2)) return &std::sqrt;
+  if (buffer == NULL) return &std::sqrt;
   MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size));
   // esp[1 * kPointerSize]: raw double input
   // esp[0 * kPointerSize]: return address
   // Move double input into registers.
   {
-    CpuFeatureScope use_sse2(&masm, SSE2);
     __ movsd(xmm0, Operand(esp, 1 * kPointerSize));
     __ sqrtsd(xmm0, xmm0);
     __ movsd(Operand(esp, 1 * kPointerSize), xmm0);
@@ -243,325 +238,264 @@ OS::MemMoveFunction CreateMemMoveFunction() {
   __ cmp(dst, src);
   __ j(equal, &pop_and_return);
 
-  if (CpuFeatures::IsSupported(SSE2)) {
-    CpuFeatureScope sse2_scope(&masm, SSE2);
-    __ prefetch(Operand(src, 0), 1);
+  __ prefetch(Operand(src, 0), 1);
+  __ cmp(count, kSmallCopySize);
+  __ j(below_equal, &small_size);
+  __ cmp(count, kMediumCopySize);
+  __ j(below_equal, &medium_size);
+  __ cmp(dst, src);
+  __ j(above, &backward);
+
+  {
+    // |dst| is a lower address than |src|. Copy front-to-back.
+    Label unaligned_source, move_last_15, skip_last_move;
+    __ mov(eax, src);
+    __ sub(eax, dst);
+    __ cmp(eax, kMinMoveDistance);
+    __ j(below, &forward_much_overlap);
+    // Copy first 16 bytes.
+    __ movdqu(xmm0, Operand(src, 0));
+    __ movdqu(Operand(dst, 0), xmm0);
+    // Determine distance to alignment: 16 - (dst & 0xF).
+    __ mov(edx, dst);
+    __ and_(edx, 0xF);
+    __ neg(edx);
+    __ add(edx, Immediate(16));
+    __ add(dst, edx);
+    __ add(src, edx);
+    __ sub(count, edx);
+    // dst is now aligned. Main copy loop.
+    __ mov(loop_count, count);
+    __ shr(loop_count, 6);
+    // Check if src is also aligned.
+    __ test(src, Immediate(0xF));
+    __ j(not_zero, &unaligned_source);
+    // Copy loop for aligned source and destination.
+    MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED);
+    // At most 15 bytes to copy. Copy 16 bytes at end of string.
+    __ bind(&move_last_15);
+    __ and_(count, 0xF);
+    __ j(zero, &skip_last_move, Label::kNear);
+    __ movdqu(xmm0, Operand(src, count, times_1, -0x10));
+    __ movdqu(Operand(dst, count, times_1, -0x10), xmm0);
+    __ bind(&skip_last_move);
+    MemMoveEmitPopAndReturn(&masm);
+
+    // Copy loop for unaligned source and aligned destination.
+    __ bind(&unaligned_source);
+    MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED);
+    __ jmp(&move_last_15);
+
+    // Less than kMinMoveDistance offset between dst and src.
+    Label loop_until_aligned, last_15_much_overlap;
+    __ bind(&loop_until_aligned);
+    __ mov_b(eax, Operand(src, 0));
+    __ inc(src);
+    __ mov_b(Operand(dst, 0), eax);
+    __ inc(dst);
+    __ dec(count);
+    __ bind(&forward_much_overlap);  // Entry point into this block.
+    __ test(dst, Immediate(0xF));
+    __ j(not_zero, &loop_until_aligned);
+    // dst is now aligned, src can't be. Main copy loop.
+    __ mov(loop_count, count);
+    __ shr(loop_count, 6);
+    MemMoveEmitMainLoop(&masm, &last_15_much_overlap,
+                        FORWARD, MOVE_UNALIGNED);
+    __ bind(&last_15_much_overlap);
+    __ and_(count, 0xF);
+    __ j(zero, &pop_and_return);
     __ cmp(count, kSmallCopySize);
     __ j(below_equal, &small_size);
-    __ cmp(count, kMediumCopySize);
-    __ j(below_equal, &medium_size);
-    __ cmp(dst, src);
-    __ j(above, &backward);
-
-    {
-      // |dst| is a lower address than |src|. Copy front-to-back.
-      Label unaligned_source, move_last_15, skip_last_move;
-      __ mov(eax, src);
-      __ sub(eax, dst);
-      __ cmp(eax, kMinMoveDistance);
-      __ j(below, &forward_much_overlap);
-      // Copy first 16 bytes.
-      __ movdqu(xmm0, Operand(src, 0));
-      __ movdqu(Operand(dst, 0), xmm0);
-      // Determine distance to alignment: 16 - (dst & 0xF).
-      __ mov(edx, dst);
-      __ and_(edx, 0xF);
-      __ neg(edx);
-      __ add(edx, Immediate(16));
-      __ add(dst, edx);
-      __ add(src, edx);
-      __ sub(count, edx);
-      // dst is now aligned. Main copy loop.
-      __ mov(loop_count, count);
-      __ shr(loop_count, 6);
-      // Check if src is also aligned.
-      __ test(src, Immediate(0xF));
-      __ j(not_zero, &unaligned_source);
-      // Copy loop for aligned source and destination.
-      MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED);
-      // At most 15 bytes to copy. Copy 16 bytes at end of string.
-      __ bind(&move_last_15);
-      __ and_(count, 0xF);
-      __ j(zero, &skip_last_move, Label::kNear);
-      __ movdqu(xmm0, Operand(src, count, times_1, -0x10));
-      __ movdqu(Operand(dst, count, times_1, -0x10), xmm0);
-      __ bind(&skip_last_move);
-      MemMoveEmitPopAndReturn(&masm);
-
-      // Copy loop for unaligned source and aligned destination.
-      __ bind(&unaligned_source);
-      MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED);
-      __ jmp(&move_last_15);
-
-      // Less than kMinMoveDistance offset between dst and src.
-      Label loop_until_aligned, last_15_much_overlap;
-      __ bind(&loop_until_aligned);
-      __ mov_b(eax, Operand(src, 0));
-      __ inc(src);
-      __ mov_b(Operand(dst, 0), eax);
-      __ inc(dst);
-      __ dec(count);
-      __ bind(&forward_much_overlap);  // Entry point into this block.
-      __ test(dst, Immediate(0xF));
-      __ j(not_zero, &loop_until_aligned);
-      // dst is now aligned, src can't be. Main copy loop.
-      __ mov(loop_count, count);
-      __ shr(loop_count, 6);
-      MemMoveEmitMainLoop(&masm, &last_15_much_overlap,
-                          FORWARD, MOVE_UNALIGNED);
-      __ bind(&last_15_much_overlap);
-      __ and_(count, 0xF);
-      __ j(zero, &pop_and_return);
-      __ cmp(count, kSmallCopySize);
-      __ j(below_equal, &small_size);
-      __ jmp(&medium_size);
-    }
+    __ jmp(&medium_size);
+  }
 
-    {
-      // |dst| is a higher address than |src|. Copy backwards.
-      Label unaligned_source, move_first_15, skip_last_move;
-      __ bind(&backward);
-      // |dst| and |src| always point to the end of what's left to copy.
-      __ add(dst, count);
-      __ add(src, count);
-      __ mov(eax, dst);
-      __ sub(eax, src);
-      __ cmp(eax, kMinMoveDistance);
-      __ j(below, &backward_much_overlap);
-      // Copy last 16 bytes.
-      __ movdqu(xmm0, Operand(src, -0x10));
-      __ movdqu(Operand(dst, -0x10), xmm0);
-      // Find distance to alignment: dst & 0xF
-      __ mov(edx, dst);
-      __ and_(edx, 0xF);
-      __ sub(dst, edx);
-      __ sub(src, edx);
-      __ sub(count, edx);
-      // dst is now aligned. Main copy loop.
-      __ mov(loop_count, count);
-      __ shr(loop_count, 6);
-      // Check if src is also aligned.
-      __ test(src, Immediate(0xF));
-      __ j(not_zero, &unaligned_source);
-      // Copy loop for aligned source and destination.
-      MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED);
-      // At most 15 bytes to copy. Copy 16 bytes at beginning of string.
-      __ bind(&move_first_15);
-      __ and_(count, 0xF);
-      __ j(zero, &skip_last_move, Label::kNear);
-      __ sub(src, count);
-      __ sub(dst, count);
-      __ movdqu(xmm0, Operand(src, 0));
-      __ movdqu(Operand(dst, 0), xmm0);
-      __ bind(&skip_last_move);
-      MemMoveEmitPopAndReturn(&masm);
-
-      // Copy loop for unaligned source and aligned destination.
-      __ bind(&unaligned_source);
-      MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED);
-      __ jmp(&move_first_15);
-
-      // Less than kMinMoveDistance offset between dst and src.
-      Label loop_until_aligned, first_15_much_overlap;
-      __ bind(&loop_until_aligned);
-      __ dec(src);
-      __ dec(dst);
-      __ mov_b(eax, Operand(src, 0));
-      __ mov_b(Operand(dst, 0), eax);
-      __ dec(count);
-      __ bind(&backward_much_overlap);  // Entry point into this block.
-      __ test(dst, Immediate(0xF));
-      __ j(not_zero, &loop_until_aligned);
-      // dst is now aligned, src can't be. Main copy loop.
-      __ mov(loop_count, count);
-      __ shr(loop_count, 6);
-      MemMoveEmitMainLoop(&masm, &first_15_much_overlap,
-                          BACKWARD, MOVE_UNALIGNED);
-      __ bind(&first_15_much_overlap);
-      __ and_(count, 0xF);
-      __ j(zero, &pop_and_return);
-      // Small/medium handlers expect dst/src to point to the beginning.
-      __ sub(dst, count);
-      __ sub(src, count);
-      __ cmp(count, kSmallCopySize);
-      __ j(below_equal, &small_size);
-      __ jmp(&medium_size);
-    }
-    {
-      // Special handlers for 9 <= copy_size < 64. No assumptions about
-      // alignment or move distance, so all reads must be unaligned and
-      // must happen before any writes.
-      Label medium_handlers, f9_16, f17_32, f33_48, f49_63;
-
-      __ bind(&f9_16);
-      __ movsd(xmm0, Operand(src, 0));
-      __ movsd(xmm1, Operand(src, count, times_1, -8));
-      __ movsd(Operand(dst, 0), xmm0);
-      __ movsd(Operand(dst, count, times_1, -8), xmm1);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f17_32);
-      __ movdqu(xmm0, Operand(src, 0));
-      __ movdqu(xmm1, Operand(src, count, times_1, -0x10));
-      __ movdqu(Operand(dst, 0x00), xmm0);
-      __ movdqu(Operand(dst, count, times_1, -0x10), xmm1);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f33_48);
-      __ movdqu(xmm0, Operand(src, 0x00));
-      __ movdqu(xmm1, Operand(src, 0x10));
-      __ movdqu(xmm2, Operand(src, count, times_1, -0x10));
-      __ movdqu(Operand(dst, 0x00), xmm0);
-      __ movdqu(Operand(dst, 0x10), xmm1);
-      __ movdqu(Operand(dst, count, times_1, -0x10), xmm2);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f49_63);
-      __ movdqu(xmm0, Operand(src, 0x00));
-      __ movdqu(xmm1, Operand(src, 0x10));
-      __ movdqu(xmm2, Operand(src, 0x20));
-      __ movdqu(xmm3, Operand(src, count, times_1, -0x10));
-      __ movdqu(Operand(dst, 0x00), xmm0);
-      __ movdqu(Operand(dst, 0x10), xmm1);
-      __ movdqu(Operand(dst, 0x20), xmm2);
-      __ movdqu(Operand(dst, count, times_1, -0x10), xmm3);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&medium_handlers);
-      __ dd(conv.address(&f9_16));
-      __ dd(conv.address(&f17_32));
-      __ dd(conv.address(&f33_48));
-      __ dd(conv.address(&f49_63));
-
-      __ bind(&medium_size);  // Entry point into this block.
-      __ mov(eax, count);
-      __ dec(eax);
-      __ shr(eax, 4);
-      if (FLAG_debug_code) {
-        Label ok;
-        __ cmp(eax, 3);
-        __ j(below_equal, &ok);
-        __ int3();
-        __ bind(&ok);
-      }
-      __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers)));
-      __ jmp(eax);
-    }
-    {
-      // Specialized copiers for copy_size <= 8 bytes.
-      Label small_handlers, f0, f1, f2, f3, f4, f5_8;
-      __ bind(&f0);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f1);
-      __ mov_b(eax, Operand(src, 0));
-      __ mov_b(Operand(dst, 0), eax);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f2);
-      __ mov_w(eax, Operand(src, 0));
-      __ mov_w(Operand(dst, 0), eax);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f3);
-      __ mov_w(eax, Operand(src, 0));
-      __ mov_b(edx, Operand(src, 2));
-      __ mov_w(Operand(dst, 0), eax);
-      __ mov_b(Operand(dst, 2), edx);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f4);
-      __ mov(eax, Operand(src, 0));
-      __ mov(Operand(dst, 0), eax);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&f5_8);
-      __ mov(eax, Operand(src, 0));
-      __ mov(edx, Operand(src, count, times_1, -4));
-      __ mov(Operand(dst, 0), eax);
-      __ mov(Operand(dst, count, times_1, -4), edx);
-      MemMoveEmitPopAndReturn(&masm);
-
-      __ bind(&small_handlers);
-      __ dd(conv.address(&f0));
-      __ dd(conv.address(&f1));
-      __ dd(conv.address(&f2));
-      __ dd(conv.address(&f3));
-      __ dd(conv.address(&f4));
-      __ dd(conv.address(&f5_8));
-      __ dd(conv.address(&f5_8));
-      __ dd(conv.address(&f5_8));
-      __ dd(conv.address(&f5_8));
-
-      __ bind(&small_size);  // Entry point into this block.
-      if (FLAG_debug_code) {
-        Label ok;
-        __ cmp(count, 8);
-        __ j(below_equal, &ok);
-        __ int3();
-        __ bind(&ok);
-      }
-      __ mov(eax, Operand(count, times_4, conv.address(&small_handlers)));
-      __ jmp(eax);
-    }
-  } else {
-    // No SSE2.
-    Label forward;
-    __ cmp(count, 0);
-    __ j(equal, &pop_and_return);
-    __ cmp(dst, src);
-    __ j(above, &backward);
-    __ jmp(&forward);
-    {
-      // Simple forward copier.
-      Label forward_loop_1byte, forward_loop_4byte;
-      __ bind(&forward_loop_4byte);
-      __ mov(eax, Operand(src, 0));
-      __ sub(count, Immediate(4));
-      __ add(src, Immediate(4));
-      __ mov(Operand(dst, 0), eax);
-      __ add(dst, Immediate(4));
-      __ bind(&forward);  // Entry point.
-      __ cmp(count, 3);
-      __ j(above, &forward_loop_4byte);
-      __ bind(&forward_loop_1byte);
-      __ cmp(count, 0);
-      __ j(below_equal, &pop_and_return);
-      __ mov_b(eax, Operand(src, 0));
-      __ dec(count);
-      __ inc(src);
-      __ mov_b(Operand(dst, 0), eax);
-      __ inc(dst);
-      __ jmp(&forward_loop_1byte);
+  {
+    // |dst| is a higher address than |src|. Copy backwards.
+    Label unaligned_source, move_first_15, skip_last_move;
+    __ bind(&backward);
+    // |dst| and |src| always point to the end of what's left to copy.
+    __ add(dst, count);
+    __ add(src, count);
+    __ mov(eax, dst);
+    __ sub(eax, src);
+    __ cmp(eax, kMinMoveDistance);
+    __ j(below, &backward_much_overlap);
+    // Copy last 16 bytes.
+    __ movdqu(xmm0, Operand(src, -0x10));
+    __ movdqu(Operand(dst, -0x10), xmm0);
+    // Find distance to alignment: dst & 0xF
+    __ mov(edx, dst);
+    __ and_(edx, 0xF);
+    __ sub(dst, edx);
+    __ sub(src, edx);
+    __ sub(count, edx);
+    // dst is now aligned. Main copy loop.
+    __ mov(loop_count, count);
+    __ shr(loop_count, 6);
+    // Check if src is also aligned.
+    __ test(src, Immediate(0xF));
+    __ j(not_zero, &unaligned_source);
+    // Copy loop for aligned source and destination.
+    MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED);
+    // At most 15 bytes to copy. Copy 16 bytes at beginning of string.
+    __ bind(&move_first_15);
+    __ and_(count, 0xF);
+    __ j(zero, &skip_last_move, Label::kNear);
+    __ sub(src, count);
+    __ sub(dst, count);
+    __ movdqu(xmm0, Operand(src, 0));
+    __ movdqu(Operand(dst, 0), xmm0);
+    __ bind(&skip_last_move);
+    MemMoveEmitPopAndReturn(&masm);
+
+    // Copy loop for unaligned source and aligned destination.
+    __ bind(&unaligned_source);
+    MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED);
+    __ jmp(&move_first_15);
+
+    // Less than kMinMoveDistance offset between dst and src.
+    Label loop_until_aligned, first_15_much_overlap;
+    __ bind(&loop_until_aligned);
+    __ dec(src);
+    __ dec(dst);
+    __ mov_b(eax, Operand(src, 0));
+    __ mov_b(Operand(dst, 0), eax);
+    __ dec(count);
+    __ bind(&backward_much_overlap);  // Entry point into this block.
+    __ test(dst, Immediate(0xF));
+    __ j(not_zero, &loop_until_aligned);
+    // dst is now aligned, src can't be. Main copy loop.
+    __ mov(loop_count, count);
+    __ shr(loop_count, 6);
+    MemMoveEmitMainLoop(&masm, &first_15_much_overlap,
+                        BACKWARD, MOVE_UNALIGNED);
+    __ bind(&first_15_much_overlap);
+    __ and_(count, 0xF);
+    __ j(zero, &pop_and_return);
+    // Small/medium handlers expect dst/src to point to the beginning.
+    __ sub(dst, count);
+    __ sub(src, count);
+    __ cmp(count, kSmallCopySize);
+    __ j(below_equal, &small_size);
+    __ jmp(&medium_size);
+  }
+  {
+    // Special handlers for 9 <= copy_size < 64. No assumptions about
+    // alignment or move distance, so all reads must be unaligned and
+    // must happen before any writes.
+    Label medium_handlers, f9_16, f17_32, f33_48, f49_63;
+
+    __ bind(&f9_16);
+    __ movsd(xmm0, Operand(src, 0));
+    __ movsd(xmm1, Operand(src, count, times_1, -8));
+    __ movsd(Operand(dst, 0), xmm0);
+    __ movsd(Operand(dst, count, times_1, -8), xmm1);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f17_32);
+    __ movdqu(xmm0, Operand(src, 0));
+    __ movdqu(xmm1, Operand(src, count, times_1, -0x10));
+    __ movdqu(Operand(dst, 0x00), xmm0);
+    __ movdqu(Operand(dst, count, times_1, -0x10), xmm1);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f33_48);
+    __ movdqu(xmm0, Operand(src, 0x00));
+    __ movdqu(xmm1, Operand(src, 0x10));
+    __ movdqu(xmm2, Operand(src, count, times_1, -0x10));
+    __ movdqu(Operand(dst, 0x00), xmm0);
+    __ movdqu(Operand(dst, 0x10), xmm1);
+    __ movdqu(Operand(dst, count, times_1, -0x10), xmm2);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f49_63);
+    __ movdqu(xmm0, Operand(src, 0x00));
+    __ movdqu(xmm1, Operand(src, 0x10));
+    __ movdqu(xmm2, Operand(src, 0x20));
+    __ movdqu(xmm3, Operand(src, count, times_1, -0x10));
+    __ movdqu(Operand(dst, 0x00), xmm0);
+    __ movdqu(Operand(dst, 0x10), xmm1);
+    __ movdqu(Operand(dst, 0x20), xmm2);
+    __ movdqu(Operand(dst, count, times_1, -0x10), xmm3);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&medium_handlers);
+    __ dd(conv.address(&f9_16));
+    __ dd(conv.address(&f17_32));
+    __ dd(conv.address(&f33_48));
+    __ dd(conv.address(&f49_63));
+
+    __ bind(&medium_size);  // Entry point into this block.
+    __ mov(eax, count);
+    __ dec(eax);
+    __ shr(eax, 4);
+    if (FLAG_debug_code) {
+      Label ok;
+      __ cmp(eax, 3);
+      __ j(below_equal, &ok);
+      __ int3();
+      __ bind(&ok);
     }
-    {
-      // Simple backward copier.
-      Label backward_loop_1byte, backward_loop_4byte, entry_shortcut;
-      __ bind(&backward);
-      __ add(src, count);
-      __ add(dst, count);
-      __ cmp(count, 3);
-      __ j(below_equal, &entry_shortcut);
-
-      __ bind(&backward_loop_4byte);
-      __ sub(src, Immediate(4));
-      __ sub(count, Immediate(4));
-      __ mov(eax, Operand(src, 0));
-      __ sub(dst, Immediate(4));
-      __ mov(Operand(dst, 0), eax);
-      __ cmp(count, 3);
-      __ j(above, &backward_loop_4byte);
-      __ bind(&backward_loop_1byte);
-      __ cmp(count, 0);
-      __ j(below_equal, &pop_and_return);
-      __ bind(&entry_shortcut);
-      __ dec(src);
-      __ dec(count);
-      __ mov_b(eax, Operand(src, 0));
-      __ dec(dst);
-      __ mov_b(Operand(dst, 0), eax);
-      __ jmp(&backward_loop_1byte);
+    __ mov(eax, Operand(eax, times_4, conv.address(&medium_handlers)));
+    __ jmp(eax);
+  }
+  {
+    // Specialized copiers for copy_size <= 8 bytes.
+    Label small_handlers, f0, f1, f2, f3, f4, f5_8;
+    __ bind(&f0);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f1);
+    __ mov_b(eax, Operand(src, 0));
+    __ mov_b(Operand(dst, 0), eax);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f2);
+    __ mov_w(eax, Operand(src, 0));
+    __ mov_w(Operand(dst, 0), eax);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f3);
+    __ mov_w(eax, Operand(src, 0));
+    __ mov_b(edx, Operand(src, 2));
+    __ mov_w(Operand(dst, 0), eax);
+    __ mov_b(Operand(dst, 2), edx);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f4);
+    __ mov(eax, Operand(src, 0));
+    __ mov(Operand(dst, 0), eax);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&f5_8);
+    __ mov(eax, Operand(src, 0));
+    __ mov(edx, Operand(src, count, times_1, -4));
+    __ mov(Operand(dst, 0), eax);
+    __ mov(Operand(dst, count, times_1, -4), edx);
+    MemMoveEmitPopAndReturn(&masm);
+
+    __ bind(&small_handlers);
+    __ dd(conv.address(&f0));
+    __ dd(conv.address(&f1));
+    __ dd(conv.address(&f2));
+    __ dd(conv.address(&f3));
+    __ dd(conv.address(&f4));
+    __ dd(conv.address(&f5_8));
+    __ dd(conv.address(&f5_8));
+    __ dd(conv.address(&f5_8));
+    __ dd(conv.address(&f5_8));
+
+    __ bind(&small_size);  // Entry point into this block.
+    if (FLAG_debug_code) {
+      Label ok;
+      __ cmp(count, 8);
+      __ j(below_equal, &ok);
+      __ int3();
+      __ bind(&ok);
     }
+    __ mov(eax, Operand(count, times_4, conv.address(&small_handlers)));
+    __ jmp(eax);
   }
 
   __ bind(&pop_and_return);
@@ -671,11 +605,8 @@ void ElementsTransitionGenerator::GenerateSmiToDouble(
   ExternalReference canonical_the_hole_nan_reference =
       ExternalReference::address_of_the_hole_nan();
   XMMRegister the_hole_nan = xmm1;
-  if (CpuFeatures::IsSupported(SSE2)) {
-    CpuFeatureScope use_sse2(masm, SSE2);
-    __ movsd(the_hole_nan,
-              Operand::StaticVariable(canonical_the_hole_nan_reference));
-  }
+  __ movsd(the_hole_nan,
+           Operand::StaticVariable(canonical_the_hole_nan_reference));
   __ jmp(&entry);
 
   // Call into runtime if GC is required.
@@ -696,17 +627,9 @@ void ElementsTransitionGenerator::GenerateSmiToDouble(
 
   // Normal smi, convert it to double and store.
   __ SmiUntag(ebx);
-  if (CpuFeatures::IsSupported(SSE2)) {
-    CpuFeatureScope fscope(masm, SSE2);
-    __ Cvtsi2sd(xmm0, ebx);
-    __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize),
-              xmm0);
-  } else {
-    __ push(ebx);
-    __ fild_s(Operand(esp, 0));
-    __ pop(ebx);
-    __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize));
-  }
+  __ Cvtsi2sd(xmm0, ebx);
+  __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize),
+           xmm0);
   __ jmp(&entry);
 
   // Found hole, store hole_nan_as_double instead.
@@ -717,14 +640,8 @@ void ElementsTransitionGenerator::GenerateSmiToDouble(
     __ Assert(equal, kObjectFoundInSmiOnlyArray);
   }
 
-  if (CpuFeatures::IsSupported(SSE2)) {
-    CpuFeatureScope use_sse2(masm, SSE2);
-    __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize),
-              the_hole_nan);
-  } else {
-    __ fld_d(Operand::StaticVariable(canonical_the_hole_nan_reference));
-    __ fstp_d(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize));
-  }
+  __ movsd(FieldOperand(eax, edi, times_4, FixedDoubleArray::kHeaderSize),
+           the_hole_nan);
 
   __ bind(&entry);
   __ sub(edi, Immediate(Smi::FromInt(1)));
@@ -826,17 +743,9 @@ void ElementsTransitionGenerator::GenerateDoubleToObject(
   // Non-hole double, copy value into a heap number.
   __ AllocateHeapNumber(edx, esi, no_reg, &gc_required);
   // edx: new heap number
-  if (CpuFeatures::IsSupported(SSE2)) {
-    CpuFeatureScope fscope(masm, SSE2);
-    __ movsd(xmm0,
-              FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize));
-    __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0);
-  } else {
-    __ mov(esi, FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize));
-    __ mov(FieldOperand(edx, HeapNumber::kValueOffset), esi);
-    __ mov(esi, FieldOperand(edi, ebx, times_4, offset));
-    __ mov(FieldOperand(edx, HeapNumber::kValueOffset + kPointerSize), esi);
-  }
+  __ movsd(xmm0,
+           FieldOperand(edi, ebx, times_4, FixedDoubleArray::kHeaderSize));
+  __ movsd(FieldOperand(edx, HeapNumber::kValueOffset), xmm0);
   __ mov(FieldOperand(eax, ebx, times_2, FixedArray::kHeaderSize), edx);
   __ mov(esi, ebx);
   __ RecordWriteArray(eax,