OLD | NEW |
1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
176 // Helper functions for CreateMemMoveFunction. | 176 // Helper functions for CreateMemMoveFunction. |
177 #undef __ | 177 #undef __ |
178 #define __ ACCESS_MASM(masm) | 178 #define __ ACCESS_MASM(masm) |
179 | 179 |
180 // Keep around global pointers to these objects so that Valgrind won't complain. | 180 // Keep around global pointers to these objects so that Valgrind won't complain. |
181 static size_t* medium_handlers = NULL; | 181 static size_t* medium_handlers = NULL; |
182 static size_t* small_handlers = NULL; | 182 static size_t* small_handlers = NULL; |
183 | 183 |
184 | 184 |
185 enum Direction { FORWARD, BACKWARD }; | 185 enum Direction { FORWARD, BACKWARD }; |
186 enum Alignment { ALIGNED, UNALIGNED }; | 186 enum Alignment { MOVE_ALIGNED, MOVE_UNALIGNED }; |
187 | 187 |
188 // Expects registers: | 188 // Expects registers: |
189 // esi - source, aligned if alignment == ALIGNED | 189 // esi - source, aligned if alignment == ALIGNED |
190 // edi - destination, always aligned | 190 // edi - destination, always aligned |
191 // ecx - count (copy size in bytes) | 191 // ecx - count (copy size in bytes) |
192 // edx - loop count (number of 64 byte chunks) | 192 // edx - loop count (number of 64 byte chunks) |
193 void MemMoveEmitMainLoop(MacroAssembler* masm, | 193 void MemMoveEmitMainLoop(MacroAssembler* masm, |
194 Label* move_last_15, | 194 Label* move_last_15, |
195 Direction direction, | 195 Direction direction, |
196 Alignment alignment) { | 196 Alignment alignment) { |
197 Register src = esi; | 197 Register src = esi; |
198 Register dst = edi; | 198 Register dst = edi; |
199 Register count = ecx; | 199 Register count = ecx; |
200 Register loop_count = edx; | 200 Register loop_count = edx; |
201 Label loop, move_last_31, move_last_63; | 201 Label loop, move_last_31, move_last_63; |
202 __ cmp(loop_count, 0); | 202 __ cmp(loop_count, 0); |
203 __ j(equal, &move_last_63); | 203 __ j(equal, &move_last_63); |
204 __ bind(&loop); | 204 __ bind(&loop); |
205 // Main loop. Copy in 64 byte chunks. | 205 // Main loop. Copy in 64 byte chunks. |
206 if (direction == BACKWARD) __ sub(src, Immediate(0x40)); | 206 if (direction == BACKWARD) __ sub(src, Immediate(0x40)); |
207 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0x00)); | 207 __ movdq(alignment == MOVE_ALIGNED, xmm0, Operand(src, 0x00)); |
208 __ movdq(alignment == ALIGNED, xmm1, Operand(src, 0x10)); | 208 __ movdq(alignment == MOVE_ALIGNED, xmm1, Operand(src, 0x10)); |
209 __ movdq(alignment == ALIGNED, xmm2, Operand(src, 0x20)); | 209 __ movdq(alignment == MOVE_ALIGNED, xmm2, Operand(src, 0x20)); |
210 __ movdq(alignment == ALIGNED, xmm3, Operand(src, 0x30)); | 210 __ movdq(alignment == MOVE_ALIGNED, xmm3, Operand(src, 0x30)); |
211 if (direction == FORWARD) __ add(src, Immediate(0x40)); | 211 if (direction == FORWARD) __ add(src, Immediate(0x40)); |
212 if (direction == BACKWARD) __ sub(dst, Immediate(0x40)); | 212 if (direction == BACKWARD) __ sub(dst, Immediate(0x40)); |
213 __ movdqa(Operand(dst, 0x00), xmm0); | 213 __ movdqa(Operand(dst, 0x00), xmm0); |
214 __ movdqa(Operand(dst, 0x10), xmm1); | 214 __ movdqa(Operand(dst, 0x10), xmm1); |
215 __ movdqa(Operand(dst, 0x20), xmm2); | 215 __ movdqa(Operand(dst, 0x20), xmm2); |
216 __ movdqa(Operand(dst, 0x30), xmm3); | 216 __ movdqa(Operand(dst, 0x30), xmm3); |
217 if (direction == FORWARD) __ add(dst, Immediate(0x40)); | 217 if (direction == FORWARD) __ add(dst, Immediate(0x40)); |
218 __ dec(loop_count); | 218 __ dec(loop_count); |
219 __ j(not_zero, &loop); | 219 __ j(not_zero, &loop); |
220 // At most 63 bytes left to copy. | 220 // At most 63 bytes left to copy. |
221 __ bind(&move_last_63); | 221 __ bind(&move_last_63); |
222 __ test(count, Immediate(0x20)); | 222 __ test(count, Immediate(0x20)); |
223 __ j(zero, &move_last_31); | 223 __ j(zero, &move_last_31); |
224 if (direction == BACKWARD) __ sub(src, Immediate(0x20)); | 224 if (direction == BACKWARD) __ sub(src, Immediate(0x20)); |
225 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0x00)); | 225 __ movdq(alignment == MOVE_ALIGNED, xmm0, Operand(src, 0x00)); |
226 __ movdq(alignment == ALIGNED, xmm1, Operand(src, 0x10)); | 226 __ movdq(alignment == MOVE_ALIGNED, xmm1, Operand(src, 0x10)); |
227 if (direction == FORWARD) __ add(src, Immediate(0x20)); | 227 if (direction == FORWARD) __ add(src, Immediate(0x20)); |
228 if (direction == BACKWARD) __ sub(dst, Immediate(0x20)); | 228 if (direction == BACKWARD) __ sub(dst, Immediate(0x20)); |
229 __ movdqa(Operand(dst, 0x00), xmm0); | 229 __ movdqa(Operand(dst, 0x00), xmm0); |
230 __ movdqa(Operand(dst, 0x10), xmm1); | 230 __ movdqa(Operand(dst, 0x10), xmm1); |
231 if (direction == FORWARD) __ add(dst, Immediate(0x20)); | 231 if (direction == FORWARD) __ add(dst, Immediate(0x20)); |
232 // At most 31 bytes left to copy. | 232 // At most 31 bytes left to copy. |
233 __ bind(&move_last_31); | 233 __ bind(&move_last_31); |
234 __ test(count, Immediate(0x10)); | 234 __ test(count, Immediate(0x10)); |
235 __ j(zero, move_last_15); | 235 __ j(zero, move_last_15); |
236 if (direction == BACKWARD) __ sub(src, Immediate(0x10)); | 236 if (direction == BACKWARD) __ sub(src, Immediate(0x10)); |
237 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0)); | 237 __ movdq(alignment == MOVE_ALIGNED, xmm0, Operand(src, 0)); |
238 if (direction == FORWARD) __ add(src, Immediate(0x10)); | 238 if (direction == FORWARD) __ add(src, Immediate(0x10)); |
239 if (direction == BACKWARD) __ sub(dst, Immediate(0x10)); | 239 if (direction == BACKWARD) __ sub(dst, Immediate(0x10)); |
240 __ movdqa(Operand(dst, 0), xmm0); | 240 __ movdqa(Operand(dst, 0), xmm0); |
241 if (direction == FORWARD) __ add(dst, Immediate(0x10)); | 241 if (direction == FORWARD) __ add(dst, Immediate(0x10)); |
242 } | 242 } |
243 | 243 |
244 | 244 |
245 void MemMoveEmitPopAndReturn(MacroAssembler* masm) { | 245 void MemMoveEmitPopAndReturn(MacroAssembler* masm) { |
246 __ pop(esi); | 246 __ pop(esi); |
247 __ pop(edi); | 247 __ pop(edi); |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
332 __ add(dst, edx); | 332 __ add(dst, edx); |
333 __ add(src, edx); | 333 __ add(src, edx); |
334 __ sub(count, edx); | 334 __ sub(count, edx); |
335 // dst is now aligned. Main copy loop. | 335 // dst is now aligned. Main copy loop. |
336 __ mov(loop_count, count); | 336 __ mov(loop_count, count); |
337 __ shr(loop_count, 6); | 337 __ shr(loop_count, 6); |
338 // Check if src is also aligned. | 338 // Check if src is also aligned. |
339 __ test(src, Immediate(0xF)); | 339 __ test(src, Immediate(0xF)); |
340 __ j(not_zero, &unaligned_source); | 340 __ j(not_zero, &unaligned_source); |
341 // Copy loop for aligned source and destination. | 341 // Copy loop for aligned source and destination. |
342 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, ALIGNED); | 342 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_ALIGNED); |
343 // At most 15 bytes to copy. Copy 16 bytes at end of string. | 343 // At most 15 bytes to copy. Copy 16 bytes at end of string. |
344 __ bind(&move_last_15); | 344 __ bind(&move_last_15); |
345 __ and_(count, 0xF); | 345 __ and_(count, 0xF); |
346 __ j(zero, &skip_last_move, Label::kNear); | 346 __ j(zero, &skip_last_move, Label::kNear); |
347 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); | 347 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); |
348 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); | 348 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); |
349 __ bind(&skip_last_move); | 349 __ bind(&skip_last_move); |
350 MemMoveEmitPopAndReturn(&masm); | 350 MemMoveEmitPopAndReturn(&masm); |
351 | 351 |
352 // Copy loop for unaligned source and aligned destination. | 352 // Copy loop for unaligned source and aligned destination. |
353 __ bind(&unaligned_source); | 353 __ bind(&unaligned_source); |
354 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, UNALIGNED); | 354 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, MOVE_UNALIGNED); |
355 __ jmp(&move_last_15); | 355 __ jmp(&move_last_15); |
356 | 356 |
357 // Less than kMinMoveDistance offset between dst and src. | 357 // Less than kMinMoveDistance offset between dst and src. |
358 Label loop_until_aligned, last_15_much_overlap; | 358 Label loop_until_aligned, last_15_much_overlap; |
359 __ bind(&loop_until_aligned); | 359 __ bind(&loop_until_aligned); |
360 __ mov_b(eax, Operand(src, 0)); | 360 __ mov_b(eax, Operand(src, 0)); |
361 __ inc(src); | 361 __ inc(src); |
362 __ mov_b(Operand(dst, 0), eax); | 362 __ mov_b(Operand(dst, 0), eax); |
363 __ inc(dst); | 363 __ inc(dst); |
364 __ dec(count); | 364 __ dec(count); |
365 __ bind(&forward_much_overlap); // Entry point into this block. | 365 __ bind(&forward_much_overlap); // Entry point into this block. |
366 __ test(dst, Immediate(0xF)); | 366 __ test(dst, Immediate(0xF)); |
367 __ j(not_zero, &loop_until_aligned); | 367 __ j(not_zero, &loop_until_aligned); |
368 // dst is now aligned, src can't be. Main copy loop. | 368 // dst is now aligned, src can't be. Main copy loop. |
369 __ mov(loop_count, count); | 369 __ mov(loop_count, count); |
370 __ shr(loop_count, 6); | 370 __ shr(loop_count, 6); |
371 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, FORWARD, UNALIGNED); | 371 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, |
| 372 FORWARD, MOVE_UNALIGNED); |
372 __ bind(&last_15_much_overlap); | 373 __ bind(&last_15_much_overlap); |
373 __ and_(count, 0xF); | 374 __ and_(count, 0xF); |
374 __ j(zero, &pop_and_return); | 375 __ j(zero, &pop_and_return); |
375 __ cmp(count, kSmallCopySize); | 376 __ cmp(count, kSmallCopySize); |
376 __ j(below_equal, &small_size); | 377 __ j(below_equal, &small_size); |
377 __ jmp(&medium_size); | 378 __ jmp(&medium_size); |
378 } | 379 } |
379 | 380 |
380 { | 381 { |
381 // |dst| is a higher address than |src|. Copy backwards. | 382 // |dst| is a higher address than |src|. Copy backwards. |
(...skipping 15 matching lines...) Expand all Loading... |
397 __ sub(dst, edx); | 398 __ sub(dst, edx); |
398 __ sub(src, edx); | 399 __ sub(src, edx); |
399 __ sub(count, edx); | 400 __ sub(count, edx); |
400 // dst is now aligned. Main copy loop. | 401 // dst is now aligned. Main copy loop. |
401 __ mov(loop_count, count); | 402 __ mov(loop_count, count); |
402 __ shr(loop_count, 6); | 403 __ shr(loop_count, 6); |
403 // Check if src is also aligned. | 404 // Check if src is also aligned. |
404 __ test(src, Immediate(0xF)); | 405 __ test(src, Immediate(0xF)); |
405 __ j(not_zero, &unaligned_source); | 406 __ j(not_zero, &unaligned_source); |
406 // Copy loop for aligned source and destination. | 407 // Copy loop for aligned source and destination. |
407 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, ALIGNED); | 408 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_ALIGNED); |
408 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. | 409 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. |
409 __ bind(&move_first_15); | 410 __ bind(&move_first_15); |
410 __ and_(count, 0xF); | 411 __ and_(count, 0xF); |
411 __ j(zero, &skip_last_move, Label::kNear); | 412 __ j(zero, &skip_last_move, Label::kNear); |
412 __ sub(src, count); | 413 __ sub(src, count); |
413 __ sub(dst, count); | 414 __ sub(dst, count); |
414 __ movdqu(xmm0, Operand(src, 0)); | 415 __ movdqu(xmm0, Operand(src, 0)); |
415 __ movdqu(Operand(dst, 0), xmm0); | 416 __ movdqu(Operand(dst, 0), xmm0); |
416 __ bind(&skip_last_move); | 417 __ bind(&skip_last_move); |
417 MemMoveEmitPopAndReturn(&masm); | 418 MemMoveEmitPopAndReturn(&masm); |
418 | 419 |
419 // Copy loop for unaligned source and aligned destination. | 420 // Copy loop for unaligned source and aligned destination. |
420 __ bind(&unaligned_source); | 421 __ bind(&unaligned_source); |
421 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, UNALIGNED); | 422 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, MOVE_UNALIGNED); |
422 __ jmp(&move_first_15); | 423 __ jmp(&move_first_15); |
423 | 424 |
424 // Less than kMinMoveDistance offset between dst and src. | 425 // Less than kMinMoveDistance offset between dst and src. |
425 Label loop_until_aligned, first_15_much_overlap; | 426 Label loop_until_aligned, first_15_much_overlap; |
426 __ bind(&loop_until_aligned); | 427 __ bind(&loop_until_aligned); |
427 __ dec(src); | 428 __ dec(src); |
428 __ dec(dst); | 429 __ dec(dst); |
429 __ mov_b(eax, Operand(src, 0)); | 430 __ mov_b(eax, Operand(src, 0)); |
430 __ mov_b(Operand(dst, 0), eax); | 431 __ mov_b(Operand(dst, 0), eax); |
431 __ dec(count); | 432 __ dec(count); |
432 __ bind(&backward_much_overlap); // Entry point into this block. | 433 __ bind(&backward_much_overlap); // Entry point into this block. |
433 __ test(dst, Immediate(0xF)); | 434 __ test(dst, Immediate(0xF)); |
434 __ j(not_zero, &loop_until_aligned); | 435 __ j(not_zero, &loop_until_aligned); |
435 // dst is now aligned, src can't be. Main copy loop. | 436 // dst is now aligned, src can't be. Main copy loop. |
436 __ mov(loop_count, count); | 437 __ mov(loop_count, count); |
437 __ shr(loop_count, 6); | 438 __ shr(loop_count, 6); |
438 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, BACKWARD, UNALIGNED); | 439 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, |
| 440 BACKWARD, MOVE_UNALIGNED); |
439 __ bind(&first_15_much_overlap); | 441 __ bind(&first_15_much_overlap); |
440 __ and_(count, 0xF); | 442 __ and_(count, 0xF); |
441 __ j(zero, &pop_and_return); | 443 __ j(zero, &pop_and_return); |
442 // Small/medium handlers expect dst/src to point to the beginning. | 444 // Small/medium handlers expect dst/src to point to the beginning. |
443 __ sub(dst, count); | 445 __ sub(dst, count); |
444 __ sub(src, count); | 446 __ sub(src, count); |
445 __ cmp(count, kSmallCopySize); | 447 __ cmp(count, kSmallCopySize); |
446 __ j(below_equal, &small_size); | 448 __ j(below_equal, &small_size); |
447 __ jmp(&medium_size); | 449 __ jmp(&medium_size); |
448 } | 450 } |
(...skipping 763 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1212 Code* stub = GetCodeAgeStub(age, parity); | 1214 Code* stub = GetCodeAgeStub(age, parity); |
1213 CodePatcher patcher(sequence, young_length); | 1215 CodePatcher patcher(sequence, young_length); |
1214 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); | 1216 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); |
1215 } | 1217 } |
1216 } | 1218 } |
1217 | 1219 |
1218 | 1220 |
1219 } } // namespace v8::internal | 1221 } } // namespace v8::internal |
1220 | 1222 |
1221 #endif // V8_TARGET_ARCH_IA32 | 1223 #endif // V8_TARGET_ARCH_IA32 |
OLD | NEW |