OLD | NEW |
1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
166 CodeDesc desc; | 166 CodeDesc desc; |
167 masm.GetCode(&desc); | 167 masm.GetCode(&desc); |
168 ASSERT(!RelocInfo::RequiresRelocation(desc)); | 168 ASSERT(!RelocInfo::RequiresRelocation(desc)); |
169 | 169 |
170 CPU::FlushICache(buffer, actual_size); | 170 CPU::FlushICache(buffer, actual_size); |
171 OS::ProtectCode(buffer, actual_size); | 171 OS::ProtectCode(buffer, actual_size); |
172 return FUNCTION_CAST<UnaryMathFunction>(buffer); | 172 return FUNCTION_CAST<UnaryMathFunction>(buffer); |
173 } | 173 } |
174 | 174 |
175 | 175 |
176 static void MemCopyWrapper(void* dest, const void* src, size_t size) { | 176 // Helper functions for CreateMemMoveFunction. |
177 memcpy(dest, src, size); | 177 #undef __ |
| 178 #define __ ACCESS_MASM(masm) |
| 179 |
| 180 // Keep around global pointers to these objects so that Valgrind won't complain. |
| 181 static size_t* medium_handlers = NULL; |
| 182 static size_t* small_handlers = NULL; |
| 183 |
| 184 |
| 185 enum Direction { FORWARD, BACKWARD }; |
| 186 enum Alignment { ALIGNED, UNALIGNED }; |
| 187 |
| 188 // Expects registers: |
| 189 // esi - source, aligned if alignment == ALIGNED |
| 190 // edi - destination, always aligned |
| 191 // ecx - count (copy size in bytes) |
| 192 // edx - loop count (number of 64 byte chunks) |
| 193 void MemMoveEmitMainLoop(MacroAssembler* masm, |
| 194 Label* move_last_15, |
| 195 Direction direction, |
| 196 Alignment alignment) { |
| 197 Register src = esi; |
| 198 Register dst = edi; |
| 199 Register count = ecx; |
| 200 Register loop_count = edx; |
| 201 Label loop, move_last_31, move_last_63; |
| 202 __ cmp(loop_count, 0); |
| 203 __ j(equal, &move_last_63); |
| 204 __ bind(&loop); |
| 205 // Main loop. Copy in 64 byte chunks. |
| 206 if (direction == BACKWARD) __ sub(src, Immediate(0x40)); |
| 207 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0x00)); |
| 208 __ movdq(alignment == ALIGNED, xmm1, Operand(src, 0x10)); |
| 209 __ movdq(alignment == ALIGNED, xmm2, Operand(src, 0x20)); |
| 210 __ movdq(alignment == ALIGNED, xmm3, Operand(src, 0x30)); |
| 211 if (direction == FORWARD) __ add(src, Immediate(0x40)); |
| 212 if (direction == BACKWARD) __ sub(dst, Immediate(0x40)); |
| 213 __ movdqa(Operand(dst, 0x00), xmm0); |
| 214 __ movdqa(Operand(dst, 0x10), xmm1); |
| 215 __ movdqa(Operand(dst, 0x20), xmm2); |
| 216 __ movdqa(Operand(dst, 0x30), xmm3); |
| 217 if (direction == FORWARD) __ add(dst, Immediate(0x40)); |
| 218 __ dec(loop_count); |
| 219 __ j(not_zero, &loop); |
| 220 // At most 63 bytes left to copy. |
| 221 __ bind(&move_last_63); |
| 222 __ test(count, Immediate(0x20)); |
| 223 __ j(zero, &move_last_31); |
| 224 if (direction == BACKWARD) __ sub(src, Immediate(0x20)); |
| 225 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0x00)); |
| 226 __ movdq(alignment == ALIGNED, xmm1, Operand(src, 0x10)); |
| 227 if (direction == FORWARD) __ add(src, Immediate(0x20)); |
| 228 if (direction == BACKWARD) __ sub(dst, Immediate(0x20)); |
| 229 __ movdqa(Operand(dst, 0x00), xmm0); |
| 230 __ movdqa(Operand(dst, 0x10), xmm1); |
| 231 if (direction == FORWARD) __ add(dst, Immediate(0x20)); |
| 232 // At most 31 bytes left to copy. |
| 233 __ bind(&move_last_31); |
| 234 __ test(count, Immediate(0x10)); |
| 235 __ j(zero, move_last_15); |
| 236 if (direction == BACKWARD) __ sub(src, Immediate(0x10)); |
| 237 __ movdq(alignment == ALIGNED, xmm0, Operand(src, 0)); |
| 238 if (direction == FORWARD) __ add(src, Immediate(0x10)); |
| 239 if (direction == BACKWARD) __ sub(dst, Immediate(0x10)); |
| 240 __ movdqa(Operand(dst, 0), xmm0); |
| 241 if (direction == FORWARD) __ add(dst, Immediate(0x10)); |
178 } | 242 } |
179 | 243 |
180 | 244 |
181 OS::MemCopyFunction CreateMemCopyFunction() { | 245 void MemMoveEmitPopAndReturn(MacroAssembler* masm) { |
| 246 __ pop(esi); |
| 247 __ pop(edi); |
| 248 __ ret(0); |
| 249 } |
| 250 |
| 251 |
| 252 #undef __ |
| 253 #define __ masm. |
| 254 |
| 255 |
| 256 OS::MemMoveFunction CreateMemMoveFunction() { |
182 size_t actual_size; | 257 size_t actual_size; |
183 // Allocate buffer in executable space. | 258 // Allocate buffer in executable space. |
184 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, | 259 byte* buffer = static_cast<byte*>(OS::Allocate(1 * KB, &actual_size, true)); |
185 &actual_size, | 260 if (buffer == NULL) return NULL; |
186 true)); | |
187 if (buffer == NULL) return &MemCopyWrapper; | |
188 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); | 261 MacroAssembler masm(NULL, buffer, static_cast<int>(actual_size)); |
189 | 262 |
190 // Generated code is put into a fixed, unmovable, buffer, and not into | 263 // Generated code is put into a fixed, unmovable buffer, and not into |
191 // the V8 heap. We can't, and don't, refer to any relocatable addresses | 264 // the V8 heap. We can't, and don't, refer to any relocatable addresses |
192 // (e.g. the JavaScript nan-object). | 265 // (e.g. the JavaScript nan-object). |
193 | 266 |
194 // 32-bit C declaration function calls pass arguments on stack. | 267 // 32-bit C declaration function calls pass arguments on stack. |
195 | 268 |
196 // Stack layout: | 269 // Stack layout: |
197 // esp[12]: Third argument, size. | 270 // esp[12]: Third argument, size. |
198 // esp[8]: Second argument, source pointer. | 271 // esp[8]: Second argument, source pointer. |
199 // esp[4]: First argument, destination pointer. | 272 // esp[4]: First argument, destination pointer. |
200 // esp[0]: return address | 273 // esp[0]: return address |
201 | 274 |
202 const int kDestinationOffset = 1 * kPointerSize; | 275 const int kDestinationOffset = 1 * kPointerSize; |
203 const int kSourceOffset = 2 * kPointerSize; | 276 const int kSourceOffset = 2 * kPointerSize; |
204 const int kSizeOffset = 3 * kPointerSize; | 277 const int kSizeOffset = 3 * kPointerSize; |
205 | 278 |
| 279 // When copying up to this many bytes, use special "small" handlers. |
| 280 const size_t kSmallCopySize = 8; |
| 281 // When copying up to this many bytes, use special "medium" handlers. |
| 282 const size_t kMediumCopySize = 63; |
| 283 // When non-overlapping region of src and dst is less than this, |
| 284 // use a more careful implementation (slightly slower). |
| 285 const size_t kMinMoveDistance = 16; |
| 286 // Note that these values are dictated by the implementation below, |
| 287 // do not just change them and hope things will work! |
| 288 |
206 int stack_offset = 0; // Update if we change the stack height. | 289 int stack_offset = 0; // Update if we change the stack height. |
207 | 290 |
208 if (FLAG_debug_code) { | 291 Label backward, backward_much_overlap; |
209 __ cmp(Operand(esp, kSizeOffset + stack_offset), | 292 Label forward_much_overlap, small_size, medium_size, pop_and_return; |
210 Immediate(OS::kMinComplexMemCopy)); | 293 __ push(edi); |
211 Label ok; | 294 __ push(esi); |
212 __ j(greater_equal, &ok); | 295 stack_offset += 2 * kPointerSize; |
213 __ int3(); | 296 Register dst = edi; |
214 __ bind(&ok); | 297 Register src = esi; |
215 } | 298 Register count = ecx; |
| 299 Register loop_count = edx; |
| 300 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); |
| 301 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); |
| 302 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); |
| 303 |
| 304 __ cmp(dst, src); |
| 305 __ j(equal, &pop_and_return); |
| 306 |
216 if (CpuFeatures::IsSupported(SSE2)) { | 307 if (CpuFeatures::IsSupported(SSE2)) { |
217 CpuFeatureScope enable(&masm, SSE2); | 308 CpuFeatureScope sse2_scope(&masm, SSE2); |
218 __ push(edi); | 309 __ prefetch(Operand(src, 0), 1); |
219 __ push(esi); | 310 __ cmp(count, kSmallCopySize); |
220 stack_offset += 2 * kPointerSize; | 311 __ j(below_equal, &small_size); |
221 Register dst = edi; | 312 __ cmp(count, kMediumCopySize); |
222 Register src = esi; | 313 __ j(below_equal, &medium_size); |
223 Register count = ecx; | 314 __ cmp(dst, src); |
224 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); | 315 __ j(above, &backward); |
225 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); | 316 |
226 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); | 317 { |
227 | 318 // |dst| is a lower address than |src|. Copy front-to-back. |
228 | 319 Label unaligned_source, move_last_15, skip_last_move; |
229 __ movdqu(xmm0, Operand(src, 0)); | 320 __ mov(eax, src); |
230 __ movdqu(Operand(dst, 0), xmm0); | 321 __ sub(eax, dst); |
231 __ mov(edx, dst); | 322 __ cmp(eax, kMinMoveDistance); |
232 __ and_(edx, 0xF); | 323 __ j(below, &forward_much_overlap); |
233 __ neg(edx); | 324 // Copy first 16 bytes. |
234 __ add(edx, Immediate(16)); | 325 __ movdqu(xmm0, Operand(src, 0)); |
235 __ add(dst, edx); | 326 __ movdqu(Operand(dst, 0), xmm0); |
236 __ add(src, edx); | 327 // Determine distance to alignment: 16 - (dst & 0xF). |
237 __ sub(count, edx); | 328 __ mov(edx, dst); |
238 | 329 __ and_(edx, 0xF); |
239 // edi is now aligned. Check if esi is also aligned. | 330 __ neg(edx); |
240 Label unaligned_source; | 331 __ add(edx, Immediate(16)); |
241 __ test(src, Immediate(0x0F)); | 332 __ add(dst, edx); |
242 __ j(not_zero, &unaligned_source); | 333 __ add(src, edx); |
243 { | 334 __ sub(count, edx); |
| 335 // dst is now aligned. Main copy loop. |
| 336 __ mov(loop_count, count); |
| 337 __ shr(loop_count, 6); |
| 338 // Check if src is also aligned. |
| 339 __ test(src, Immediate(0xF)); |
| 340 __ j(not_zero, &unaligned_source); |
244 // Copy loop for aligned source and destination. | 341 // Copy loop for aligned source and destination. |
245 __ mov(edx, count); | 342 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, ALIGNED); |
246 Register loop_count = ecx; | |
247 Register count = edx; | |
248 __ shr(loop_count, 5); | |
249 { | |
250 // Main copy loop. | |
251 Label loop; | |
252 __ bind(&loop); | |
253 __ prefetch(Operand(src, 0x20), 1); | |
254 __ movdqa(xmm0, Operand(src, 0x00)); | |
255 __ movdqa(xmm1, Operand(src, 0x10)); | |
256 __ add(src, Immediate(0x20)); | |
257 | |
258 __ movdqa(Operand(dst, 0x00), xmm0); | |
259 __ movdqa(Operand(dst, 0x10), xmm1); | |
260 __ add(dst, Immediate(0x20)); | |
261 | |
262 __ dec(loop_count); | |
263 __ j(not_zero, &loop); | |
264 } | |
265 | |
266 // At most 31 bytes to copy. | |
267 Label move_less_16; | |
268 __ test(count, Immediate(0x10)); | |
269 __ j(zero, &move_less_16); | |
270 __ movdqa(xmm0, Operand(src, 0)); | |
271 __ add(src, Immediate(0x10)); | |
272 __ movdqa(Operand(dst, 0), xmm0); | |
273 __ add(dst, Immediate(0x10)); | |
274 __ bind(&move_less_16); | |
275 | |
276 // At most 15 bytes to copy. Copy 16 bytes at end of string. | 343 // At most 15 bytes to copy. Copy 16 bytes at end of string. |
| 344 __ bind(&move_last_15); |
277 __ and_(count, 0xF); | 345 __ and_(count, 0xF); |
| 346 __ j(zero, &skip_last_move, Label::kNear); |
278 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); | 347 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); |
279 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); | 348 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); |
280 | 349 __ bind(&skip_last_move); |
281 __ mov(eax, Operand(esp, stack_offset + kDestinationOffset)); | 350 MemMoveEmitPopAndReturn(&masm); |
282 __ pop(esi); | 351 |
283 __ pop(edi); | |
284 __ ret(0); | |
285 } | |
286 __ Align(16); | |
287 { | |
288 // Copy loop for unaligned source and aligned destination. | 352 // Copy loop for unaligned source and aligned destination. |
289 // If source is not aligned, we can't read it as efficiently. | |
290 __ bind(&unaligned_source); | 353 __ bind(&unaligned_source); |
291 __ mov(edx, ecx); | 354 MemMoveEmitMainLoop(&masm, &move_last_15, FORWARD, UNALIGNED); |
292 Register loop_count = ecx; | 355 __ jmp(&move_last_15); |
293 Register count = edx; | 356 |
294 __ shr(loop_count, 5); | 357 // Less than kMinMoveDistance offset between dst and src. |
295 { | 358 Label loop_until_aligned, last_15_much_overlap; |
296 // Main copy loop | 359 __ bind(&loop_until_aligned); |
297 Label loop; | 360 __ mov_b(eax, Operand(src, 0)); |
298 __ bind(&loop); | 361 __ inc(src); |
299 __ prefetch(Operand(src, 0x20), 1); | 362 __ mov_b(Operand(dst, 0), eax); |
300 __ movdqu(xmm0, Operand(src, 0x00)); | 363 __ inc(dst); |
301 __ movdqu(xmm1, Operand(src, 0x10)); | 364 __ dec(count); |
302 __ add(src, Immediate(0x20)); | 365 __ bind(&forward_much_overlap); // Entry point into this block. |
303 | 366 __ test(dst, Immediate(0xF)); |
304 __ movdqa(Operand(dst, 0x00), xmm0); | 367 __ j(not_zero, &loop_until_aligned); |
305 __ movdqa(Operand(dst, 0x10), xmm1); | 368 // dst is now aligned, src can't be. Main copy loop. |
306 __ add(dst, Immediate(0x20)); | 369 __ mov(loop_count, count); |
307 | 370 __ shr(loop_count, 6); |
308 __ dec(loop_count); | 371 MemMoveEmitMainLoop(&masm, &last_15_much_overlap, FORWARD, UNALIGNED); |
309 __ j(not_zero, &loop); | 372 __ bind(&last_15_much_overlap); |
| 373 __ and_(count, 0xF); |
| 374 __ j(zero, &pop_and_return); |
| 375 __ cmp(count, kSmallCopySize); |
| 376 __ j(below_equal, &small_size); |
| 377 __ jmp(&medium_size); |
| 378 } |
| 379 |
| 380 { |
| 381 // |dst| is a higher address than |src|. Copy backwards. |
| 382 Label unaligned_source, move_first_15, skip_last_move; |
| 383 __ bind(&backward); |
| 384 // |dst| and |src| always point to the end of what's left to copy. |
| 385 __ add(dst, count); |
| 386 __ add(src, count); |
| 387 __ mov(eax, dst); |
| 388 __ sub(eax, src); |
| 389 __ cmp(eax, kMinMoveDistance); |
| 390 __ j(below, &backward_much_overlap); |
| 391 // Copy last 16 bytes. |
| 392 __ movdqu(xmm0, Operand(src, -0x10)); |
| 393 __ movdqu(Operand(dst, -0x10), xmm0); |
| 394 // Find distance to alignment: dst & 0xF |
| 395 __ mov(edx, dst); |
| 396 __ and_(edx, 0xF); |
| 397 __ sub(dst, edx); |
| 398 __ sub(src, edx); |
| 399 __ sub(count, edx); |
| 400 // dst is now aligned. Main copy loop. |
| 401 __ mov(loop_count, count); |
| 402 __ shr(loop_count, 6); |
| 403 // Check if src is also aligned. |
| 404 __ test(src, Immediate(0xF)); |
| 405 __ j(not_zero, &unaligned_source); |
| 406 // Copy loop for aligned source and destination. |
| 407 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, ALIGNED); |
| 408 // At most 15 bytes to copy. Copy 16 bytes at beginning of string. |
| 409 __ bind(&move_first_15); |
| 410 __ and_(count, 0xF); |
| 411 __ j(zero, &skip_last_move, Label::kNear); |
| 412 __ sub(src, count); |
| 413 __ sub(dst, count); |
| 414 __ movdqu(xmm0, Operand(src, 0)); |
| 415 __ movdqu(Operand(dst, 0), xmm0); |
| 416 __ bind(&skip_last_move); |
| 417 MemMoveEmitPopAndReturn(&masm); |
| 418 |
| 419 // Copy loop for unaligned source and aligned destination. |
| 420 __ bind(&unaligned_source); |
| 421 MemMoveEmitMainLoop(&masm, &move_first_15, BACKWARD, UNALIGNED); |
| 422 __ jmp(&move_first_15); |
| 423 |
| 424 // Less than kMinMoveDistance offset between dst and src. |
| 425 Label loop_until_aligned, first_15_much_overlap; |
| 426 __ bind(&loop_until_aligned); |
| 427 __ dec(src); |
| 428 __ dec(dst); |
| 429 __ mov_b(eax, Operand(src, 0)); |
| 430 __ mov_b(Operand(dst, 0), eax); |
| 431 __ dec(count); |
| 432 __ bind(&backward_much_overlap); // Entry point into this block. |
| 433 __ test(dst, Immediate(0xF)); |
| 434 __ j(not_zero, &loop_until_aligned); |
| 435 // dst is now aligned, src can't be. Main copy loop. |
| 436 __ mov(loop_count, count); |
| 437 __ shr(loop_count, 6); |
| 438 MemMoveEmitMainLoop(&masm, &first_15_much_overlap, BACKWARD, UNALIGNED); |
| 439 __ bind(&first_15_much_overlap); |
| 440 __ and_(count, 0xF); |
| 441 __ j(zero, &pop_and_return); |
| 442 // Small/medium handlers expect dst/src to point to the beginning. |
| 443 __ sub(dst, count); |
| 444 __ sub(src, count); |
| 445 __ cmp(count, kSmallCopySize); |
| 446 __ j(below_equal, &small_size); |
| 447 __ jmp(&medium_size); |
| 448 } |
| 449 { |
| 450 // Special handlers for 9 <= copy_size < 64. No assumptions about |
| 451 // alignment or move distance, so all reads must be unaligned and |
| 452 // must happen before any writes. |
| 453 Label f9_16, f17_32, f33_48, f49_63; |
| 454 |
| 455 __ bind(&f9_16); |
| 456 __ movdbl(xmm0, Operand(src, 0)); |
| 457 __ movdbl(xmm1, Operand(src, count, times_1, -8)); |
| 458 __ movdbl(Operand(dst, 0), xmm0); |
| 459 __ movdbl(Operand(dst, count, times_1, -8), xmm1); |
| 460 MemMoveEmitPopAndReturn(&masm); |
| 461 |
| 462 __ bind(&f17_32); |
| 463 __ movdqu(xmm0, Operand(src, 0)); |
| 464 __ movdqu(xmm1, Operand(src, count, times_1, -0x10)); |
| 465 __ movdqu(Operand(dst, 0x00), xmm0); |
| 466 __ movdqu(Operand(dst, count, times_1, -0x10), xmm1); |
| 467 MemMoveEmitPopAndReturn(&masm); |
| 468 |
| 469 __ bind(&f33_48); |
| 470 __ movdqu(xmm0, Operand(src, 0x00)); |
| 471 __ movdqu(xmm1, Operand(src, 0x10)); |
| 472 __ movdqu(xmm2, Operand(src, count, times_1, -0x10)); |
| 473 __ movdqu(Operand(dst, 0x00), xmm0); |
| 474 __ movdqu(Operand(dst, 0x10), xmm1); |
| 475 __ movdqu(Operand(dst, count, times_1, -0x10), xmm2); |
| 476 MemMoveEmitPopAndReturn(&masm); |
| 477 |
| 478 __ bind(&f49_63); |
| 479 __ movdqu(xmm0, Operand(src, 0x00)); |
| 480 __ movdqu(xmm1, Operand(src, 0x10)); |
| 481 __ movdqu(xmm2, Operand(src, 0x20)); |
| 482 __ movdqu(xmm3, Operand(src, count, times_1, -0x10)); |
| 483 __ movdqu(Operand(dst, 0x00), xmm0); |
| 484 __ movdqu(Operand(dst, 0x10), xmm1); |
| 485 __ movdqu(Operand(dst, 0x20), xmm2); |
| 486 __ movdqu(Operand(dst, count, times_1, -0x10), xmm3); |
| 487 MemMoveEmitPopAndReturn(&masm); |
| 488 |
| 489 medium_handlers = new size_t[4]; |
| 490 medium_handlers[0] = reinterpret_cast<intptr_t>(buffer) + f9_16.pos(); |
| 491 medium_handlers[1] = reinterpret_cast<intptr_t>(buffer) + f17_32.pos(); |
| 492 medium_handlers[2] = reinterpret_cast<intptr_t>(buffer) + f33_48.pos(); |
| 493 medium_handlers[3] = reinterpret_cast<intptr_t>(buffer) + f49_63.pos(); |
| 494 |
| 495 __ bind(&medium_size); // Entry point into this block. |
| 496 __ mov(eax, count); |
| 497 __ dec(eax); |
| 498 __ shr(eax, 4); |
| 499 if (FLAG_debug_code) { |
| 500 Label ok; |
| 501 __ cmp(eax, 3); |
| 502 __ j(below_equal, &ok); |
| 503 __ int3(); |
| 504 __ bind(&ok); |
310 } | 505 } |
311 | 506 __ mov(eax, Operand(eax, times_4, |
312 // At most 31 bytes to copy. | 507 reinterpret_cast<intptr_t>(medium_handlers))); |
313 Label move_less_16; | 508 __ jmp(eax); |
314 __ test(count, Immediate(0x10)); | 509 } |
315 __ j(zero, &move_less_16); | 510 { |
316 __ movdqu(xmm0, Operand(src, 0)); | 511 // Specialized copiers for copy_size <= 8 bytes. |
317 __ add(src, Immediate(0x10)); | 512 Label f0, f1, f2, f3, f4, f5_8; |
318 __ movdqa(Operand(dst, 0), xmm0); | 513 __ bind(&f0); |
319 __ add(dst, Immediate(0x10)); | 514 MemMoveEmitPopAndReturn(&masm); |
320 __ bind(&move_less_16); | 515 |
321 | 516 __ bind(&f1); |
322 // At most 15 bytes to copy. Copy 16 bytes at end of string. | 517 __ mov_b(eax, Operand(src, 0)); |
323 __ and_(count, 0x0F); | 518 __ mov_b(Operand(dst, 0), eax); |
324 __ movdqu(xmm0, Operand(src, count, times_1, -0x10)); | 519 MemMoveEmitPopAndReturn(&masm); |
325 __ movdqu(Operand(dst, count, times_1, -0x10), xmm0); | 520 |
326 | 521 __ bind(&f2); |
327 __ mov(eax, Operand(esp, stack_offset + kDestinationOffset)); | 522 __ mov_w(eax, Operand(src, 0)); |
328 __ pop(esi); | 523 __ mov_w(Operand(dst, 0), eax); |
329 __ pop(edi); | 524 MemMoveEmitPopAndReturn(&masm); |
330 __ ret(0); | 525 |
331 } | 526 __ bind(&f3); |
332 | 527 __ mov_w(eax, Operand(src, 0)); |
| 528 __ mov_b(edx, Operand(src, 2)); |
| 529 __ mov_w(Operand(dst, 0), eax); |
| 530 __ mov_b(Operand(dst, 2), edx); |
| 531 MemMoveEmitPopAndReturn(&masm); |
| 532 |
| 533 __ bind(&f4); |
| 534 __ mov(eax, Operand(src, 0)); |
| 535 __ mov(Operand(dst, 0), eax); |
| 536 MemMoveEmitPopAndReturn(&masm); |
| 537 |
| 538 __ bind(&f5_8); |
| 539 __ mov(eax, Operand(src, 0)); |
| 540 __ mov(edx, Operand(src, count, times_1, -4)); |
| 541 __ mov(Operand(dst, 0), eax); |
| 542 __ mov(Operand(dst, count, times_1, -4), edx); |
| 543 MemMoveEmitPopAndReturn(&masm); |
| 544 |
| 545 small_handlers = new size_t[9]; |
| 546 small_handlers[0] = reinterpret_cast<intptr_t>(buffer) + f0.pos(); |
| 547 small_handlers[1] = reinterpret_cast<intptr_t>(buffer) + f1.pos(); |
| 548 small_handlers[2] = reinterpret_cast<intptr_t>(buffer) + f2.pos(); |
| 549 small_handlers[3] = reinterpret_cast<intptr_t>(buffer) + f3.pos(); |
| 550 small_handlers[4] = reinterpret_cast<intptr_t>(buffer) + f4.pos(); |
| 551 small_handlers[5] = reinterpret_cast<intptr_t>(buffer) + f5_8.pos(); |
| 552 small_handlers[6] = reinterpret_cast<intptr_t>(buffer) + f5_8.pos(); |
| 553 small_handlers[7] = reinterpret_cast<intptr_t>(buffer) + f5_8.pos(); |
| 554 small_handlers[8] = reinterpret_cast<intptr_t>(buffer) + f5_8.pos(); |
| 555 |
| 556 __ bind(&small_size); // Entry point into this block. |
| 557 if (FLAG_debug_code) { |
| 558 Label ok; |
| 559 __ cmp(count, 8); |
| 560 __ j(below_equal, &ok); |
| 561 __ int3(); |
| 562 __ bind(&ok); |
| 563 } |
| 564 __ mov(eax, Operand(count, times_4, |
| 565 reinterpret_cast<intptr_t>(small_handlers))); |
| 566 __ jmp(eax); |
| 567 } |
333 } else { | 568 } else { |
334 // SSE2 not supported. Unlikely to happen in practice. | 569 // No SSE2. |
335 __ push(edi); | 570 Label forward; |
336 __ push(esi); | 571 __ cmp(count, 0); |
337 stack_offset += 2 * kPointerSize; | 572 __ j(equal, &pop_and_return); |
338 __ cld(); | 573 __ cmp(dst, src); |
339 Register dst = edi; | 574 __ j(above, &backward); |
340 Register src = esi; | 575 __ jmp(&forward); |
341 Register count = ecx; | 576 { |
342 __ mov(dst, Operand(esp, stack_offset + kDestinationOffset)); | 577 // Simple forward copier. |
343 __ mov(src, Operand(esp, stack_offset + kSourceOffset)); | 578 Label forward_loop_1byte, forward_loop_4byte; |
344 __ mov(count, Operand(esp, stack_offset + kSizeOffset)); | 579 __ bind(&forward_loop_4byte); |
345 | 580 __ mov(eax, Operand(src, 0)); |
346 // Copy the first word. | 581 __ sub(count, Immediate(4)); |
347 __ mov(eax, Operand(src, 0)); | 582 __ add(src, Immediate(4)); |
348 __ mov(Operand(dst, 0), eax); | 583 __ mov(Operand(dst, 0), eax); |
349 | 584 __ add(dst, Immediate(4)); |
350 // Increment src,dstso that dst is aligned. | 585 __ bind(&forward); // Entry point. |
351 __ mov(edx, dst); | 586 __ cmp(count, 3); |
352 __ and_(edx, 0x03); | 587 __ j(above, &forward_loop_4byte); |
353 __ neg(edx); | 588 __ bind(&forward_loop_1byte); |
354 __ add(edx, Immediate(4)); // edx = 4 - (dst & 3) | 589 __ cmp(count, 0); |
355 __ add(dst, edx); | 590 __ j(below_equal, &pop_and_return); |
356 __ add(src, edx); | 591 __ mov_b(eax, Operand(src, 0)); |
357 __ sub(count, edx); | 592 __ dec(count); |
358 // edi is now aligned, ecx holds number of remaning bytes to copy. | 593 __ inc(src); |
359 | 594 __ mov_b(Operand(dst, 0), eax); |
360 __ mov(edx, count); | 595 __ inc(dst); |
361 count = edx; | 596 __ jmp(&forward_loop_1byte); |
362 __ shr(ecx, 2); // Make word count instead of byte count. | 597 } |
363 __ rep_movs(); | 598 { |
364 | 599 // Simple backward copier. |
365 // At most 3 bytes left to copy. Copy 4 bytes at end of string. | 600 Label backward_loop_1byte, backward_loop_4byte, entry_shortcut; |
366 __ and_(count, 3); | 601 __ bind(&backward); |
367 __ mov(eax, Operand(src, count, times_1, -4)); | 602 __ add(src, count); |
368 __ mov(Operand(dst, count, times_1, -4), eax); | 603 __ add(dst, count); |
369 | 604 __ cmp(count, 3); |
370 __ mov(eax, Operand(esp, stack_offset + kDestinationOffset)); | 605 __ j(below_equal, &entry_shortcut); |
371 __ pop(esi); | 606 |
372 __ pop(edi); | 607 __ bind(&backward_loop_4byte); |
373 __ ret(0); | 608 __ sub(src, Immediate(4)); |
| 609 __ sub(count, Immediate(4)); |
| 610 __ mov(eax, Operand(src, 0)); |
| 611 __ sub(dst, Immediate(4)); |
| 612 __ mov(Operand(dst, 0), eax); |
| 613 __ cmp(count, 3); |
| 614 __ j(above, &backward_loop_4byte); |
| 615 __ bind(&backward_loop_1byte); |
| 616 __ cmp(count, 0); |
| 617 __ j(below_equal, &pop_and_return); |
| 618 __ bind(&entry_shortcut); |
| 619 __ dec(src); |
| 620 __ dec(count); |
| 621 __ mov_b(eax, Operand(src, 0)); |
| 622 __ dec(dst); |
| 623 __ mov_b(Operand(dst, 0), eax); |
| 624 __ jmp(&backward_loop_1byte); |
| 625 } |
374 } | 626 } |
375 | 627 |
| 628 __ bind(&pop_and_return); |
| 629 MemMoveEmitPopAndReturn(&masm); |
| 630 |
376 CodeDesc desc; | 631 CodeDesc desc; |
377 masm.GetCode(&desc); | 632 masm.GetCode(&desc); |
378 ASSERT(!RelocInfo::RequiresRelocation(desc)); | 633 ASSERT(!RelocInfo::RequiresRelocation(desc)); |
379 | |
380 CPU::FlushICache(buffer, actual_size); | 634 CPU::FlushICache(buffer, actual_size); |
381 OS::ProtectCode(buffer, actual_size); | 635 OS::ProtectCode(buffer, actual_size); |
382 return FUNCTION_CAST<OS::MemCopyFunction>(buffer); | 636 return FUNCTION_CAST<OS::MemMoveFunction>(buffer); |
383 } | 637 } |
384 | 638 |
| 639 |
385 #undef __ | 640 #undef __ |
386 | 641 |
387 // ------------------------------------------------------------------------- | 642 // ------------------------------------------------------------------------- |
388 // Code generators | 643 // Code generators |
389 | 644 |
390 #define __ ACCESS_MASM(masm) | 645 #define __ ACCESS_MASM(masm) |
391 | 646 |
392 | 647 |
393 void ElementsTransitionGenerator::GenerateMapChangeElementsTransition( | 648 void ElementsTransitionGenerator::GenerateMapChangeElementsTransition( |
394 MacroAssembler* masm, AllocationSiteMode mode, | 649 MacroAssembler* masm, AllocationSiteMode mode, |
(...skipping 562 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
957 Code* stub = GetCodeAgeStub(age, parity); | 1212 Code* stub = GetCodeAgeStub(age, parity); |
958 CodePatcher patcher(sequence, young_length); | 1213 CodePatcher patcher(sequence, young_length); |
959 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); | 1214 patcher.masm()->call(stub->instruction_start(), RelocInfo::NONE32); |
960 } | 1215 } |
961 } | 1216 } |
962 | 1217 |
963 | 1218 |
964 } } // namespace v8::internal | 1219 } } // namespace v8::internal |
965 | 1220 |
966 #endif // V8_TARGET_ARCH_IA32 | 1221 #endif // V8_TARGET_ARCH_IA32 |
OLD | NEW |