| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved. | 2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved. |
| 3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). | 3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). |
| 4 * | 4 * |
| 5 * This library is free software; you can redistribute it and/or | 5 * This library is free software; you can redistribute it and/or |
| 6 * modify it under the terms of the GNU Library General Public | 6 * modify it under the terms of the GNU Library General Public |
| 7 * License as published by the Free Software Foundation; either | 7 * License as published by the Free Software Foundation; either |
| 8 * version 2 of the License, or (at your option) any later version. | 8 * version 2 of the License, or (at your option) any later version. |
| 9 * | 9 * |
| 10 * This library is distributed in the hope that it will be useful, | 10 * This library is distributed in the hope that it will be useful, |
| (...skipping 21 matching lines...) Expand all Loading... |
| 32 #include <emmintrin.h> | 32 #include <emmintrin.h> |
| 33 #endif | 33 #endif |
| 34 | 34 |
| 35 namespace WTF { | 35 namespace WTF { |
| 36 | 36 |
| 37 // Assuming that a pointer is the size of a "machine word", then | 37 // Assuming that a pointer is the size of a "machine word", then |
| 38 // uintptr_t is an integer type that is also a machine word. | 38 // uintptr_t is an integer type that is also a machine word. |
| 39 typedef uintptr_t MachineWord; | 39 typedef uintptr_t MachineWord; |
| 40 const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1; | 40 const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1; |
| 41 | 41 |
| 42 inline bool isAlignedToMachineWord(const void* pointer) | 42 inline bool isAlignedToMachineWord(const void* pointer) { |
| 43 { | 43 return !(reinterpret_cast<uintptr_t>(pointer) & machineWordAlignmentMask); |
| 44 return !(reinterpret_cast<uintptr_t>(pointer) & machineWordAlignmentMask); | |
| 45 } | 44 } |
| 46 | 45 |
| 47 template<typename T> inline T* alignToMachineWord(T* pointer) | 46 template <typename T> |
| 48 { | 47 inline T* alignToMachineWord(T* pointer) { |
| 49 return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & ~machineW
ordAlignmentMask); | 48 return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & |
| 49 ~machineWordAlignmentMask); |
| 50 } | 50 } |
| 51 | 51 |
| 52 template<size_t size, typename CharacterType> struct NonASCIIMask; | 52 template <size_t size, typename CharacterType> |
| 53 template<> struct NonASCIIMask<4, UChar> { | 53 struct NonASCIIMask; |
| 54 static inline uint32_t value() { return 0xFF80FF80U; } | 54 template <> |
| 55 struct NonASCIIMask<4, UChar> { |
| 56 static inline uint32_t value() { return 0xFF80FF80U; } |
| 55 }; | 57 }; |
| 56 template<> struct NonASCIIMask<4, LChar> { | 58 template <> |
| 57 static inline uint32_t value() { return 0x80808080U; } | 59 struct NonASCIIMask<4, LChar> { |
| 60 static inline uint32_t value() { return 0x80808080U; } |
| 58 }; | 61 }; |
| 59 template<> struct NonASCIIMask<8, UChar> { | 62 template <> |
| 60 static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; } | 63 struct NonASCIIMask<8, UChar> { |
| 64 static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; } |
| 61 }; | 65 }; |
| 62 template<> struct NonASCIIMask<8, LChar> { | 66 template <> |
| 63 static inline uint64_t value() { return 0x8080808080808080ULL; } | 67 struct NonASCIIMask<8, LChar> { |
| 68 static inline uint64_t value() { return 0x8080808080808080ULL; } |
| 64 }; | 69 }; |
| 65 | 70 |
| 66 | 71 template <typename CharacterType> |
| 67 template<typename CharacterType> | 72 inline bool isAllASCII(MachineWord word) { |
| 68 inline bool isAllASCII(MachineWord word) | 73 return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value()); |
| 69 { | |
| 70 return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value()); | |
| 71 } | 74 } |
| 72 | 75 |
| 73 // Note: This function assume the input is likely all ASCII, and | 76 // Note: This function assume the input is likely all ASCII, and |
| 74 // does not leave early if it is not the case. | 77 // does not leave early if it is not the case. |
| 75 template<typename CharacterType> | 78 template <typename CharacterType> |
| 76 inline bool charactersAreAllASCII(const CharacterType* characters, size_t length
) | 79 inline bool charactersAreAllASCII(const CharacterType* characters, |
| 77 { | 80 size_t length) { |
| 78 MachineWord allCharBits = 0; | 81 MachineWord allCharBits = 0; |
| 79 const CharacterType* end = characters + length; | 82 const CharacterType* end = characters + length; |
| 80 | 83 |
| 81 // Prologue: align the input. | 84 // Prologue: align the input. |
| 82 while (!isAlignedToMachineWord(characters) && characters != end) { | 85 while (!isAlignedToMachineWord(characters) && characters != end) { |
| 83 allCharBits |= *characters; | 86 allCharBits |= *characters; |
| 84 ++characters; | 87 ++characters; |
| 85 } | 88 } |
| 86 | 89 |
| 87 // Compare the values of CPU word size. | 90 // Compare the values of CPU word size. |
| 88 const CharacterType* wordEnd = alignToMachineWord(end); | 91 const CharacterType* wordEnd = alignToMachineWord(end); |
| 89 const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType); | 92 const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType); |
| 90 while (characters < wordEnd) { | 93 while (characters < wordEnd) { |
| 91 allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters)); | 94 allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters)); |
| 92 characters += loopIncrement; | 95 characters += loopIncrement; |
| 93 } | 96 } |
| 94 | 97 |
| 95 // Process the remaining bytes. | 98 // Process the remaining bytes. |
| 96 while (characters != end) { | 99 while (characters != end) { |
| 97 allCharBits |= *characters; | 100 allCharBits |= *characters; |
| 98 ++characters; | 101 ++characters; |
| 99 } | 102 } |
| 100 | 103 |
| 101 MachineWord nonASCIIBitMask = NonASCIIMask<sizeof(MachineWord), CharacterTyp
e>::value(); | 104 MachineWord nonASCIIBitMask = |
| 102 return !(allCharBits & nonASCIIBitMask); | 105 NonASCIIMask<sizeof(MachineWord), CharacterType>::value(); |
| 106 return !(allCharBits & nonASCIIBitMask); |
| 103 } | 107 } |
| 104 | 108 |
| 105 inline void copyLCharsFromUCharSource(LChar* destination, const UChar* source, s
ize_t length) | 109 inline void copyLCharsFromUCharSource(LChar* destination, |
| 106 { | 110 const UChar* source, |
| 111 size_t length) { |
| 107 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64)) | 112 #if OS(MACOSX) && (CPU(X86) || CPU(X86_64)) |
| 108 const uintptr_t memoryAccessSize = 16; // Memory accesses on 16 byte (128 bi
t) alignment | 113 const uintptr_t memoryAccessSize = |
| 114 16; // Memory accesses on 16 byte (128 bit) alignment |
| 115 const uintptr_t memoryAccessMask = memoryAccessSize - 1; |
| 116 |
| 117 size_t i = 0; |
| 118 for (; i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) { |
| 119 ASSERT(!(source[i] & 0xff00)); |
| 120 destination[i] = static_cast<LChar>(source[i]); |
| 121 } |
| 122 |
| 123 const uintptr_t sourceLoadSize = |
| 124 32; // Process 32 bytes (16 UChars) each iteration |
| 125 const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar); |
| 126 if (length > ucharsPerLoop) { |
| 127 const size_t endLength = length - ucharsPerLoop + 1; |
| 128 for (; i < endLength; i += ucharsPerLoop) { |
| 129 #if ENABLE(ASSERT) |
| 130 for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkIndex) |
| 131 ASSERT(!(source[i + checkIndex] & 0xff00)); |
| 132 #endif |
| 133 __m128i first8UChars = |
| 134 _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i])); |
| 135 __m128i second8UChars = |
| 136 _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i + 8])); |
| 137 __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars); |
| 138 _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), |
| 139 packedChars); |
| 140 } |
| 141 } |
| 142 |
| 143 for (; i < length; ++i) { |
| 144 ASSERT(!(source[i] & 0xff00)); |
| 145 destination[i] = static_cast<LChar>(source[i]); |
| 146 } |
| 147 #elif COMPILER(GCC) && CPU(ARM_NEON) && \ |
| 148 !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN)) && defined(NDEBUG) |
| 149 const LChar* const end = destination + length; |
| 150 const uintptr_t memoryAccessSize = 8; |
| 151 |
| 152 if (length >= (2 * memoryAccessSize) - 1) { |
| 153 // Prefix: align dst on 64 bits. |
| 109 const uintptr_t memoryAccessMask = memoryAccessSize - 1; | 154 const uintptr_t memoryAccessMask = memoryAccessSize - 1; |
| 155 while (!isAlignedTo<memoryAccessMask>(destination)) |
| 156 *destination++ = static_cast<LChar>(*source++); |
| 110 | 157 |
| 111 size_t i = 0; | 158 // Vector interleaved unpack, we only store the lower 8 bits. |
| 112 for (;i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) { | 159 const uintptr_t lengthLeft = end - destination; |
| 113 ASSERT(!(source[i] & 0xff00)); | 160 const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize); |
| 114 destination[i] = static_cast<LChar>(source[i]); | 161 do { |
| 115 } | 162 asm("vld2.8 { d0-d1 }, [%[SOURCE]] !\n\t" |
| 163 "vst1.8 { d0 }, [%[DESTINATION],:64] !\n\t" |
| 164 : [SOURCE] "+r"(source), [DESTINATION] "+r"(destination) |
| 165 : |
| 166 : "memory", "d0", "d1"); |
| 167 } while (destination != simdEnd); |
| 168 } |
| 116 | 169 |
| 117 const uintptr_t sourceLoadSize = 32; // Process 32 bytes (16 UChars) each it
eration | 170 while (destination != end) |
| 118 const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar); | 171 *destination++ = static_cast<LChar>(*source++); |
| 119 if (length > ucharsPerLoop) { | |
| 120 const size_t endLength = length - ucharsPerLoop + 1; | |
| 121 for (; i < endLength; i += ucharsPerLoop) { | |
| 122 #if ENABLE(ASSERT) | |
| 123 for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkInd
ex) | |
| 124 ASSERT(!(source[i+checkIndex] & 0xff00)); | |
| 125 #endif | |
| 126 __m128i first8UChars = _mm_load_si128(reinterpret_cast<const __m128i
*>(&source[i])); | |
| 127 __m128i second8UChars = _mm_load_si128(reinterpret_cast<const __m128
i*>(&source[i+8])); | |
| 128 __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars); | |
| 129 _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), packed
Chars); | |
| 130 } | |
| 131 } | |
| 132 | |
| 133 for (; i < length; ++i) { | |
| 134 ASSERT(!(source[i] & 0xff00)); | |
| 135 destination[i] = static_cast<LChar>(source[i]); | |
| 136 } | |
| 137 #elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN))
&& defined(NDEBUG) | |
| 138 const LChar* const end = destination + length; | |
| 139 const uintptr_t memoryAccessSize = 8; | |
| 140 | |
| 141 if (length >= (2 * memoryAccessSize) - 1) { | |
| 142 // Prefix: align dst on 64 bits. | |
| 143 const uintptr_t memoryAccessMask = memoryAccessSize - 1; | |
| 144 while (!isAlignedTo<memoryAccessMask>(destination)) | |
| 145 *destination++ = static_cast<LChar>(*source++); | |
| 146 | |
| 147 // Vector interleaved unpack, we only store the lower 8 bits. | |
| 148 const uintptr_t lengthLeft = end - destination; | |
| 149 const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize); | |
| 150 do { | |
| 151 asm("vld2.8 { d0-d1 }, [%[SOURCE]] !\n\t" | |
| 152 "vst1.8 { d0 }, [%[DESTINATION],:64] !\n\t" | |
| 153 : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination) | |
| 154 : | |
| 155 : "memory", "d0", "d1"); | |
| 156 } while (destination != simdEnd); | |
| 157 } | |
| 158 | |
| 159 while (destination != end) | |
| 160 *destination++ = static_cast<LChar>(*source++); | |
| 161 #else | 172 #else |
| 162 for (size_t i = 0; i < length; ++i) { | 173 for (size_t i = 0; i < length; ++i) { |
| 163 ASSERT(!(source[i] & 0xff00)); | 174 ASSERT(!(source[i] & 0xff00)); |
| 164 destination[i] = static_cast<LChar>(source[i]); | 175 destination[i] = static_cast<LChar>(source[i]); |
| 165 } | 176 } |
| 166 #endif | 177 #endif |
| 167 } | 178 } |
| 168 | 179 |
| 169 } // namespace WTF | 180 } // namespace WTF |
| 170 | 181 |
| 171 #endif // ASCIIFastPath_h | 182 #endif // ASCIIFastPath_h |
| OLD | NEW |