OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved. | |
3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). | |
4 * | |
5 * This library is free software; you can redistribute it and/or | |
6 * modify it under the terms of the GNU Library General Public | |
7 * License as published by the Free Software Foundation; either | |
8 * version 2 of the License, or (at your option) any later version. | |
9 * | |
10 * This library is distributed in the hope that it will be useful, | |
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Library General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Library General Public License | |
16 * along with this library; see the file COPYING.LIB. If not, write to | |
17 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, | |
18 * Boston, MA 02110-1301, USA. | |
19 * | |
20 */ | |
21 | |
22 #ifndef ASCIIFastPath_h | |
23 #define ASCIIFastPath_h | |
24 | |
25 #if OS(DARWIN) && (CPU(X86) || CPU(X86_64)) | |
26 #include <emmintrin.h> | |
27 #endif | |
28 #include <stdint.h> | |
29 #include <wtf/Alignment.h> | |
30 #include <wtf/StdLibExtras.h> | |
31 #include <wtf/unicode/Unicode.h> | |
32 | |
33 namespace WTF { | |
34 | |
35 // Assuming that a pointer is the size of a "machine word", then | |
36 // uintptr_t is an integer type that is also a machine word. | |
37 typedef uintptr_t MachineWord; | |
38 const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1; | |
39 | |
40 inline bool isAlignedToMachineWord(const void* pointer) | |
41 { | |
42 return !(reinterpret_cast<uintptr_t>(pointer) & machineWordAlignmentMask); | |
43 } | |
44 | |
45 template<typename T> inline T* alignToMachineWord(T* pointer) | |
46 { | |
47 return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & ~machineW
ordAlignmentMask); | |
48 } | |
49 | |
50 template<size_t size, typename CharacterType> struct NonASCIIMask; | |
51 template<> struct NonASCIIMask<4, UChar> { | |
52 static inline uint32_t value() { return 0xFF80FF80U; } | |
53 }; | |
54 template<> struct NonASCIIMask<4, LChar> { | |
55 static inline uint32_t value() { return 0x80808080U; } | |
56 }; | |
57 template<> struct NonASCIIMask<8, UChar> { | |
58 static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; } | |
59 }; | |
60 template<> struct NonASCIIMask<8, LChar> { | |
61 static inline uint64_t value() { return 0x8080808080808080ULL; } | |
62 }; | |
63 | |
64 | |
65 template<typename CharacterType> | |
66 inline bool isAllASCII(MachineWord word) | |
67 { | |
68 return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value()); | |
69 } | |
70 | |
71 // Note: This function assume the input is likely all ASCII, and | |
72 // does not leave early if it is not the case. | |
73 template<typename CharacterType> | |
74 inline bool charactersAreAllASCII(const CharacterType* characters, size_t length
) | |
75 { | |
76 MachineWord allCharBits = 0; | |
77 const CharacterType* end = characters + length; | |
78 | |
79 // Prologue: align the input. | |
80 while (!isAlignedToMachineWord(characters) && characters != end) { | |
81 allCharBits |= *characters; | |
82 ++characters; | |
83 } | |
84 | |
85 // Compare the values of CPU word size. | |
86 const CharacterType* wordEnd = alignToMachineWord(end); | |
87 const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType); | |
88 while (characters < wordEnd) { | |
89 allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters)); | |
90 characters += loopIncrement; | |
91 } | |
92 | |
93 // Process the remaining bytes. | |
94 while (characters != end) { | |
95 allCharBits |= *characters; | |
96 ++characters; | |
97 } | |
98 | |
99 MachineWord nonASCIIBitMask = NonASCIIMask<sizeof(MachineWord), CharacterTyp
e>::value(); | |
100 return !(allCharBits & nonASCIIBitMask); | |
101 } | |
102 | |
103 inline void copyLCharsFromUCharSource(LChar* destination, const UChar* source, s
ize_t length) | |
104 { | |
105 #if OS(DARWIN) && (CPU(X86) || CPU(X86_64)) | |
106 const uintptr_t memoryAccessSize = 16; // Memory accesses on 16 byte (128 bi
t) alignment | |
107 const uintptr_t memoryAccessMask = memoryAccessSize - 1; | |
108 | |
109 size_t i = 0; | |
110 for (;i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) { | |
111 ASSERT(!(source[i] & 0xff00)); | |
112 destination[i] = static_cast<LChar>(source[i]); | |
113 } | |
114 | |
115 const uintptr_t sourceLoadSize = 32; // Process 32 bytes (16 UChars) each it
eration | |
116 const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar); | |
117 if (length > ucharsPerLoop) { | |
118 const size_t endLength = length - ucharsPerLoop + 1; | |
119 for (; i < endLength; i += ucharsPerLoop) { | |
120 #ifndef NDEBUG | |
121 for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkInd
ex) | |
122 ASSERT(!(source[i+checkIndex] & 0xff00)); | |
123 #endif | |
124 __m128i first8UChars = _mm_load_si128(reinterpret_cast<const __m128i
*>(&source[i])); | |
125 __m128i second8UChars = _mm_load_si128(reinterpret_cast<const __m128
i*>(&source[i+8])); | |
126 __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars); | |
127 _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), packed
Chars); | |
128 } | |
129 } | |
130 | |
131 for (; i < length; ++i) { | |
132 ASSERT(!(source[i] & 0xff00)); | |
133 destination[i] = static_cast<LChar>(source[i]); | |
134 } | |
135 #elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN))
&& defined(NDEBUG) | |
136 const LChar* const end = destination + length; | |
137 const uintptr_t memoryAccessSize = 8; | |
138 | |
139 if (length >= (2 * memoryAccessSize) - 1) { | |
140 // Prefix: align dst on 64 bits. | |
141 const uintptr_t memoryAccessMask = memoryAccessSize - 1; | |
142 while (!isAlignedTo<memoryAccessMask>(destination)) | |
143 *destination++ = static_cast<LChar>(*source++); | |
144 | |
145 // Vector interleaved unpack, we only store the lower 8 bits. | |
146 const uintptr_t lengthLeft = end - destination; | |
147 const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize); | |
148 do { | |
149 asm("vld2.8 { d0-d1 }, [%[SOURCE]] !\n\t" | |
150 "vst1.8 { d0 }, [%[DESTINATION],:64] !\n\t" | |
151 : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination) | |
152 : | |
153 : "memory", "d0", "d1"); | |
154 } while (destination != simdEnd); | |
155 } | |
156 | |
157 while (destination != end) | |
158 *destination++ = static_cast<LChar>(*source++); | |
159 #else | |
160 for (size_t i = 0; i < length; ++i) { | |
161 ASSERT(!(source[i] & 0xff00)); | |
162 destination[i] = static_cast<LChar>(source[i]); | |
163 } | |
164 #endif | |
165 } | |
166 | |
167 } // namespace WTF | |
168 | |
169 #endif // ASCIIFastPath_h | |
OLD | NEW |