OLD | NEW |
1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
112 } | 112 } |
113 | 113 |
114 | 114 |
115 void UTF16Buffer::PushBack(uc32 ch) { | 115 void UTF16Buffer::PushBack(uc32 ch) { |
116 pushback_buffer()->Add(last_); | 116 pushback_buffer()->Add(last_); |
117 last_ = ch; | 117 last_ = ch; |
118 pos_--; | 118 pos_--; |
119 } | 119 } |
120 | 120 |
121 | 121 |
122 static inline bool IsByteOrderMark(uc32 c) { | |
123 // The Unicode value U+FFFE is guaranteed never to be assigned as a | |
124 // Unicode character; this implies that in a Unicode context the | |
125 // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF | |
126 // character expressed in little-endian byte order (since it could | |
127 // not be a U+FFFE character expressed in big-endian byte | |
128 // order). Nevertheless, we check for it to be compatible with | |
129 // Spidermonkey. | |
130 return c == 0xFEFF || c == 0xFFFE; | |
131 } | |
132 | |
133 | |
134 uc32 UTF16Buffer::Advance() { | 122 uc32 UTF16Buffer::Advance() { |
135 // NOTE: It is of importance to Persian / Farsi resources that we do | 123 // NOTE: It is of importance to Persian / Farsi resources that we do |
136 // *not* strip format control characters in the scanner; see | 124 // *not* strip format control characters in the scanner; see |
137 // | 125 // |
138 // https://bugzilla.mozilla.org/show_bug.cgi?id=274152 | 126 // https://bugzilla.mozilla.org/show_bug.cgi?id=274152 |
139 // | 127 // |
140 // So, even though ECMA-262, section 7.1, page 11, dictates that we | 128 // So, even though ECMA-262, section 7.1, page 11, dictates that we |
141 // must remove Unicode format-control characters, we only remove the BOM. | 129 // must remove Unicode format-control characters, we do not. This is |
142 // This is in line with how Safari handles it. | 130 // in line with how IE and SpiderMonkey handles it. |
143 if (!pushback_buffer()->is_empty()) { | 131 if (!pushback_buffer()->is_empty()) { |
144 pos_++; | 132 pos_++; |
145 return last_ = pushback_buffer()->RemoveLast(); | 133 return last_ = pushback_buffer()->RemoveLast(); |
| 134 } else if (stream_->has_more()) { |
| 135 pos_++; |
| 136 uc32 next = stream_->GetNext(); |
| 137 return last_ = next; |
146 } else { | 138 } else { |
147 while (stream_->has_more()) { | |
148 pos_++; | |
149 uc32 next = stream_->GetNext(); | |
150 if (!IsByteOrderMark(next)) return last_ = next; | |
151 } | |
152 // note: currently the following increment is necessary to avoid a | 139 // note: currently the following increment is necessary to avoid a |
153 // test-parser problem! | 140 // test-parser problem! |
154 pos_++; | 141 pos_++; |
155 return last_ = static_cast<uc32>(-1); | 142 return last_ = static_cast<uc32>(-1); |
156 } | 143 } |
157 } | 144 } |
158 | 145 |
159 | 146 |
160 void UTF16Buffer::SeekForward(int pos) { | 147 void UTF16Buffer::SeekForward(int pos) { |
161 pos_ = pos; | 148 pos_ = pos; |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
240 c0_ = source_.Advance(); | 227 c0_ = source_.Advance(); |
241 } | 228 } |
242 | 229 |
243 | 230 |
244 void Scanner::PushBack(uc32 ch) { | 231 void Scanner::PushBack(uc32 ch) { |
245 source_.PushBack(ch); | 232 source_.PushBack(ch); |
246 c0_ = ch; | 233 c0_ = ch; |
247 } | 234 } |
248 | 235 |
249 | 236 |
| 237 static inline bool IsByteOrderMark(uc32 c) { |
| 238 // The Unicode value U+FFFE is guaranteed never to be assigned as a |
| 239 // Unicode character; this implies that in a Unicode context the |
| 240 // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF |
| 241 // character expressed in little-endian byte order (since it could |
| 242 // not be a U+FFFE character expressed in big-endian byte |
| 243 // order). Nevertheless, we check for it to be compatible with |
| 244 // Spidermonkey. |
| 245 return c == 0xFEFF || c == 0xFFFE; |
| 246 } |
| 247 |
| 248 |
250 void Scanner::SkipWhiteSpace(bool initial) { | 249 void Scanner::SkipWhiteSpace(bool initial) { |
251 has_line_terminator_before_next_ = initial; | 250 has_line_terminator_before_next_ = initial; |
252 | 251 |
253 while (true) { | 252 while (true) { |
254 while (kIsWhiteSpace.get(c0_)) { | 253 // We treat byte-order marks (BOMs) as whitespace for better |
| 254 // compatibility with Spidermonkey and other JavaScript engines. |
| 255 while (kIsWhiteSpace.get(c0_) || IsByteOrderMark(c0_)) { |
255 // IsWhiteSpace() includes line terminators! | 256 // IsWhiteSpace() includes line terminators! |
256 if (kIsLineTerminator.get(c0_)) | 257 if (kIsLineTerminator.get(c0_)) |
257 // Ignore line terminators, but remember them. This is necessary | 258 // Ignore line terminators, but remember them. This is necessary |
258 // for automatic semicolon insertion. | 259 // for automatic semicolon insertion. |
259 has_line_terminator_before_next_ = true; | 260 has_line_terminator_before_next_ = true; |
260 Advance(); | 261 Advance(); |
261 } | 262 } |
262 | 263 |
263 // If there is an HTML comment end '-->' at the beginning of a | 264 // If there is an HTML comment end '-->' at the beginning of a |
264 // line (with only whitespace in front of it), we treat the rest | 265 // line (with only whitespace in front of it), we treat the rest |
(...skipping 565 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
830 StartLiteral(); | 831 StartLiteral(); |
831 while (kIsIdentifierPart.get(c0_)) | 832 while (kIsIdentifierPart.get(c0_)) |
832 AddCharAdvance(); | 833 AddCharAdvance(); |
833 TerminateLiteral(); | 834 TerminateLiteral(); |
834 | 835 |
835 next_.location.end_pos = source_pos() - 1; | 836 next_.location.end_pos = source_pos() - 1; |
836 return true; | 837 return true; |
837 } | 838 } |
838 | 839 |
839 } } // namespace v8::internal | 840 } } // namespace v8::internal |
OLD | NEW |