| OLD | NEW |
| 1 // A frame for simple tokenizers. Takes care of newlines and | 1 // A frame for simple tokenizers. Takes care of newlines and |
| 2 // white-space, and of getting the text from the source stream into | 2 // white-space, and of getting the text from the source stream into |
| 3 // the token object. A state is a function of two arguments -- a | 3 // the token object. A state is a function of two arguments -- a |
| 4 // string stream and a setState function. The second can be used to | 4 // string stream and a setState function. The second can be used to |
| 5 // change the tokenizer's state, and can be ignored for stateless | 5 // change the tokenizer's state, and can be ignored for stateless |
| 6 // tokenizers. This function should advance the stream over a token | 6 // tokenizers. This function should advance the stream over a token |
| 7 // and return a string or object containing information about the next | 7 // and return a string or object containing information about the next |
| 8 // token, or null to pass and have the (new) state be called to finish | 8 // token, or null to pass and have the (new) state be called to finish |
| 9 // the token. When a string is given, it is wrapped in a {style, type} | 9 // the token. When a string is given, it is wrapped in a {style, type} |
| 10 // object. In the resulting object, the characters consumed are stored | 10 // object. In the resulting object, the characters consumed are stored |
| (...skipping 25 matching lines...) Expand all Loading... |
| 36 }, | 36 }, |
| 37 | 37 |
| 38 next: function () { | 38 next: function () { |
| 39 if (!source.more()) throw StopIteration; | 39 if (!source.more()) throw StopIteration; |
| 40 | 40 |
| 41 var type; | 41 var type; |
| 42 if (source.equals("\n")) { | 42 if (source.equals("\n")) { |
| 43 source.next(); | 43 source.next(); |
| 44 return this.take("whitespace"); | 44 return this.take("whitespace"); |
| 45 } | 45 } |
| 46 | 46 |
| 47 if (source.applies(isWhiteSpace)) | 47 if (source.applies(isWhiteSpace)) |
| 48 type = "whitespace"; | 48 type = "whitespace"; |
| 49 else | 49 else |
| 50 while (!type) | 50 while (!type) |
| 51 type = this.state(source, function(s) {tokenizer.state = s;}); | 51 type = this.state(source, function(s) {tokenizer.state = s;}); |
| 52 | 52 |
| 53 return this.take(type); | 53 return this.take(type); |
| 54 } | 54 } |
| 55 }; | 55 }; |
| 56 return tokenizer; | 56 return tokenizer; |
| 57 } | 57 } |
| OLD | NEW |