OLD | NEW |
1 // A frame for simple tokenizers. Takes care of newlines and | 1 // A frame for simple tokenizers. Takes care of newlines and |
2 // white-space, and of getting the text from the source stream into | 2 // white-space, and of getting the text from the source stream into |
3 // the token object. A state is a function of two arguments -- a | 3 // the token object. A state is a function of two arguments -- a |
4 // string stream and a setState function. The second can be used to | 4 // string stream and a setState function. The second can be used to |
5 // change the tokenizer's state, and can be ignored for stateless | 5 // change the tokenizer's state, and can be ignored for stateless |
6 // tokenizers. This function should advance the stream over a token | 6 // tokenizers. This function should advance the stream over a token |
7 // and return a string or object containing information about the next | 7 // and return a string or object containing information about the next |
8 // token, or null to pass and have the (new) state be called to finish | 8 // token, or null to pass and have the (new) state be called to finish |
9 // the token. When a string is given, it is wrapped in a {style, type} | 9 // the token. When a string is given, it is wrapped in a {style, type} |
10 // object. In the resulting object, the characters consumed are stored | 10 // object. In the resulting object, the characters consumed are stored |
(...skipping 25 matching lines...) Expand all Loading... |
36 }, | 36 }, |
37 | 37 |
38 next: function () { | 38 next: function () { |
39 if (!source.more()) throw StopIteration; | 39 if (!source.more()) throw StopIteration; |
40 | 40 |
41 var type; | 41 var type; |
42 if (source.equals("\n")) { | 42 if (source.equals("\n")) { |
43 source.next(); | 43 source.next(); |
44 return this.take("whitespace"); | 44 return this.take("whitespace"); |
45 } | 45 } |
46 | 46 |
47 if (source.applies(isWhiteSpace)) | 47 if (source.applies(isWhiteSpace)) |
48 type = "whitespace"; | 48 type = "whitespace"; |
49 else | 49 else |
50 while (!type) | 50 while (!type) |
51 type = this.state(source, function(s) {tokenizer.state = s;}); | 51 type = this.state(source, function(s) {tokenizer.state = s;}); |
52 | 52 |
53 return this.take(type); | 53 return this.take(type); |
54 } | 54 } |
55 }; | 55 }; |
56 return tokenizer; | 56 return tokenizer; |
57 } | 57 } |
OLD | NEW |