2 // --- Scope -----------------
\r
3 // Lang : Language expressions
\r
6 * Constructs a new Tokenizer.
\r
7 * @exports ProtoBuf.DotProto.Tokenizer
\r
8 * @class prototype tokenizer
\r
9 * @param {string} proto Proto to tokenize
\r
12 var Tokenizer = function(proto) {
\r
19 this.source = proto+"";
\r
37 * @type {!Array.<string>}
\r
43 * Opening character of the current string read, if any.
\r
47 this._stringOpen = null;
\r
51 * @alias ProtoBuf.DotProto.Tokenizer.prototype
\r
54 var TokenizerPrototype = Tokenizer.prototype;
\r
57 * Reads a string beginning at the current index.
\r
61 TokenizerPrototype._readString = function() {
\r
62 var re = this._stringOpen === '"'
\r
65 re.lastIndex = this.index - 1; // Include the open quote
\r
66 var match = re.exec(this.source);
\r
68 throw Error("unterminated string");
\r
69 this.index = re.lastIndex;
\r
70 this.stack.push(this._stringOpen);
\r
71 this._stringOpen = null;
\r
76 * Gets the next token and advances by one.
\r
77 * @return {?string} Token or `null` on EOF
\r
80 TokenizerPrototype.next = function() {
\r
81 if (this.stack.length > 0)
\r
82 return this.stack.shift();
\r
83 if (this.index >= this.source.length)
\r
85 if (this._stringOpen !== null)
\r
86 return this._readString();
\r
94 // Strip white spaces
\r
95 while (Lang.WHITESPACE.test(next = this.source.charAt(this.index))) {
\r
98 if (++this.index === this.source.length)
\r
103 if (this.source.charAt(this.index) === '/') {
\r
105 if (this.source.charAt(this.index) === '/') { // Line
\r
106 while (this.source.charAt(++this.index) !== '\n')
\r
107 if (this.index == this.source.length)
\r
112 } else if ((next = this.source.charAt(this.index)) === '*') { /* Block */
\r
116 if (++this.index === this.source.length)
\r
119 next = this.source.charAt(this.index);
\r
120 } while (prev !== '*' || next !== '/');
\r
128 if (this.index === this.source.length)
\r
131 // Read the next token
\r
132 var end = this.index;
\r
133 Lang.DELIM.lastIndex = 0;
\r
134 var delim = Lang.DELIM.test(this.source.charAt(end++));
\r
136 while(end < this.source.length && !Lang.DELIM.test(this.source.charAt(end)))
\r
138 var token = this.source.substring(this.index, this.index = end);
\r
139 if (token === '"' || token === "'")
\r
140 this._stringOpen = token;
\r
145 * Peeks for the next token.
\r
146 * @return {?string} Token or `null` on EOF
\r
149 TokenizerPrototype.peek = function() {
\r
150 if (this.stack.length === 0) {
\r
151 var token = this.next();
\r
152 if (token === null)
\r
154 this.stack.push(token);
\r
156 return this.stack[0];
\r
160 * Skips a specific token and throws if it differs.
\r
161 * @param {string} expected Expected token
\r
162 * @throws {Error} If the actual token differs
\r
164 TokenizerPrototype.skip = function(expected) {
\r
165 var actual = this.next();
\r
166 if (actual !== expected)
\r
167 throw Error("illegal '"+actual+"', '"+expected+"' expected");
\r
171 * Omits an optional token.
\r
172 * @param {string} expected Expected optional token
\r
173 * @returns {boolean} `true` if the token exists
\r
175 TokenizerPrototype.omit = function(expected) {
\r
176 if (this.peek() === expected) {
\r
184 * Returns a string representation of this object.
\r
185 * @return {string} String representation as of "Tokenizer(index/length)"
\r
188 TokenizerPrototype.toString = function() {
\r
189 return "Tokenizer ("+this.index+"/"+this.source.length+" at line "+this.line+")";
\r