001 /*
002 // $Id: //open/mondrian/src/main/mondrian/olap/Scanner.java#24 $
003 // This software is subject to the terms of the Common Public License
004 // Agreement, available at the following URL:
005 // http://www.opensource.org/licenses/cpl.html.
006 // Copyright (C) 1998-2002 Kana Software, Inc.
007 // Copyright (C) 2001-2008 Julian Hyde and others
008 // All Rights Reserved.
009 // You must accept the terms of that agreement to use this software.
010 //
011 // jhyde, 20 January, 1999
012 */
013
014 package mondrian.olap;
015
016 import java_cup.runtime.Symbol;
017 import org.apache.log4j.Logger;
018 import java.util.List;
019 import java.util.ArrayList;
020 import java.util.Enumeration;
021 import java.util.Hashtable;
022 import java.io.IOException;
023 import java.math.BigDecimal;
024
025 /**
026 * Lexical analyzer for MDX.
027 */
028 public class Scanner {
029 private static final Logger LOGGER = Logger.getLogger(Scanner.class);
030
031 /** single lookahead character */
032 protected int nextChar;
033 /** next lookahead character */
034 private int lookaheadChars[] = new int[16];
035 private int firstLookaheadChar = 0;
036 private int lastLookaheadChar = 0;
037 private Hashtable<String, Integer> m_resWordsTable;
038 private int iMaxResword;
039 private String m_aResWords[];
040 protected boolean debug;
041
042 /** lines[x] is the start of the x'th line */
043 private List<Integer> lines;
044
045 /** number of times advance() has been called */
046 private int iChar;
047
048 /** end of previous token */
049 private int iPrevChar;
050
051 /** previous symbol returned */
052 private int previousSymbol;
053 private boolean inFormula;
054
055 /**
056 * Comment delimiters. Modify this list to support other comment styles.
057 */
058 private static final String[][] commentDelim = {
059 new String[] {"//", null},
060 new String[] {"--", null},
061 new String[] {"/*", "*/"}
062 };
063
064 /**
065 * Whether to allow nested comments.
066 */
067 private static final boolean allowNestedComments = true;
068
069 /**
070 * The {@link java.math.BigDecimal} value 0.
071 * Note that BigDecimal.ZERO does not exist until JDK 1.5.
072 */
073 private static final BigDecimal BigDecimalZero = BigDecimal.valueOf(0);
074
075 /**
076 * Creates a Scanner.
077 *
078 * @param debug Whether to emit debug messages.
079 */
080 Scanner(boolean debug) {
081 this.debug = debug;
082 }
083
084 /**
085 * Returns the current nested comments state.
086 */
087 public static boolean getNestedCommentsState() {
088 return allowNestedComments;
089 }
090
091 /**
092 * Returns the list of comment delimiters.
093 */
094 public static String[][] getCommentDelimiters() {
095 return commentDelim;
096 }
097
098 /**
099 * Advance input by one character, setting {@link #nextChar}.
100 */
101 private void advance()
102 throws java.io.IOException {
103
104 if (firstLookaheadChar == lastLookaheadChar) {
105 // We have nothing in the lookahead buffer.
106 nextChar = getChar();
107 } else {
108 // We have called lookahead(); advance to the next character it got.
109 nextChar = lookaheadChars[firstLookaheadChar++];
110 if (firstLookaheadChar == lastLookaheadChar) {
111 firstLookaheadChar = 0;
112 lastLookaheadChar = 0;
113 }
114 }
115 if (nextChar == '\012') {
116 lines.add(iChar);
117 }
118 iChar++;
119 }
120
121 /** Peek at the character after {@link #nextChar} without advancing. */
122 private int lookahead()
123 throws java.io.IOException {
124
125 return lookahead(1);
126 }
127
128 /**
129 * Peeks at the character n after {@link #nextChar} without advancing.
130 * lookahead(0) returns the current char (nextChar).
131 * lookahead(1) returns the next char (was lookaheadChar, same as lookahead());
132 */
133 private int lookahead(int n)
134 throws java.io.IOException {
135
136 if (n == 0) {
137 return nextChar;
138 } else {
139 // if the desired character not in lookahead buffer, read it in
140 if (n > lastLookaheadChar - firstLookaheadChar) {
141 int len = lastLookaheadChar - firstLookaheadChar;
142 int t[];
143
144 // make sure we do not go off the end of the buffer
145 if (n + firstLookaheadChar > lookaheadChars.length) {
146 if (n > lookaheadChars.length) {
147 // the array is too small; make it bigger and shift
148 // everything to the beginning.
149 t = new int[n * 2];
150 } else {
151 // the array is big enough, so just shift everything
152 // to the beginning of it.
153 t = lookaheadChars;
154 }
155
156 System.arraycopy(
157 lookaheadChars, firstLookaheadChar, t, 0, len);
158 lookaheadChars = t;
159 firstLookaheadChar = 0;
160 lastLookaheadChar = len;
161 }
162
163 // read ahead enough
164 while (n > lastLookaheadChar - firstLookaheadChar) {
165 lookaheadChars[lastLookaheadChar++] = getChar();
166 }
167 }
168
169 return lookaheadChars[n - 1 + firstLookaheadChar];
170 }
171 }
172
173 /** Read a character from input, returning -1 if end of input. */
174 protected int getChar()
175 throws java.io.IOException {
176
177 return System.in.read();
178 }
179
180 /** Initialize the scanner */
181 public void init()
182 throws java.io.IOException {
183
184 initReswords();
185 lines = new ArrayList<Integer>();
186 iChar = iPrevChar = 0;
187 advance();
188 }
189
190 /**
191 * Deduces the line and column (0-based) of a symbol.
192 * Called by {@link Parser#syntax_error}.
193 */
194 void getLocation(Symbol symbol, int[] loc) {
195 int iTarget = symbol.left;
196 int iLine = -1;
197 int iLineEnd = 0;
198 int iLineStart;
199 do {
200 iLine++;
201 iLineStart = iLineEnd;
202 iLineEnd = Integer.MAX_VALUE;
203 if (iLine < lines.size()) {
204 iLineEnd = lines.get(iLine);
205 }
206 } while (iLineEnd < iTarget);
207
208 loc[0] = iLine; // line
209 loc[1] = iTarget - iLineStart; // column
210 }
211
212 private Symbol trace(Symbol s) {
213 if (debug) {
214 String name = null;
215 if (s.sym < m_aResWords.length) {
216 name = m_aResWords[s.sym];
217 }
218
219 LOGGER.error("Scanner returns #" + s.sym +
220 (name == null ? "" : ":" + name) +
221 (s.value == null ? "" : "(" + s.value.toString() + ")"));
222 }
223 return s;
224 }
225
226 private void initResword(int id, String s) {
227 m_resWordsTable.put(s, id);
228 if (id > iMaxResword) {
229 iMaxResword = id;
230 }
231 }
232
233 /**
234 * Initializes the table of reserved words.
235 */
236 private void initReswords() {
237 // This list generated by piping the 'terminal' declaration in mdx.cup
238 // through:
239 // grep -list // |
240 // sed -e 's/,//' |
241 // awk '{printf "initResword(%20s,%c%s%c);",$1,34,$1,34}'
242 m_resWordsTable = new Hashtable<String, Integer>();
243 iMaxResword = 0;
244 // initResword(ParserSym.ALL ,"ALL");
245 initResword(ParserSym.AND ,"AND");
246 initResword(ParserSym.AS ,"AS");
247 // initResword(ParserSym.ASC ,"ASC");
248 initResword(ParserSym.AXIS ,"AXIS");
249 // initResword(ParserSym.BACK_COLOR ,"BACK_COLOR");
250 // initResword(ParserSym.BASC ,"BASC");
251 // initResword(ParserSym.BDESC ,"BDESC");
252 initResword(ParserSym.CAST ,"CAST"); // mondrian extension
253 initResword(ParserSym.CASE ,"CASE");
254 initResword(ParserSym.CELL ,"CELL");
255 // initResword(ParserSym.CELL_ORDINAL ,"CELL_ORDINAL");
256 // initResword(ParserSym.CHAPTERS ,"CHAPTERS");
257 // initResword(ParserSym.CHILDREN ,"CHILDREN");
258 // initResword(ParserSym.COLUMNS ,"COLUMNS");
259 // initResword(ParserSym.DESC ,"DESC");
260 initResword(ParserSym.DIMENSION ,"DIMENSION");
261 initResword(ParserSym.ELSE ,"ELSE");
262 initResword(ParserSym.EMPTY ,"EMPTY");
263 initResword(ParserSym.END ,"END");
264 // initResword(ParserSym.FIRSTCHILD ,"FIRSTCHILD");
265 // initResword(ParserSym.FIRSTSIBLING ,"FIRSTSIBLING");
266 // initResword(ParserSym.FONT_FLAGS ,"FONT_FLAGS");
267 // initResword(ParserSym.FONT_NAME ,"FONT_NAME");
268 // initResword(ParserSym.FONT_SIZE ,"FONT_SIZE");
269 // initResword(ParserSym.FORE_COLOR ,"FORE_COLOR");
270 // initResword(ParserSym.FORMATTED_VALUE ,"FORMATTED_VALUE");
271 // initResword(ParserSym.FORMAT_STRING ,"FORMAT_STRING");
272 initResword(ParserSym.FROM ,"FROM");
273 initResword(ParserSym.IS ,"IS");
274 initResword(ParserSym.IN ,"IN");
275 // initResword(ParserSym.LAG ,"LAG");
276 // initResword(ParserSym.LASTCHILD ,"LASTCHILD");
277 // initResword(ParserSym.LASTSIBLING ,"LASTSIBLING");
278 // initResword(ParserSym.LEAD ,"LEAD");
279 initResword(ParserSym.MATCHES ,"MATCHES");
280 initResword(ParserSym.MEMBER ,"MEMBER");
281 // initResword(ParserSym.MEMBERS ,"MEMBERS");
282 // initResword(ParserSym.NEXTMEMBER ,"NEXTMEMBER");
283 initResword(ParserSym.NON ,"NON");
284 initResword(ParserSym.NOT ,"NOT");
285 initResword(ParserSym.NULL ,"NULL");
286 initResword(ParserSym.ON ,"ON");
287 initResword(ParserSym.OR ,"OR");
288 // initResword(ParserSym.PAGES ,"PAGES");
289 // initResword(ParserSym.PARENT ,"PARENT");
290 // initResword(ParserSym.PREVMEMBER ,"PREVMEMBER");
291 initResword(ParserSym.PROPERTIES ,"PROPERTIES");
292 // initResword(ParserSym.RECURSIVE ,"RECURSIVE");
293 // initResword(ParserSym.ROWS ,"ROWS");
294 // initResword(ParserSym.SECTIONS ,"SECTIONS");
295 initResword(ParserSym.SELECT ,"SELECT");
296 initResword(ParserSym.SET ,"SET");
297 // initResword(ParserSym.SOLVE_ORDER ,"SOLVE_ORDER");
298 initResword(ParserSym.THEN ,"THEN");
299 // initResword(ParserSym.VALUE ,"VALUE");
300 initResword(ParserSym.WHEN ,"WHEN");
301 initResword(ParserSym.WHERE ,"WHERE");
302 initResword(ParserSym.WITH ,"WITH");
303 initResword(ParserSym.XOR ,"XOR");
304
305 m_aResWords = new String[iMaxResword + 1];
306 Enumeration<String> e = m_resWordsTable.keys();
307 while (e.hasMoreElements()) {
308 Object o = e.nextElement();
309 String s = (String) o;
310 int i = (m_resWordsTable.get(s)).intValue();
311 m_aResWords[i] = s;
312 }
313 }
314
315 /** return the name of the reserved word whose token code is "i" */
316 public String lookupReserved(int i) {
317 return m_aResWords[i];
318 }
319
320 private Symbol makeSymbol(int id, Object o) {
321 int iPrevPrevChar = iPrevChar;
322 this.iPrevChar = iChar;
323 this.previousSymbol = id;
324 return trace(new Symbol(id, iPrevPrevChar, iChar, o));
325 }
326
327 /**
328 * Creates a token representing a numeric literal.
329 *
330 * @param mantissa The digits of the number
331 * @param exponent The base-10 exponent of the number
332 * @return number literal token
333 */
334 private Symbol makeNumber(BigDecimal mantissa, int exponent) {
335 double d = mantissa.movePointRight(exponent).doubleValue();
336 return makeSymbol(ParserSym.NUMBER, d);
337 }
338
339 private Symbol makeId(String s, boolean quoted, boolean ampersand) {
340 return makeSymbol(
341 quoted && ampersand ? ParserSym.AMP_QUOTED_ID :
342 quoted ? ParserSym.QUOTED_ID :
343 ParserSym.ID,
344 s);
345 }
346
347 /**
348 * Creates a token representing a reserved word.
349 *
350 * @param i Token code
351 * @return Token
352 */
353 private Symbol makeRes(int i) {
354 return makeSymbol(i, m_aResWords[i]);
355 }
356
357 /**
358 * Creates a token.
359 *
360 * @param i Token code
361 * @param s Text of the token
362 * @return Token
363 */
364 private Symbol makeToken(int i, String s) {
365 return makeSymbol(i, s);
366 }
367
368 /**
369 * Creates a token representing a string literal.
370 *
371 * @param s String
372 * @return String token
373 */
374 private Symbol makeString(String s) {
375 if (inFormula) {
376 inFormula = false;
377 return makeSymbol(ParserSym.FORMULA_STRING, s);
378 } else {
379 return makeSymbol(ParserSym.STRING, s);
380 }
381 }
382
383 /**
384 * Discards all characters until the end of the current line.
385 */
386 private void skipToEOL() throws IOException {
387 while (nextChar != -1 && nextChar != '\012') {
388 advance();
389 }
390 }
391
392 /**
393 * Eats a delimited comment.
394 * The type of delimiters are kept in commentDelim. The current
395 * comment type is indicated by commentType.
396 * end of file terminates a comment without error.
397 */
398 private void skipComment(
399 final String startDelim,
400 final String endDelim) throws IOException {
401
402 int depth = 1;
403
404 // skip the starting delimiter
405 for (int x = 0; x < startDelim.length(); x++) {
406 advance();
407 }
408
409 for (;;) {
410 if (nextChar == -1) {
411 return;
412 } else if (checkForSymbol(endDelim)) {
413 // eat the end delimiter
414 for (int x = 0; x < endDelim.length(); x++) {
415 advance();
416 }
417 if (--depth == 0) {
418 return;
419 }
420 } else if (allowNestedComments && checkForSymbol(startDelim)) {
421 // eat the nested start delimiter
422 for (int x = 0; x < startDelim.length(); x++) {
423 advance();
424 }
425 depth++;
426 } else {
427 advance();
428 }
429 }
430 }
431
432 /**
433 * If the next tokens are comments, skip over them.
434 */
435 private void searchForComments() throws IOException {
436
437 // eat all following comments
438 boolean foundComment;
439 do {
440 foundComment = false;
441 for (String[] aCommentDelim : commentDelim) {
442 if (checkForSymbol(aCommentDelim[0])) {
443 if (aCommentDelim[1] == null) {
444 foundComment = true;
445 skipToEOL();
446 } else {
447 foundComment = true;
448 skipComment(aCommentDelim[0], aCommentDelim[1]);
449 }
450 }
451 }
452 } while (foundComment);
453 }
454
455 /**
456 * Checks if the next symbol is the supplied string
457 */
458 private boolean checkForSymbol(final String symb) throws IOException {
459 for (int x = 0; x < symb.length(); x++) {
460 if (symb.charAt(x) != lookahead(x)) {
461 return false;
462 }
463 }
464 return true;
465 }
466
467 /**
468 * Recognizes and returns the next complete token.
469 */
470 public Symbol next_token() throws IOException {
471
472 StringBuilder id;
473 boolean ampersandId = false;
474 for (;;) {
475 searchForComments();
476 switch (nextChar) {
477 case '.':
478 switch (lookahead()) {
479 case '0': case '1': case '2': case '3': case '4':
480 case '5': case '6': case '7': case '8': case '9':
481 // We're looking at the '.' on the start of a number,
482 // e.g. .1; fall through to parse a number.
483 break;
484 default:
485 advance();
486 return makeToken(ParserSym.DOT, ".");
487 }
488 // fall through
489
490 case '0': case '1': case '2': case '3': case '4':
491 case '5': case '6': case '7': case '8': case '9':
492
493 // Parse a number. Valid examples include 1, 1.2, 0.1, .1,
494 // 1e2, 1E2, 1e-2, 1e+2. Invalid examples include e2, 1.2.3,
495 // 1e2e3, 1e2.3.
496 //
497 // Signs preceding numbers (e.g. -1, + 1E-5) are valid, but are
498 // handled by the parser.
499 //
500 BigDecimal n = BigDecimalZero;
501 int digitCount = 0, exponent = 0;
502 boolean positive = true;
503 BigDecimal mantissa = BigDecimalZero;
504 State state = State.leftOfPoint;
505
506 for (;;) {
507 switch (nextChar) {
508 case '.':
509 switch (state) {
510 case leftOfPoint:
511 state = State.rightOfPoint;
512 mantissa = n;
513 n = BigDecimalZero;
514 digitCount = 0;
515 positive = true;
516 advance();
517 break;
518 // Error: we are seeing a point in the exponent
519 // (e.g. 1E2.3 or 1.2E3.4) or a second point in the
520 // mantissa (e.g. 1.2.3). Return what we've got
521 // and let the parser raise the error.
522 case rightOfPoint:
523 mantissa =
524 mantissa.add(
525 n.movePointRight(-digitCount));
526 return makeNumber(mantissa, exponent);
527 case inExponent:
528 if (!positive) {
529 n = n.negate();
530 }
531 exponent = n.intValue();
532 return makeNumber(mantissa, exponent);
533 }
534 break;
535
536 case 'E':
537 case 'e':
538 switch (state) {
539 case inExponent:
540 // Error: we are seeing an 'e' in the exponent
541 // (e.g. 1.2e3e4). Return what we've got and let
542 // the parser raise the error.
543 if (!positive) {
544 n = n.negate();
545 }
546 exponent = n.intValue();
547 return makeNumber(mantissa, exponent);
548 case leftOfPoint:
549 mantissa = n;
550 break;
551 default:
552 mantissa =
553 mantissa.add(
554 n.movePointRight(-digitCount));
555 break;
556 }
557
558 digitCount = 0;
559 n = BigDecimalZero;
560 positive = true;
561 advance();
562 state = State.inExponent;
563 break;
564
565 case'0': case'1': case'2': case'3': case'4':
566 case'5': case'6': case'7': case'8': case'9':
567 n = n.movePointRight(1);
568 n = n.add(BigDecimal.valueOf(nextChar - '0'));
569 digitCount++;
570 advance();
571 break;
572
573 case '+':
574 case '-':
575 if (state == State.inExponent &&
576 digitCount == 0) {
577 // We're looking at the sign after the 'e'.
578 positive = !positive;
579 advance();
580 break;
581 }
582 // fall through - end of number
583
584 default:
585 // Reached end of number.
586 switch (state) {
587 case leftOfPoint:
588 mantissa = n;
589 break;
590 case rightOfPoint:
591 mantissa =
592 mantissa.add(
593 n.movePointRight(-digitCount));
594 break;
595 default:
596 if (!positive) {
597 n = n.negate();
598 }
599 exponent = n.intValue();
600 break;
601 }
602 return makeNumber(mantissa, exponent);
603 }
604 }
605
606 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
607 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
608 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
609 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
610 case 'y': case 'z':
611 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
612 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
613 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
614 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
615 case 'Y': case 'Z':
616 /* parse an identifier */
617 id = new StringBuilder();
618 for (;;) {
619 id.append((char)nextChar);
620 advance();
621 switch (nextChar) {
622 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
623 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
624 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
625 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
626 case 'y': case 'z':
627 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
628 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
629 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
630 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
631 case 'Y': case 'Z':
632 case '0': case '1': case '2': case '3': case '4':
633 case '5': case '6': case '7': case '8': case '9':
634 case '_':
635 break;
636 default:
637 String strId = id.toString();
638 Integer i = m_resWordsTable.get(
639 strId.toUpperCase());
640 if (i == null) {
641 // identifier
642 return makeId(strId, false, false);
643 } else {
644 // reserved word
645 return makeRes(i);
646 }
647 }
648 }
649
650 case '&':
651 advance();
652 if (nextChar == '[') {
653 ampersandId = true;
654 // fall through
655 } else {
656 return makeToken(ParserSym.UNKNOWN, "&");
657 }
658
659 case '[':
660 /* parse a delimited identifier */
661 id = new StringBuilder();
662 for (;;) {
663 advance();
664 switch (nextChar) {
665 case ']':
666 advance();
667 if (nextChar == ']') {
668 // ] escaped with ] - just take one
669 id.append(']');
670 break;
671 } else {
672 // end of identifier
673 if (ampersandId) {
674 ampersandId = false;
675 return makeId(id.toString(), true, true);
676 } else {
677 return makeId(id.toString(), true, false);
678 }
679 }
680 case -1:
681 if (ampersandId) {
682 ampersandId = false;
683 return makeId(id.toString(), true, true);
684 } else {
685 return makeId(id.toString(), true, false);
686 }
687 default:
688 id.append((char)nextChar);
689 }
690 }
691
692 case ':':
693 advance();
694 return makeToken(ParserSym.COLON, ":");
695 case ',':
696 advance();
697 return makeToken(ParserSym.COMMA, ",");
698 case '=':
699 advance();
700 return makeToken(ParserSym.EQ, "=");
701 case '<':
702 advance();
703 switch (nextChar) {
704 case '>':
705 advance();
706 return makeToken(ParserSym.NE, "<>");
707 case '=':
708 advance();
709 return makeToken(ParserSym.LE, "<=");
710 default:
711 return makeToken(ParserSym.LT, "<");
712 }
713 case '>':
714 advance();
715 switch (nextChar) {
716 case '=':
717 advance();
718 return makeToken(ParserSym.GE, ">=");
719 default:
720 return makeToken(ParserSym.GT, ">");
721 }
722 case '{':
723 advance();
724 return makeToken(ParserSym.LBRACE, "{");
725 case '(':
726 advance();
727 return makeToken(ParserSym.LPAREN, "(");
728 case '}':
729 advance();
730 return makeToken(ParserSym.RBRACE, "}");
731 case ')':
732 advance();
733 return makeToken(ParserSym.RPAREN, ")");
734 case '+':
735 advance();
736 return makeToken(ParserSym.PLUS, "+");
737 case '-':
738 advance();
739 return makeToken(ParserSym.MINUS, "-");
740 case '*':
741 advance();
742 return makeToken(ParserSym.ASTERISK, "*");
743 case '/':
744 advance();
745 return makeToken(ParserSym.SOLIDUS, "/");
746 case '!':
747 advance();
748 return makeToken(ParserSym.BANG, "!");
749 case '|':
750 advance();
751 switch (nextChar) {
752 case '|':
753 advance();
754 return makeToken(ParserSym.CONCAT, "||");
755 default:
756 return makeToken(ParserSym.UNKNOWN, "|");
757 }
758
759 case '"':
760 /* parse a double-quoted string */
761 id = new StringBuilder();
762 for (;;) {
763 advance();
764 switch (nextChar) {
765 case '"':
766 advance();
767 if (nextChar == '"') {
768 // " escaped with "
769 id.append('"');
770 break;
771 } else {
772 // end of string
773 return makeString(id.toString());
774 }
775 case -1:
776 return makeString(id.toString());
777 default:
778 id.append((char)nextChar);
779 }
780 }
781
782 case '\'':
783 if (previousSymbol == ParserSym.AS) {
784 inFormula = true;
785 }
786
787 /* parse a single-quoted string */
788 id = new StringBuilder();
789 for (;;) {
790 advance();
791 switch (nextChar) {
792 case '\'':
793 advance();
794 if (nextChar == '\'') {
795 // " escaped with "
796 id.append('\'');
797 break;
798 } else {
799 // end of string
800 return makeString(id.toString());
801 }
802 case -1:
803 return makeString(id.toString());
804 default:
805 id.append((char)nextChar);
806 }
807 }
808
809 case -1:
810 // we're done
811 return makeToken(ParserSym.EOF, "EOF");
812
813 default:
814 // If it's whitespace, skip over it.
815 // (When we switch to JDK 1.5, use Character.isWhitespace(int);
816 // til then, there's just Character.isWhitespace(char).)
817 if (nextChar <= Character.MAX_VALUE &&
818 Character.isWhitespace((char) nextChar)) {
819 // fall through
820 } else {
821 // everything else is an error
822 throw new RuntimeException(
823 "Unexpected character '" + (char) nextChar + "'");
824 }
825
826 case ' ':
827 case '\t':
828 case '\n':
829 case '\r':
830 // whitespace can be ignored
831 iPrevChar = iChar;
832 advance();
833 break;
834 }
835 }
836 }
837
838 private enum State {
839 leftOfPoint,
840 rightOfPoint,
841 inExponent,
842 }
843 }
844
845 // End Scanner.java