From c885b918e307973894e7dbb706952e03b1a6fc71 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Mon, 17 Mar 2003 19:48:49 +0000 Subject: [PATCH] Changed EOF token into a backtrack exception. --- core/org.eclipse.cdt.core/parser/ChangeLog | 4 + .../cdt/internal/core/parser/IScanner.java | 2 +- .../cdt/internal/core/parser/Parser.java | 74 ++++++++++--------- .../cdt/internal/core/parser/Scanner.java | 33 +++++---- .../cdt/internal/core/parser/Token.java | 3 - .../cdt/core/parser/tests/ExprEvalTest.java | 12 +-- .../core/parser/tests/ScannerTestCase.java | 63 +++++++++++----- 7 files changed, 113 insertions(+), 78 deletions(-) diff --git a/core/org.eclipse.cdt.core/parser/ChangeLog b/core/org.eclipse.cdt.core/parser/ChangeLog index 42536943a57..6194b2eee15 100644 --- a/core/org.eclipse.cdt.core/parser/ChangeLog +++ b/core/org.eclipse.cdt.core/parser/ChangeLog @@ -1,3 +1,7 @@ +2003-03-17 Doug Schaefer + Changed EOF to be a Backtrack exception instead of a token so simplify + error handling. + 2003-03-13 John Camelon Moved ## token to IScanner from Scanner. Updated IParserCallback and implementations to deal with Elaborated Type Specifiers. diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java index 759aa3897ec..35609664422 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java @@ -26,7 +26,7 @@ public interface IScanner { public void addIncludePath(String includePath); public void overwriteIncludePath( List newIncludePaths ); - public Token nextToken() throws ScannerException; + public Token nextToken() throws ScannerException, Parser.EndOfFile; public void setQuickScan(boolean qs); public void setCallback(IParserCallback c); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java index 3ea19643631..04d95c21858 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java @@ -32,7 +32,6 @@ public class Parser { quickParse = quick; scanner.setQuickScan(quick); scanner.setCallback(c); - //fetchToken(); } public Parser(IScanner s, IParserCallback c) throws Exception { @@ -78,13 +77,16 @@ c, quick); public void translationUnit() throws Exception { Object translationUnit = callback.translationUnitBegin(); Token lastBacktrack = null; - Token lastToken = null; - while (LT(1) != Token.tEOF) { + Token lastToken; + while (true) { try { - lastToken = currToken; + lastToken = LA(1); declaration( translationUnit ); - if( currToken == lastToken ) + if( LA(1) == lastToken ) consumeToNextSemicolon(); + } catch (EndOfFile e) { + // Good + break; } catch (Backtrack b) { // Mark as failure and try to reach a recovery point parsePassed = false; @@ -102,12 +104,11 @@ c, quick); callback.translationUnitEnd(translationUnit); } - protected void consumeToNextSemicolon() { - for (int t = LT(1); t != Token.tEOF; t = LT(1)) { + protected void consumeToNextSemicolon() throws EndOfFile { + consume(); + // TODO - we should really check for matching braces too + while (LT(1) != Token.tSEMI) { consume(); - // TO DO: we should really check for matching braces too - if (t == Token.tSEMI) - break; } } @@ -205,9 +206,7 @@ c, quick); // and look for the left brace; consume(); while (LT(1) != Token.tLBRACE) { - if (consume().getType() == Token.tEOF) - // Oops, couldn't find it - throw backtrack; + consume(); } // Falling through on purpose case Token.tLBRACE: @@ -225,9 +224,6 @@ c, quick); case Token.tLBRACE: ++depth; break; - case Token.tEOF: - // Oops, no match - throw backtrack; } } } else { @@ -461,9 +457,6 @@ c, quick); case Token.tLBRACE: ++depth; break; - case Token.tEOF: - // Oops, no match - throw backtrack; } } } @@ -529,6 +522,7 @@ c, quick); // parameterDeclarationClause Object clause = callback.argumentsBegin(declarator); consume(); + boolean seenParameter = false; parameterDeclarationLoop: for (;;) { switch (LT(1)) { @@ -540,9 +534,13 @@ c, quick); break; case Token.tCOMMA: consume(); + seenParameter = false; break; default: - parameterDeclaration( clause ); + if (seenParameter) + throw backtrack; + parameterDeclaration( clause ); + seenParameter = true; } } callback.argumentsEnd(clause); @@ -633,9 +631,6 @@ c, quick); case Token.tLBRACE: ++depth; break; - case Token.tEOF: - // Oops, no match - throw backtrack; } } } @@ -1362,24 +1357,32 @@ c, quick); } // Backtracking - private static class Backtrack extends Exception { + public static class Backtrack extends Exception { } private static Backtrack backtrack = new Backtrack(); + // End of file generally causes backtracking + public static class EndOfFile extends Backtrack { + } + + public static EndOfFile endOfFile = new EndOfFile(); + // Token management private IScanner scanner; private Token currToken; - private Token fetchToken() { + private Token fetchToken() throws EndOfFile { try { return scanner.nextToken(); + } catch (EndOfFile e) { + throw e; } catch (Exception e) { return null; } } - protected Token LA(int i) { + protected Token LA(int i) throws EndOfFile { if (i < 1) // can't go backwards return null; @@ -1390,34 +1393,37 @@ c, quick); Token retToken = currToken; for (; i > 1; --i) { - if (retToken.getNext() == null) - fetchToken(); retToken = retToken.getNext(); + if (retToken == null) + retToken = fetchToken(); } return retToken; } - protected int LT(int i) { + protected int LT(int i) throws EndOfFile { return LA(i).type; } - protected Token consume() { - if (currToken.getNext() == null) - fetchToken(); + protected Token consume() throws EndOfFile { + if (currToken == null) + currToken = fetchToken(); + Token retToken = currToken; currToken = currToken.getNext(); return retToken; } - protected Token consume(int type) throws Exception { + protected Token consume(int type) throws Backtrack { if (LT(1) == type) return consume(); else throw backtrack; } - protected Token mark() { + protected Token mark() throws EndOfFile { + if (currToken == null) + currToken = fetchToken(); return currToken; } diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java index 6cfb5ac011d..c6bf539e1c5 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java @@ -366,6 +366,9 @@ public class Scanner implements IScanner { private int getChar() { int c = NOCHAR; + if (currentContext == null) + // past the end of file + return c; boolean done; do { @@ -436,12 +439,12 @@ public class Scanner implements IScanner { - public Token nextToken() throws ScannerException { + public Token nextToken() throws ScannerException, Parser.EndOfFile { return nextToken( true ); } - protected Token nextToken( boolean pasting ) throws ScannerException + protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile { count++; @@ -1102,10 +1105,9 @@ public class Scanner implements IScanner { break; } - return newToken( - Token.tEOF, - "Bad Char: " + (char) c, - currentContext); + // Bad character + // TODO - does this need it's own exception + throw Parser.endOfFile; } } @@ -1113,9 +1115,7 @@ public class Scanner implements IScanner { throw new ScannerException("End of file encountered without terminating #endif"); // we're done - if (currentToken != null) - currentToken.setNext(Token.EOF); - return Token.EOF; + throw Parser.endOfFile; } static { @@ -1254,7 +1254,8 @@ public class Scanner implements IScanner { ExpressionEvaluator evaluator = new ExpressionEvaluator(); Scanner trial = new Scanner( - new StringReader(expression), + // Semicolon makes this valid C (hopefully) + new StringReader(expression + ";"), EXPRESSION, definitions); Parser parser = new Parser(trial, evaluator); @@ -1373,7 +1374,7 @@ public class Scanner implements IScanner { handleInclusion(f.trim(), useIncludePath ); } - protected void poundDefine() throws ScannerException { + protected void poundDefine() throws ScannerException, Parser.EndOfFile { skipOverWhitespace(); // definition String key = getNextIdentifier(); @@ -1424,9 +1425,13 @@ public class Scanner implements IScanner { null); Token t = helperScanner.nextToken(false); - while (t.type != Token.tEOF) { - macroReplacementTokens.add(t); - t = helperScanner.nextToken(false); + try { + while (true) { + macroReplacementTokens.add(t); + t = helperScanner.nextToken(false); + } + } catch (Parser.EndOfFile e) { + // Good } IMacroDescriptor descriptor = new MacroDescriptor(); diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java index f273e536276..3cc0699e0e0 100644 --- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java +++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java @@ -24,8 +24,6 @@ public class Token { image = i; } - public static Token EOF = new Token(Token.tEOF, ""); - public String toString() { return "Token type=" + type + " image =" + image + " offset=" + offset; @@ -47,7 +45,6 @@ public class Token { public void setNext(Token t) { next = t; } // Token types - static public final int tEOF = 0; static public final int tIDENTIFIER = 1; static public final int tINTEGER = 2; static public final int tCOLONCOLON = 3; diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java index fd330147e7e..d395014858a 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java @@ -25,17 +25,17 @@ public class ExprEvalTest extends TestCase { } public void testInteger() throws Exception { - runTest("5", 5); + runTest("5;", 5); } public void testRelational() throws Exception { - runTest("1 < 2", 1); - runTest("2 < 1", 0); - runTest("2 == 1 + 1", 1); - runTest("2 != 1 + 1", 0); + runTest("1 < 2;", 1); + runTest("2 < 1;", 0); + runTest("2 == 1 + 1;", 1); + runTest("2 != 1 + 1;", 0); } public void testBracketed() throws Exception { - runTest("2 * (3 + 4)", 14); + runTest("2 * (3 + 4);", 14); } } diff --git a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java index e0afc8e9673..b51743bed52 100644 --- a/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java +++ b/core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java @@ -8,6 +8,7 @@ import junit.framework.TestCase; import junit.framework.TestSuite; import org.eclipse.cdt.internal.core.parser.IMacroDescriptor; +import org.eclipse.cdt.internal.core.parser.Parser; import org.eclipse.cdt.internal.core.parser.Scanner; import org.eclipse.cdt.internal.core.parser.ScannerException; import org.eclipse.cdt.internal.core.parser.Token; @@ -176,16 +177,19 @@ public class ScannerTestCase extends TestCase try { Token t= scanner.nextToken(); - while ((t != null) && (t.type != Token.tEOF)) + while (t != null) { if (verbose) System.out.println("Token t = " + t); - if ((t.type < Token.tEOF) || (t.type > Token.tLAST)) + if ((t.type < 1) || (t.type > Token.tLAST)) System.out.println("Unknown type for token " + t); t= scanner.nextToken(); } } + catch (Parser.EndOfFile e) + { + } catch (ScannerException se) { throw se; @@ -245,7 +249,7 @@ public class ScannerTestCase extends TestCase validateToken(Token.tASSIGN); validateInteger("5"); validateToken(Token.tSEMI); - validateToken(Token.tEOF); + validateEOF(); // the case we were failing against in ctype.h // this is a definition, not a macro! @@ -445,7 +449,7 @@ public class ScannerTestCase extends TestCase validateEOF(); initializeScanner("#define SYMBOL 5\n#ifndef SYMBOL\nint counter(SYMBOL);\n#endif"); - validateToken(Token.tEOF); + validateEOF(); initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;"); validateToken(Token.t_int); @@ -522,7 +526,7 @@ public class ScannerTestCase extends TestCase validateBalance(); initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif"); - validateToken(Token.tEOF); + validateEOF(); validateBalance(); validateDefinition("FOO", "4"); @@ -833,7 +837,7 @@ public class ScannerTestCase extends TestCase } } - public void testQuickScan() + public void testQuickScan() throws Parser.EndOfFile { try { @@ -970,29 +974,45 @@ public class ScannerTestCase extends TestCase public void validateIdentifier(String expectedImage) throws ScannerException { - Token t= scanner.nextToken(); - assertTrue(t.type == Token.tIDENTIFIER); - assertTrue(t.image.equals(expectedImage)); + try { + Token t= scanner.nextToken(); + assertTrue(t.type == Token.tIDENTIFIER); + assertTrue(t.image.equals(expectedImage)); + } catch (Parser.EndOfFile e) { + assertTrue(false); + } } public void validateInteger(String expectedImage) throws ScannerException { - Token t= scanner.nextToken(); - assertTrue(t.type == Token.tINTEGER); - assertTrue(t.image.equals(expectedImage)); + try { + Token t= scanner.nextToken(); + assertTrue(t.type == Token.tINTEGER); + assertTrue(t.image.equals(expectedImage)); + } catch (Parser.EndOfFile e) { + assertTrue(false); + } } public void validateString(String expectedImage) throws ScannerException { - Token t= scanner.nextToken(); - assertTrue(t.type == Token.tSTRING); - assertTrue(t.image.equals(expectedImage)); + try { + Token t= scanner.nextToken(); + assertTrue(t.type == Token.tSTRING); + assertTrue(t.image.equals(expectedImage)); + } catch (Parser.EndOfFile e) { + assertTrue(false); + } } public void validateToken(int tokenType) throws ScannerException { - Token t= scanner.nextToken(); - assertTrue(t.type == tokenType); + try { + Token t= scanner.nextToken(); + assertTrue(t.type == tokenType); + } catch (Parser.EndOfFile e) { + assertTrue(false); + } } public void validateBalance(int expected) @@ -1007,7 +1027,10 @@ public class ScannerTestCase extends TestCase public void validateEOF() throws ScannerException { - validateToken(Token.tEOF); + try { + assertNull(scanner.nextToken()); + } catch (Parser.EndOfFile e) { + } } public void validateDefinition(String name, String value) @@ -1022,7 +1045,7 @@ public class ScannerTestCase extends TestCase { String definition= null; definition= (String) scanner.getDefinition(name); - this.assertNotNull(definition); + assertNotNull(definition); int intValue= (Integer.valueOf((String) definition)).intValue(); assertEquals(value, intValue); } @@ -1105,7 +1128,7 @@ public class ScannerTestCase extends TestCase validateInteger( "0"); validateToken( Token.tSEMI ); validateToken( Token.tRBRACE ); - validateToken( Token.tEOF ); + validateEOF(); } } catch( ScannerException se ) {