Changed EOF token into a backtrack exception.

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -1,3 +1,7 @@
+2003-03-17 Doug Schaefer
+	Changed EOF to be a Backtrack exception instead of a token so simplify
+	error handling.
+	
 2003-03-13 John Camelon
 	Moved ## token to IScanner from Scanner.
 	Updated IParserCallback and implementations to deal with Elaborated Type Specifiers. 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java

@@ -26,7 +26,7 @@ public interface IScanner {
 	public void addIncludePath(String includePath); 
 	public void overwriteIncludePath( List newIncludePaths );
 	
-	public Token nextToken() throws ScannerException;
+	public Token nextToken() throws ScannerException, Parser.EndOfFile;
 	 
 	public void setQuickScan(boolean qs);
 	public void setCallback(IParserCallback c);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java

@@ -32,7 +32,6 @@ public class Parser {
 		quickParse = quick;
 		scanner.setQuickScan(quick);
 		scanner.setCallback(c);
-		//fetchToken();
 	}
 
 	public Parser(IScanner s, IParserCallback c) throws Exception {
@@ -78,13 +77,16 @@ c, quick);
 	public void translationUnit() throws Exception {
 		Object translationUnit = callback.translationUnitBegin();
 		Token lastBacktrack = null;
-		Token lastToken = null;
+		Token lastToken;
-		while (LT(1) != Token.tEOF) {
+		while (true) {
 			try {
-				lastToken = currToken;
+				lastToken = LA(1);
 				declaration( translationUnit );
-				if( currToken == lastToken )
+				if( LA(1) == lastToken )
 					consumeToNextSemicolon();
+			} catch (EndOfFile e) {
+				// Good
+				break;
 			} catch (Backtrack b) {
 				// Mark as failure and try to reach a recovery point
 				parsePassed = false;
@@ -102,12 +104,11 @@ c, quick);
 		callback.translationUnitEnd(translationUnit);
 	}
 
-	protected void consumeToNextSemicolon() {
+	protected void consumeToNextSemicolon() throws EndOfFile {
-		for (int t = LT(1); t != Token.tEOF; t = LT(1)) {
+		consume();
+		// TODO - we should really check for matching braces too
+		while (LT(1) != Token.tSEMI) {
 			consume();
-			// TO DO: we should really check for matching braces too
-			if (t == Token.tSEMI)
-				break;
 		}
 	}
 	
@@ -205,9 +206,7 @@ c, quick);
 				// and look for the left brace;
 				consume();
 				while (LT(1) != Token.tLBRACE) {
-					if (consume().getType() == Token.tEOF)
+					consume();
-						// Oops, couldn't find it
-						throw backtrack;
 				}
 				// Falling through on purpose
 			case Token.tLBRACE:
@@ -225,9 +224,6 @@ c, quick);
 							case Token.tLBRACE:
 								++depth;
 								break;
-							case Token.tEOF:
-								// Oops, no match
-								throw backtrack;
 						}
 					}
 				} else {
@@ -461,9 +457,6 @@ c, quick);
 						case Token.tLBRACE:
 							++depth;
 							break;
-						case Token.tEOF:
-							// Oops, no match
-							throw backtrack;
 					}
 				}
 			}
@@ -529,6 +522,7 @@ c, quick);
 							// parameterDeclarationClause
 							Object clause = callback.argumentsBegin(declarator);
 							consume();
+							boolean seenParameter = false;
 							parameterDeclarationLoop:
 							for (;;) {
 								switch (LT(1)) {
@@ -540,9 +534,13 @@ c, quick);
 										break;
 									case Token.tCOMMA:
 										consume();
+										seenParameter = false;
 										break;
 									default:
+										if (seenParameter)
+											throw backtrack;
 										parameterDeclaration( clause );
+										seenParameter = true;
 								}
 							}
 							callback.argumentsEnd(clause);
@@ -633,9 +631,6 @@ c, quick);
 					case Token.tLBRACE:
 						++depth;
 						break;
-					case Token.tEOF:
-						// Oops, no match
-						throw backtrack;
 				}
 			}
 		}
@@ -1362,24 +1357,32 @@ c, quick);
 	}
 	
 	// Backtracking
-	private static class Backtrack extends Exception {
+	public static class Backtrack extends Exception {
 	}
 	
 	private static Backtrack backtrack = new Backtrack();
 	
+	// End of file generally causes backtracking
+	public static class EndOfFile extends Backtrack {
+	}
+	
+	public static EndOfFile endOfFile = new EndOfFile();
+	
 	// Token management
 	private IScanner scanner;
 	private Token currToken;
 	
-	private Token fetchToken() {
+	private Token fetchToken() throws EndOfFile {
 		try {
 			return scanner.nextToken();
+		} catch (EndOfFile e) {
+			throw e;
 		} catch (Exception e) {
 			return null;
 		}
 	}
 
-	protected Token LA(int i) {
+	protected Token LA(int i) throws EndOfFile {
 		if (i < 1)
 			// can't go backwards
 			return null;
@@ -1390,34 +1393,37 @@ c, quick);
 		Token retToken = currToken;
 		 
 		for (; i > 1; --i) {
-			if (retToken.getNext() == null)
-				fetchToken();
 			retToken = retToken.getNext();
+			if (retToken == null)
+				retToken = fetchToken();
 		}
 		
 		return retToken;
 	}
 
-	protected int LT(int i) {
+	protected int LT(int i) throws EndOfFile {
 		return LA(i).type;
 	}
 	
-	protected Token consume() {
+	protected Token consume() throws EndOfFile {
-		if (currToken.getNext() == null)
+		if (currToken == null)
-			fetchToken();
+			currToken = fetchToken();
+
 		Token retToken = currToken;
 		currToken = currToken.getNext();
 		return retToken;
 	}
 	
-	protected Token consume(int type) throws Exception {
+	protected Token consume(int type) throws Backtrack {
 		if (LT(1) == type)
 			return consume();
 		else
 			throw backtrack;
 	}
 	
-	protected Token mark() {
+	protected Token mark() throws EndOfFile {
+		if (currToken == null)
+			currToken = fetchToken();
 		return currToken;
 	}
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -366,6 +366,9 @@ public class Scanner implements IScanner {
 
 	private int getChar() {
 		int c = NOCHAR;
+		if (currentContext == null)
+			// past the end of file
+			return c;
 
 		boolean done;
 		do {
@@ -436,12 +439,12 @@ public class Scanner implements IScanner {
 
 	
 
-	public Token nextToken() throws ScannerException {
+	public Token nextToken() throws ScannerException, Parser.EndOfFile {
 		return nextToken( true ); 
 	}
 
 
-	protected Token nextToken( boolean pasting ) throws ScannerException
+	protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile
 	{
 	
 		count++;
@@ -1102,10 +1105,9 @@ public class Scanner implements IScanner {
 						break;
 				}
 
-				return newToken(
+				// Bad character
-					Token.tEOF,
+				// TODO - does this need it's own exception
-					"Bad Char: " + (char) c,
+				throw Parser.endOfFile;
-					currentContext);
 			}
 		}
 
@@ -1113,9 +1115,7 @@ public class Scanner implements IScanner {
 			throw new ScannerException("End of file encountered without terminating #endif");
 
 		// we're done
-		if (currentToken != null)
+		throw Parser.endOfFile;
-			currentToken.setNext(Token.EOF);
-		return Token.EOF;
 	}
 
 	static {
@@ -1254,7 +1254,8 @@ public class Scanner implements IScanner {
 				ExpressionEvaluator evaluator = new ExpressionEvaluator();
 				Scanner trial =
 					new Scanner(
-						new StringReader(expression),
+						// Semicolon makes this valid C (hopefully)
+						new StringReader(expression + ";"),
 						EXPRESSION,
 						definitions);
 				Parser parser = new Parser(trial, evaluator);
@@ -1373,7 +1374,7 @@ public class Scanner implements IScanner {
 			handleInclusion(f.trim(), useIncludePath );
 	}
 
-	protected void poundDefine() throws ScannerException {
+	protected void poundDefine() throws ScannerException, Parser.EndOfFile {
 		skipOverWhitespace();
 		// definition 
 		String key = getNextIdentifier();
@@ -1424,9 +1425,13 @@ public class Scanner implements IScanner {
 				null);
 			Token t = helperScanner.nextToken(false);
 
-			while (t.type != Token.tEOF) {
+			try {
-				macroReplacementTokens.add(t);
+				while (true) {
-				t = helperScanner.nextToken(false);
+					macroReplacementTokens.add(t);
+					t = helperScanner.nextToken(false);
+				}
+			} catch (Parser.EndOfFile e) {
+				// Good
 			}
 
 			IMacroDescriptor descriptor = new MacroDescriptor();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java

@@ -24,8 +24,6 @@ public class Token {
 		image = i;
 	}
 	
-	public static Token EOF = new Token(Token.tEOF, "<EOF>");
-	
 	public String toString()
 	{
 		return "Token type=" + type + "  image =" + image + " offset=" + offset; 	
@@ -47,7 +45,6 @@ public class Token {
 	public void setNext(Token t) { next = t; }
 
 	// Token types
-	static public final int tEOF = 0;
 	static public final int tIDENTIFIER = 1;
 	static public final int tINTEGER = 2;
 	static public final int tCOLONCOLON = 3;

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java

@@ -25,17 +25,17 @@ public class ExprEvalTest extends TestCase {
 	}
 	
 	public void testInteger() throws Exception {
-		runTest("5", 5);
+		runTest("5;", 5);
 	}
 	
 	public void testRelational() throws Exception {
-		runTest("1 < 2", 1);
+		runTest("1 < 2;", 1);
-		runTest("2 < 1", 0);
+		runTest("2 < 1;", 0);
-		runTest("2 == 1 + 1", 1);
+		runTest("2 == 1 + 1;", 1);
-		runTest("2 != 1 + 1", 0);
+		runTest("2 != 1 + 1;", 0);
 	}
 	
 	public void testBracketed() throws Exception {
-		runTest("2 * (3 + 4)", 14);
+		runTest("2 * (3 + 4);", 14);
 	}
 }

core/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java

@@ -8,6 +8,7 @@ import junit.framework.TestCase;
 import junit.framework.TestSuite;
 
 import org.eclipse.cdt.internal.core.parser.IMacroDescriptor;
+import org.eclipse.cdt.internal.core.parser.Parser;
 import org.eclipse.cdt.internal.core.parser.Scanner;
 import org.eclipse.cdt.internal.core.parser.ScannerException;
 import org.eclipse.cdt.internal.core.parser.Token;
@@ -176,16 +177,19 @@ public class ScannerTestCase extends TestCase
 		try
 		{
 			Token t= scanner.nextToken();
-			while ((t != null) && (t.type != Token.tEOF))
+			while (t != null)
 			{
 				if (verbose)
 					System.out.println("Token t = " + t);
 
-				if ((t.type < Token.tEOF) || (t.type > Token.tLAST))
+				if ((t.type < 1) || (t.type > Token.tLAST))
 					System.out.println("Unknown type for token " + t);
 				t= scanner.nextToken();
 			}
 		}
+		catch (Parser.EndOfFile e)
+		{
+		}
 		catch (ScannerException se)
 		{
 			throw se;
@@ -245,7 +249,7 @@ public class ScannerTestCase extends TestCase
 			validateToken(Token.tASSIGN);
 			validateInteger("5");
 			validateToken(Token.tSEMI);
-			validateToken(Token.tEOF);
+			validateEOF();
 
 			// the case we were failing against in ctype.h
 			// this is a definition, not a macro!
@@ -445,7 +449,7 @@ public class ScannerTestCase extends TestCase
 			validateEOF();
 
 			initializeScanner("#define SYMBOL 5\n#ifndef SYMBOL\nint counter(SYMBOL);\n#endif");
-			validateToken(Token.tEOF);
+			validateEOF();
 
 			initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;");
 			validateToken(Token.t_int);
@@ -522,7 +526,7 @@ public class ScannerTestCase extends TestCase
 			validateBalance();
 
 			initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif");
-			validateToken(Token.tEOF);
+			validateEOF();
 			validateBalance();
 			validateDefinition("FOO", "4");
 
@@ -833,7 +837,7 @@ public class ScannerTestCase extends TestCase
 		}
 	}
 
-	public void testQuickScan()
+	public void testQuickScan() throws Parser.EndOfFile
 	{
 		try
 		{
@@ -970,29 +974,45 @@ public class ScannerTestCase extends TestCase
 
 	public void validateIdentifier(String expectedImage) throws ScannerException
 	{
-		Token t= scanner.nextToken();
+		try {
-		assertTrue(t.type == Token.tIDENTIFIER);
+			Token t= scanner.nextToken();
-		assertTrue(t.image.equals(expectedImage));
+			assertTrue(t.type == Token.tIDENTIFIER);
+			assertTrue(t.image.equals(expectedImage));
+		} catch (Parser.EndOfFile e) {
+			assertTrue(false);
+		}
 	}
 
 	public void validateInteger(String expectedImage) throws ScannerException
 	{
-		Token t= scanner.nextToken();
+		try {
-		assertTrue(t.type == Token.tINTEGER);
+			Token t= scanner.nextToken();
-		assertTrue(t.image.equals(expectedImage));
+			assertTrue(t.type == Token.tINTEGER);
+			assertTrue(t.image.equals(expectedImage));
+		} catch (Parser.EndOfFile e) {
+			assertTrue(false);
+		}
 	}
 
 	public void validateString(String expectedImage) throws ScannerException
 	{
-		Token t= scanner.nextToken();
+		try {
-		assertTrue(t.type == Token.tSTRING);
+			Token t= scanner.nextToken();
-		assertTrue(t.image.equals(expectedImage));
+			assertTrue(t.type == Token.tSTRING);
+			assertTrue(t.image.equals(expectedImage));
+		} catch (Parser.EndOfFile e) {
+			assertTrue(false);
+		}
 	}
 
 	public void validateToken(int tokenType) throws ScannerException
 	{
-		Token t= scanner.nextToken();
+		try {
-		assertTrue(t.type == tokenType);
+			Token t= scanner.nextToken();
+			assertTrue(t.type == tokenType);
+		} catch (Parser.EndOfFile e) {
+			assertTrue(false);
+		}
 	}
 
 	public void validateBalance(int expected)
@@ -1007,7 +1027,10 @@ public class ScannerTestCase extends TestCase
 
 	public void validateEOF() throws ScannerException
 	{
-		validateToken(Token.tEOF);
+		try {
+			assertNull(scanner.nextToken());
+		} catch (Parser.EndOfFile e) {
+		}
 	}
 
 	public void validateDefinition(String name, String value)
@@ -1022,7 +1045,7 @@ public class ScannerTestCase extends TestCase
 	{
 		String definition= null;
 		definition= (String) scanner.getDefinition(name);
-		this.assertNotNull(definition);
+		assertNotNull(definition);
 		int intValue= (Integer.valueOf((String) definition)).intValue();
 		assertEquals(value, intValue);
 	}
@@ -1105,7 +1128,7 @@ public class ScannerTestCase extends TestCase
 				validateInteger( "0"); 
 				validateToken( Token.tSEMI ); 
 				validateToken( Token.tRBRACE ); 
-				validateToken( Token.tEOF );
+				validateEOF();
 			}
 		} catch( ScannerException se )
 		{