scanner performance improvements from Dave Daoust

core/org.eclipse.cdt.core/parser/ChangeLog-parser

@@ -1,3 +1,8 @@
+2004-03-16 Dave Daoust
+	refactor scanner for performance improvements
+	-changes to accountForUndo, now named readFromStream
+	-changes to checking for valid macros
+
 2004-03-15 Andrew Niefer
 	Templates:
 	- added ISourceElementRequestor.acceptTemplateParameterReference

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java

@@ -82,11 +82,11 @@ public class GCCScannerExtension implements IScannerExtension {
 		{
 			scannerData.getLogService().traceLog( "GCCScannerExtension handling #include_next directive" ); //$NON-NLS-1$
 			// figure out the name of the current file and its path
-			IScannerContext context = scannerData.getContextStack().getCurrentContext();
+//			IScannerContext context = scannerData.getContextStack().getCurrentContext();
-			if( context.getKind() != IScannerContext.ContextKind.INCLUSION ) 
+//			if( context.getKind() != IScannerContext.ContextKind.INCLUSION ) 
-			{
+//			{
-				//handle appropriate error
+//				//handle appropriate error
-			}
+//			}
 //			String fullInclusionPath = context.getFilename();
 //			IASTInclusion inclusion = context.getExtension();
 			

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java

@@ -721,7 +721,7 @@ public class Scanner implements IScanner {
 		return getChar();
 	}
 	
-	int getChar() throws ScannerException
+	final int getChar() throws ScannerException
 	{
 		return getChar( false );
 	}
@@ -731,12 +731,14 @@ public class Scanner implements IScanner {
 		
 		lastContext = scannerData.getContextStack().getCurrentContext();
 		
-		if (scannerData.getContextStack().getCurrentContext() == null)
+		if (lastContext == null)
 			// past the end of file
 			return c;
 
-        c = accountForUndo(c);
+       	if (lastContext.undoStackSize() != 0 )
-    	int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
+    		c = lastContext.popUndo();
+    	else
+    		c = readFromStream();
 		
 		if (enableTrigraphReplacement && (!insideString || enableTrigraphReplacementInStrings)) {
 			// Trigraph processing
@@ -747,39 +749,39 @@ public class Scanner implements IScanner {
 					c = getChar(insideString);
 					switch (c) {
 						case '(':
-							expandDefinition("??(", "[", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??(", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case ')':
-							expandDefinition("??)", "]", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??)", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '<':
-							expandDefinition("??<", "{", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??<", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '>':
-							expandDefinition("??>", "}", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '=':
-							expandDefinition("??=", "#", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??=", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '/':
-							expandDefinition("??/", "\\", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??/", "\\", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '\'':
-							expandDefinition("??\'", "^", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??\'", "^", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '!':
-							expandDefinition("??!", "|", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??!", "|", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '-':
-							expandDefinition("??-", "~", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??-", "~", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						default:
@@ -805,10 +807,10 @@ public class Scanner implements IScanner {
 				if (c == '<') {
 					c = getChar(false);
 					if (c == '%') {
-						expandDefinition("<%", "{", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("<%", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else if (c == ':') {
-						expandDefinition("<:", "[", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("<:", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -818,7 +820,7 @@ public class Scanner implements IScanner {
 				} else if (c == ':') {
 					c = getChar(false);
 					if (c == '>') {
-						expandDefinition(":>", "]", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition(":>", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -828,10 +830,10 @@ public class Scanner implements IScanner {
 				} else if (c == '%') {
 					c = getChar(false);
 					if (c == '>') {
-						expandDefinition("%>", "}", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("%>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else if (c == ':') {
-						expandDefinition("%:", "#", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("%:", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -845,38 +847,29 @@ public class Scanner implements IScanner {
 		return c;
 	}
 
-    protected int accountForUndo(int c)
+    protected int readFromStream()
     {
-        boolean done;
+    	int c;
-        do {
-        	done = true;
-        
-        	if (scannerData.getContextStack().getCurrentContext().undoStackSize() != 0 ) {
-        		c = scannerData.getContextStack().getCurrentContext().popUndo();
-        	} else {
     	try {
     		c = scannerData.getContextStack().getCurrentContext().read();
-        			if (c == NOCHAR) {
+    	}
-        				if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false) {
+    	catch (IOException e) {
     		c = NOCHAR;
-        					break;
-        				} else {
-        					done = false;
-        				}
-        			}
-        		} catch (IOException e) {
-        			if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false) {
-        				c = NOCHAR;
-        			} else {
-        				done = false;
-        			}
-        		}
-        	}
-        } while (!done);
-        return c;
     	}
     	
-	void ungetChar(int c) throws ScannerException{
+    	if (c != NOCHAR)
+    		return c;
+    	
+    	if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false)
+    		return NOCHAR;
+    	
+    	if (scannerData.getContextStack().getCurrentContext().undoStackSize() != 0 )
+    		return scannerData.getContextStack().getCurrentContext().popUndo();
+    	
+    	return readFromStream();
+    }
+
+	final void ungetChar(int c) throws ScannerException{
 		scannerData.getContextStack().getCurrentContext().pushUndo(c);
 		try
 		{
@@ -1220,9 +1213,12 @@ public class Scanner implements IScanner {
 		if (directive == null) {
 			if( scannerExtension.canHandlePreprocessorDirective( token ) )
 				scannerExtension.handlePreprocessorDirective( token, getRestOfPreprocessorLine() );
+			else
+			{
 				StringBuffer buffer = new StringBuffer( "#"); //$NON-NLS-1$
 				buffer.append( token );
 				handleProblem( IProblem.PREPROCESSOR_INVALID_DIRECTIVE, buffer.toString(), beginningOffset, false, true );
+			}
 			return null;
 		}
 
@@ -1432,12 +1428,12 @@ public class Scanner implements IScanner {
 
 		c = getChar();				
 		
+		// do the least expensive tests first!
 		while (
-				Character.isUnicodeIdentifierPart( (char)c)
+			((c >= 'a') && (c <= 'z'))
-//			((c >= 'a') && (c <= 'z'))
+			|| ((c >= 'A') && (c <= 'Z'))
-//			|| ((c >= 'A') && (c <= 'Z'))
+			|| ((c >= '0') && (c <= '9'))
-//			|| ((c >= '0') && (c <= '9'))
+			|| (c == '_') || Character.isUnicodeIdentifierPart( (char)c)
-//			|| (c == '_')
 			) {
 			buff.append((char) c);
 			c = getChar();
@@ -2545,6 +2541,36 @@ public class Scanner implements IScanner {
 		return macroReplacementTokens;
 	}
 	
+	protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) {
+		StringBuffer signatureBuffer  = new StringBuffer();
+		signatureBuffer.append( key );
+		signatureBuffer.append( ' ' );
+		signatureBuffer.append( value );
+
+//		List macroReplacementTokens;
+//		if (value.trim().equals( "" ))
+//			macroReplacementTokens = new ArrayList();
+//		else
+//			macroReplacementTokens = tokenizeReplacementString( NO_OFFSET_LIMIT, key, value, null ); 
+		
+		List macroReplacementTokens = new ArrayList();
+		if( !value.trim().equals( "" ) )  //$NON-NLS-1$
+		{	
+
+			Token t = new Token(
+					IToken.tIDENTIFIER, 
+					value, 
+					scannerData.getContextStack().getCurrentContext(),
+					scannerData.getContextStack().getCurrentLineNumber()
+					);
+			macroReplacementTokens.add( t );
+		}
+	
+		return new ObjectMacroDescriptor( key, 
+				signatureBuffer.toString(), 
+				macroReplacementTokens, 
+				value);
+	}
 
 	protected void poundDefine(int beginning, int beginningLine ) throws ScannerException, EndOfFileException {
 		StringBuffer potentialErrorMessage = new StringBuffer( POUND_DEFINE );
@@ -2636,8 +2662,9 @@ public class Scanner implements IScanner {
 		}
 		else if ((c == '\n') || (c == '\r'))
 		{
-			checkValidMacroRedefinition(key, previousDefinition, "", beginning);				 //$NON-NLS-1$
+			descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
-			addDefinition( key, "" ); //$NON-NLS-1$
+			checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
+			addDefinition( key, descriptor ); 
 		}
 		else if ((c == ' ') || (c == '\t') ) {
 			// this is a simple definition 
@@ -2646,8 +2673,9 @@ public class Scanner implements IScanner {
 			// get what we are to map the name to and add it to the definitions list
 			String value = getRestOfPreprocessorLine();
 			
-			checkValidMacroRedefinition(key, previousDefinition, value, beginning);
+			descriptor = createObjectMacroDescriptor(key, value);
-			addDefinition( key, value ); 
+			checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
+			addDefinition( key, descriptor ); 
 		
 		} else if (c == '/') {
 			// this could be a comment	
@@ -2655,20 +2683,23 @@ public class Scanner implements IScanner {
 			if (c == '/') // one line comment
 				{
 				skipOverSinglelineComment();
-				checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
+				descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
-				addDefinition(key, ""); //$NON-NLS-1$
+				checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
+				addDefinition(key, descriptor); 
 			} else if (c == '*') // multi-line comment
 				{
 				if (skipOverMultilineComment()) {
 					// we have gone over a newline
 					// therefore, this symbol was defined to an empty string
-					checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
+					descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
-					addDefinition(key, ""); //$NON-NLS-1$
+					checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning); 
+					addDefinition(key, descriptor);
 				} else {
 					String value = getRestOfPreprocessorLine();
 					
-					checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
+					descriptor = createObjectMacroDescriptor(key, value);
-					addDefinition(key, value);
+					checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning); 
+					addDefinition(key, descriptor);
 				}
 			} else {
 				// this is not a comment 
@@ -2699,40 +2730,6 @@ public class Scanner implements IScanner {
         } 
 	}
 
-	protected void checkValidMacroRedefinition(
-			String key,
-			IMacroDescriptor previousDefinition,
-			String newDefinition, int beginningOffset )
-	throws ScannerException 
-	{
-		StringBuffer buffer = new StringBuffer(key);
-		buffer.append( ' ');
-		buffer.append(newDefinition);
-		IMacroDescriptor newMacro = new ObjectMacroDescriptor( key, buffer.toString(), 
-				tokenizeReplacementString( NO_OFFSET_LIMIT, key, newDefinition, null ), newDefinition );
-		checkValidMacroRedefinition( key, previousDefinition, newMacro, beginningOffset );
-	}
-	
-	
-	protected void checkValidMacroRedefinition(
-			String key,
-			String previousDefinition,
-			String newDefinition, int beginningOffset )
-	throws ScannerException 
-	{
-		StringBuffer oldMacro = new StringBuffer( key );
-		oldMacro.append( ' ');
-		StringBuffer newMacro = new StringBuffer( oldMacro.toString() );
-		oldMacro.append( previousDefinition );
-		newMacro.append( newDefinition );
-		IMacroDescriptor prevMacroDescriptor = new ObjectMacroDescriptor( key, oldMacro.toString(), 
-				tokenizeReplacementString( NO_OFFSET_LIMIT, key, previousDefinition, null ), previousDefinition );
-		IMacroDescriptor newMacroDescriptor = new ObjectMacroDescriptor( key, newMacro.toString(), 
-				tokenizeReplacementString( NO_OFFSET_LIMIT, key, newDefinition, null ), newDefinition );
-		checkValidMacroRedefinition( key, prevMacroDescriptor, newMacroDescriptor, beginningOffset );
-	}
-	
-
 	protected void checkValidMacroRedefinition(
 		String key,
 		IMacroDescriptor previousDefinition,

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java

@@ -12,7 +12,6 @@ package org.eclipse.cdt.internal.core.parser.scanner;
 
 import java.io.IOException;
 import java.io.Reader;
-import java.util.Stack;
 
 import org.eclipse.cdt.core.parser.ast.IASTInclusion;
 
@@ -24,7 +23,6 @@ public class ScannerContext implements IScannerContext
     private int macroLength = -1;
 	private int line = 1;
 	private int offset;
-	private Stack undo = new Stack(); 
 	private ContextKind kind; 
 				
     /* (non-Javadoc)
@@ -108,18 +106,15 @@ public class ScannerContext implements IScannerContext
 		return line;
 	}
 
-	/**
+	/*	 there are never more than 2 elements in the unget stack! 
-	 * Returns the reader.
+	 *   trigraphs may involve 2, but in general there is a single element
-	 * @return Reader
+	 *	 I have made room for 10 -- just in case :-) 
 	 */
-	public final Reader getReader()
+	private int pos = 0;
-	{
+	private int undo[] = new int[10];  
-		return reader;
-	}
-
 	public final int undoStackSize()
 	{
-		return undo.size();
+		return pos;
 	}
 
 	/**
@@ -128,7 +123,7 @@ public class ScannerContext implements IScannerContext
 	 */
 	public final int popUndo()
 	{
-		int c = ((Integer)undo.pop()).intValue();
+		int c = undo[--pos];
 		if ((char)c == '\n') line++;
 		return c;
 	}
@@ -137,10 +132,18 @@ public class ScannerContext implements IScannerContext
 	 * Sets the undo.
 	 * @param undo The undo to set
 	 */
-	public void pushUndo(int undo)
+	public final void pushUndo(int c)
 	{
-		if ((char)undo == '\n') line--;
+		if ((char)c == '\n') line--;
-		this.undo.push( new Integer( undo )); 
+		undo[pos++] = c; 
+	}
+	/**
+	 * Returns the reader.
+	 * @return Reader
+	 */
+	public final Reader getReader()
+	{
+		return reader;
 	}