Patch for David Daoust. <BR>

Refactored the scanner to make Scanner Contexts more specific for macro expansions and inclusions, yielding performance and memory improvements.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java

@@ -1153,10 +1153,10 @@ public class QuickParseASTTests extends BaseASTTest
 		}
 	}
 	
-	public void testPreprocessor() throws Exception
+	public void testPreprocessor() throws Exception {
-	{
+		
-		Writer code = new StringWriter(); 
+		String code = "#include <stdio.h>\n#define DEF VALUE\n"; //$NON-NLS-1$
-		code.write( "#include <stdio.h>\n#define DEF VALUE\n"); //$NON-NLS-1$
+		
 		IASTCompilationUnit tu = parse( code.toString()  );
 		assertFalse( tu.getDeclarations().hasNext());
 		Iterator inclusions = quickParseCallback.getInclusions();
@@ -1173,9 +1173,9 @@ public class QuickParseASTTests extends BaseASTTest
 		
 		IASTMacro m = (IASTMacro)macros.next();
 		assertEquals( m.getName(), "DEF" );  //$NON-NLS-1$
-		assertEquals( m.getStartingOffset(), 19 );
+		assertEquals( m.getStartingOffset(), code.indexOf("#define") ); //$NON-NLS-1$
-		assertEquals( m.getNameOffset(), 27 );
+		assertEquals( m.getNameOffset(), code.indexOf("DEF") );
-		assertEquals( m.getEndingOffset(), 18 + 19);
+		assertEquals( m.getEndingOffset(), code.indexOf("VALUE") + 5);
 	}
 	
 	public void testTemplateDeclarationOfFunction() throws Exception

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ContextStack.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2003 IBM Corporation and others.
+ * Copyright (c) 2003, 2004 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials 
  * are made available under the terms of the Common Public License v0.5 
  * which accompanies this distribution, and is available at
@@ -11,7 +11,6 @@
 
 package org.eclipse.cdt.internal.core.parser.scanner;
 
-import java.io.IOException;
 import java.io.Reader;
 
 import org.eclipse.cdt.core.parser.IParserLogService;
@@ -30,25 +29,17 @@ import org.eclipse.cdt.internal.core.parser.util.TraceUtil;
 public class ContextStack {
 
 	private static class SentinelContext implements IScannerContext {
-		public int read() throws IOException { return '\n'; }
+		public int getChar() { return '\n'; }
-		public String getFilename() { return ""; } //$NON-NLS-1$
+		public String getContextName() { return ""; } //$NON-NLS-1$
-		public int getMacroOffset() { return -1; }
-		public int getMacroLength() { return -1; }
 		public int getOffset() { return 0; }
-		public int getRelativeOffset() { return 0; }
+		public void ungetChar(int undo) { }
-		public Reader getReader() { return null; }
-		public void pushUndo(int undo) { }
 		public int getKind() { return IScannerContext.ContextKind.SENTINEL; }
-		public void setKind(int kind) { }
+		public void close() { }
-		public IASTInclusion getExtension() { return null; }
-		public void setExtension(IASTInclusion ext) { }
-		public int getLine() { return -1; }
-		public int undoStackSize() { return 0; }  
-		public int popUndo() { return '\n'; }
-		public int getFilenameIndex() {	return -1;	}
 	}
 	private final IParserLogService log;
 	private int current_size = 8;
+	
+	private int lastFileContext = 0;
 	 
 	private IScannerContext [] cs = new IScannerContext[current_size];
 	private int cs_pos = 0;
@@ -59,7 +50,6 @@ public class ContextStack {
 	private static final String EMPTY_STRING = "";  //$NON-NLS-1$
 	
 	
-	
 	public final String getInclusionFilename( int index )
 	{
 		try
@@ -121,81 +111,62 @@ public class ContextStack {
 		scanner.setScannerContext(sentinel);
 	}
 
-    public void updateContext(Reader reader, String filename, int type, IASTInclusion inclusion, ISourceElementRequestor requestor) throws ContextException {
+    public void updateInclusionContext(Reader reader, String filename, IASTInclusion inclusion, ISourceElementRequestor requestor) throws ContextException {
-        updateContext(reader, filename, type, inclusion, requestor, -1, -1);
+    	addInclusionFilename( filename );
+    	ScannerContextInclusion context = new ScannerContextInclusion( reader, filename, inclusion, currentInclusionIndex - 1 );
+    	
+    	if( isCircularInclusion( context.getContextName() ) )
+			throw new ContextException( IProblem.PREPROCESSOR_CIRCULAR_INCLUSION );
+		
+		TraceUtil.outputTrace(log, "Scanner::ContextStack: entering inclusion ", null, context.getContextName(), null, null ); //$NON-NLS-1$
+		context.getExtension().enterScope( requestor );	
+		cs_push(context);
+		lastFileContext++;
     }
   
-	public void updateContext(Reader reader, String filename, int type, IASTInclusion inclusion, ISourceElementRequestor requestor, int macroOffset, int macroLength) throws ContextException 
+	public void updateMacroContext(String reader, String filename, ISourceElementRequestor requestor, int macroOffset, int macroLength) throws ContextException 
-    {
+    {		
-		int startLine = 1;
-		int index = -1;
-		
         // If we expand a macro within a macro, then keep offsets of the top-level one,
         // as only the top level macro identifier is properly positioned    
-        if (type == IScannerContext.ContextKind.MACROEXPANSION) {
+        if (getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION) {
-            if (getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION) {
+            macroOffset = ((ScannerContextMacro)getCurrentContext()).getOffset();
-                macroOffset = getCurrentContext().getMacroOffset();
+            macroLength = ((ScannerContextMacro)getCurrentContext()).getMacroLength();
-                macroLength = getCurrentContext().getMacroLength();
-            }
-            
-			startLine = getCurrentContext().getLine();
         }
-        else if( type == IScannerContext.ContextKind.INCLUSION )
+
-        {
+		cs_push(new ScannerContextMacro( 
-        	addInclusionFilename( filename );
+				reader, 
-        	index = currentInclusionIndex - 1;
+				filename, 
-        }
+				macroOffset, macroLength));	
-        	
-		IScannerContext context = new ScannerContext( reader, filename, type, null, macroOffset, macroLength, startLine, index );
-		context.setExtension(inclusion); 
-		push( context, requestor );	
 	}
 	
-	protected void push( IScannerContext context, ISourceElementRequestor requestor ) throws ContextException
+	protected void pushInitialContext( IScannerContext context ) throws ContextException
 	{
-		if( context.getKind() == IScannerContext.ContextKind.INCLUSION ) {
+		addInclusionFilename( context.getContextName() );
-			if( isCircularInclusion( context.getFilename() ) )
+		lastFileContext++;
-				throw new ContextException( IProblem.PREPROCESSOR_CIRCULAR_INCLUSION );
-			
-			TraceUtil.outputTrace(log, "Scanner::ContextStack: entering inclusion ", null, context.getFilename(), null, null ); //$NON-NLS-1$
-			context.getExtension().enterScope( requestor );				
-		}
-        else if( context.getKind() == IScannerContext.ContextKind.TOP )
-        {
-        	addInclusionFilename( context.getFilename() );
-        }
-		
-//		This could be replaced with a check for shouldExpandMacro -- but it is called by 
-//		the scanner before this point
-//		else if( context.getKind() == IScannerContext.ContextKind.MACROEXPANSION )
-//		{
-//			if( !defines.add( context.getFilename() ) )
-//				throw new ContextException( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN );
-//		}
 		cs_push(context);
 	}
 	
 	public boolean rollbackContext(ISourceElementRequestor requestor) {
 		IScannerContext context = getCurrentContext();
-		try {
+			context.close();
-			context.getReader().close();
-		} catch (IOException ie) {
-			TraceUtil.outputTrace( log, "ContextStack : Error closing reader "); //$NON-NLS-1$
-		}
 
 		if( context.getKind() == IScannerContext.ContextKind.INCLUSION )
 		{
-			TraceUtil.outputTrace(log, "Scanner::ContextStack: ending inclusion ", null, context.getFilename(), null, null); //$NON-NLS-1$
+			TraceUtil.outputTrace(log, "Scanner::ContextStack: ending inclusion ", null, context.getContextName(), null, null); //$NON-NLS-1$
-			context.getExtension().exitScope( requestor );
+			((ScannerContextInclusion)context).getExtension().exitScope( requestor );
+			lastFileContext--;
 		}
 		cs_pop();
 		return cs_pos != 0;
 	}
 	
 	public void undoRollback( IScannerContext undoTo, ISourceElementRequestor requestor ) {
+		IScannerContext context;
 		while (getCurrentContext() != undoTo ) {
-			//cs_pos++;
+			context = cs[cs_pos++];
-			scanner.setScannerContext(cs[cs_pos++]);
+			if(context.getKind() == IScannerContext.ContextKind.INCLUSION)
+				lastFileContext++;
+			scanner.setScannerContext(context);
 		}
 	}
 	
@@ -213,7 +184,7 @@ public class ContextStack {
 	{
 		for(int i = cs_pos-1; i >= 0; i--)
 			if (cs[i].getKind() == IScannerContext.ContextKind.MACROEXPANSION
-					&& cs[i].getFilename().equals(symbol))
+					&& cs[i].getContextName().equals(symbol))
 				return false;
 		return true;
 	}
@@ -222,29 +193,24 @@ public class ContextStack {
 	{
 		for(int i = cs_pos-1; i >= 0; i--)
 			if (cs[i].getKind() == IScannerContext.ContextKind.INCLUSION &&
-					cs[i].getFilename().equals(symbol))
+					cs[i].getContextName().equals(symbol))
 				return true;
 		return false;
 	}
 	
 	public final IScannerContext getCurrentContext(){
-		//return (cs_pos == 0) ? sentinel : cs[cs_pos -1];
 		return cs[cs_pos -1];
 	}
 
-	public IScannerContext getMostRelevantFileContext()
+	public ScannerContextInclusion getMostRelevantFileContext()
 	{
-		IScannerContext context = sentinel;
+		return (ScannerContextInclusion)cs[lastFileContext];
-		for( int i = cs_pos - 1; i >= 0; --i )
-		{
-			context = cs[i];
-			if( context.getKind() == IScannerContext.ContextKind.INCLUSION 
-					|| context.getKind() == IScannerContext.ContextKind.TOP )
-				break;
-		}
-		return context;
 	}
 	
+	public int getMostRelevantFileContextIndex()
+	{
+		return getMostRelevantFileContext().getFilenameIndex();
+	}
 	public int getCurrentLineNumber()
 	{
 		return getMostRelevantFileContext().getLine();

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java

@@ -1,5 +1,5 @@
 /**********************************************************************
- * Copyright (c) 2002,2003 Rational Software Corporation and others.
+ * Copyright (c) 2002,2004 IBM Rational Software and others.
  * All rights reserved.   This program and the accompanying materials
  * are made available under the terms of the Common Public License v0.5
  * which accompanies this distribution, and is available at
@@ -168,8 +168,8 @@ public class GCCScannerExtension implements IScannerExtension {
 			if( context == null || context.getKind() != IScannerContext.ContextKind.INCLUSION ) 
 				return;
 			
-			String fullInclusionPath = context.getFilename();
+			String fullInclusionPath = context.getContextName();
-			IASTInclusion inclusion = context.getExtension();
+			IASTInclusion inclusion = ((ScannerContextInclusion)context).getExtension();
 			
 			Iterator iter = scannerData.getIncludePathNames().iterator();
 			
@@ -199,7 +199,7 @@ public class GCCScannerExtension implements IScannerExtension {
 			{
 				try			
 				{
-					scannerData.getContextStack().updateContext(duple.getUnderlyingReader(), duple.getFilename(), ScannerContext.ContextKind.INCLUSION, inclusion, scannerData.getClientRequestor() );
+					scannerData.getContextStack().updateInclusionContext(duple.getUnderlyingReader(), duple.getFilename(), inclusion, scannerData.getClientRequestor() );
 					TraceUtil.outputTrace( scannerData.getLogService(), "GCCScannerExtension handling #include_next directive successfully pushed on new include file" ); //$NON-NLS-1$
 				}
 				catch (ContextException e1)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/IScannerContext.java

@@ -1,15 +1,22 @@
-package org.eclipse.cdt.internal.core.parser.scanner;
+/*******************************************************************************
-import java.io.IOException;
+ * Copyright (c) 2003, 2004 IBM - Rational Software and others.
-import java.io.Reader;
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM - Rational Software - initial implementation
+ ******************************************************************************/
+
+package org.eclipse.cdt.internal.core.parser.scanner;
 
-import org.eclipse.cdt.core.parser.ast.IASTInclusion;
 /**
  * @author jcamelon
  *
  */
 public interface IScannerContext {
 	
-	
 	public static class ContextKind  
 	{
 		public static int SENTINEL =  0;
@@ -17,58 +24,13 @@ public interface IScannerContext {
 		public static int INCLUSION = 2; 
 		public static int MACROEXPANSION = 3; 
 	}
-
-    /**
-     * This initializer is used for scanner contexts which are macro expansions.
-     * 
-     * @param macroOffset   Offset of the expanding macro
-     * @param macroLength   Length of the macro identifier
-     * @return
-     */
-	public int read() throws IOException;
-	public String getFilename();
-    
-    /**
-     * Returns macro offset (the offset of the top expanded macro).
-     * @return int
-     */
-    public int getMacroOffset();
-    
-    /**
-     * Returns macro length (the length of the top expanded macro identifier).
-     * @return int
-     */
-    public int getMacroLength();
-    
-    /**
-     * Returns the offset.
-     * @return int
-     */
-	public int getOffset();
-    
-    /**
-     * Returns relative offset (relative to the beginning of the ScannerContext).
-     * @return int
-     */
-    public int getRelativeOffset();
-    
-	public Reader getReader();
-	
-	public int undoStackSize();  
-	public int popUndo();
-	public void pushUndo(int undo);
-	
 	public int getKind(); 
-	public void setKind( int kind ); 
-
-	public IASTInclusion getExtension(); 
-	public void setExtension( IASTInclusion ext );
 	
-	public int getFilenameIndex();
+	public int getChar();
-
+	public void ungetChar(int undo);
-	/**
-	 * @return
-	 */
-	public int getLine();	
 	
+	public String getContextName();
+	public int getOffset();   
+	public void close();
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/LimitedScannerContext.java

@@ -1,23 +1,23 @@
-/**********************************************************************
+/*******************************************************************************
- * Copyright (c) 2002,2003 Rational Software Corporation and others.
+ * Copyright (c) 2002, 2004 IBM Corporation and others.
- * All rights reserved.   This program and the accompanying materials
+ * All rights reserved. This program and the accompanying materials 
- * are made available under the terms of the Common Public License v0.5
+ * are made available under the terms of the Common Public License v0.5 
  * which accompanies this distribution, and is available at
  * http://www.eclipse.org/legal/cpl-v05.html
  * 
  * Contributors: 
  * IBM Rational Software - Initial API and implementation
 ***********************************************************************/
+
 package org.eclipse.cdt.internal.core.parser.scanner;
 
-import java.io.IOException;
 import java.io.Reader;
 
 /**
  * @author jcamelon
  */
 public class LimitedScannerContext
-	extends ScannerContext
+	extends ScannerContextInclusion
 	implements IScannerContext {
 
 	private final int limit;
@@ -30,8 +30,8 @@ public class LimitedScannerContext
 	 * @param object
 	 * @param offsetLimit
 	 */
-	public LimitedScannerContext(Scanner scanner, Reader reader, String string, int kind, int offsetLimit, int index ) {
+	public LimitedScannerContext(Scanner scanner, Reader reader, String string, int offsetLimit, int index ) {
-		super( reader, string, kind, null, index );
+		super( reader, string, null, index );
 		this.scanner = scanner;
 		limit = offsetLimit;
 	}
@@ -41,13 +41,16 @@ public class LimitedScannerContext
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#read()
 	 */
-	public int read() throws IOException {
+	public int getChar()  {
 		if( getOffset() == limit )
 		{
 			scanner.setOffsetLimitReached(true);
-			throw new IOException();
+			return -1;
 		}
-		return super.read();
+		return super.getChar();
 	}
 
+	public int getKind() {
+		return ScannerContextInclusion.ContextKind.TOP;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java

@@ -1,17 +1,17 @@
 /*******************************************************************************
- * Copyright (c) 2001 Rational Software Corp. and others.
+ * Copyright (c) 2001, 2004 IBM Rational Software and others.
  * All rights reserved. This program and the accompanying materials 
  * are made available under the terms of the Common Public License v0.5 
  * which accompanies this distribution, and is available at
  * http://www.eclipse.org/legal/cpl-v05.html
  * 
  * Contributors:
- *     Rational Software - initial implementation
+ *     IBM - Rational Software
  ******************************************************************************/
+
 package org.eclipse.cdt.internal.core.parser.scanner;
 
 import java.io.File;
-import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.ArrayList;
@@ -202,7 +202,7 @@ public class Scanner implements IScanner {
 			addDefinition(  __FILE__, 
 					new DynamicMacroDescriptor( __FILE__, new DynamicMacroEvaluator() {
 						public String execute() {
-							return scannerData.getContextStack().getMostRelevantFileContext().getFilename();
+							return scannerData.getContextStack().getMostRelevantFileContext().getContextName();
 						}				
 					} ) );
 		
@@ -286,10 +286,10 @@ public class Scanner implements IScanner {
     	try
     	{
     		if( offsetLimit == NO_OFFSET_LIMIT )
-    			context = new ScannerContext(scannerData.getInitialReader(), resolvedFilename, ScannerContext.ContextKind.TOP, null, 0 );
+    			context = new ScannerContextTop(scannerData.getInitialReader(), resolvedFilename);
     		else
-    			context = new LimitedScannerContext( this, scannerData.getInitialReader(), resolvedFilename, ScannerContext.ContextKind.TOP, offsetLimit, 0 );
+    			context = new LimitedScannerContext( this, scannerData.getInitialReader(), resolvedFilename, offsetLimit, 0 );
-    		scannerData.getContextStack().push( context, scannerData.getClientRequestor() ); 
+    		scannerData.getContextStack().pushInitialContext( context ); 
     	} catch( ContextException  ce )
     	{
     		handleInternalError();
@@ -534,7 +534,7 @@ public class Scanner implements IScanner {
 			}
 			else // local inclusion
 			{
-				duple = ScannerUtility.createReaderDuple( new File( currentContext.getFilename() ).getParentFile().getAbsolutePath(), fileName, scannerData.getClientRequestor(), scannerData.getWorkingCopies() );
+				duple = ScannerUtility.createReaderDuple( new File( currentContext.getContextName() ).getParentFile().getAbsolutePath(), fileName, scannerData.getClientRequestor(), scannerData.getWorkingCopies() );
 				if( duple != null )
 					break totalLoop;
 				useIncludePaths = true;
@@ -563,10 +563,9 @@ public class Scanner implements IScanner {
 			
 			try
 			{
-				scannerData.getContextStack().updateContext(
+				scannerData.getContextStack().updateInclusionContext(
 					duple.getUnderlyingReader(), 
-					duple.getFilename(), 
+					duple.getFilename(),
-					ScannerContext.ContextKind.INCLUSION, 
 					inclusion, 
 					scannerData.getClientRequestor() );
 			}
@@ -715,11 +714,8 @@ public class Scanner implements IScanner {
 		if (lastContext.getKind() == IScannerContext.ContextKind.SENTINEL)
 			// past the end of file
 			return c;
-
+		
-       	if (lastContext.undoStackSize() != 0 )
+    	c = readFromStream();
-    		c = lastContext.popUndo();
-    	else
-    		c = readFromStream();
 		
 		if (enableTrigraphReplacement && (!insideString || enableTrigraphReplacementInStrings)) {
 			// Trigraph processing
@@ -730,39 +726,39 @@ public class Scanner implements IScanner {
 					c = getChar(insideString);
 					switch (c) {
 						case '(':
-							expandDefinition("??(", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??(", "[", lastContext.getOffset() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case ')':
-							expandDefinition("??)", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??)", "]", lastContext.getOffset() - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '<':
-							expandDefinition("??<", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??<", "{", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '>':
-							expandDefinition("??>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??>", "}", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '=':
-							expandDefinition("??=", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??=", "#", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '/':
-							expandDefinition("??/", "\\", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??/", "\\", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '\'':
-							expandDefinition("??\'", "^", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??\'", "^", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '!':
-							expandDefinition("??!", "|", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??!", "|", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						case '-':
-							expandDefinition("??-", "~", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+							expandDefinition("??-", "~", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 							c = getChar(insideString);
 							break;
 						default:
@@ -788,10 +784,10 @@ public class Scanner implements IScanner {
 				if (c == '<') {
 					c = getChar(false);
 					if (c == '%') {
-						expandDefinition("<%", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("<%", "{", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else if (c == ':') {
-						expandDefinition("<:", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("<:", "[", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -801,7 +797,7 @@ public class Scanner implements IScanner {
 				} else if (c == ':') {
 					c = getChar(false);
 					if (c == '>') {
-						expandDefinition(":>", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition(":>", "]", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -811,10 +807,10 @@ public class Scanner implements IScanner {
 				} else if (c == '%') {
 					c = getChar(false);
 					if (c == '>') {
-						expandDefinition("%>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("%>", "}", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else if (c == ':') {
-						expandDefinition("%:", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
+						expandDefinition("%:", "#", lastContext.getOffset()  - 1); //$NON-NLS-1$ //$NON-NLS-2$
 						c = getChar(false);
 					} else {
 						// Not a digraph
@@ -830,13 +826,7 @@ public class Scanner implements IScanner {
 
     protected int readFromStream()
     {
-    	int c;
+    	int c = currentContext.getChar();
-    	try {
-    		c = currentContext.read();
-    	}
-    	catch (IOException e) {
-    		c = NOCHAR;
-    	}
     	
     	if (c != NOCHAR)
     		return c;
@@ -844,16 +834,13 @@ public class Scanner implements IScanner {
     	if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false)
     		return NOCHAR;
     	
-    	if (currentContext.undoStackSize() != 0 )
-    		return currentContext.popUndo();
-    	
     	return readFromStream();
     }
 
-	final void ungetChar(int c) throws ScannerException{
+	final void ungetChar(int c) {
-		currentContext.pushUndo(c);
+		currentContext.ungetChar(c);
-		scannerData.getContextStack().undoRollback( lastContext, scannerData.getClientRequestor() );
+		if( lastContext != currentContext)
-
+			scannerData.getContextStack().undoRollback( lastContext, scannerData.getClientRequestor() );
 	}
 
 	protected boolean lookAheadForTokenPasting() throws ScannerException
@@ -905,16 +892,14 @@ public class Scanner implements IScanner {
 			storageBuffer.append( buff.toString() );
 			try
 			{
-				scannerData.getContextStack().updateContext( 
+				scannerData.getContextStack().updateMacroContext( 
-					new StringReader( storageBuffer.toString()), 
+					storageBuffer.toString(), 
-					PASTING, 
+					PASTING,
-					IScannerContext.ContextKind.MACROEXPANSION, 
+					scannerData.getClientRequestor(), -1, -1 );
-					null, 
-					scannerData.getClientRequestor() );
 			}
 			catch (ContextException e)
 			{
-				handleProblem( e.getId(), currentContext.getFilename(), getCurrentOffset(), false, true  );
+				handleProblem( e.getId(), currentContext.getContextName(), getCurrentOffset(), false, true  );
 			}
 			storageBuffer = null; 
 			return true;
@@ -1482,7 +1467,7 @@ public class Scanner implements IScanner {
 
 	protected IToken processKeywordOrIdentifier(StringBuffer buff, boolean pasting) throws ScannerException, EndOfFileException
 	{ 
-        int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
+        int baseOffset = lastContext.getOffset() - 1;
 				
 		// String buffer is slow, we need a better way such as memory mapped files
 		int c = getChar();				
@@ -1917,6 +1902,68 @@ public class Scanner implements IScanner {
 						continue;
 					}
 					return token;
+				case 'a':
+				case 'b':
+				case 'c':
+				case 'd':
+				case 'e':
+				case 'f':
+				case 'g':
+				case 'h':
+				case 'i':
+				case 'j':
+				case 'k':
+				case 'l':
+				case 'm':
+				case 'n':
+				case 'o':
+				case 'p':
+				case 'q':
+				case 'r':
+				case 's':
+				case 't':
+				case 'u':
+				case 'v':
+				case 'w':
+				case 'x':
+				case 'y':
+				case 'z':
+				case 'A':
+				case 'B':
+				case 'C':
+				case 'D':
+				case 'E':
+				case 'F':
+				case 'G':
+				case 'H':
+				case 'I':
+				case 'J':
+				case 'K':
+					// 'L' is handled elsewhere
+				case 'M':
+				case 'N':
+				case 'O':
+				case 'P':
+				case 'Q':
+				case 'R':
+				case 'S':
+				case 'T':
+				case 'U':
+				case 'V':
+				case 'W':
+				case 'X':
+				case 'Y':
+				case 'Z':
+				case '_':
+					StringBuffer sBuffer = new StringBuffer( );
+					sBuffer.append( (char) c );
+					token = processKeywordOrIdentifier(sBuffer, pasting);
+					if (token == null) 
+					{
+						c = getChar();
+						continue;
+					}
+					return token;
 				case '"' :
 					token = processStringLiteral(false);
 					if (token == null) 
@@ -2030,9 +2077,9 @@ public class Scanner implements IScanner {
 		int completionPoint = expression.length() + 2;
 		IASTCompletionNode.CompletionKind kind = IASTCompletionNode.CompletionKind.MACRO_REFERENCE;
 		
-		String prefix = ""; //$NON-NLS-1$
+		String prefix = EMPTY_STRING;
 		
-		if( ! expression.trim().equals("")) //$NON-NLS-1$
+		if( ! expression.trim().equals(EMPTY_STRING))
 		{	
 			IScanner subScanner = new Scanner( 
 					new StringReader(expression), 
@@ -2149,7 +2196,7 @@ public class Scanner implements IScanner {
 
     protected String getCurrentFile()
 	{
-		return scannerData.getContextStack().getMostRelevantFileContext() != null ? scannerData.getContextStack().getMostRelevantFileContext().getFilename() : ""; //$NON-NLS-1$
+		return scannerData.getContextStack().getMostRelevantFileContext() != null ? scannerData.getContextStack().getMostRelevantFileContext().getContextName() : ""; //$NON-NLS-1$
 	}
 
 
@@ -2530,7 +2577,7 @@ public class Scanner implements IScanner {
 
 	protected void poundInclude( int beginningOffset, int startLine ) throws ScannerException, EndOfFileException {
 		skipOverWhitespace();				
-		int baseOffset = lastContext.getOffset() - lastContext.undoStackSize();
+		int baseOffset = lastContext.getOffset() ;
 		int nameLine = scannerData.getContextStack().getCurrentLineNumber();
 		String includeLine = getRestOfPreprocessorLine();
 		if( isLimitReached() )
@@ -2693,7 +2740,7 @@ public class Scanner implements IScanner {
 	protected void poundDefine(int beginning, int beginningLine ) throws ScannerException, EndOfFileException {
 		// definition 
 		String key = getNextIdentifier();
-		int offset = currentContext.getOffset() - key.length() - currentContext.undoStackSize();
+		int offset = currentContext.getOffset() - key.length();
 		int nameLine = scannerData.getContextStack().getCurrentLineNumber();
 
 		// store the previous definition to check against later
@@ -2968,17 +3015,16 @@ public class Scanner implements IScanner {
 			String replacementValue = expansion.getExpansionSignature();
 			try
 			{
-				scannerData.getContextStack().updateContext( 
+				scannerData.getContextStack().updateMacroContext( 
-					new StringReader(replacementValue), 
+					replacementValue, 
-					symbol, ScannerContext.ContextKind.MACROEXPANSION, 
+					symbol,
-					null, 
 					scannerData.getClientRequestor(), 
 					symbolOffset, 
 					symbol.length());
 			}
 			catch (ContextException e)
 			{
-				handleProblem( e.getId(), currentContext.getFilename(), getCurrentOffset(), false, true );
+				handleProblem( e.getId(), currentContext.getContextName(), getCurrentOffset(), false, true );
 				consumeUntilOutOfMacroExpansion();
 				return;
 			}
@@ -3004,7 +3050,7 @@ public class Scanner implements IScanner {
 				}
                 
                 // Position of the closing ')'
-                int endMacroOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
+                int endMacroOffset = lastContext.getOffset()  - 1;
 				
 				String betweenTheBrackets = buffer.toString().trim();
                 
@@ -3119,18 +3165,16 @@ public class Scanner implements IScanner {
 				String finalString = buffer.toString();
 				try
 				{
-					scannerData.getContextStack().updateContext(
+					scannerData.getContextStack().updateMacroContext(
-						new StringReader(finalString),
+						finalString,
-						expansion.getName(), 
+						expansion.getName(),
-						ScannerContext.ContextKind.MACROEXPANSION, 
-						null, 
 						scannerData.getClientRequestor(), 
 						symbolOffset, 
 						endMacroOffset - symbolOffset + 1 );
 				}
 				catch (ContextException e)
 				{
-					handleProblem( e.getId(), currentContext.getFilename(), getCurrentOffset(), false, true );
+					handleProblem( e.getId(), currentContext.getContextName(), getCurrentOffset(), false, true );
 					consumeUntilOutOfMacroExpansion();
 					return;
 				}
@@ -3247,10 +3291,8 @@ public class Scanner implements IScanner {
 	 * @see org.eclipse.cdt.core.parser.IFilenameProvider#getCurrentFileIndex()
 	 */
 	public int getCurrentFileIndex() {
-		IScannerContext mostRelevantFileContext = scannerData.getContextStack().getMostRelevantFileContext();
+		return scannerData.getContextStack().getMostRelevantFileContextIndex();
-		return (( mostRelevantFileContext == null ) ? -1 : 	mostRelevantFileContext.getFilenameIndex() );
+}
-	}
-
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.IFilenameProvider#getFilenameForIndex(int)
 	 */

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java

@@ -1,203 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2001 Rational Software Corp. and others.
- * All rights reserved. This program and the accompanying materials 
- * are made available under the terms of the Common Public License v0.5 
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/cpl-v05.html
- * 
- * Contributors:
- *     Rational Software - initial implementation
- ******************************************************************************/
-package org.eclipse.cdt.internal.core.parser.scanner;
-
-import java.io.IOException;
-import java.io.Reader;
-
-import org.eclipse.cdt.core.parser.ast.IASTInclusion;
-
-public class ScannerContext implements IScannerContext
-{
-	private Reader reader;
-	private String filename;
-    private int macroOffset = -1;
-    private int macroLength = -1;
-	private int line = 1;
-	private int offset;
-	private int kind; 
-				
-    /* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion, int, int, int)
-     */
-	public ScannerContext(Reader r, String f, int k, IASTInclusion i, int mO, int mL, int l, int index)
-	{
-		reader = r;
-		filename = f;
-		offset = 0;
-		kind = k; 
-		inc = i;
-        macroOffset = mO;
-        macroLength = mL;
-        line = l;
-        this.index = index;
-	}
-    
-    /* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion)
-     */
-    public ScannerContext(Reader r, String f, int k, IASTInclusion i, int index)
-    {
-        this(r, f, k, i, -1, -1, 1, index);
-    }
-		
-	public int read() throws IOException {
-		++offset;
-		int c = reader.read();
-		if ((char)c == '\n') line++;
-		return c;
-	}
-	
-	/**
-	 * Returns the filename.
-	 * @return String
-	 */
-	public final String getFilename()
-	{
-		return filename;
-	}
-    
-    /* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getExtension()
-     */
-    public final int getMacroOffset() 
-    {
-        return macroOffset;
-    }
-    
-    /* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getMacroLength()
-     */
-    public final int getMacroLength() 
-    {
-        return macroLength;
-    }
-
-	/* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getOffset()
-     */
-	public final int getOffset()
-	{
-        // All the tokens generated by the macro expansion 
-        // will have dimensions (offset and length) equal to the expanding symbol.
-		return (macroOffset < 0) ? offset : macroOffset;
-	}
-    
-    /* (non-Javadoc)
-     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getRelativeOffset()
-     */
-    public final int getRelativeOffset()
-    {
-        return offset;
-    }
-    
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getLine()
-	 */
-	public final int getLine()
-	{
-		return line;
-	}
-
-	/*	 there are never more than 2 elements in the unget stack! 
-	 *   trigraphs may involve 2, but in general there is a single element
-	 *	 I have made room for 10 -- just in case :-) 
-	 */
-	private int pos = 0;
-	private int undo[] = new int[10];  
-	public final int undoStackSize()
-	{
-		return pos;
-	}
-
-	/**
-	 * Returns the undo.
-	 * @return int
-	 */
-	public final int popUndo()
-	{
-		int c = undo[--pos];
-		if ((char)c == '\n') line++;
-		return c;
-	}
-
-	/**
-	 * Sets the undo.
-	 * @param undo The undo to set
-	 */
-	public final void pushUndo(int c)
-	{
-		if ((char)c == '\n') line--;
-		undo[pos++] = c; 
-	}
-	/**
-	 * Returns the reader.
-	 * @return Reader
-	 */
-	public final Reader getReader()
-	{
-		return reader;
-	}
-
-
-	/**
-	 * Returns the kind.
-	 * @return int
-	 */
-	public int getKind() {
-		return kind;
-	}
-
-	/**
-	 * Sets the kind.
-	 * @param kind The kind to set
-	 */
-	public void setKind(int kind) {
-		this.kind = kind;
-	}
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getExtension()
-	 */
-	public IASTInclusion getExtension() {
-		return inc;
-	}
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#setExtension(org.eclipse.cdt.core.parser.ast.IASTInclusion)
-	 */
-	public void setExtension(IASTInclusion ext) {
-		inc = ext;
-	}
-	
-	private IASTInclusion inc = null;
-	private final int index;
-	
-	
-
-
-	/* (non-Javadoc)
-	 * @see java.lang.Object#toString()
-	 */
-	public String toString() {
-		StringBuffer buffer = new StringBuffer();
-		buffer.append( "file "); //$NON-NLS-1$
-		buffer.append( getFilename() );
-		buffer.append( ":"); //$NON-NLS-1$
-		buffer.append( getLine() );
-		return buffer.toString();
-	}
-
-	/* (non-Javadoc)
-	 * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerContext#getFilenameIndex()
-	 */
-	public int getFilenameIndex() {
-		return index;
-	}
-}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextInclusion.java

@@ -0,0 +1,114 @@
+/*******************************************************************************
+ * Copyright (c) 2001, 2004 IBM - Rational Software and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM - Rational Software - initial implementation
+ ******************************************************************************/
+
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.eclipse.cdt.core.parser.ast.IASTInclusion;
+
+public class ScannerContextInclusion implements IScannerContext
+{
+	protected Reader reader;
+	private String filename;
+	private IASTInclusion inc;
+	private final int index;
+	private int line;
+	protected int offset = 0;
+
+    /* (non-Javadoc)
+     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion)
+     */
+    public ScannerContextInclusion(Reader r, String f, IASTInclusion i, int index) {
+    	reader = r;
+		filename = f;
+		line = 1;
+		inc = i;
+        this.index = index;
+    }
+		
+	public final String getContextName()
+	{
+		return filename;
+	}
+	public int getOffset()
+	{
+		return offset - pos;
+	}
+	public void close() {
+		try {
+			reader.close();
+		}
+		catch (IOException ie) {
+		}
+	}
+	protected int pos = 0;
+	protected int undo[] = new int[2];  
+	public final void ungetChar(int c) {
+		undo[pos++] = c; 
+	}
+	
+	public int getChar() {
+		if (pos > 0)
+			return undo[--pos];
+		try {
+			++offset;
+			int c = reader.read();
+			if ((char)c == '\n') line++;
+			return c;
+		}
+	   	catch (IOException e) {
+    		return -1;
+    	}
+	}
+	
+	/**
+	 * Returns the kind.
+	 * @return int
+	 */
+	public int getKind() {
+		return ScannerContextInclusion.ContextKind.INCLUSION;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getExtension()
+	 */
+	public IASTInclusion getExtension() {
+		return inc;
+	}
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.scanner.IScannerContext#getFilenameIndex()
+	 */
+	public int getFilenameIndex() {
+		return index;
+	}
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getLine()
+	 */
+	public final int getLine()
+	{
+		return line;
+	}
+
+	/* (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	public String toString() {
+		StringBuffer buffer = new StringBuffer();
+		buffer.append( "file "); //$NON-NLS-1$
+		buffer.append( getContextName() );
+		buffer.append( ":"); //$NON-NLS-1$
+		buffer.append( getLine() );
+		return buffer.toString();
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextMacro.java

@@ -0,0 +1,88 @@
+/*******************************************************************************
+ * Copyright (c) 2001, 2004 IBM - Rational Software and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM - Rational Software - initial implementation
+ ******************************************************************************/
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+
+public class ScannerContextMacro implements IScannerContext
+{
+    private int macroOffset;
+    private int macroLength;
+    int position=0;
+    int length;
+    char [] reader;
+    String macroName;
+				
+    /* (non-Javadoc)
+     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion, int, int, int)
+     */
+	public ScannerContextMacro(String in, String macro, int mO, int mL)
+	{
+		macroName = macro;
+		reader = in.toCharArray();
+		length = in.length();
+        macroOffset = mO;
+        macroLength = mL;
+	}
+    
+	public final String getContextName() {
+		return macroName;
+	}
+	public int getChar() {
+		if (position < length)
+			return reader[position++];
+		return -1;
+	}
+	
+	public void close() {
+		
+	}
+	
+	public final void ungetChar(int c)
+	{
+		position--;
+		// may want to assert that reader.charAt(position) == c
+	}
+
+    /* (non-Javadoc)
+     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getMacroLength()
+     */
+    public final int getMacroLength() 
+    {
+        return macroLength;
+    }
+
+	/* (non-Javadoc)
+     * @see org.eclipse.cdt.internal.core.parser.IScannerContext#getOffset()
+     */
+	public int getOffset()
+	{
+        // All the tokens generated by the macro expansion 
+        // will have dimensions (offset and length) equal to the expanding symbol.
+		return macroOffset;
+	}
+
+	/**
+	 * Returns the kind.
+	 * @return int
+	 */
+	public int getKind() {
+		return IScannerContext.ContextKind.MACROEXPANSION;
+	}
+	/* (non-Javadoc)
+	 * @see java.lang.Object#toString()
+	 */
+	public String toString() {
+		StringBuffer buffer = new StringBuffer();
+		buffer.append( "Macro "); //$NON-NLS-1$
+		buffer.append( getContextName() );
+		return buffer.toString();
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContextTop.java

@@ -0,0 +1,25 @@
+/*******************************************************************************
+ * Copyright (c) 2001, 2004 IBM - Rational Software and others.
+ * All rights reserved. This program and the accompanying materials 
+ * are made available under the terms of the Common Public License v0.5 
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/cpl-v05.html
+ * 
+ * Contributors:
+ *     IBM - Rational Software - initial implementation
+ ******************************************************************************/
+
+package org.eclipse.cdt.internal.core.parser.scanner;
+
+import java.io.Reader;
+
+public class ScannerContextTop extends ScannerContextInclusion
+{
+	ScannerContextTop(Reader r, String f) {
+		super(r,f,null, 0);
+	}
+	
+	public int getKind() {
+		return ScannerContextInclusion.ContextKind.TOP;
+	}
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedExpansionToken.java

@@ -13,6 +13,7 @@ package org.eclipse.cdt.internal.core.parser.token;
 import org.eclipse.cdt.core.parser.IToken;
 import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
 import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerContextMacro;
 
 /**
  * @author johnc
@@ -34,8 +35,9 @@ public class ImagedExpansionToken extends ImagedToken implements IToken {
 	 * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
 	 */
 	protected void setOffsetAndLength(IScannerContext context) {
-		offset = context.getMacroOffset();
+		ScannerContextMacro m = (ScannerContextMacro) context;
-		length = context.getMacroLength();
+		offset = m.getOffset();
+		length = m.getMacroLength();
 	}
 	
 	/* (non-Javadoc)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/ImagedToken.java

@@ -55,7 +55,7 @@ public class ImagedToken extends SimpleToken {
 	 */
 	protected void setOffsetAndLength(IScannerContext context) {
 		if( getImage() == null ) return;
-		offset = context.getOffset() - getImage().length() - context.undoStackSize();		
+		offset = context.getOffset() - getImage().length();		
 		if( getType() == tSTRING || getType() == tCHAR )
 			offset--;
 		else if( getType() == tLSTRING || getType() == tLCHAR )

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleExpansionToken.java

@@ -13,6 +13,7 @@ package org.eclipse.cdt.internal.core.parser.token;
 import org.eclipse.cdt.core.parser.IToken;
 import org.eclipse.cdt.internal.core.parser.scanner.ContextStack;
 import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
+import org.eclipse.cdt.internal.core.parser.scanner.ScannerContextMacro;
 
 /**
  * @author johnc
@@ -34,8 +35,9 @@ public class SimpleExpansionToken extends SimpleToken implements IToken {
 	 * @see org.eclipse.cdt.internal.core.parser.token.SimpleToken#setOffsetAndLength(org.eclipse.cdt.internal.core.parser.scanner.IScannerContext)
 	 */
 	protected void setOffsetAndLength(IScannerContext context) {
-		offset = context.getMacroOffset();
+		ScannerContextMacro m = (ScannerContextMacro) context;
-		length = context.getMacroLength();
+		offset = m.getOffset();
+		length = m.getMacroLength();
 	}
 	
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/SimpleToken.java

@@ -44,7 +44,7 @@ public class SimpleToken extends AbstractToken implements IToken {
 	 * @param context
 	 */
 	protected void setOffsetAndLength(IScannerContext context) {
-		offset = context.getOffset() - getLength() - context.undoStackSize();		
+		offset = context.getOffset() - getLength();		
 	}
 	
 	public String getImage() { 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/token/TokenFactory.java

@@ -11,6 +11,7 @@
 package org.eclipse.cdt.internal.core.parser.token;
 
 import org.eclipse.cdt.core.parser.IToken;
+import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
 import org.eclipse.cdt.internal.core.parser.scanner.IScannerData;
 
 /**
@@ -22,7 +23,7 @@ public class TokenFactory {
 	{
 		if( value.length() > 15 )
 			return createUniquelyImagedToken( IToken.tINTEGER, value, scannerData );
-		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 			return new IntegerExpansionToken( IToken.tINTEGER, Integer.parseInt(value ), scannerData.getContextStack() );
 			
 		return new IntegerToken( IToken.tINTEGER, Integer.parseInt( value ), scannerData.getContextStack() );		
@@ -33,7 +34,7 @@ public class TokenFactory {
 		if( value.length() > 15 )
 			return createUniquelyImagedToken( IToken.tHEXINT, value, scannerData );
 		
-		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 			return new HexIntegerExpansionToken( IToken.tHEXINT, value, scannerData.getContextStack() );
 			
 		return new HexIntegerToken( IToken.tHEXINT, value, scannerData.getContextStack() );		
@@ -43,7 +44,7 @@ public class TokenFactory {
 	
 	public static IToken createToken( int tokenType, IScannerData scannerData )
 	{
-		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 			return new SimpleExpansionToken( tokenType, scannerData.getContextStack() );
 		
 		return new SimpleToken(	tokenType, scannerData.getContextStack() );
@@ -56,7 +57,7 @@ public class TokenFactory {
 	 * @return
 	 */
 	public static IToken createUniquelyImagedToken(int type, String image, IScannerData scannerData) {
-		if( scannerData.getContextStack().getCurrentContext().getMacroOffset() >= 0 )
+		if( scannerData.getContextStack().getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 			return new ImagedExpansionToken( type, scannerData.getContextStack(), image );
 
 		return new ImagedToken(type, scannerData.getContextStack(), image );