CORE

Removed some warnings. 
	Fixed Bug 39678 : Scanner doesn't support concatenation of different-type string literals (GCC) 
	Refactored ScannerContext to use constructors rather than initializers.  
	Refactored IScannerContext to use enumeration-esque kinds.  
	Added code assist/selection search support to Scanner.  

TESTS
	Removed some warnings.  
	Moved testBug39678() from ASTFailedTests to QuickParseASTTests.

core/org.eclipse.cdt.core.tests/ChangeLog

@@ -1,3 +1,7 @@
+2003-12-04 John Camelon
+	Removed some warnings.  
+	Moved testBug39678() from ASTFailedTests to QuickParseASTTests. 
+
 2003-12-03 Andrew Niefer
 	-modified FailedCompleteParseASTTest.testPMDotStarPointerToMemberFunction_Bug43242
 										.testPMArrowStarPointerToMemberFunction_Bug43242

core/org.eclipse.cdt.core.tests/failures/org/eclipse/cdt/core/parser/failedTests/ASTFailedTests.java

@@ -12,9 +12,7 @@ package org.eclipse.cdt.core.parser.failedTests;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.io.Writer;
-import java.util.Iterator;
 
-import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.core.parser.ast.IASTAbstractTypeSpecifierDeclaration;
 import org.eclipse.cdt.core.parser.ast.IASTClassSpecifier;
 import org.eclipse.cdt.core.parser.ast.IASTFunction;
@@ -48,10 +46,7 @@ public class ASTFailedTests extends BaseASTTest
     {
         assertCodeFailsParse("B::B() : a(({ 1; })) {}");
     }
-    public void testBug39678() throws Exception
+
-    {
-        assertCodeFailsParse("char *s = L\"a\" \"b\";");
-    }
     public void testBug39679() throws Exception
     {
         assertCodeFailsParse("Foo blat() return f(4) {}");

core/org.eclipse.cdt.core.tests/model/org/eclipse/cdt/core/model/tests/IIncludeTests.java

@@ -86,7 +86,6 @@ public class IIncludeTests extends IntegratedCModelTest {
 		for( int i=0; i<getIncludeNameList.length; i++ )
 		{
 			IInclude inc1 = theIncludes[i];
-			
 			assertEquals( getIncludeNameList[i], inc1.getIncludeName() );
 		}
 //		checkLineNumbers((CElement)inc1, 2, 2);

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/QuickParseASTTests.java

@@ -2015,4 +2015,11 @@ public class QuickParseASTTests extends BaseASTTest
 		assertSimpleType( variable, IASTSimpleTypeSpecifier.Type._BOOL );
 	}
 
+	public void testBug39678() throws Exception
+	{
+		IASTVariable variable = (IASTVariable) assertSoleDeclaration("char *s = L\"a\" \"b\";");
+		IASTExpression exp = variable.getInitializerClause().getAssigmentExpression();
+		assertEquals( exp.getLiteralString(), "ab");
+	}
+	
 }

core/org.eclipse.cdt.core/parser/ChangeLog

@@ -1,3 +1,10 @@
+2003-12-04 John Camelon
+	Removed some warnings. 
+	Fixed Bug 39678 : Scanner doesn't support concatenation of different-type string literals (GCC) 
+	Refactored ScannerContext to use constructors rather than initializers.  
+	Refactored IScannerContext to use enumeration-esque kinds.  
+	Added code assist/selection search support to Scanner.  
+
 2003-12-03 Andrew Niefer
 	- Symbol table - modify prefix lookup handling of ambiguities
 	               - fix up qualified lookup

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IScanner.java

@@ -11,6 +11,8 @@ public interface IScanner  {
 	public static final int tPOUNDPOUND = -6;
 	public static final int tPOUND      = -7;
 	
+	public void setOffsetBoundary( int offset );
+	
 	public void setASTFactory( IASTFactory f );
 	public void addDefinition(String key, IMacroDescriptor macroToBeAdded );
 	public void addDefinition(String key, String value); 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/core/parser/IToken.java

@@ -26,6 +26,7 @@ public interface IToken {
 	public abstract int getDelta(IToken other);
 	public abstract IToken getNext();
 	public abstract void setNext(IToken t);
+	public abstract void setType(int i);	
 	public abstract boolean looksLikeExpression();
 	public abstract boolean isPointer();
 	public abstract boolean isOperator();
@@ -301,4 +302,5 @@ public interface IToken {
 	static public final int t_restrict = 137;
 
 	static public final int tLAST = t_restrict;
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ContextStack.java

@@ -22,6 +22,7 @@ import org.eclipse.cdt.core.parser.IParserLogService;
 import org.eclipse.cdt.core.parser.IProblem;
 import org.eclipse.cdt.core.parser.ISourceElementRequestor;
 import org.eclipse.cdt.core.parser.ast.IASTInclusion;
+import org.eclipse.cdt.internal.core.parser.IScannerContext.ContextKind;
 
 /**
  * @author aniefer
@@ -37,18 +38,18 @@ public class ContextStack {
 		log = l;
 	}
 
-    public void updateContext(Reader reader, String filename, int type, IASTInclusion inclusion, ISourceElementRequestor requestor) throws ContextException {
+    public void updateContext(Reader reader, String filename, ContextKind type, IASTInclusion inclusion, ISourceElementRequestor requestor) throws ContextException {
         updateContext(reader, filename, type, inclusion, requestor, -1, -1);
     }
   
-	public void updateContext(Reader reader, String filename, int type, IASTInclusion inclusion, ISourceElementRequestor requestor, int macroOffset, int macroLength) throws ContextException 
+	public void updateContext(Reader reader, String filename, ContextKind type, IASTInclusion inclusion, ISourceElementRequestor requestor, int macroOffset, int macroLength) throws ContextException 
     {
 		int startLine = 1;
 		
         // If we expand a macro within a macro, then keep offsets of the top-level one,
         // as only the top level macro identifier is properly positioned    
-        if (type == IScannerContext.MACROEXPANSION) {
+        if (type == IScannerContext.ContextKind.MACROEXPANSION) {
-            if (currentContext.getKind() == IScannerContext.MACROEXPANSION) {
+            if (currentContext.getKind() == IScannerContext.ContextKind.MACROEXPANSION) {
                 macroOffset = currentContext.getMacroOffset();
                 macroLength = currentContext.getMacroLength();
             }
@@ -57,20 +58,20 @@ public class ContextStack {
         }
 
 		undoStack.clear();
-		IScannerContext context = new ScannerContext().initialize(reader, filename, type, null, macroOffset, macroLength, startLine );
+		IScannerContext context = new ScannerContext( reader, filename, type, null, macroOffset, macroLength, startLine );
 		context.setExtension(inclusion); 
 		push( context, requestor );	
 	}
 	
 	protected void push( IScannerContext context, ISourceElementRequestor requestor ) throws ContextException
 	{
-		if( context.getKind() == IScannerContext.INCLUSION )
+		if( context.getKind() == IScannerContext.ContextKind.INCLUSION )
 		{
 			if( !inclusions.add( context.getFilename() ) )
 				throw new ContextException( IProblem.PREPROCESSOR_CIRCULAR_INCLUSION );
 			context.getExtension().enterScope( requestor );				
 
-		} else if( context.getKind() == IScannerContext.MACROEXPANSION )
+		} else if( context.getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 		{
 			if( !defines.add( context.getFilename() ) )
 				throw new ContextException( IProblem.PREPROCESSOR_INVALID_MACRO_DEFN );
@@ -79,7 +80,7 @@ public class ContextStack {
 			contextStack.push(currentContext);
 		
 		currentContext = context;
-		if( context.getKind() == IScannerContext.TOP )
+		if( context.getKind() == IScannerContext.ContextKind.TOP )
 			topContext = context;
 	}
 	
@@ -90,11 +91,11 @@ public class ContextStack {
 			log.traceLog("ContextStack : Error closing reader ");
 		}
 
-		if( currentContext.getKind() == IScannerContext.INCLUSION )
+		if( currentContext.getKind() == IScannerContext.ContextKind.INCLUSION )
 		{
 			inclusions.remove( currentContext.getFilename() );
 			currentContext.getExtension().exitScope( requestor );
-		} else if( currentContext.getKind() == IScannerContext.MACROEXPANSION )
+		} else if( currentContext.getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 		{
 			defines.remove( currentContext.getFilename() );
 		}
@@ -164,15 +165,15 @@ public class ContextStack {
 	{
 		if( currentContext != null )
 		{
-			if( currentContext.getKind() == IScannerContext.TOP ) return currentContext;
+			if( currentContext.getKind() == IScannerContext.ContextKind.TOP ) return currentContext;
-			if( currentContext.getKind() == IScannerContext.INCLUSION ) return currentContext;
+			if( currentContext.getKind() == IScannerContext.ContextKind.INCLUSION ) return currentContext;
 		}
 				
 		IScannerContext context = null;
 		for( int i = contextStack.size() - 1; i >= 0; --i )
 		{
 			context = (IScannerContext)contextStack.get(i);
-			if( context.getKind() == IScannerContext.INCLUSION || context.getKind() == IScannerContext.TOP )
+			if( context.getKind() == IScannerContext.ContextKind.INCLUSION || context.getKind() == IScannerContext.ContextKind.TOP )
 				break;
 			if( i == 0 ) context = null;
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScannerContext.java

@@ -2,6 +2,7 @@ package org.eclipse.cdt.internal.core.parser;
 import java.io.IOException;
 import java.io.Reader;
 
+import org.eclipse.cdt.core.parser.Enum;
 import org.eclipse.cdt.core.parser.ast.IASTInclusion;
 /**
  * @author jcamelon
@@ -9,10 +10,22 @@ import org.eclipse.cdt.core.parser.ast.IASTInclusion;
  */
 public interface IScannerContext {
 	
-	public static int SENTINEL = 0;
+	
-	public static int TOP = 1; 
+	public static class ContextKind extends Enum  
-	public static int INCLUSION = 2; 
+	{
-	public static int MACROEXPANSION = 3; 
+		public static ContextKind SENTINEL = new ContextKind( 0 );
+		public static ContextKind TOP = new ContextKind( 1 ); 
+		public static ContextKind INCLUSION = new ContextKind( 2 ); 
+		public static ContextKind MACROEXPANSION = new ContextKind( 3 ); 
+		
+		/**
+		 * @param enumValue
+		 */
+		protected ContextKind(int enumValue) {
+			super(enumValue);
+			// 
+		}
+	}
 
     /**
      * This initializer is used for scanner contexts which are macro expansions.
@@ -21,9 +34,6 @@ public interface IScannerContext {
      * @param macroLength   Length of the macro identifier
      * @return
      */
-    public IScannerContext initialize(Reader r, String f, int k, IASTInclusion i, int macroOffset, int macroLength, int line );
-    
-	public IScannerContext initialize(Reader r, String f, int k, IASTInclusion i);
 	public int read() throws IOException;
 	public String getFilename();
     
@@ -57,8 +67,8 @@ public interface IScannerContext {
 	public int popUndo();
 	public void pushUndo(int undo);
 	
-	public int getKind(); 
+	public ContextKind getKind(); 
-	public void setKind( int kind ); 
+	public void setKind( ContextKind kind ); 
 
 	public IASTInclusion getExtension(); 
 	public void setExtension( IASTInclusion ext );

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/LimitedScannerContext.java

@@ -0,0 +1,46 @@
+/*
+ * Created on Dec 4, 2003
+ *
+ * To change the template for this generated file go to
+ * Window - Preferences - Java - Code Generation - Code and Comments
+ */
+package org.eclipse.cdt.internal.core.parser;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * @author jcamelon
+ *
+ * To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Generation - Code and Comments
+ */
+public class LimitedScannerContext
+	extends ScannerContext
+	implements IScannerContext {
+
+	private final int limit;
+
+	/**
+	 * @param reader
+	 * @param string
+	 * @param i
+	 * @param object
+	 * @param offsetLimit
+	 */
+	public LimitedScannerContext(Reader reader, String string, ContextKind kind, int offsetLimit) {
+		super( reader, string, kind, null );
+		limit = offsetLimit;
+	}
+
+	
+	
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.internal.core.parser.IScannerContext#read()
+	 */
+	public int read() throws IOException {
+		if( getOffset() == limit ) throw new IOException();
+		return super.read();
+	}
+
+}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java

@@ -61,6 +61,9 @@ public class Scanner implements IScanner {
 	private final static String SCRATCH = "<scratch>";
 	private Reader backupReader;
 	private IProblemFactory problemFactory = new ScannerProblemFactory();
+	private boolean initialContextInitialized = false;
+	private final String filename;
+	private final Reader reader;
 
 	protected void handleProblem( int problemID, String argument, int beginningOffset, boolean warning, boolean error ) throws ScannerException
 	{
@@ -84,6 +87,8 @@ public class Scanner implements IScanner {
     	this.log = log;
 		this.requestor = requestor;
 		this.mode = parserMode;
+		this.filename = filename;
+		this.reader = reader;
 		this.language = language;
 		astFactory = ParserFactory.createASTFactory( mode, language );
 		this.backupReader = reader;
@@ -91,15 +96,11 @@ public class Scanner implements IScanner {
 		try {
 			//this is a hack to get around a sudden EOF experience
 			contextStack.push(
-						new ScannerContext().initialize(
+						new ScannerContext(
 						new StringReader("\n"),
 						START,
-						ScannerContext.SENTINEL, null), requestor);
+						ScannerContext.ContextKind.SENTINEL, null), requestor);
 
-			if (filename == null)
-				contextStack.push( new ScannerContext().initialize(reader, TEXT, ScannerContext.TOP, null ), requestor ); 
-			else
-				contextStack.push( new ScannerContext().initialize(reader, filename, ScannerContext.TOP, null ), requestor );
 		} catch( ContextException ce ) {
 			//won't happen since we aren't adding an include or a macro
 		} 
@@ -110,9 +111,27 @@ public class Scanner implements IScanner {
 			
 		if( info.getIncludePaths() != null )
 			overwriteIncludePath( info.getIncludePaths() );
-            
+
+		
     }
 
+    private void setupInitialContext()
+    {
+    	String resolvedFilename = filename == null ? TEXT : filename;
+    	IScannerContext context = null;
+    	try
+    	{
+    		if( offsetLimit == NO_OFFSET_LIMIT )
+    			context = new ScannerContext(reader, resolvedFilename, ScannerContext.ContextKind.TOP, null );
+    		else
+    			context = new LimitedScannerContext( reader, resolvedFilename, ScannerContext.ContextKind.TOP, offsetLimit );
+    		contextStack.push( context, requestor ); 
+    	} catch( ContextException  ce )
+    	{
+    		// should never occur
+    	}
+    	initialContextInitialized = true;   	
+    }
 	public void addIncludePath(String includePath) {
 		includePathNames.add(includePath);
 		includePaths.add( new File( includePath ) );
@@ -392,7 +411,7 @@ public class Scanner implements IScanner {
 			
 			try
 			{
-				contextStack.updateContext(inclusionReader, newPath, ScannerContext.INCLUSION, inclusion, requestor );
+				contextStack.updateContext(inclusionReader, newPath, ScannerContext.ContextKind.INCLUSION, inclusion, requestor );
 			}
 			catch (ContextException e1)
 			{
@@ -449,13 +468,20 @@ public class Scanner implements IScanner {
 	
 	private final ParserMode mode;
 	
-
+	public int getCharacter() throws ScannerException
+	{
+		if( ! initialContextInitialized )
+			setupInitialContext();
+		
+		return getChar();
+	}
+	
 	private int getChar() throws ScannerException
 	{
 		return getChar( false );
 	}
 
-	private int getChar( boolean insideString ) throws ScannerException {
+	private int getChar( boolean insideString ) throws ScannerException {	
 		int c = NOCHAR;
 		
 		lastContext = contextStack.getCurrentContext();
@@ -661,7 +687,7 @@ public class Scanner implements IScanner {
 
 	protected void consumeUntilOutOfMacroExpansion() throws ScannerException
 	{
-		while( contextStack.getCurrentContext().getKind() == IScannerContext.MACROEXPANSION )
+		while( contextStack.getCurrentContext().getKind() == IScannerContext.ContextKind.MACROEXPANSION )
 			getChar();
 	}
 
@@ -675,6 +701,9 @@ public class Scanner implements IScanner {
 
 	public IToken nextToken( boolean pasting, boolean lookingForNextAlready ) throws ScannerException, EndOfFile
 	{
+		if( ! initialContextInitialized )
+			setupInitialContext();
+		
 		if( cachedToken != null ){
 			setCurrentToken( cachedToken );
 			cachedToken = null;
@@ -768,7 +797,8 @@ public class Scanner implements IScanner {
 							next = null;
 						}
 						
-						while( next != null && next.getType()  == returnToken.getType() ){
+						while( next != null && ( next.getType()  == IToken.tSTRING || 
+							next.getType() == IToken.tLSTRING ) ){
 							returnToken.setImage( returnToken.getImage() + next.getImage() ); 
 							returnToken.setNext( null );
 							currentToken = returnToken; 
@@ -866,7 +896,7 @@ public class Scanner implements IScanner {
 							storageBuffer.append( ident );
 							try
 							{
-								contextStack.updateContext( new StringReader( storageBuffer.toString()), PASTING, IScannerContext.MACROEXPANSION, null, requestor );
+								contextStack.updateContext( new StringReader( storageBuffer.toString()), PASTING, IScannerContext.ContextKind.MACROEXPANSION, null, requestor );
 							}
 							catch (ContextException e)
 							{
@@ -1020,7 +1050,7 @@ public class Scanner implements IScanner {
 						{
 							try
 							{
-								contextStack.updateContext( new StringReader( buff.toString()), PASTING, IScannerContext.MACROEXPANSION, null, requestor );
+								contextStack.updateContext( new StringReader( buff.toString()), PASTING, IScannerContext.ContextKind.MACROEXPANSION, null, requestor );
 							}
 							catch (ContextException e)
 							{
@@ -2390,7 +2420,7 @@ public class Scanner implements IScanner {
         
         try {
             while (true) {
-				int c = tokenizer.getChar();
+				int c = tokenizer.getCharacter();
 				if ((c != ' ') && (c != '\t') && (c != '\r') && (c != '\n')) {
 					space = false;
 				}
@@ -2436,7 +2466,7 @@ public class Scanner implements IScanner {
 			String replacementValue = (String) expansion;
 			try
 			{
-				contextStack.updateContext( new StringReader(replacementValue), (POUND_DEFINE + symbol ), ScannerContext.MACROEXPANSION, null, requestor, symbolOffset, symbol.length());
+				contextStack.updateContext( new StringReader(replacementValue), (POUND_DEFINE + symbol ), ScannerContext.ContextKind.MACROEXPANSION, null, requestor, symbolOffset, symbol.length());
 			}
 			catch (ContextException e)
 			{
@@ -2567,7 +2597,7 @@ public class Scanner implements IScanner {
 				{
 					contextStack.updateContext(
 						new StringReader(finalString),
-						POUND_DEFINE + macro.getSignature(), ScannerContext.MACROEXPANSION, null, requestor, symbolOffset, endMacroOffset - symbolOffset + 1 );
+						POUND_DEFINE + macro.getSignature(), ScannerContext.ContextKind.MACROEXPANSION, null, requestor, symbolOffset, endMacroOffset - symbolOffset + 1 );
 				}
 				catch (ContextException e)
 				{
@@ -2635,7 +2665,9 @@ public class Scanner implements IScanner {
 	}
 	
 	private final ISourceElementRequestor requestor;
-	private IASTFactory astFactory = null; 
+	private IASTFactory astFactory = null;
+	private static final int NO_OFFSET_LIMIT = -1;
+	private int offsetLimit = NO_OFFSET_LIMIT; 
 	
 	/* (non-Javadoc)
 	 * @see org.eclipse.cdt.core.parser.IScanner#setASTFactory(org.eclipse.cdt.internal.core.parser.ast.IASTFactory)
@@ -2652,4 +2684,11 @@ public class Scanner implements IScanner {
         ILineOffsetReconciler reconciler = ParserFactory.createLineOffsetReconciler( backupReader );
         return reconciler.getLineNumberForOffset(i);
     }
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IScanner#setOffsetBoundary(int)
+	 */
+	public void setOffsetBoundary(int offset) {
+		offsetLimit = offset;
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ScannerContext.java

@@ -15,6 +15,7 @@ import java.io.Reader;
 import java.util.Stack;
 
 import org.eclipse.cdt.core.parser.ast.IASTInclusion;
+import org.eclipse.cdt.internal.core.parser.IScannerContext.ContextKind;
 
 public class ScannerContext implements IScannerContext
 {
@@ -25,14 +26,12 @@ public class ScannerContext implements IScannerContext
 	private int line = 1;
 	private int offset;
 	private Stack undo = new Stack(); 
-	private int kind; 
+	private ContextKind kind; 
 				
-	public ScannerContext(){}
-    
     /* (non-Javadoc)
      * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion, int, int, int)
      */
-	public IScannerContext initialize(Reader r, String f, int k, IASTInclusion i, int mO, int mL, int l)
+	public ScannerContext(Reader r, String f, ContextKind k, IASTInclusion i, int mO, int mL, int l)
 	{
 		reader = r;
 		filename = f;
@@ -42,15 +41,14 @@ public class ScannerContext implements IScannerContext
         macroOffset = mO;
         macroLength = mL;
         line = l;
-		return this;
 	}
     
     /* (non-Javadoc)
      * @see org.eclipse.cdt.internal.core.parser.IScannerContext#initialize(Reader, String, int, IASTInclusion)
      */
-    public IScannerContext initialize(Reader r, String f, int k, IASTInclusion i)
+    public ScannerContext(Reader r, String f, ContextKind k, IASTInclusion i)
     {
-        return initialize(r, f, k, i, -1, -1, 1);
+        this(r, f, k, i, -1, -1, 1);
     }
 		
 	public int read() throws IOException {
@@ -151,7 +149,7 @@ public class ScannerContext implements IScannerContext
 	 * Returns the kind.
 	 * @return int
 	 */
-	public int getKind() {
+	public ContextKind getKind() {
 		return kind;
 	}
 
@@ -159,7 +157,7 @@ public class ScannerContext implements IScannerContext
 	 * Sets the kind.
 	 * @param kind The kind to set
 	 */
-	public void setKind(int kind) {
+	public void setKind(ContextKind kind) {
 		this.kind = kind;
 	}
 	/* (non-Javadoc)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java

@@ -168,4 +168,11 @@ public class Token implements IToken {
 		return true;
 	}
 
+	/* (non-Javadoc)
+	 * @see org.eclipse.cdt.core.parser.IToken#setType(int)
+	 */
+	public void setType(int i) {
+		type = i;
+	}
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/ast/complete/ASTExpression.java

@@ -10,7 +10,6 @@
 ***********************************************************************/
 package org.eclipse.cdt.internal.core.parser.ast.complete;
 
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;