Fix for 177154: [patch] single-line comments incorrectly retained in macros (Patch by Ed Swartz)

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2Tests.java

@@ -44,6 +44,7 @@ import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTNamedTypeSpecifier;
 import org.eclipse.cdt.core.dom.ast.IASTNullStatement;
 import org.eclipse.cdt.core.dom.ast.IASTParameterDeclaration;
+import org.eclipse.cdt.core.dom.ast.IASTPreprocessorMacroDefinition;
 import org.eclipse.cdt.core.dom.ast.IASTProblem;
 import org.eclipse.cdt.core.dom.ast.IASTProblemDeclaration;
 import org.eclipse.cdt.core.dom.ast.IASTReturnStatement;
@@ -3526,4 +3527,56 @@ public class AST2Tests extends AST2BaseTest {
     	StringBuffer sb= getContents(1)[0];
     	parse(sb.toString(), ParserLanguage.C, false, false);
     }
+    
+    /**
+     * Bug in not removing single-line comments from macros.
+     * @throws Exception
+     */
+	public void testMacroCommentsBug_177154() throws Exception {
+		// simple case
+		String simple = 
+			"#define LIT 1  // my value\r\n" + 
+			"int func(int x) {\r\n" + 
+			"}\r\n" + 
+			"int main() {\r\n" + 
+			"  return func(LIT);   // fails to parse\r\n" + 
+			"}\r\n"; 
+		
+    	IASTTranslationUnit tu = parse( simple, ParserLanguage.CPP, true, true );
+
+    	// actual reduced test case, plus extra cases
+    	String text =
+    		"#define KBOOT		1 //0x00000002\r\n" + 
+    		"#define KBOOT2	 /* value */	1  /* another */ //0x00000002\r\n" + 
+    		"#define KBOOT3	 /* value \r\n" +
+    		" multi line\r\n"+
+    		" comment */	1  \\\r\n"+
+    		"/* another */ + \\\r\n"+
+    		"2 //0x00000002\r\n" + 
+    		"#define DEBUGNUM(x) (KDebugNum(x))\r\n" + 
+    		"bool KDebugNum(int aBitNum);\r\n" + 
+    		"#define __KTRACE_OPT(a,p) {if((DEBUGNUM(a)))p;}\r\n" +
+    		"void fail();\r\n"+
+    		"void test() {\r\n"+
+    		"__KTRACE_OPT(KBOOT,fail());\r\n" + 
+    		"__KTRACE_OPT(KBOOT2,fail());\r\n" + 
+    		"}\r\n"
+    		;
+    	
+    	// essential test: this code should be parseable
+    	tu = parse( text, ParserLanguage.CPP, true, true );
+
+    	// verify macros
+    	IASTPreprocessorMacroDefinition[] macroDefinitions = tu.getMacroDefinitions();
+    	assertEquals(5, macroDefinitions.length);
+    	assertEquals("1", macroDefinitions[0].getExpansion());
+    	assertEquals("1", macroDefinitions[1].getExpansion());
+    	// regression test for #64268 and #71733 which also handle comments
+    	assertEquals("1   + 2", macroDefinitions[2].getExpansion());
+    	assertEquals("(KDebugNum(x))", macroDefinitions[3].getExpansion());
+    	assertEquals("{if((DEBUGNUM(a)))p;}", macroDefinitions[4].getExpansion());
+    	
+    	// TODO: exhaustive macro testing
+	}
+
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java

@@ -2127,7 +2127,7 @@ abstract class BaseScanner implements IScanner {
         int textend = textstart - 1;
         int varArgDefinitionInd = -1;
 
-        boolean encounteredMultilineComment = false;
+        boolean encounteredComment = false;
         boolean usesVarArgInDefinition = false;
         while (bufferPos[bufferStackPos] + 1 < limit
                 && buffer[bufferPos[bufferStackPos] + 1] != '\n') {
@@ -2146,7 +2146,7 @@ abstract class BaseScanner implements IScanner {
             if (arglist != null && !skipOverNonWhiteSpace(true)) {
                 ++bufferPos[bufferStackPos]; //advances us to the #
                 if (skipOverWhiteSpace())
-                    encounteredMultilineComment = true;
+                    encounteredComment = true;
 
                 boolean isArg = false;
                 if (bufferPos[bufferStackPos] + 1 < limit) {
@@ -2195,7 +2195,7 @@ abstract class BaseScanner implements IScanner {
             }
             textend = bufferPos[bufferStackPos];
             if (skipOverWhiteSpace())
-                encounteredMultilineComment = true;
+                encounteredComment = true;
         }
 
         int textlen = textend - textstart + 1;
@@ -2208,8 +2208,8 @@ abstract class BaseScanner implements IScanner {
 //            System.out.println(countIt);
         }
 
-        if (encounteredMultilineComment)
+        if (encounteredComment)
-            text = removeMultilineCommentFromBuffer(text);
+            text = removeCommentFromBuffer(text);
         text = removedEscapedNewline(text, 0, text.length);
 
         IMacro result = null;
@@ -2333,16 +2333,22 @@ abstract class BaseScanner implements IScanner {
      * @param text
      * @return
      */
-    protected char[] removeMultilineCommentFromBuffer(char[] text) {
+    protected char[] removeCommentFromBuffer(char[] text) {
         char[] result = new char[text.length];
         Arrays.fill(result, ' ');
         int resultCount = 0;
+        // either a single-line or multi-line comment was found
         for (int i = 0; i < text.length; ++i) {
-            if (text[i] == '/' && (i + 1 < text.length) && text[i + 1] == '*') {
+            if (text[i] == '/' && (i + 1 < text.length) && (text[i + 1] == '*' || text[i + 1] == '/')) {
-                i += 2;
+            	if (text[i + 1] == '/') {
-                while (i < text.length
+            		// done
-                        && !(text[i] == '*' && i + 1 < text.length && text[i + 1] == '/'))
+            		break;
-                    ++i;
+            	} else {
+	                i += 2;
+	                while (i < text.length
+	                        && !(text[i] == '*' && i + 1 < text.length && text[i + 1] == '/'))
+	                    ++i;
+            	}
                 ++i;
             } else
                 result[resultCount++] = text[i];
@@ -2619,7 +2625,7 @@ abstract class BaseScanner implements IScanner {
         //		if( pos > 0 && pos < limit && buffer[pos] == '\n')
         //			return false;
 
-        boolean encounteredMultiLineComment = false;
+        boolean encounteredComment = false;
         while (++bufferPos[bufferStackPos] < limit) {
             pos = bufferPos[bufferStackPos];
             switch (buffer[pos]) {
@@ -2632,7 +2638,8 @@ abstract class BaseScanner implements IScanner {
                     if (buffer[pos + 1] == '/') {
                         // C++ comment, skip rest of line
                         skipToNewLine(true);
-                        return false;
+                        encounteredComment = true;
+                        return encounteredComment;
                     } else if (buffer[pos + 1] == '*') {
                         // C comment, find closing */
                         for (bufferPos[bufferStackPos] += 2; bufferPos[bufferStackPos] < limit; ++bufferPos[bufferStackPos]) {
@@ -2640,7 +2647,7 @@ abstract class BaseScanner implements IScanner {
                             if (buffer[pos] == '*' && pos + 1 < limit
                                     && buffer[pos + 1] == '/') {
                                 ++bufferPos[bufferStackPos];
-                                encounteredMultiLineComment = true;
+                                encounteredComment = true;
                                 break;
                             }
                         }
@@ -2665,10 +2672,10 @@ abstract class BaseScanner implements IScanner {
 
             // fell out of switch without continuing, we're done
             --bufferPos[bufferStackPos];
-            return encounteredMultiLineComment;
+            return encounteredComment;
         }
         --bufferPos[bufferStackPos];
-        return encounteredMultiLineComment;
+        return encounteredComment;
     }
 
     protected int indexOfNextNonWhiteSpace(char[] buffer, int start, int limit) {