Bug 323971 - [C++0x] Editor support for raw string literals

core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/formatter/CodeFormatterVisitor.java

@@ -19,6 +19,7 @@ import java.util.List;
 import java.util.Stack;
 
 import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.core.dom.ast.ASTVisitor;
 import org.eclipse.cdt.core.dom.ast.ExpansionOverlapsBoundaryException;
 import org.eclipse.cdt.core.dom.ast.IASTASMDeclaration;
 import org.eclipse.cdt.core.dom.ast.IASTArrayDeclarator;
@@ -95,7 +96,7 @@ import org.eclipse.cdt.core.dom.ast.c.ICASTCompositeTypeSpecifier;
 import org.eclipse.cdt.core.dom.ast.c.ICASTDesignatedInitializer;
 import org.eclipse.cdt.core.dom.ast.c.ICASTDesignator;
 import org.eclipse.cdt.core.dom.ast.c.ICASTTypeIdInitializerExpression;
-import org.eclipse.cdt.core.dom.ast.cpp.CPPASTVisitor;
+import org.eclipse.cdt.core.dom.ast.c.ICASTVisitor;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCastExpression;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCatchHandler;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCompositeTypeSpecifier;
@@ -128,6 +129,7 @@ import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTTryBlockStatement;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTUsingDeclaration;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTUsingDirective;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTVisibilityLabel;
+import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTVisitor;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTWhileStatement;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCompositeTypeSpecifier.ICPPASTBaseSpecifier;
 import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator;
@@ -152,7 +154,7 @@ import org.eclipse.text.edits.TextEdit;
  * 
  * @since 4.0
  */
-public class CodeFormatterVisitor extends CPPASTVisitor {
+public class CodeFormatterVisitor extends ASTVisitor implements ICPPASTVisitor, ICASTVisitor {
 
 	private static boolean DEBUG = "true".equalsIgnoreCase(Platform.getDebugOption("org.eclipse.cdt.core/debug/formatter")); //$NON-NLS-1$ //$NON-NLS-2$
 
@@ -2201,11 +2203,11 @@ public class CodeFormatterVisitor extends CPPASTVisitor {
 			final int line= scribe.line;
 			boolean indented= false;
 			try {
-				int[] stringLiterals = { Token.tSTRING, Token.tLSTRING };
+				int[] stringLiterals = { Token.tSTRING, Token.tLSTRING, Token.tRSTRING };
 				while (true) {
 					scribe.printNextToken(stringLiterals, needSpace);
 					token= peekNextToken();
-					if (token != Token.tSTRING && token != Token.tLSTRING) {
+					if (token != Token.tSTRING && token != Token.tLSTRING && token != Token.tRSTRING) {
 						break;
 					}
 					scribe.printCommentPreservingNewLines();

core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/formatter/scanner/SimpleScanner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2004, 2009 IBM Corporation and others.
+ * Copyright (c) 2004, 2010 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -243,10 +243,14 @@ public class SimpleScanner {
 	            } while ((c == ' ') || (c == '\r') || (c == '\t') || (c == '\n'));
 	            ungetChar(c);
 	            return newToken(Token.tWHITESPACE);
-	        } else if (c == '"' || (c == 'L' && !madeMistake)) {
 	            
-	            boolean wideString = false;
+	        } else if (c == '"') {
-	            if (c == 'L') {
+	
+	            matchStringLiteral();
+	            return newToken(Token.tSTRING);
+
+	        } else if (c == 'L' && !madeMistake) {
+	        	
                 int oldChar = c;
                 c = getChar();
                 if (c != '"') {
@@ -255,14 +259,25 @@ public class SimpleScanner {
                     c = oldChar;
                     madeMistake = true;
                     continue;
-	                } else {
-	                    wideString = true;
-	                }
                 }
 	
 	            matchStringLiteral();
-	            int type = wideString ? Token.tLSTRING : Token.tSTRING;
+	            return newToken(Token.tLSTRING);
-	            return newToken(type);
+
+	        } else if (c == 'R' && !madeMistake) {
+	        	
+                int oldChar = c;
+                c = getChar();
+                if (c != '"') {
+                    // we have made a mistake
+                    ungetChar(c);
+                    c = oldChar;
+                    madeMistake = true;
+                    continue;
+                }
+	
+	            matchRawStringLiteral();
+	            return newToken(Token.tRSTRING);
 
 	        } else if (((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || (c == '_') || (c > 255 && Character.isUnicodeIdentifierStart(c))) {
 	
@@ -669,6 +684,40 @@ public class SimpleScanner {
 	    }
 	}
 
+	private void matchRawStringLiteral() {
+	    // raw-string R"<delim-opt>(string)<delim-opt>";
+	    int c = getChar(false);
+		StringBuilder delim = new StringBuilder(12);
+	    while (c != '(') {
+	    	if (c == EOFCHAR) {
+	    		return;
+	    	}
+	    	delim.append((char) c);
+	    	c = getChar(false);
+	    }
+	    int delimLen = delim.length();
+    	c = getChar(false);
+	    LOOP:
+	    for (;;) {
+	        if (c == EOFCHAR)
+	            break;
+	        if (c == ')') {
+		        c = getChar(false);
+		        int idx = 0;
+		        while (idx < delimLen) {
+		        	if (c != delim.charAt(idx)) {
+		        		continue LOOP;
+		        	}
+		        	++idx;
+			        c = getChar(false);
+		        }
+		        if (c == '"')
+		        	break;
+	        }
+	        c = getChar(false);
+	    }
+	}
+
 	/**
 	 * Matches a preprocesser directive.
 	 * 

core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/formatter/scanner/Token.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2004, 2008 IBM Corporation and others.
+ * Copyright (c) 2004, 2010 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -487,6 +487,7 @@ public class Token {
     static public final int tFLOATINGPT = 130;
     static public final int tLSTRING = 131;
     static public final int tCHAR = 132;
+    static public final int tRSTRING = 133;
     static public final int t_restrict = 136;
     static public final int t_interface = 200;
     static public final int t_import = 201;

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/CPartitionerTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- *  Copyright (c) 2000, 2009 IBM Corporation and others.
+ *  Copyright (c) 2000, 2010 IBM Corporation and others.
  *  All rights reserved. This program and the accompanying materials
  *  are made available under the terms of the Eclipse Public License v1.0
  *  which accompanies this distribution, and is available at
@@ -1164,4 +1164,242 @@ public class CPartitionerTest extends TestCase {
 			assertTrue(false);
 		}
 	}
+
+	public void testString1() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "\"[string]\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  fDocument.getLength(),  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "\"string1\" \"string2\"");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  9,  ICPartitions.C_STRING),
+				new TypedRegion(9,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(10,  9,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testRawString1() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "R\"(line 1\n/*line 2*/\nline 3\n)\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(1,  fDocument.getLength() - 1,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "R\"()\"//comment");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(1,  fDocument.getLength() - 10,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 9,  9,  ICPartitions.C_SINGLE_LINE_COMMENT),
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "R\"delimiter(line 1\n()delimitex\nline 3\n)delimiter\"");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(1,  fDocument.getLength() - 1,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testRawString2() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "/***/R\"(line 1\nline 2\nline 3\n)\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 6,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "#define X x\nR\"delimiter(line 1\n()delimitex\nline 3\n)delimiter\"");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  12,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(12,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(13,  fDocument.getLength() - 13,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testRawString3() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "/***/R\"(line 1\nline 2\nline 3\n)\" \"str\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			fDocument.replace(0, fDocument.getLength(), "#define X x\nR\"delimiter(line 1\n()delimitex\nline 3\n)del)delimiter\" \"str\"");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  12,  ICPartitions.C_PREPROCESSOR),
+				new TypedRegion(12,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(13,  fDocument.getLength() - 19,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testEditingRawString1() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "/***/R\"(line 1\nline 2\nline 3\n)\" \"str\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			// insert line
+			fDocument.replace(8, 0, "line 0\n");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			// delete text
+			fDocument.replace(12, 8, "");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testEditingRawString2() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "/***/R\"(line 1\nline 2\nline 3\n)\" \"str\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			// insert opening delimiter
+			fDocument.replace(7, 0, "***");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,   5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 6,  ICPartitions.C_STRING)
+			};
+			checkPartitioning(expectation, result);
+
+			// insert closing delimiter
+			fDocument.replace(fDocument.getLength() - 7, 0, "***");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			// invalidate closing delimiter
+			fDocument.replace(fDocument.getLength() - 7, 1, "");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 6,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
+
+	public void testEditingRawString3() {
+		try {
+
+			fDocument.replace(0, fDocument.getLength(), "/***/R\"(line 1\nline 2\nline 3\n)\" \"str\"");
+			ITypedRegion[] result= fDocument.computePartitioning(0, fDocument.getLength());
+			TypedRegion[] expectation= {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 12,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 6, 1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+			// insert text after closing quote
+			fDocument.replace(fDocument.getLength() - 6, 0, " ");
+			result= fDocument.computePartitioning(0, fDocument.getLength());
+			expectation= new TypedRegion[] {
+				new TypedRegion(0,  5,  ICPartitions.C_MULTI_LINE_COMMENT),
+				new TypedRegion(5,  1,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(6,  fDocument.getLength() - 13,  ICPartitions.C_STRING),
+				new TypedRegion(fDocument.getLength() - 7, 2,  IDocument.DEFAULT_CONTENT_TYPE),
+				new TypedRegion(fDocument.getLength() - 5, 5,  ICPartitions.C_STRING),
+			};
+			checkPartitioning(expectation, result);
+
+		} catch (BadLocationException x) {
+			assertTrue(false);
+		}
+	}
 }

core/org.eclipse.cdt.ui.tests/ui/org/eclipse/cdt/ui/tests/text/PartitionTokenScannerTest.java

@@ -40,9 +40,10 @@ public class PartitionTokenScannerTest extends TestCase {
 		super(name);	
 	}
 
+	@Override
 	protected void setUp() {
 		fReference= new CPartitionScanner();
-		fTestee= new FastCPartitionScanner(true, null);
+		fTestee= new FastCPartitionScanner();
 	}
 
 	// read sample C file
@@ -51,7 +52,7 @@ public class PartitionTokenScannerTest extends TestCase {
 			InputStream stream= getClass().getResourceAsStream(name);
 			BufferedReader reader= new BufferedReader(new InputStreamReader(stream));
 			
-			StringBuffer buffer= new StringBuffer();
+			StringBuilder buffer= new StringBuilder();
 			String line= reader.readLine();
 			while (line != null) {
 				buffer.append(line);
@@ -67,7 +68,7 @@ public class PartitionTokenScannerTest extends TestCase {
 	
 	private static IDocument getRandomDocument(int size) {
 		final char[] characters= {'/', '*', '\'', '"', '\r', '\n', '\\'};
-		final StringBuffer buffer= new StringBuffer();
+		final StringBuilder buffer= new StringBuilder();
 		
 		for (int i= 0; i < size; i++) {
 			final int randomIndex= (int) (Math.random() * characters.length);
@@ -142,7 +143,7 @@ public class PartitionTokenScannerTest extends TestCase {
 	}
 	
 	private void testConformance(final IDocument document) {
-		final StringBuffer message= new StringBuffer();
+		final StringBuilder message= new StringBuilder();
 		
 		fReference.setRange(document, 0, document.getLength());
 		fTestee.setRange(document, 0, document.getLength());
@@ -201,7 +202,7 @@ public class PartitionTokenScannerTest extends TestCase {
 	}
 		
 	private static String extractString(IDocument document, int offset) {
-		final StringBuffer buffer= new StringBuffer();
+		final StringBuilder buffer= new StringBuilder();
 
 		try {
 			IRegion region= document.getLineInformationOfOffset(offset);
@@ -229,7 +230,7 @@ public class PartitionTokenScannerTest extends TestCase {
 	 * Escapes CR, LF and TAB in a string.
 	 */
 	private static String escape(String string) {
-		final StringBuffer buffer= new StringBuffer();
+		final StringBuilder buffer= new StringBuilder();
 		
 		final int length= string.length();
 		for (int i= 0; i < length; i++) {

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/actions/IndentAction.java

@@ -214,6 +214,9 @@ public class IndentAction extends TextEditorAction {
 				indent= computeCommentIndent(document, line, scanner, startingPartition);
 			} else if (startingPartition.getType().equals(ICPartitions.C_PREPROCESSOR)) {
 				indent= computePreprocessorIndent(document, line, startingPartition);
+			} else if (startingPartition.getType().equals(ICPartitions.C_STRING) && offset > startingPartition.getOffset()) {
+				// don't indent inside (raw-)string
+				indent = ""; //$NON-NLS-1$
 			} else if (!fIsTabAction && startingPartition.getOffset() == offset && startingPartition.getType().equals(ICPartitions.C_SINGLE_LINE_COMMENT)) {
 				// line comment starting at position 0 -> indent inside
 				if (indentInsideLineComments()) {

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/CStringAutoIndentStrategy.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2008 IBM Corporation and others.
+ * Copyright (c) 2000, 2010 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -141,6 +141,9 @@ public class CStringAutoIndentStrategy extends DefaultIndentLineAutoEditStrategy
 		if (command.offset == offset + length && document.getChar(offset + length - 1) == '\"')
 			return;
 
+		if (offset > 0 && document.getChar(offset - 1) == 'R')  // raw string
+			return;
+		
 		CHeuristicScanner scanner = new CHeuristicScanner(document);
 		CIndenter indenter = new CIndenter(document, scanner, fProject);
 		StringBuilder indentation = indenter.computeContinuationLineIndentation(offset);

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java

@@ -37,11 +37,21 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private static final int MULTI_LINE_COMMENT= 2;
 	private static final int CHARACTER= 3;
 	private static final int STRING= 4;
-	private static final int PREPROCESSOR= 5;
+	private static final int RAW_STRING= 5;
-	private static final int PREPROCESSOR_MULTI_LINE_COMMENT= 6;
+	private static final int PREPROCESSOR= 6;
-	private static final int PREPROCESSOR_STRING= 7;
+	private static final int PREPROCESSOR_MULTI_LINE_COMMENT= 7;
-	private static final int SINGLE_LINE_DOC_COMMENT= 8;
+	private static final int PREPROCESSOR_STRING= 8;
-	private static final int MULTI_LINE_DOC_COMMENT= 9;
+	private static final int SINGLE_LINE_DOC_COMMENT= 9;
+	private static final int MULTI_LINE_DOC_COMMENT= 10;
+	
+	/**
+	 * Sub state for raw strings.
+	 */
+	private enum RawStringState {
+		OPEN_DELIMITER,
+		CONTENT,
+		CLOSE_DELIMITER
+	}
 	
 	// beginning of prefixes and postfixes
 	private static final int NONE= 0;
@@ -52,6 +62,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
 	private static final int BACKSLASH_CR= 6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
 	private static final int BACKSLASH_BACKSLASH= 7; // postfix for STRING, CHARACTER
+	private static final int RAW_STRING_R= 8; // prefix for RAW_STRING
 	
 	/** The scanner. */
 	private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);	// faster implementation
@@ -69,13 +80,9 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	private int fPrefixLength;
 	/** Indicate whether current char is first non-whitespace char on the line*/
 	private boolean fFirstCharOnLine= true;
-	/** An optional (possibly null) comment owner for detecting documentation-comments **/
+	/** An optional (possibly null) comment owner for detecting documentation-comments */
-	private IDocCommentOwner fOwner;
+	private final IDocCommentOwner fOwner;
 	
-	// emulate CPartitionScanner
-	private final boolean fEmulate;
-	private int fCCodeOffset;
-	private int fCCodeLength;
 	private IDocument fDocument;
 	
 	private final IToken[] fTokens= new IToken[] {
@@ -84,6 +91,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		new Token(C_MULTI_LINE_COMMENT),
 		new Token(C_CHARACTER),
 		new Token(C_STRING),
+		new Token(C_STRING),
 		new Token(C_PREPROCESSOR),
 		new Token(C_MULTI_LINE_COMMENT),
 		new Token(C_PREPROCESSOR),
@@ -91,17 +99,14 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		new Token(C_MULTI_LINE_DOC_COMMENT)
 	};
 	
-	public FastCPartitionScanner(boolean emulate, IDocCommentOwner owner) {
+	private final StringBuilder fRawStringDelimiter = new StringBuilder(12);
-	    fEmulate= emulate;
-	    fOwner= owner;
-	}
 
 	public FastCPartitionScanner(IDocCommentOwner owner) {
-	    this(false, owner);
+	   fOwner = owner;
 	}
 	
 	public FastCPartitionScanner() {
-	    this(false, null);
+	    this(null);
 	}
 
 	/*
@@ -109,52 +114,43 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	 */
 	public IToken nextToken() {
 
-		// emulate CPartitionScanner
-		if (fEmulate) {
-			if (fCCodeOffset != -1 && fTokenOffset + fTokenLength != fCCodeOffset + fCCodeLength) {
-				fTokenOffset += fTokenLength;
-				return fTokens[CCODE];
-			} else {
-				fCCodeOffset= -1;
-				fCCodeLength= 0;
-			}
-		}
-
 		fTokenOffset += fTokenLength;
 		fTokenLength= fPrefixLength;
 
+		RawStringState rawStringState = RawStringState.OPEN_DELIMITER;
+		int rawStringDelimiterIdx = 0;
+		
 		while (true) {
 			final int ch= fScanner.read();
 			
 			final boolean isFirstCharOnLine= fFirstCharOnLine;
-			if (fFirstCharOnLine && ch != ' ' && ch != '\t') {
+			if (isFirstCharOnLine && ch != ' ' && ch != '\t') {
 				fFirstCharOnLine= false;
 			}
 			// characters
 	 		switch (ch) {
 	 		case ICharacterScanner.EOF:
-		 		if (fTokenLength > 0) {
 	 			fLast= NONE; // ignore last
+		 		if (fTokenLength > 0) {
 		 			return preFix(fState, CCODE, NONE, 0);
 
 		 		} else {
-		 			fLast= NONE;
 		 			fPrefixLength= 0;
 					return Token.EOF;
 		 		}
 
 	 		case '\r':
 	 			fFirstCharOnLine= true;
-	 			if (!fEmulate && fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
+	 			if (fLast == BACKSLASH || fLast == BACKSLASH_BACKSLASH) {
 	 				fLast= BACKSLASH_CR;
 					fTokenLength++;
  					continue;
-	 			} else if (!fEmulate && fLast != CARRIAGE_RETURN) {
+	 			} else if (fLast != CARRIAGE_RETURN) {
 	 				fLast= CARRIAGE_RETURN;
 	 				fTokenLength++;
 	 				continue;
 	 			} else {
-	 				// fEmulate || fLast == CARRIAGE_RETURN
+	 				// fLast == CARRIAGE_RETURN
 					switch (fState) {
 					case SINGLE_LINE_COMMENT:
 					case CHARACTER:
@@ -162,16 +158,8 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 					case PREPROCESSOR:
 						if (fTokenLength > 0) {
 							IToken token= fTokens[fState];
-
-				 			// emulate CPartitionScanner
-							if (fEmulate) {
-								fTokenLength++;
-								fLast= NONE;
-								fPrefixLength= 0;
-							} else {
 							fLast= CARRIAGE_RETURN;
 							fPrefixLength= 1;
-							}
 
 							fState= CCODE;
 							return token;
@@ -222,8 +210,15 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 					continue;
 				}
 
+	 		case 'R':
+	 			if (fState == CCODE) {
+	 				fLast = RAW_STRING_R;
+	 			}
+	 			fTokenLength++;
+	 			continue;
+
 			default:
-				if (!fEmulate && fLast == CARRIAGE_RETURN) {
+				if (fLast == CARRIAGE_RETURN) {
 					switch (fState) {
 					case SINGLE_LINE_COMMENT:
 					case CHARACTER:
@@ -349,18 +344,24 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 					}
 
 				case '"':
+					int newState = STRING;
+					if (fLast == RAW_STRING_R) {
+						newState = RAW_STRING;
+						rawStringState = RawStringState.OPEN_DELIMITER;
+						fRawStringDelimiter.setLength(0);
+					}
 					fLast= NONE; // ignore fLast
 					if (fTokenLength > 0 ) {
-						return preFix(CCODE, STRING, NONE, 1);
+						return preFix(CCODE, newState, NONE, 1);
 					} else {
-						preFix(CCODE, STRING, NONE, 1);
+						preFix(CCODE, newState, NONE, 1);
 						fTokenOffset += fTokenLength;
 						fTokenLength= fPrefixLength;
 						break;
 					}
 					
 				case '#':
-					if (!fEmulate && isFirstCharOnLine) {
+					if (isFirstCharOnLine) {
 						int column= fScanner.getColumn() - 1;
 						fTokenLength -= column;
 						if (fTokenLength > 0) {
@@ -372,7 +373,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 							break;
 						}
 					}
-					// fallthrough
 					consume();
 					break;
 				default:
@@ -398,7 +398,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 							break;
 						}
 					} else {
-						consume();
+						fTokenLength++;
 						fLast= SLASH;
 					}
 					break;
@@ -493,7 +493,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 				case '\"':
 	 				if (fLast != BACKSLASH) {
 	 					return postFix(STRING);
-
 		 			} else {
 		 				consume();
 		 				break;
@@ -505,12 +504,47 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 	 			}
 	 			break;
 
+	 		case RAW_STRING:
+	 			switch (rawStringState) {
+	 			case OPEN_DELIMITER:
+	 				if (ch == '(') {
+	 					rawStringState = RawStringState.CONTENT;
+	 				} else {
+	 					fRawStringDelimiter.append((char) ch);
+	 				}
+		 			consume();
+	 				break;
+	 			case CONTENT:
+	 				if (ch == ')') {
+	 					rawStringState = RawStringState.CLOSE_DELIMITER;
+	 					rawStringDelimiterIdx = 0;
+	 				}
+		 			consume();
+	 				break;
+	 			case CLOSE_DELIMITER:
+	 				if (ch == ')') {
+	 					rawStringDelimiterIdx = 0;
+	 				} else if (rawStringDelimiterIdx < fRawStringDelimiter.length()) {
+	 					if (fRawStringDelimiter.charAt(rawStringDelimiterIdx) != ch) {
+	 						rawStringState = RawStringState.CONTENT;
+	 					} else {
+	 						++rawStringDelimiterIdx;
+	 					}
+	 				} else if (ch == '"') {
+	 					return postFix(RAW_STRING);
+	 				} else {
+	 					rawStringState = RawStringState.CONTENT;
+	 				}
+ 					consume();
+	 				break;
+	 			}
+	 			break;
+
 	 		case CHARACTER:
 	 			switch (ch) {
 	 			case '\'':
 	 				if (fLast != BACKSLASH) {
 	 					return postFix(CHARACTER);
-	
 	 				} else {
 		 				consume();
 		 				break;
@@ -537,6 +571,7 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		case BACKSLASH:
 		case SLASH:
 		case STAR:
+		case RAW_STRING_R:
 			return 1;
 
 		case SLASH_STAR:
@@ -566,25 +601,12 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 
 
 	private final IToken preFix(int state, int newState, int last, int prefixLength) {
-		// emulate CPartitionScanner
-		if (fEmulate && state == CCODE && (fTokenLength - getLastLength(fLast) > 0)) {
-			fTokenLength -= getLastLength(fLast);
-			fCCodeOffset= fTokenOffset;
-			fCCodeLength= fTokenLength;
-			fTokenLength= 1;
-			fState= newState;
-			fPrefixLength= prefixLength;
-			fLast= last;
-			return fTokens[interceptTokenState(state)];
-
-		} else {
 		fTokenLength -= getLastLength(fLast);
 		fLast= last;
 		fPrefixLength= prefixLength;
 		fState= newState;
 		return fTokens[interceptTokenState(state)];
 	}
-	}
 
 	private static int getState(String contentType) {
 
@@ -627,11 +649,26 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		fPrefixLength= offset - partitionOffset;
 		fLast= NONE;
 		
+		fState= getState(contentType);
+		if (fState == STRING) {
+			// raw string is a special case: need to restart from partition offset
+			try {
+				if (partitionOffset > 0 && fDocument.getChar(partitionOffset - 1) == 'R') {
+					fState = RAW_STRING;
+					int endOffset = offset + length;
+					offset = partitionOffset + 1;
+					length = endOffset - offset;
+					fScanner.setRange(document, offset, length);
+					fPrefixLength = offset - partitionOffset;
+					fRawStringDelimiter.setLength(0);
+				}
+			} catch (BadLocationException exc) {
+				// cannot happen
+			}
+		}
 		if (offset == partitionOffset) {
 			// restart at beginning of partition
 			fState= CCODE;
-		} else {
-			fState= getState(contentType);
 		}
 
 		try {
@@ -640,12 +677,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		} catch (BadLocationException exc) {
 			fFirstCharOnLine= true;
 		}
-		
-		// emulate CPartitionScanner
-		if (fEmulate) {
-			fCCodeOffset= -1;
-			fCCodeLength= 0;
-		}
 	}
 
 	/*
@@ -666,12 +697,6 @@ public final class FastCPartitionScanner implements IPartitionTokenScanner, ICPa
 		} catch (BadLocationException exc) {
 			fFirstCharOnLine= true;
 		}
-		
-		// emulate CPartitionScanner
-		if (fEmulate) {
-			fCCodeOffset= -1;
-			fCCodeLength= 0;
-		}
 	}
 
 	/*