Move to new DefaultPartitioner and add a fast C partitioner based on

Java one.

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/BufferedDocumentScanner.java

@@ -0,0 +1,152 @@
+package org.eclipse.cdt.internal.ui.text;
+
+/*
+ * (c) Copyright IBM Corp. 2000, 2001.
+ * All Rights Reserved.
+ */
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.rules.ICharacterScanner;
+import org.eclipse.jface.util.Assert;
+
+/**
+ * A buffered document scanner. The buffer always contains a section 
+ * of a fixed size of the document to be scanned.
+ */
+public final class BufferedDocumentScanner implements ICharacterScanner {
+
+	/** The document being scanned. */
+	private IDocument fDocument;
+	/** The offset of the document range to scan. */
+	private int fRangeOffset;
+	/** The length of the document range to scan. */
+	private int fRangeLength;
+	/** The delimiters of the document. */
+	private char[][] fDelimiters;
+	
+	/** The buffer. */
+	private final char[] fBuffer;
+	/** The offset of the buffer within the document. */
+	private int fBufferOffset;
+	/** The valid length of the buffer for access. */
+	private int fBufferLength;
+	/** The offset of the scanner within the buffer. */
+	private int fOffset;
+	
+	/**
+	 * Creates a new buffered document scanner.
+	 * The buffer size is set to the given number of characters.
+	 *
+	 * @param size the buffer size
+	 */
+	public BufferedDocumentScanner(int size) {
+		Assert.isTrue(size >= 1);
+		fBuffer= new char[size];
+	}
+	
+	/**
+	 * Fills the buffer with the contens of the document starting at the given offset.
+	 *
+	 * @param offset the document offset at which the buffer starts
+	 */
+	private final void updateBuffer(int offset) {
+
+		fBufferOffset= offset;
+
+		if (fBufferOffset + fBuffer.length > fRangeLength)
+			fBufferLength= fRangeLength - fBufferOffset;
+		else
+			fBufferLength= fBuffer.length;
+				
+		try {
+			final String content= fDocument.get(fBufferOffset, fBufferLength);
+			content.getChars(0, fBufferLength, fBuffer, 0);
+			
+		} catch (BadLocationException e) {
+		}
+	}
+	
+	/**
+	 * Configures the scanner by providing access to the document range over which to scan.
+	 *
+	 * @param document the document to scan
+	 * @param offset the offset of the document range to scan
+	 * @param length the length of the document range to scan
+	 */
+	public final void setRange(IDocument document, int offset, int length) {
+
+		fDocument= document;
+		fRangeOffset= offset;
+		fRangeLength= length;
+
+		String[] delimiters= document.getLegalLineDelimiters();
+		fDelimiters= new char[delimiters.length][];
+		for (int i= 0; i < delimiters.length; i++)
+			fDelimiters[i]= delimiters[i].toCharArray();
+
+		updateBuffer(offset);
+		fOffset= 0;
+	}
+	
+	/*
+	 * @see ICharacterScanner#read()
+	 */
+	public final int read() {
+		
+		if (fOffset == fBufferLength) {
+			if (fBufferOffset + fBufferLength == fDocument.getLength())
+				return EOF;
+			else {
+				updateBuffer(fBufferOffset + fBufferLength);
+				fOffset= 0;
+			}
+		}
+					
+		return fBuffer[fOffset++];
+	}
+	
+	/*
+	 * @see ICharacterScanner#unread
+	 */
+	public final void unread() {
+
+		if (fOffset == 0) {
+			if (fBufferOffset == fRangeOffset) {
+				// error: BOF
+			} else {
+				updateBuffer(fBufferOffset - fBuffer.length);
+				fOffset= fBuffer.length - 1;
+			}
+		} else {			
+			--fOffset;
+		}
+	}
+
+	/*
+	 * @see ICharacterScanner#getColumn()
+	 */
+	public final int getColumn() {
+
+		try {
+			final int offset= fBufferOffset + fOffset;
+			final int line= fDocument.getLineOfOffset(offset);
+			final int start= fDocument.getLineOffset(line);
+				
+			return offset - start;
+				
+		} catch (BadLocationException e) {
+		}
+
+		return -1;
+	}
+
+	/*
+	 * @see ICharacterScanner#getLegalLineDelimiters()
+	 */
+	public final char[][] getLegalLineDelimiters() {
+		return fDelimiters;
+	}
+
+}
+
+

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/CSourceViewerConfiguration.java

@@ -27,6 +27,7 @@ import org.eclipse.jface.text.presentation.IPresentationReconciler;
 import org.eclipse.jface.text.presentation.PresentationReconciler;
 import org.eclipse.jface.text.reconciler.IReconciler;
 import org.eclipse.jface.text.reconciler.Reconciler;
+import org.eclipse.jface.text.rules.DefaultPartitioner;
 import org.eclipse.jface.text.rules.RuleBasedPartitioner;
 import org.eclipse.jface.text.rules.RuleBasedScanner;
 import org.eclipse.jface.text.source.IAnnotationHover;
@@ -327,7 +328,7 @@ public class CSourceViewerConfiguration extends SourceViewerConfiguration {
 	 */
 	public IContentFormatter getContentFormatter(ISourceViewer sourceViewer) {
 		String[] types= new String[] {
-			RuleBasedPartitioner.CONTENT_TYPES_CATEGORY
+			DefaultPartitioner.CONTENT_TYPES_CATEGORY
 		};
 		
 		ContentFormatter formatter= new ContentFormatter();

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/CTextTools.java

@@ -10,6 +10,8 @@ import org.eclipse.cdt.internal.ui.text.util.CColorManager;
 
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.text.IDocumentPartitioner;
+import org.eclipse.jface.text.rules.DefaultPartitioner;
+import org.eclipse.jface.text.rules.IPartitionTokenScanner;
 import org.eclipse.jface.text.rules.RuleBasedPartitioner;
 import org.eclipse.jface.text.rules.RuleBasedScanner;
 import org.eclipse.jface.util.IPropertyChangeListener;
@@ -35,7 +37,7 @@ public class CTextTools {
 	/** The C++ source code scanner */
 	private CppCodeScanner fCppCodeScanner;
 	/** The C partitions scanner */
-	private CPartitionScanner fPartitionScanner;
+	private FastCPartitionScanner fPartitionScanner;
 	/** The Java multiline comment scanner */
 	private SingleTokenCScanner fMultilineCommentScanner;
 	/** The Java singleline comment scanner */
@@ -63,7 +65,7 @@ public class CTextTools {
 		fColorManager= new CColorManager();
 		fCodeScanner= new CCodeScanner(fColorManager, store);
 		fCppCodeScanner= new CppCodeScanner(fColorManager, store);
-		fPartitionScanner= new CPartitionScanner();
+		fPartitionScanner= new FastCPartitionScanner();
 		
 		fMultilineCommentScanner= new SingleTokenCScanner(fColorManager, store, ICColorConstants.C_MULTI_LINE_COMMENT);
 		fSinglelineCommentScanner= new SingleTokenCScanner(fColorManager, store, ICColorConstants.C_SINGLE_LINE_COMMENT);
@@ -125,9 +127,13 @@ public class CTextTools {
 	}
 	
 	/**
-	 * Gets the partition scanner used.
+	 * Returns a scanner which is configured to scan 
+	 * C-specific partitions, which are multi-line comments,
+	 * and regular C source code.
+	 *
+	 * @return a C partition scanner
 	 */
-	public RuleBasedScanner getPartitionScanner() {
+	public IPartitionTokenScanner getPartitionScanner() {
 		return fPartitionScanner;
 	}
 	
@@ -142,7 +148,7 @@ public class CTextTools {
 			CPartitionScanner.C_STRING
 		};
 		
-		return new RuleBasedPartitioner(getPartitionScanner(), types);
+		return new DefaultPartitioner(getPartitionScanner(), types);
 	}
 	
 	/**

core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/FastCPartitionScanner.java

@@ -0,0 +1,485 @@
+package org.eclipse.cdt.internal.ui.text;
+
+
+/*
+ * (c) Copyright IBM Corp. 2000, 2001.
+ * All Rights Reserved.
+ */
+
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.rules.ICharacterScanner;
+import org.eclipse.jface.text.rules.IPartitionTokenScanner;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.jface.text.rules.Token;
+
+
+/**
+ * This scanner recognizes the C multi line comments, C single line comments,
+ * C strings and C characters.
+ */
+public class FastCPartitionScanner implements IPartitionTokenScanner {
+
+	private final static String SKIP= "__skip"; //$NON-NLS-1$	
+	public final static String C_STRING= "c_string"; //$NON-NLS-1$
+	public final static String C_SINGLE_LINE_COMMENT= "c_single_line_comment"; //$NON-NLS-1$
+	public final static String C_MULTI_LINE_COMMENT= "c_multi_line_comment"; //$NON-NLS-1$
+
+	// states
+	private static final int CCODE= 0;	
+	private static final int SINGLE_LINE_COMMENT= 1;
+	private static final int MULTI_LINE_COMMENT= 2;
+	private static final int CHARACTER= 3;
+	private static final int STRING= 4;
+	
+	// beginning of prefixes and postfixes
+	private static final int NONE= 0;
+	private static final int BACKSLASH= 1; // postfix for STRING and CHARACTER
+	private static final int SLASH= 2; // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
+	private static final int SLASH_STAR= 3; // prefix for MULTI_LINE_COMMENT or JAVADOC
+	private static final int STAR= 4; // postfix for MULTI_LINE_COMMENT or JAVADOC
+	private static final int CARRIAGE_RETURN=5; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
+	
+	/** The scanner. */
+//	private final BufferedRuleBasedScanner fScanner= new BufferedRuleBasedScanner(1000);
+	private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);	// faster implementation
+	
+	/** The offset of the last returned token. */
+	private int fTokenOffset;
+	/** The length of the last returned token. */
+	private int fTokenLength;
+	
+	/** The state of the scanner. */	
+	private int fState;
+	/** The last significant characters read. */
+	private int fLast;
+	/** The amount of characters already read on first call to nextToken(). */
+	private int fPrefixLength;
+	
+	// emulate CPartitionScanner
+	private static final boolean fgEmulate= false;
+	private int fCOffset;
+	private int fCLength;
+	
+	private final IToken[] fTokens= new IToken[] {
+		new Token(null),
+		new Token(C_SINGLE_LINE_COMMENT),
+		new Token(C_MULTI_LINE_COMMENT),
+		new Token(SKIP),
+		new Token(C_STRING)
+	};
+
+	/*
+	 * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
+	 */
+	public IToken nextToken() {
+		
+		// emulate CPartitionScanner
+		if (fgEmulate) {
+			if (fCOffset != -1 && fTokenOffset + fTokenLength != fCOffset + fCLength) {
+				fTokenOffset += fTokenLength;		
+				return fTokens[CCODE];	
+			} else {
+				fCOffset= -1;
+				fCLength= 0;	
+			}
+		}		
+
+		fTokenOffset += fTokenLength;
+		fTokenLength= fPrefixLength;
+
+		while (true) {
+			final int ch= fScanner.read();
+			
+			// characters
+	 		switch (ch) {
+	 		case ICharacterScanner.EOF:
+		 		if (fTokenLength > 0) {
+		 			fLast= NONE; // ignore last
+		 			return preFix(fState, CCODE, NONE, 0);
+
+		 		} else {
+		 			fLast= NONE;
+		 			fPrefixLength= 0;
+					return Token.EOF;
+		 		}
+
+	 		case '\r':
+	 			// emulate CPartitionScanner
+	 			if (!fgEmulate && fLast != CARRIAGE_RETURN) {
+						fLast= CARRIAGE_RETURN;
+						fTokenLength++;
+	 					continue;
+
+	 			} else {
+	 				
+					switch (fState) {
+					case SINGLE_LINE_COMMENT:
+					case CHARACTER:
+					case STRING:
+						if (fTokenLength > 0) {
+							IToken token= fTokens[fState];
+							
+				 			// emulate CPartitionScanner
+							if (fgEmulate) {
+								fTokenLength++;
+								fLast= NONE;
+								fPrefixLength= 0;
+							} else {								
+								fLast= CARRIAGE_RETURN;	
+								fPrefixLength= 1;
+							}
+							
+							fState= CCODE;
+							return token;
+
+						} else {
+							consume();
+							continue;	
+						}
+
+					default:
+						consume();
+						continue;
+					}	 				
+	 			}
+	
+	 		case '\n': 		 		
+				switch (fState) {
+				case SINGLE_LINE_COMMENT:
+				case CHARACTER:
+				case STRING:				
+					// assert(fTokenLength > 0);
+					return postFix(fState);
+
+				default:
+					consume();
+					continue;
+				}
+
+			default:
+				if (!fgEmulate && fLast == CARRIAGE_RETURN) {			
+					switch (fState) {
+					case SINGLE_LINE_COMMENT:
+					case CHARACTER:
+					case STRING:
+
+						int last;
+						int newState;
+						switch (ch) {
+						case '/':
+							last= SLASH;
+							newState= CCODE;
+							break;
+
+						case '*':
+							last= STAR;
+							newState= CCODE;
+							break;
+						
+						case '\'':
+							last= NONE;
+							newState= CHARACTER;
+							break;
+
+						case '"':
+							last= NONE;
+							newState= STRING;
+							break;
+
+						case '\r':
+							last= CARRIAGE_RETURN;
+							newState= CCODE;
+							break;
+
+						case '\\':
+							last= BACKSLASH;
+							newState= CCODE;
+							break;
+
+						default:
+							last= NONE;
+							newState= CCODE;
+							break;
+						}
+						
+						fLast= NONE; // ignore fLast
+						return preFix(fState, newState, last, 1);
+	
+					default:
+						break;
+					}
+				}
+			}
+
+			// states	 
+	 		switch (fState) {
+	 		case CCODE:
+				switch (ch) {
+				case '/':
+					if (fLast == SLASH) {
+						if (fTokenLength - getLastLength(fLast) > 0) {
+							return preFix(CCODE, SINGLE_LINE_COMMENT, NONE, 2);
+						} else {							
+							preFix(CCODE, SINGLE_LINE_COMMENT, NONE, 2);
+							fTokenOffset += fTokenLength;
+							fTokenLength= fPrefixLength;
+							break;
+						}
+	
+					} else {
+						fTokenLength++;
+						fLast= SLASH;
+						break;
+					}
+	
+				case '*':
+					if (fLast == SLASH) {
+						if (fTokenLength - getLastLength(fLast) > 0)
+							return preFix(CCODE, MULTI_LINE_COMMENT, SLASH_STAR, 2);
+						else {
+							preFix(CCODE, MULTI_LINE_COMMENT, SLASH_STAR, 2);
+							fTokenOffset += fTokenLength;
+							fTokenLength= fPrefixLength;
+							break;
+						}
+
+					} else {
+						consume();
+						break;
+					}
+					
+				case '\'':
+					fLast= NONE; // ignore fLast
+					if (fTokenLength > 0)
+						return preFix(CCODE, CHARACTER, NONE, 1);
+					else {						
+						preFix(CCODE, CHARACTER, NONE, 1);
+						fTokenOffset += fTokenLength;
+						fTokenLength= fPrefixLength;
+						break;
+					}
+
+				case '"':
+					fLast= NONE; // ignore fLast				
+					if (fTokenLength > 0)
+						return preFix(CCODE, STRING, NONE, 1);
+					else {
+						preFix(CCODE, STRING, NONE, 1);
+						fTokenOffset += fTokenLength;
+						fTokenLength= fPrefixLength;
+						break;
+					}
+	
+				default:
+					consume();
+					break;
+				}
+				break;
+	
+	 		case SINGLE_LINE_COMMENT:
+				consume();
+				break;
+	
+	 		case MULTI_LINE_COMMENT:
+				switch (ch) {
+				case '*':
+					fTokenLength++;
+					fLast= STAR;
+					break;
+	
+				case '/':
+					if (fLast == STAR) {
+						return postFix(MULTI_LINE_COMMENT);
+					} else {
+						consume();
+						break;
+					}
+	
+				default:
+					consume();
+					break;			
+				}
+				break;
+				
+	 		case STRING:
+	 			switch (ch) {
+	 			case '\\':
+					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
+					fTokenLength++;
+					break;
+					
+				case '\"':	 			 			
+	 				if (fLast != BACKSLASH) {
+	 					return postFix(STRING);
+
+		 			} else {
+						consume();
+						break; 					
+	 				}
+		 		
+		 		default:
+					consume();
+	 				break;
+	 			}
+	 			break;
+	
+	 		case CHARACTER:
+	 			switch (ch) {
+				case '\\':
+					fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
+					fTokenLength++;
+					break;
+	
+	 			case '\'':
+	 				if (fLast != BACKSLASH) {
+	 					return postFix(CHARACTER);
+	
+	 				} else {
+						consume();
+		 				break;
+	 				}
+	
+	 			default:
+					consume();
+	 				break;
+	 			}
+	 			break;
+	 		}
+		} 
+ 	}		
+
+	private static final int getLastLength(int last) {
+		switch (last) {
+		default:
+			return -1;
+
+		case NONE:
+			return 0;
+			
+		case CARRIAGE_RETURN:
+		case BACKSLASH:
+		case SLASH:
+		case STAR:
+			return 1;
+
+		case SLASH_STAR:
+			return 2;
+
+		}	
+	}
+
+	private final void consume() {
+		fTokenLength++;
+		fLast= NONE;	
+	}
+	
+	private final IToken postFix(int state) {
+		fTokenLength++;
+		fLast= NONE;
+		fState= CCODE;
+		fPrefixLength= 0;		
+		return fTokens[state];
+	}
+
+	private final IToken preFix(int state, int newState, int last, int prefixLength) {
+		// emulate CPartitionScanner
+		if (fgEmulate && state == CCODE && (fTokenLength - getLastLength(fLast) > 0)) {
+			fTokenLength -= getLastLength(fLast);
+			fCOffset= fTokenOffset;
+			fCLength= fTokenLength;
+			fTokenLength= 1;
+			fState= newState;
+			fPrefixLength= prefixLength;
+			fLast= last;
+			return fTokens[state];
+
+		} else {
+			fTokenLength -= getLastLength(fLast);
+			fLast= last;
+			fPrefixLength= prefixLength;
+			IToken token= fTokens[state];		
+			fState= newState;
+			return token;
+		}
+	}
+
+	private static int getState(String contentType) {
+
+		if (contentType == null)
+			return CCODE;
+
+		else if (contentType.equals(C_SINGLE_LINE_COMMENT))
+			return SINGLE_LINE_COMMENT;
+
+		else if (contentType.equals(C_MULTI_LINE_COMMENT))
+			return MULTI_LINE_COMMENT;
+
+		else if (contentType.equals(C_STRING))
+			return STRING;
+
+		else if (contentType.equals(SKIP))
+			return CHARACTER;
+			
+		else
+			return CCODE;
+	}
+
+	/*
+	 * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String, int)
+	 */
+	public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
+
+		fScanner.setRange(document, offset, length);
+		fTokenOffset= partitionOffset;
+		fTokenLength= 0;
+		fPrefixLength= offset - partitionOffset;
+		fLast= NONE;
+		
+		if (offset == partitionOffset) {
+			// restart at beginning of partition
+			fState= CCODE;
+		} else {
+			fState= getState(contentType);			
+		}
+
+		// emulate CPartitionScanner
+		if (fgEmulate) {
+			fCOffset= -1;
+			fCLength= 0;
+		}
+	}
+
+	/*
+	 * @see ITokenScanner#setRange(IDocument, int, int)
+	 */
+	public void setRange(IDocument document, int offset, int length) {
+
+		fScanner.setRange(document, offset, length);
+		fTokenOffset= offset;
+		fTokenLength= 0;		
+		fPrefixLength= 0;
+		fLast= NONE;
+		fState= CCODE;
+
+		// emulate CPartitionScanner
+		if (fgEmulate) {
+			fCOffset= -1;
+			fCLength= 0;
+		}
+	}
+
+	/*
+	 * @see ITokenScanner#getTokenLength()
+	 */
+	public int getTokenLength() {
+		return fTokenLength;
+	}
+
+	/*
+	 * @see ITokenScanner#getTokenOffset()
+	 */
+	public int getTokenOffset() {
+		return fTokenOffset;
+	}
+
+}