mirrors/cdt
1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-06-06 01:06:01 +02:00
Code Issues Wiki Activity

scanner performance improvements from Dave Daoust

Browse source
This commit is contained in:
Andrew Niefer 2004-03-16 22:23:36 +00:00
parent 1661bb8022
commit 4b5850c062
4 changed files with 124 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
core/org.eclipse.cdt.core/parser/ChangeLog-parser
View file

@ -1,3 +1,8 @@
2004-03-16 Dave Daoust
refactor scanner for performance improvements
-changes to accountForUndo, now named readFromStream
-changes to checking for valid macros
2004-03-15 Andrew Niefer
Templates:
- added ISourceElementRequestor.acceptTemplateParameterReference

10
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/GCCScannerExtension.java
View file

@ -82,11 +82,11 @@ public class GCCScannerExtension implements IScannerExtension {
{
scannerData.getLogService().traceLog( "GCCScannerExtension handling #include_next directive" ); //$NON-NLS-1$
// figure out the name of the current file and its path
IScannerContext context = scannerData.getContextStack().getCurrentContext();
if( context.getKind() != IScannerContext.ContextKind.INCLUSION )
{
//handle appropriate error
}
// IScannerContext context = scannerData.getContextStack().getCurrentContext();
// if( context.getKind() != IScannerContext.ContextKind.INCLUSION )
// {
// //handle appropriate error
// }
// String fullInclusionPath = context.getFilename();
// IASTInclusion inclusion = context.getExtension();

195
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/Scanner.java
View file

@ -721,7 +721,7 @@ public class Scanner implements IScanner {
return getChar();
}
int getChar() throws ScannerException
final int getChar() throws ScannerException
{
return getChar( false );
}
@ -731,12 +731,14 @@ public class Scanner implements IScanner {
lastContext = scannerData.getContextStack().getCurrentContext();
if (scannerData.getContextStack().getCurrentContext() == null)
if (lastContext == null)
// past the end of file
return c;
c = accountForUndo(c);
int baseOffset = lastContext.getOffset() - lastContext.undoStackSize() - 1;
if (lastContext.undoStackSize() != 0 )
c = lastContext.popUndo();
else
c = readFromStream();
if (enableTrigraphReplacement && (!insideString || enableTrigraphReplacementInStrings)) {
// Trigraph processing
@ -747,39 +749,39 @@ public class Scanner implements IScanner {
c = getChar(insideString);
switch (c) {
case '(':
expandDefinition("??(", "[", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??(", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case ')':
expandDefinition("??)", "]", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??)", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '<':
expandDefinition("??<", "{", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??<", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '>':
expandDefinition("??>", "}", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '=':
expandDefinition("??=", "#", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??=", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '/':
expandDefinition("??/", "\\", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??/", "\\", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '\'':
expandDefinition("??\'", "^", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??\'", "^", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '!':
expandDefinition("??!", "|", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??!", "|", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
case '-':
expandDefinition("??-", "~", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("??-", "~", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(insideString);
break;
default:
@ -805,10 +807,10 @@ public class Scanner implements IScanner {
if (c == '<') {
c = getChar(false);
if (c == '%') {
expandDefinition("<%", "{", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("<%", "{", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(false);
} else if (c == ':') {
expandDefinition("<:", "[", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("<:", "[", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(false);
} else {
// Not a digraph
@ -818,7 +820,7 @@ public class Scanner implements IScanner {
} else if (c == ':') {
c = getChar(false);
if (c == '>') {
expandDefinition(":>", "]", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition(":>", "]", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(false);
} else {
// Not a digraph
@ -828,10 +830,10 @@ public class Scanner implements IScanner {
} else if (c == '%') {
c = getChar(false);
if (c == '>') {
expandDefinition("%>", "}", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("%>", "}", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(false);
} else if (c == ':') {
expandDefinition("%:", "#", baseOffset); //$NON-NLS-1$ //$NON-NLS-2$
expandDefinition("%:", "#", lastContext.getOffset() - lastContext.undoStackSize() - 1); //$NON-NLS-1$ //$NON-NLS-2$
c = getChar(false);
} else {
// Not a digraph
@ -845,38 +847,29 @@ public class Scanner implements IScanner {
return c;
}
protected int accountForUndo(int c)
protected int readFromStream()
{
boolean done;
do {
done = true;
if (scannerData.getContextStack().getCurrentContext().undoStackSize() != 0 ) {
c = scannerData.getContextStack().getCurrentContext().popUndo();
} else {
try {
c = scannerData.getContextStack().getCurrentContext().read();
if (c == NOCHAR) {
if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false) {
c = NOCHAR;
break;
} else {
done = false;
}
}
} catch (IOException e) {
if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false) {
c = NOCHAR;
} else {
done = false;
}
}
}
} while (!done);
return c;
int c;
try {
c = scannerData.getContextStack().getCurrentContext().read();
}
catch (IOException e) {
c = NOCHAR;
}
if (c != NOCHAR)
return c;
if (scannerData.getContextStack().rollbackContext(scannerData.getClientRequestor()) == false)
return NOCHAR;
if (scannerData.getContextStack().getCurrentContext().undoStackSize() != 0 )
return scannerData.getContextStack().getCurrentContext().popUndo();
return readFromStream();
}
void ungetChar(int c) throws ScannerException{
final void ungetChar(int c) throws ScannerException{
scannerData.getContextStack().getCurrentContext().pushUndo(c);
try
{
@ -1220,9 +1213,12 @@ public class Scanner implements IScanner {
if (directive == null) {
if( scannerExtension.canHandlePreprocessorDirective( token ) )
scannerExtension.handlePreprocessorDirective( token, getRestOfPreprocessorLine() );
StringBuffer buffer = new StringBuffer( "#"); //$NON-NLS-1$
buffer.append( token );
handleProblem( IProblem.PREPROCESSOR_INVALID_DIRECTIVE, buffer.toString(), beginningOffset, false, true );
else
{
StringBuffer buffer = new StringBuffer( "#"); //$NON-NLS-1$
buffer.append( token );
handleProblem( IProblem.PREPROCESSOR_INVALID_DIRECTIVE, buffer.toString(), beginningOffset, false, true );
}
return null;
}
@ -1432,12 +1428,12 @@ public class Scanner implements IScanner {
c = getChar();
// do the least expensive tests first!
while (
Character.isUnicodeIdentifierPart( (char)c)
// ((c >= 'a') && (c <= 'z'))
// || ((c >= 'A') && (c <= 'Z'))
// || ((c >= '0') && (c <= '9'))
// || (c == '_')
((c >= 'a') && (c <= 'z'))
|| ((c >= 'A') && (c <= 'Z'))
|| ((c >= '0') && (c <= '9'))
|| (c == '_') || Character.isUnicodeIdentifierPart( (char)c)
) {
buff.append((char) c);
c = getChar();
@ -2545,7 +2541,37 @@ public class Scanner implements IScanner {
return macroReplacementTokens;
}
protected IMacroDescriptor createObjectMacroDescriptor(String key, String value ) {
StringBuffer signatureBuffer = new StringBuffer();
signatureBuffer.append( key );
signatureBuffer.append( ' ' );
signatureBuffer.append( value );
// List macroReplacementTokens;
// if (value.trim().equals( "" ))
// macroReplacementTokens = new ArrayList();
// else
// macroReplacementTokens = tokenizeReplacementString( NO_OFFSET_LIMIT, key, value, null );
List macroReplacementTokens = new ArrayList();
if( !value.trim().equals( "" ) ) //$NON-NLS-1$
{
Token t = new Token(
IToken.tIDENTIFIER,
value,
scannerData.getContextStack().getCurrentContext(),
scannerData.getContextStack().getCurrentLineNumber()
);
macroReplacementTokens.add( t );
}
return new ObjectMacroDescriptor( key,
signatureBuffer.toString(),
macroReplacementTokens,
value);
}
protected void poundDefine(int beginning, int beginningLine ) throws ScannerException, EndOfFileException {
StringBuffer potentialErrorMessage = new StringBuffer( POUND_DEFINE );
skipOverWhitespace();
@ -2636,8 +2662,9 @@ public class Scanner implements IScanner {
}
else if ((c == '\n') || (c == '\r'))
{
checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
addDefinition( key, "" ); //$NON-NLS-1$
descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
addDefinition( key, descriptor );
}
else if ((c == ' ') || (c == '\t') ) {
// this is a simple definition
@ -2646,8 +2673,9 @@ public class Scanner implements IScanner {
// get what we are to map the name to and add it to the definitions list
String value = getRestOfPreprocessorLine();
checkValidMacroRedefinition(key, previousDefinition, value, beginning);
addDefinition( key, value );
descriptor = createObjectMacroDescriptor(key, value);
checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
addDefinition( key, descriptor );
} else if (c == '/') {
// this could be a comment
@ -2655,20 +2683,23 @@ public class Scanner implements IScanner {
if (c == '/') // one line comment
{
skipOverSinglelineComment();
checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
addDefinition(key, ""); //$NON-NLS-1$
descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
addDefinition(key, descriptor);
} else if (c == '*') // multi-line comment
{
if (skipOverMultilineComment()) {
// we have gone over a newline
// therefore, this symbol was defined to an empty string
checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
addDefinition(key, ""); //$NON-NLS-1$
descriptor = createObjectMacroDescriptor(key, ""); //$NON-NLS-1$
checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
addDefinition(key, descriptor);
} else {
String value = getRestOfPreprocessorLine();
checkValidMacroRedefinition(key, previousDefinition, "", beginning); //$NON-NLS-1$
addDefinition(key, value);
descriptor = createObjectMacroDescriptor(key, value);
checkValidMacroRedefinition(key, previousDefinition, descriptor, beginning);
addDefinition(key, descriptor);
}
} else {
// this is not a comment
@ -2699,40 +2730,6 @@ public class Scanner implements IScanner {
}
}
protected void checkValidMacroRedefinition(
String key,
IMacroDescriptor previousDefinition,
String newDefinition, int beginningOffset )
throws ScannerException
{
StringBuffer buffer = new StringBuffer(key);
buffer.append( ' ');
buffer.append(newDefinition);
IMacroDescriptor newMacro = new ObjectMacroDescriptor( key, buffer.toString(),
tokenizeReplacementString( NO_OFFSET_LIMIT, key, newDefinition, null ), newDefinition );
checkValidMacroRedefinition( key, previousDefinition, newMacro, beginningOffset );
}
protected void checkValidMacroRedefinition(
String key,
String previousDefinition,
String newDefinition, int beginningOffset )
throws ScannerException
{
StringBuffer oldMacro = new StringBuffer( key );
oldMacro.append( ' ');
StringBuffer newMacro = new StringBuffer( oldMacro.toString() );
oldMacro.append( previousDefinition );
newMacro.append( newDefinition );
IMacroDescriptor prevMacroDescriptor = new ObjectMacroDescriptor( key, oldMacro.toString(),
tokenizeReplacementString( NO_OFFSET_LIMIT, key, previousDefinition, null ), previousDefinition );
IMacroDescriptor newMacroDescriptor = new ObjectMacroDescriptor( key, newMacro.toString(),
tokenizeReplacementString( NO_OFFSET_LIMIT, key, newDefinition, null ), newDefinition );
checkValidMacroRedefinition( key, prevMacroDescriptor, newMacroDescriptor, beginningOffset );
}
protected void checkValidMacroRedefinition(
String key,
IMacroDescriptor previousDefinition,

33
core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner/ScannerContext.java
View file

@ -12,7 +12,6 @@ package org.eclipse.cdt.internal.core.parser.scanner;
import java.io.IOException;
import java.io.Reader;
import java.util.Stack;
import org.eclipse.cdt.core.parser.ast.IASTInclusion;
@ -24,7 +23,6 @@ public class ScannerContext implements IScannerContext
private int macroLength = -1;
private int line = 1;
private int offset;
private Stack undo = new Stack();
private ContextKind kind;
/* (non-Javadoc)
@ -108,18 +106,15 @@ public class ScannerContext implements IScannerContext
return line;
}
/**
* Returns the reader.
* @return Reader
/* there are never more than 2 elements in the unget stack!
* trigraphs may involve 2, but in general there is a single element
* I have made room for 10 -- just in case :-)
*/
public final Reader getReader()
{
return reader;
}
private int pos = 0;
private int undo[] = new int[10];
public final int undoStackSize()
{
return undo.size();
return pos;
}
/**
@ -128,7 +123,7 @@ public class ScannerContext implements IScannerContext
*/
public final int popUndo()
{
int c = ((Integer)undo.pop()).intValue();
int c = undo[--pos];
if ((char)c == '\n') line++;
return c;
}
@ -137,10 +132,18 @@ public class ScannerContext implements IScannerContext
* Sets the undo.
* @param undo The undo to set
*/
public void pushUndo(int undo)
public final void pushUndo(int c)
{
if ((char)undo == '\n') line--;
this.undo.push( new Integer( undo ));
if ((char)c == '\n') line--;
undo[pos++] = c;
}
/**
* Returns the reader.
* @return Reader
*/
public final Reader getReader()
{
return reader;
}