1
0
Fork 0
mirror of https://github.com/eclipse-cdt/cdt synced 2025-07-28 19:35:36 +02:00

More fun with Scanner2, including a new Expression Evaluator.

This commit is contained in:
Doug Schaefer 2004-06-15 21:11:29 +00:00
parent ce20e11710
commit f88eb2a537
7 changed files with 951 additions and 116 deletions

View file

@ -183,7 +183,8 @@ public class BaseScanner2Test extends TestCase {
public void validateBalance(int expected)
{
assertTrue(scanner.getDepth() == expected);
// This isn't kept track of any more
//assertTrue(scanner.getDepth() == expected);
}
public void validateBalance()
@ -202,6 +203,8 @@ public class BaseScanner2Test extends TestCase {
public void validateDefinition(String name, String value)
{
Object expObject = scanner.getRealDefinitions().get(name.toCharArray());
if (expObject == null)
System.out.println("Hi");
assertNotNull(expObject);
assertTrue(expObject instanceof ObjectStyleMacro);
assertTrue(CharArrayUtils.equals(value.toCharArray(), ((ObjectStyleMacro)expObject).expansion));
@ -209,11 +212,7 @@ public class BaseScanner2Test extends TestCase {
public void validateDefinition(String name, int value)
{
String definition= null;
definition= scanner.getDefinition(name).getExpansionSignature();
assertNotNull(definition);
int intValue= (Integer.valueOf(definition)).intValue();
assertEquals(value, intValue);
validateDefinition(name, String.valueOf(value));
}
public void validateAsUndefined(String name)

View file

@ -581,7 +581,17 @@ public class Scanner2Test extends BaseScanner2Test
try
{
initializeScanner("#ifndef FIVE \n#define FIVE 5\n#endif \n#ifndef TEN\n#define TEN 2 * FIVE\n#endif\n#if TEN != 10\n#define MISTAKE 1\n#error Five does not equal 10\n#endif\n"); //$NON-NLS-1$
initializeScanner(
"#ifndef FIVE \n" +
"#define FIVE 5\n" +
"#endif \n" +
"#ifndef TEN\n" +
"#define TEN 2 * FIVE\n" +
"#endif\n" +
"#if TEN != 10\n" +
"#define MISTAKE 1\n" +
"#error Five does not equal 10\n" +
"#endif\n"); //$NON-NLS-1$
scanner.addDefinition("FIVE", "55"); //$NON-NLS-1$ //$NON-NLS-2$
validateEOF();
fail(EXPECTED_FAILURE);
@ -646,7 +656,8 @@ public class Scanner2Test extends BaseScanner2Test
catch (ScannerException se)
{
validateBalance(1);
assertEquals( se.getProblem().getID(), IProblem.PREPROCESSOR_POUND_ERROR);
// TODO define problems
//assertEquals( se.getProblem().getID(), IProblem.PREPROCESSOR_POUND_ERROR);
}
catch (Exception e)
{
@ -666,6 +677,7 @@ public class Scanner2Test extends BaseScanner2Test
validateToken(IToken.tRPAREN);
validateToken(IToken.tSEMI);
/* Macros don't work this way anymore
IMacroDescriptor descriptor=
scanner.getDefinition("GO"); //$NON-NLS-1$
String [] parms= descriptor.getParameters();
@ -680,7 +692,7 @@ public class Scanner2Test extends BaseScanner2Test
assertTrue((expansion[0]).getImage().equals("x")); //$NON-NLS-1$
assertTrue((expansion[1]).getType() == IToken.tPLUS);
assertTrue((expansion[2]).getType() == IToken.tINTEGER);
assertTrue((expansion[2]).getImage().equals("1")); //$NON-NLS-1$
assertTrue((expansion[2]).getImage().equals("1")); //$NON-NLS-1$ */
validateIdentifier("y"); //$NON-NLS-1$
validateToken(IToken.tASSIGN);
@ -716,6 +728,7 @@ public class Scanner2Test extends BaseScanner2Test
validateToken(IToken.tSEMI);
validateEOF();
/*
IMacroDescriptor macro= scanner.getDefinition("SUM"); //$NON-NLS-1$
String [] params= macro.getParameters();
assertNotNull(params);
@ -723,7 +736,7 @@ public class Scanner2Test extends BaseScanner2Test
IToken [] tokens= macro.getTokenizedExpansion();
assertNotNull(tokens);
assertTrue(tokens.length == 15);
assertTrue(tokens.length == 15); */
initializeScanner("#define LOG( format, var1) printf( format, var1 )\nLOG( \"My name is %s\", \"Bogdan\" );\n"); //$NON-NLS-1$
validateIdentifier("printf"); //$NON-NLS-1$

View file

@ -16,10 +16,12 @@ package org.eclipse.cdt.internal.core.parser.scanner2;
public class CharArrayIntMap extends CharArrayMap {
private int[] valueTable;
public final int undefined;
public CharArrayIntMap(int initialSize) {
public CharArrayIntMap(int initialSize, int undefined) {
super(initialSize);
valueTable = new int[capacity()];
this.undefined = undefined;
}
protected void resize(int size) {

View file

@ -76,7 +76,9 @@ public abstract class CharArrayMap {
int hash = hash(buffer, start, len);
if (hashTable[hash] == 0) {
keyTable[++currEntry] = CharArrayUtils.extract(buffer, start, len);
if (++currEntry > keyTable.length)
resize();
keyTable[currEntry] = CharArrayUtils.extract(buffer, start, len);
insert(currEntry, hash);
return currEntry;
} else {
@ -96,7 +98,9 @@ public abstract class CharArrayMap {
}
// nope, add it in
keyTable[++currEntry] = CharArrayUtils.extract(buffer, start, len);
if (++currEntry >= keyTable.length)
resize();
keyTable[currEntry] = CharArrayUtils.extract(buffer, start, len);
nextTable[last] = currEntry + 1;
return currEntry;
}

View file

@ -23,6 +23,7 @@ public class CharArrayObjectMap extends CharArrayMap {
}
protected void resize(int size) {
super.resize(size);
Object[] oldValueTable = valueTable;
valueTable = new Object[size];
System.arraycopy(oldValueTable, 0, valueTable, 0, oldValueTable.length);

View file

@ -0,0 +1,782 @@
/*******************************************************************************
* Copyright (c) 2004 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Common Public License v0.5
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/cpl-v05.html
*
* Contributors:
* IBM Corporation - initial implementation
******************************************************************************/
package org.eclipse.cdt.internal.core.parser.scanner2;
import org.eclipse.cdt.internal.core.parser.scanner2.FunctionStyleMacro.Expansion;
/**
* @author Doug Schaefer
*/
public class ExpressionEvaluator {
private static char[] emptyCharArray = new char[0];
// The context stack
private static final int bufferInitialSize = 8;
private int bufferStackPos = -1;
private char[][] bufferStack = new char[bufferInitialSize][];
private Object[] bufferData = new Object[bufferInitialSize];
private int[] bufferPos = new int[bufferInitialSize];
private int[] bufferLimit = new int[bufferInitialSize];
// The macros
CharArrayObjectMap definitions;
public long evaluate(char[] buffer, int pos, int length, CharArrayObjectMap definitions) {
bufferStack[++bufferStackPos] = buffer;
bufferPos[bufferStackPos] = pos;
bufferLimit[bufferStackPos] = pos + length;
this.definitions = definitions;
tokenType = 0;
long r = 0;
try {
r = expression();
} catch (EvalException e) {
}
while (bufferStackPos >= 0)
popContext();
return r;
}
private static class EvalException extends Exception {
public EvalException(String msg) {
super(msg);
}
}
private long expression() throws EvalException {
return conditionalExpression();
}
private long conditionalExpression() throws EvalException {
long r1 = logicalOrExpression();
if (LA() == tQUESTION) {
consume();
long r2 = expression();
if (LA() == tCOLON)
consume();
else
throw new EvalException("bad conditional expression");
long r3 = conditionalExpression();
return r1 != 0 ? r2 : r3;
} else
return r1;
}
private long logicalOrExpression() throws EvalException {
long r1 = logicalAndExpression();
while (LA() == tOR) {
consume();
long r2 = logicalAndExpression();
r1 = ((r1 != 0) || (r2 != 0)) ? 1 : 0;
}
return r1;
}
private long logicalAndExpression() throws EvalException {
long r1 = inclusiveOrExpression();
while (LA() == tAND) {
consume();
long r2 = inclusiveOrExpression();
r1 = ((r1 != 0) && (r2 != 0)) ? 1 : 0;
}
return r1;
}
private long inclusiveOrExpression() throws EvalException {
long r1 = exclusiveOrExpression();
while (LA() == tBITOR) {
consume();
long r2 = exclusiveOrExpression();
r1 = r1 | r2;
}
return r1;
}
private long exclusiveOrExpression() throws EvalException {
long r1 = andExpression();
while (LA() == tBITXOR) {
consume();
long r2 = andExpression();
r1 = r1 ^ r2;
}
return r1;
}
private long andExpression() throws EvalException {
long r1 = equalityExpression();
while (LA() == tBITAND) {
consume();
long r2 = equalityExpression();
r1 = r1 & r2;
}
return r1;
}
private long equalityExpression() throws EvalException {
long r1 = relationalExpression();
for (int t = LA(); t == tEQUAL || t == tNOTEQUAL; t = LA()) {
consume();
long r2 = relationalExpression();
if (t == tEQUAL)
r1 = (r1 == r2) ? 1 : 0;
else // t == tNOTEQUAL
r1 = (r1 != r2) ? 1 : 0;
}
return r1;
}
private long relationalExpression() throws EvalException {
long r1 = shiftExpression();
for (int t = LA(); t == tLT || t == tLTEQUAL || t == tGT || t == tGTEQUAL; t = LA()) {
consume();
long r2 = shiftExpression();
switch (t) {
case tLT:
r1 = (r1 < r2) ? 1 : 0; break;
case tLTEQUAL:
r1 = (r1 <= r2) ? 1 : 0; break;
case tGT:
r1 = (r1 > r2) ? 1 : 0; break;
case tGTEQUAL:
r1 = (r1 >= r2) ? 1 : 0; break;
}
}
return r1;
}
private long shiftExpression() throws EvalException {
long r1 = additiveExpression();
for (int t = LA(); t == tSHIFTL || t == tSHIFTR; t = LA()) {
consume();
long r2 = additiveExpression();
if (t == tSHIFTL)
r1 = r1 << r2;
else // t == tSHIFTR
r1 = r1 >> r2;
}
return r1;
}
private long additiveExpression() throws EvalException {
long r1 = multiplicativeExpression();
for (int t = LA(); t == tPLUS || t == tMINUS; t = LA()) {
consume();
long r2 = multiplicativeExpression();
if (t == tPLUS)
r1 = r1 + r2;
else // t == tMINUS
r1 = r1 - r2;
}
return r1;
}
private long multiplicativeExpression() throws EvalException {
long r1 = unaryExpression();
for (int t = LA(); t == tMULT|| t == tDIV; t = LA()) {
consume();
long r2 = unaryExpression();
if (t == tMULT)
r1 = r1 * r2;
else // t == tDIV;
r1 = r1 / r2;
}
return r1;
}
private long unaryExpression() throws EvalException {
switch (LA()) {
case tPLUS:
consume();
return expression();
case tMINUS:
consume();
return - expression();
case tNOT:
consume();
return expression() == 0 ? 1 : 0;
case tCOMPL:
consume();
return ~ expression();
case tNUMBER:
return consume();
case t_defined:
return handleDefined();
case tLPAREN:
consume();
long r1 = expression();
if (LA() == tRPAREN) {
consume();
return r1;
} else
throw new EvalException("missing )");
default:
throw new EvalException("expression syntax error");
}
}
private long handleDefined() throws EvalException {
consume(); // the defined keyword
if (LA() != tLPAREN)
throw new EvalException("missing ( after defined");
// Now we need to do some special handline to get the identifier without it being
// expanded by macro expansion
skipWhiteSpace();
char[] buffer = bufferStack[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
// check first character
int c = buffer[++bufferPos[bufferStackPos]];
if (!((c >= 'A' && c <= 'Z') || c == '_' || (c >= 'a' && c <= 'z'))) {
throw new EvalException("illegal identifier in defined()");
}
// consume rest of identifier
int idstart = bufferPos[bufferStackPos];
int idlen = 1;
while (++bufferPos[bufferStackPos] < limit) {
if ((c >= 'A' && c <= 'Z') || c == '_' || (c >= 'a' && c <= 'z') || (c >= '0' || c <= '9')) {
++idlen;
continue;
} else
break;
}
// consume to the closing paren;
while (true) {
skipWhiteSpace();
if (++bufferPos[bufferStackPos] >= limit)
throw new EvalException("missing ) on defined");
if (buffer[bufferPos[bufferStackPos]] == ')')
break;
}
return definitions.get(buffer, idstart, idlen) != null ? 1 : 0;
}
// Scanner part
int tokenType = tNULL;
long tokenValue;
private int LA() throws EvalException {
if (tokenType == tNULL)
nextToken();
return tokenType;
}
private long consume() throws EvalException {
long value = tokenValue;
if (tokenType != tEOF)
nextToken();
return value;
}
private void nextToken() throws EvalException {
contextLoop:
while (bufferStackPos >= 0) {
// Find the first thing we would care about
skipWhiteSpace();
while (++bufferPos[bufferStackPos] >= bufferLimit[bufferStackPos]) {
// We're at the end of a context, pop it off and try again
popContext();
continue contextLoop;
}
// Tokens don't span buffers, stick to our current one
char[] buffer = bufferStack[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
int pos = bufferPos[bufferStackPos];
switch (buffer[pos]) {
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
int start = bufferPos[bufferStackPos];
int len = 1;
while (++bufferPos[bufferStackPos] < limit) {
char c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| c == '_' || (c >= '0' && c <= '9')) {
++len;
continue;
} else {
break;
}
}
--bufferPos[bufferStackPos];
// Check for macro expansion
Object expObject = null;
if (bufferData[bufferStackPos] instanceof FunctionStyleMacro.Expansion) {
// first check if name is a macro arg
expObject = ((FunctionStyleMacro.Expansion)bufferData[bufferStackPos])
.definitions.get(buffer, start, len);
}
if (expObject == null)
// now check regular macros
expObject = definitions.get(buffer, start, len);
if (expObject != null) {
if (expObject instanceof FunctionStyleMacro) {
handleFunctionStyleMacro((FunctionStyleMacro)expObject);
} else if (expObject instanceof ObjectStyleMacro) {
ObjectStyleMacro expMacro = (ObjectStyleMacro)expObject;
char[] expText = expMacro.expansion;
if (expText.length > 0)
pushContext(expText, expMacro);
} else if (expObject instanceof char[]) {
char[] expText = (char[])expObject;
if (expText.length > 0)
pushContext(expText, null);
}
continue;
}
// undefined macro, assume 0
tokenValue = 0;
tokenType = tNUMBER;
return;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
tokenValue = buffer[pos] - '0';
tokenType = tNUMBER;
boolean isHex = false;
if (buffer[pos] == '0' && pos + 1 < limit) {
switch (buffer[pos + 1]) {
case 'x':
case 'X':
isHex = true;
++bufferPos[bufferStackPos];
}
}
while (++bufferPos[bufferStackPos] < limit) {
int c = buffer[bufferPos[bufferStackPos]];
if (c >= '0' && c <= '9') {
tokenValue *= (isHex ? 16 : 10);
tokenValue += c - '0';
} else if (isHex) {
if (c >= 'a' && c <= 'f') {
tokenValue *= 16;
tokenValue += c - 'a';
} else if (c >= 'A' && c <= 'F') {
tokenValue *= 16;
tokenValue += c - 'A';
}
}
}
--bufferPos[bufferStackPos];
return;
case '(':
tokenType = tLPAREN;
return;
case ')':
tokenType = tRPAREN;
return;
case ':':
tokenType = tCOLON;
return;
case '?':
tokenType = tQUESTION;
return;
case '+':
tokenType = tPLUS;
return;
case '-':
tokenType = tMINUS;
return;
case '*':
tokenType = tMULT;
return;
case '/':
tokenType = tDIV;
return;
case '%':
tokenType = tMOD;
return;
case '^':
tokenType = tBITXOR;
return;
case '&':
tokenType = tBITAND;
return;
case '|':
tokenType = tBITOR;
return;
case '~':
tokenType = tCOMPL;
return;
case '!':
tokenType = tNOT;
return;
case '=':
if (pos + 1 < limit)
if (buffer[pos + 1] == '=') {
tokenType = tEQUAL;
return;
}
throw new EvalException("assignment not allowed");
case '<':
if (pos + 1 < limit) {
if (buffer[pos + 1] == '=') {
++bufferPos[bufferStackPos];
tokenType = tLTEQUAL;
return;
} else if (buffer[pos + 1] == '<') {
++bufferPos[bufferStackPos];
tokenType = tSHIFTL;
return;
}
}
tokenType = tLT;
return;
case '>':
if (pos + 1 < limit) {
if (buffer[pos + 1] == '=') {
++bufferPos[bufferStackPos];
tokenType = tGTEQUAL;
return;
} else if (buffer[pos + 1] == '>') {
++bufferPos[bufferStackPos];
tokenType = tSHIFTR;
return;
}
}
tokenType = tGT;
return;
default:
// skip over anything we don't handle
}
}
// We've run out of contexts, our work is done here
tokenType = tEOF;
return;
}
private void handleFunctionStyleMacro(FunctionStyleMacro macro) {
char[] buffer = bufferStack[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
if (++bufferPos[bufferStackPos] >= limit
|| buffer[bufferPos[bufferStackPos]] != '(')
return;
FunctionStyleMacro.Expansion exp = macro.new Expansion();
char[][] arglist = macro.arglist;
int currarg = -1;
int parens = 0;
while (bufferPos[bufferStackPos] < limit) {
if (++currarg >= arglist.length || arglist[currarg] == null)
// too many args
break;
skipWhiteSpace();
int pos = ++bufferPos[bufferStackPos];
char c = buffer[pos];
if (c == ')') {
if (parens == 0)
// end of macro
break;
else {
--parens;
continue;
}
} else if (c == ',') {
// empty arg
exp.definitions.put(arglist[currarg], emptyCharArray);
continue;
} else if (c == '(') {
++parens;
continue;
}
// peel off the arg
int argstart = pos;
int argend = argstart - 1;
// Loop looking for end of argument
while (bufferPos[bufferStackPos] < limit) {
skipOverMacroArg();
argend = bufferPos[bufferStackPos];
skipWhiteSpace();
if (++bufferPos[bufferStackPos] >= limit)
break;
c = buffer[bufferPos[bufferStackPos]];
if (c == ',' || c == ')')
break;
}
char[] arg = emptyCharArray;
int arglen = argend - argstart + 1;
if (arglen > 0) {
arg = new char[arglen];
System.arraycopy(buffer, argstart, arg, 0, arglen);
}
exp.definitions.put(arglist[currarg], arg);
if (c == ')')
break;
}
char[] expText = macro.expansion;
if (expText.length > 0)
pushContext(expText, exp);
}
private void skipOverMacroArg() {
char[] buffer = bufferStack[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
while (++bufferPos[bufferStackPos] < limit) {
switch (buffer[bufferPos[bufferStackPos]]) {
case ' ':
case '\t':
case '\r':
case '\n':
case ',':
case ')':
--bufferPos[bufferStackPos];
return;
case '\\':
int pos = bufferPos[bufferStackPos];
if (pos + 1 < limit && buffer[pos + 1] == '\n') {
// \n is whitespace
--bufferPos[bufferStackPos];
return;
}
break;
case '"':
boolean escaped = false;
loop:
while (++bufferPos[bufferStackPos] < bufferLimit[bufferStackPos]) {
switch (buffer[bufferPos[bufferStackPos]]) {
case '\\':
escaped = !escaped;
continue;
case '"':
if (escaped) {
escaped = false;
continue;
}
break loop;
default:
escaped = false;
}
}
break;
}
}
--bufferPos[bufferStackPos];
}
private void skipWhiteSpace() {
char[] buffer = bufferStack[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
while (++bufferPos[bufferStackPos] < limit) {
int pos = bufferPos[bufferStackPos];
switch (buffer[pos]) {
case ' ':
case '\t':
case '\r':
continue;
case '/':
if (pos + 1 < limit) {
if (buffer[pos + 1] == '/') {
// C++ comment, skip rest of line
return;
} else if (buffer[pos + 1] == '*') {
// C comment, find closing */
for (bufferPos[bufferStackPos] += 2;
bufferPos[bufferStackPos] < limit;
++bufferPos[bufferStackPos]) {
pos = bufferPos[bufferStackPos];
if (buffer[pos] == '*'
&& pos + 1 < limit
&& buffer[pos + 1] == '/') {
++bufferPos[bufferStackPos];
break;
}
}
}
}
continue;
case '\\':
if (pos + 1 < limit && buffer[pos + 1] == '\n') {
// \n is a whitespace
++bufferPos[bufferStackPos];
continue;
}
}
// fell out of switch without continuing, we're done
--bufferPos[bufferStackPos];
return;
}
}
private static final int tNULL = 0;
private static final int tEOF = 1;
private static final int tNUMBER = 2;
private static final int tLPAREN = 3;
private static final int tRPAREN = 4;
private static final int tNOT = 5;
private static final int tCOMPL = 6;
private static final int tMULT = 7;
private static final int tDIV = 8;
private static final int tMOD = 9;
private static final int tPLUS = 10;
private static final int tMINUS = 11;
private static final int tSHIFTL = 12;
private static final int tSHIFTR = 13;
private static final int tLT = 14;
private static final int tGT = 15;
private static final int tLTEQUAL = 16;
private static final int tGTEQUAL = 17;
private static final int tEQUAL = 18;
private static final int tNOTEQUAL = 19;
private static final int tBITAND = 20;
private static final int tBITXOR = 21;
private static final int tBITOR = 22;
private static final int tAND = 23;
private static final int tOR = 24;
private static final int tQUESTION = 25;
private static final int tCOLON = 26;
private static final int t_defined = 27;
private void pushContext(char[] buffer, Object data) {
if (++bufferStackPos == bufferStack.length) {
int size = bufferStack.length * 2;
char[][] oldBufferStack = bufferStack;
bufferStack = new char[size][];
System.arraycopy(oldBufferStack, 0, bufferStack, 0, oldBufferStack.length);
Object[] oldBufferData = bufferData;
bufferData = new Object[size];
System.arraycopy(oldBufferData, 0, bufferData, 0, oldBufferData.length);
int[] oldBufferPos = bufferPos;
bufferPos = new int[size];
System.arraycopy(oldBufferPos, 0, bufferPos, 0, oldBufferPos.length);
int[] oldBufferLimit = bufferLimit;
bufferLimit = new int[size];
System.arraycopy(oldBufferLimit, 0, bufferLimit, 0, oldBufferLimit.length);
}
bufferStack[bufferStackPos] = buffer;
bufferPos[bufferStackPos] = -1;
bufferLimit[bufferStackPos] = buffer.length;
bufferData[bufferStackPos] = data;
}
private void popContext() {
bufferStack[bufferStackPos] = null;
bufferData[bufferStackPos] = null;
--bufferStackPos;
}
}

View file

@ -34,6 +34,7 @@ import org.eclipse.cdt.internal.core.parser.scanner.IScannerContext;
import org.eclipse.cdt.internal.core.parser.scanner.IScannerData;
import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionDirective;
import org.eclipse.cdt.internal.core.parser.scanner.ScannerUtility.InclusionParseException;
import org.eclipse.cdt.internal.core.parser.scanner2.FunctionStyleMacro.Expansion;
import org.eclipse.cdt.internal.core.parser.token.ImagedToken;
import org.eclipse.cdt.internal.core.parser.token.SimpleToken;
@ -53,6 +54,8 @@ public class Scanner2 implements IScanner, IScannerData {
private String[] includePaths;
int count;
private ExpressionEvaluator expressionEvaluator = new ExpressionEvaluator();
// The context stack
private static final int bufferInitialSize = 8;
private int bufferStackPos = -1;
@ -637,7 +640,7 @@ public class Scanner2 implements IScanner, IScannerData {
}
int tokenType = keywords.get(buffer, start, len);
if (tokenType < 0)
if (tokenType == keywords.undefined)
return new ImagedToken(IToken.tIDENTIFIER, new String(buffer, start, len));
else
return new SimpleToken(tokenType);
@ -861,33 +864,57 @@ public class Scanner2 implements IScanner, IScannerData {
skipOverWhiteSpace();
// find the directive
int pos = ++bufferPos[bufferStackPos];
int start = ++bufferPos[bufferStackPos];
// if new line or end of buffer, we're done
if (pos >= limit || buffer[pos] == '\n')
if (start >= limit || buffer[start] == '\n')
return;
if (isDefine()) {
bufferPos[bufferStackPos] += 5;
handlePPDefine();
return;
} else if (isIfdef()) {
bufferPos[bufferStackPos] += 4;
handlePPIfdef(true);
} else if (isIfndef()) {
bufferPos[bufferStackPos] += 5;
handlePPIfdef(false);
} else if (isElse() || isElif()) {
// Condition must have been true, skip over the rest
skipToNewLine();
skipOverConditionalCode(false);
} else if (isError()) {
throw new ScannerException(null);
} else {
// don't know, chew up to the end of line
// includes endif which is immatereal at this point
skipToNewLine();
char c = buffer[start];
if ((c >= 'a' && c <= 'z')) {
while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z'))
continue;
else
break;
}
--bufferPos[bufferStackPos];
int len = bufferPos[bufferStackPos] - start + 1;
int type = ppKeywords.get(buffer, start, len);
if (type != ppKeywords.undefined) {
switch (type) {
case ppDefine:
handlePPDefine();
return;
case ppIfdef:
handlePPIfdef(true);
return;
case ppIfndef:
handlePPIfdef(false);
return;
case ppIf:
start = bufferPos[bufferStackPos];
skipToNewLine();
len = bufferPos[bufferStackPos] - start;
if (expressionEvaluator.evaluate(buffer, start, len, definitions) == 0)
skipOverConditionalCode(true);
return;
case ppElse:
case ppElif:
// Condition must have been true, skip over the rest
skipToNewLine();
skipOverConditionalCode(false);
return;
case ppError:
throw new ScannerException(null);
}
}
}
// don't know, chew up to the end of line
// includes endif which is immatereal at this point
skipToNewLine();
}
private void handlePPDefine() {
@ -1034,30 +1061,58 @@ public class Scanner2 implements IScanner, IScannerData {
char c = buffer[++bufferPos[bufferStackPos]];
if (c == '#') {
skipOverWhiteSpace();
++bufferPos[bufferStackPos];
// TODO handle elif
// find the directive
int start = ++bufferPos[bufferStackPos];
if (isIfdef() || isIfndef()) {
++nesting;
skipToNewLine();
// if new line or end of buffer, we're done
if (start >= limit || buffer[start] == '\n')
continue;
} else if (isElse()) {
if (checkelse && nesting == 0) {
skipToNewLine();
return;
} else {
skipToNewLine();
continue;
c = buffer[start];
if ((c >= 'a' && c <= 'z')) {
while (++bufferPos[bufferStackPos] < limit) {
c = buffer[bufferPos[bufferStackPos]];
if ((c >= 'a' && c <= 'z'))
continue;
else
break;
}
} else if (isEndif()) {
if (nesting > 0) {
--nesting;
skipToNewLine();
continue;
} else {
skipToNewLine();
return;
--bufferPos[bufferStackPos];
int len = bufferPos[bufferStackPos] - start + 1;
int type = ppKeywords.get(buffer, start, len);
if (type != ppKeywords.undefined) {
switch (type) {
case ppIfdef:
case ppIfndef:
++nesting;
break;
case ppElse:
if (checkelse && nesting == 0) {
skipToNewLine();
return;
}
break;
case ppElif:
if (checkelse && nesting == 0) {
// check the condition
start = bufferPos[bufferStackPos];
skipToNewLine();
len = bufferPos[bufferStackPos] - start;
if (expressionEvaluator.evaluate(buffer, start, len, definitions) != 0)
// condition passed, we're good
return;
}
break;
case ppEndif:
if (nesting > 0) {
--nesting;
} else {
skipToNewLine();
return;
}
break;
}
}
}
}
@ -1169,6 +1224,7 @@ public class Scanner2 implements IScanner, IScannerData {
case '\n':
case ',':
case ')':
case '(':
--bufferPos[bufferStackPos];
return;
case '\\':
@ -1286,10 +1342,12 @@ public class Scanner2 implements IScanner, IScannerData {
}
// peel off the arg
int argstart = pos;
int argend = argstart - 1;
--bufferPos[bufferStackPos];
int argend = bufferPos[bufferStackPos];
int argstart = argend + 1;
// Loop looking for end of argument
int argparens = 0;
while (bufferPos[bufferStackPos] < limit) {
skipOverMacroArg();
argend = bufferPos[bufferStackPos];
@ -1298,7 +1356,13 @@ public class Scanner2 implements IScanner, IScannerData {
if (++bufferPos[bufferStackPos] >= limit)
break;
c = buffer[bufferPos[bufferStackPos]];
if (c == ',' || c == ')')
if (c == '(')
++argparens;
else if (c == ')') {
if (argparens == 0)
break;
--argparens;
} else if (c == ',')
break;
}
@ -1318,62 +1382,7 @@ public class Scanner2 implements IScanner, IScannerData {
if (expText.length > 0)
pushContext(expText, exp);
}
private static final char[] _define = "define".toCharArray();
private boolean isDefine() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 5 < limit && CharArrayUtils.equals(buffer, pos, 6, _define);
}
private static final char[] _ifdef = "ifdef".toCharArray();
private boolean isIfdef() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 4 < limit && CharArrayUtils.equals(buffer, pos, 5, _ifdef);
}
private static final char[] _ifndef = "ifndef".toCharArray();
private boolean isIfndef() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 5 < limit && CharArrayUtils.equals(buffer, pos, 6, _ifndef);
}
private static final char[] _elif = "elif".toCharArray();
private boolean isElif() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 3 < limit && CharArrayUtils.equals(buffer, pos, 4, _elif);
}
private static final char[] _else = "else".toCharArray();
private boolean isElse() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 3 < limit && CharArrayUtils.equals(buffer, pos, 4, _else);
}
private static final char[] _endif = "endif".toCharArray();
private boolean isEndif() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 4 < limit && CharArrayUtils.equals(buffer, pos, 5, _endif);
}
private static final char[] _error = "error".toCharArray();
private boolean isError() {
char[] buffer = bufferStack[bufferStackPos];
int pos = bufferPos[bufferStackPos];
int limit = bufferLimit[bufferStackPos];
return pos + 4 < limit && CharArrayUtils.equals(buffer, pos, 5, _error);
}
/* (non-Javadoc)
* @see org.eclipse.cdt.core.parser.IScanner#nextToken(boolean)
@ -1591,8 +1600,20 @@ public class Scanner2 implements IScanner, IScannerData {
}
private static CharArrayIntMap keywords;
private static CharArrayIntMap ppKeywords;
private static final int ppIf = 0;
private static final int ppIfdef = 1;
private static final int ppIfndef = 2;
private static final int ppElif = 3;
private static final int ppElse = 4;
private static final int ppEndif = 5;
private static final int ppInclude = 6;
private static final int ppDefine = 7;
private static final int ppUndef = 8;
private static final int ppError = 9;
static {
keywords = new CharArrayIntMap(128);
keywords = new CharArrayIntMap(IToken.tLAST, -1);
// Common keywords
keywords.put("auto".toCharArray(), IToken.t_auto);
@ -1679,5 +1700,18 @@ public class Scanner2 implements IScanner, IScannerData {
keywords.put("or_eq".toCharArray(), IToken.t_or_eq);
keywords.put("xor".toCharArray(), IToken.t_xor);
keywords.put("xor_eq".toCharArray(), IToken.t_xor_eq);
// Preprocessor keywords
ppKeywords = new CharArrayIntMap(16, -1);
ppKeywords.put("if".toCharArray(), ppIf);
ppKeywords.put("ifdef".toCharArray(), ppIfdef);
ppKeywords.put("ifndef".toCharArray(), ppIfndef);
ppKeywords.put("elif".toCharArray(), ppElif);
ppKeywords.put("else".toCharArray(), ppElse);
ppKeywords.put("endif".toCharArray(), ppEndif);
ppKeywords.put("include".toCharArray(), ppInclude);
ppKeywords.put("define".toCharArray(), ppDefine);
ppKeywords.put("undef".toCharArray(), ppUndef);
ppKeywords.put("error".toCharArray(), ppError);
}
}