Indexer to respect ordering of directives, bug 270806.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexBugsTests.java

@@ -32,6 +32,8 @@ import org.eclipse.cdt.core.dom.ast.IASTImplicitName;
 import org.eclipse.cdt.core.dom.ast.IASTImplicitNameOwner;
 import org.eclipse.cdt.core.dom.ast.IASTName;
 import org.eclipse.cdt.core.dom.ast.IASTNodeSelector;
+import org.eclipse.cdt.core.dom.ast.IASTPreprocessorIfdefStatement;
+import org.eclipse.cdt.core.dom.ast.IASTPreprocessorStatement;
 import org.eclipse.cdt.core.dom.ast.IASTSimpleDeclaration;
 import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
 import org.eclipse.cdt.core.dom.ast.IBasicType;
@@ -50,6 +52,7 @@ import org.eclipse.cdt.core.dom.ast.cpp.ICPPClassType;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPFunction;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPMethod;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPTemplateInstance;
+import org.eclipse.cdt.core.dom.ast.cpp.ICPPUsingDirective;
 import org.eclipse.cdt.core.dom.ast.cpp.ICPPVariable;
 import org.eclipse.cdt.core.index.IIndex;
 import org.eclipse.cdt.core.index.IIndexBinding;
@@ -1972,4 +1975,111 @@ public class IndexBugsTests extends BaseTestCase {
 			index.releaseReadLock();
 		}
 	}
+	
+	//  // a.h
+	//	#undef XXX
+	
+	//  // b.h
+	//  #include "a.h"
+	//	#define XXX
+
+	//  // source.c
+	//	#include "b.h"
+	//  #ifdef XXX
+	//  int ok;
+	//  #endif
+	public void testPreprocessingStatementOrder_270806_1() throws Exception {
+		waitForIndexer();
+		String[] testData = getContentsForTest(3);
+		TestSourceReader.createFile(fCProject.getProject(), "a.h", testData[0]);
+		TestSourceReader.createFile(fCProject.getProject(), "b.h", testData[1]);
+		IFile s= TestSourceReader.createFile(fCProject.getProject(), "s1.c", testData[2]);
+		final IIndexManager indexManager = CCorePlugin.getIndexManager();
+		indexManager.reindex(fCProject);
+		waitForIndexer();
+		IIndex index= indexManager.getIndex(fCProject);
+		index.acquireReadLock();
+		try {
+			IASTTranslationUnit tu = TestSourceReader.createIndexBasedAST(index, fCProject, s);
+			IASTPreprocessorStatement[] pstmts= tu.getAllPreprocessorStatements();
+			IASTPreprocessorStatement ifndef= pstmts[1];
+			assertInstance(ifndef, IASTPreprocessorIfdefStatement.class);
+			assertTrue(((IASTPreprocessorIfdefStatement) ifndef).taken());
+		} finally {
+			index.releaseReadLock();
+		}
+	}
+
+	//  // a.h
+	//	#undef XXX
+	
+	//  // b.h
+	//	#define XXX
+	//  #include "a.h"
+
+	//  // source.c
+	//	#include "b.h"
+	//  #ifdef XXX
+	//  int bug;
+	//  #endif
+	public void testPreprocessingStatementOrder_270806_2() throws Exception {
+		waitForIndexer();
+		String[] testData = getContentsForTest(3);
+		TestSourceReader.createFile(fCProject.getProject(), "a.h", testData[0]);
+		TestSourceReader.createFile(fCProject.getProject(), "b.h", testData[1]);
+		IFile s= TestSourceReader.createFile(fCProject.getProject(), "s1.c", testData[2]);
+		final IIndexManager indexManager = CCorePlugin.getIndexManager();
+		indexManager.reindex(fCProject);
+		waitForIndexer();
+		IIndex index= indexManager.getIndex(fCProject);
+		index.acquireReadLock();
+		try {
+			IASTTranslationUnit tu = TestSourceReader.createIndexBasedAST(index, fCProject, s);
+			IASTPreprocessorStatement[] pstmts= tu.getAllPreprocessorStatements();
+			IASTPreprocessorStatement ifndef= pstmts[1];
+			assertInstance(ifndef, IASTPreprocessorIfdefStatement.class);
+			assertFalse(((IASTPreprocessorIfdefStatement) ifndef).taken());
+		} finally {
+			index.releaseReadLock();
+		}
+	}
+
+	//	namespace X {}
+	//	namespace Y {}
+	//	#define XXX
+	//	#define YYY
+	//  #include "inc.h"
+	//  #include <inc.h>
+	//  using namespace X;
+	//  using namespace Y;
+	public void testPreprocessingStatementOrder_270806_3() throws Exception {
+		waitForIndexer();
+		String[] testData = getContentsForTest(1);
+		IFile f= TestSourceReader.createFile(fCProject.getProject(), "a.cpp", testData[0]);
+		final IIndexManager indexManager = CCorePlugin.getIndexManager();
+		waitUntilFileIsIndexed(f, 4000);
+		IIndex index= indexManager.getIndex(fCProject);
+		index.acquireReadLock();
+		try {
+			IIndexFile file= index.getFile(ILinkage.CPP_LINKAGE_ID, IndexLocationFactory.getWorkspaceIFL(f));
+			// check order of includes
+			IIndexInclude[] incs = file.getIncludes();
+			assertEquals(2, incs.length);
+			assertFalse(incs[0].isSystemInclude());
+			assertTrue(incs[1].isSystemInclude());
+			// check order of macros
+			IIndexMacro[] macros = file.getMacros();
+			assertEquals(2, macros.length);
+			assertEquals("XXX", macros[0].getName());
+			assertEquals("YYY", macros[1].getName());
+			// check order of using directives
+			ICPPUsingDirective[] uds = file.getUsingDirectives();
+			assertEquals(2, uds.length);
+			assertEquals("X", new String(uds[0].getNominatedScope().getScopeName().getSimpleID()));
+			assertEquals("Y", new String(uds[1].getNominatedScope().getScopeName().getSimpleID()));
+			assertTrue(uds[0].getPointOfDeclaration() < uds[1].getPointOfDeclaration());
+		} finally {
+			index.releaseReadLock();
+		}
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/index/IndexBasedCodeReaderFactory.java

@@ -17,11 +17,8 @@ package org.eclipse.cdt.internal.core.index;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import org.eclipse.cdt.core.CCorePlugin;
@@ -143,20 +140,15 @@ public final class IndexBasedCodeReaderFactory extends AbstractCodeReaderFactory
 			IIndexFile file= fIndex.getFile(fLinkage, ifl);
 			if (file != null) {
 				try {
-					LinkedHashMap<IIndexFileLocation, FileContent> fileContentMap= new LinkedHashMap<IIndexFileLocation, FileContent>();
 					List<IIndexFile> files= new ArrayList<IIndexFile>();
-					collectFileContent(file, fileContentMap, files, false);
+					List<IIndexMacro> macros= new ArrayList<IIndexMacro>();
-					ArrayList<IIndexMacro> allMacros= new ArrayList<IIndexMacro>();
+					List<ICPPUsingDirective> directives= new ArrayList<ICPPUsingDirective>();
-					ArrayList<ICPPUsingDirective> allDirectives= new ArrayList<ICPPUsingDirective>();
+					Set<IIndexFileLocation> ifls= new HashSet<IIndexFileLocation>();
-					for (Map.Entry<IIndexFileLocation,FileContent> entry : fileContentMap.entrySet()) {
+					collectFileContent(file, ifls, files, macros, directives, false);
-						final FileContent content= entry.getValue();
+					// add included files only, if no exception was thrown
-						allMacros.addAll(Arrays.asList(content.fMacros));
+					fIncludedFiles.addAll(ifls);
-						allDirectives.addAll(Arrays.asList(content.fDirectives));
+					return new IncludeFileContent(path, macros, directives, files);
-						fIncludedFiles.add(entry.getKey());
+				} catch (NeedToParseException e) {
-					}
-					return new IncludeFileContent(path, allMacros, allDirectives, files);
-				}
-				catch (NeedToParseException e) {
 				}
 			}
 		}
@@ -180,9 +172,11 @@ public final class IndexBasedCodeReaderFactory extends AbstractCodeReaderFactory
 	}
 
 
-	private void collectFileContent(IIndexFile file, Map<IIndexFileLocation, FileContent> macroMap, List<IIndexFile> files, boolean checkIncluded) throws CoreException, NeedToParseException {
+	private void collectFileContent(IIndexFile file, Set<IIndexFileLocation> ifls, List<IIndexFile> files,
+			List<IIndexMacro> macros, List<ICPPUsingDirective> usingDirectives, boolean checkIncluded)
+			throws CoreException, NeedToParseException {
 		IIndexFileLocation ifl= file.getLocation();
-		if (macroMap.containsKey(ifl) || (checkIncluded && fIncludedFiles.contains(ifl))) {
+		if (!ifls.add(ifl) || (checkIncluded && fIncludedFiles.contains(ifl))) {
 			return;
 		}
 		FileContent content;
@@ -191,21 +185,23 @@ public final class IndexBasedCodeReaderFactory extends AbstractCodeReaderFactory
 			if (content == null) {
 				throw new NeedToParseException();
 			}
-		}
+		} else {
-		else {
 			content= new FileContent();
-			content.fMacros= file.getMacros();
+			content.setPreprocessorDirectives(file.getIncludes(), file.getMacros());
-			content.fDirectives= file.getUsingDirectives();
+			content.setUsingDirectives(file.getUsingDirectives());
 		}
-		macroMap.put(ifl, content); // prevent recursion
-		files.add(file);
 		
-		// follow the includes
+		files.add(file);
-		IIndexInclude[] includeDirectives= file.getIncludes();
+		usingDirectives.addAll(Arrays.asList(content.getUsingDirectives()));
-		for (final IIndexInclude indexInclude : includeDirectives) {
+		Object[] dirs= content.getPreprocessingDirectives();
-			IIndexFile includedFile= fIndex.resolveInclude(indexInclude);
+		for (Object d : dirs) {
+			if (d instanceof IIndexMacro) {
+				macros.add((IIndexMacro) d);
+			} else if (d instanceof IIndexInclude) {
+				IIndexFile includedFile= fIndex.resolveInclude((IIndexInclude) d);
 				if (includedFile != null) {
-				collectFileContent(includedFile, macroMap, files, true);
+					collectFileContent(includedFile, ifls, files, macros, usingDirectives, true);
+				}
 			}
 		}
 	}
@@ -242,7 +238,6 @@ public final class IndexBasedCodeReaderFactory extends AbstractCodeReaderFactory
 			for (IIndexFile file : filesIncluded) {
 				fIncludedFiles.add(file.getLocation());
 			}
-			Collections.reverse(macros);
 			return new IncludeFileContent(GAP, macros, directives, new ArrayList<IIndexFile>(filesIncluded));
 		}
 		catch (CoreException e) {
@@ -279,40 +274,36 @@ public final class IndexBasedCodeReaderFactory extends AbstractCodeReaderFactory
 			return false;
 		}
 
-		final IIndexInclude[] includeDirectives= from.getIncludes();
+		final IIndexInclude[] ids= from.getIncludes();
+		final IIndexMacro[] ms= from.getMacros();
+		final Object[] dirs= FileContent.merge(ids, ms);
 		IIndexInclude success= null;
-		for (IIndexInclude indexInclude : includeDirectives) {
+		for (Object d : dirs) {
-			IIndexFile includedFile= fIndex.resolveInclude(indexInclude);
+			if (d instanceof IIndexMacro) {
+				macros.add((IIndexMacro) d);
+			} else if (d instanceof IIndexInclude) {
+				IIndexFile includedFile= fIndex.resolveInclude((IIndexInclude) d);
 				if (includedFile != null) {
 					if (collectFileContentForGap(includedFile, to, filesIncluded, macros, directives)) {
-					success= indexInclude;
+						success= (IIndexInclude) d;
 						break;
 					}
 				}
 			}
+		}
 
-		IIndexMacro[] mymacros= from.getMacros();
+		final ICPPUsingDirective[] uds= from.getUsingDirectives();
-		ICPPUsingDirective[] mydirectives= from.getUsingDirectives();
-		int startm, startd;
 		if (success == null) {
-			startm= mymacros.length-1;
+			directives.addAll(Arrays.asList(uds));
-			startd= mydirectives.length-1;
+			return false;
 		}
-		else {
+			
-			startm= startd= -1;
 		final int offset= success.getNameOffset();
-			for (IIndexMacro macro : from.getMacros()) {
+		for (ICPPUsingDirective ud : uds) {
-				if (macro.getFileLocation().getNodeOffset() < offset) {
+			if (ud.getPointOfDeclaration() > offset)
-					startm++;
+				break;
+			directives.add(ud);
 		}
-			}
+		return true;
-		}
-		for (int i= startm; i >= 0; i--) {
-			macros.add(mymacros[i]);
-		}
-		for (int i= startd; i >= 0; i--) {
-			directives.add(mydirectives[i]);
-		}
-		return success != null;
 	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/AbstractIndexerTask.java

@@ -87,8 +87,71 @@ public abstract class AbstractIndexerTask extends PDOMWriter {
 		private boolean fRequestUpdate= false;
 		private boolean fRequestIsCounted= true;
 		private boolean fIsUpdated= false;
-		public IIndexMacro[] fMacros;
+		private Object[] fPreprocessingDirectives;
-		public ICPPUsingDirective[] fDirectives;
+		private ICPPUsingDirective[] fDirectives;
+
+		public Object[] getPreprocessingDirectives() throws CoreException {
+			if (fPreprocessingDirectives == null) {
+				if (fIndexFile == null)
+					return new Object[0];
+				setPreprocessorDirectives(fIndexFile.getIncludes(), fIndexFile.getMacros());
+			}
+			return fPreprocessingDirectives;
+		}
+		
+		public ICPPUsingDirective[] getUsingDirectives() throws CoreException {
+			if (fDirectives == null) {
+				if (fIndexFile == null)
+					return ICPPUsingDirective.EMPTY_ARRAY;
+				setUsingDirectives(fIndexFile.getUsingDirectives());
+			}
+			return fDirectives;
+		}
+
+		public void setPreprocessorDirectives(IIndexInclude[] includes, IIndexMacro[] macros) throws CoreException {
+			fPreprocessingDirectives= merge(includes, macros);
+		}
+
+		public void setUsingDirectives(ICPPUsingDirective[] usingDirectives) {
+			fDirectives= usingDirectives;
+		}
+
+		public void clearCaches() {
+			fPreprocessingDirectives= null;
+			fDirectives= null;
+		}
+		
+		public static Object[] merge(IIndexInclude[] includes, IIndexMacro[] macros) throws CoreException {
+			Object[] merged= new Object[includes.length+macros.length];
+			int i=0;
+			int m=0;
+			int ioffset= getOffset(includes, i);
+			int moffset= getOffset(macros, m);
+			for (int k = 0; k < merged.length; k++) {
+				if (ioffset <= moffset) {
+					merged[k]= includes[i];
+					ioffset= getOffset(includes, ++i);
+				} else {
+					merged[k]= macros[m];
+					moffset= getOffset(macros, ++m);
+				}
+			}
+			return merged;
+		}
+
+		private static int getOffset(IIndexMacro[] macros, int m) throws CoreException {
+			if (m < macros.length) {
+				return macros[m].getFileLocation().getNodeOffset();
+			}
+			return Integer.MAX_VALUE;
+		}
+
+		private static int getOffset(IIndexInclude[] includes, int i) throws CoreException {
+			if (i < includes.length) {
+				return includes[i].getNameOffset();
+			}
+			return Integer.MAX_VALUE;
+		}
 	}
 	
 	protected enum MessageKind {parsingFileTask, errorWhileParsing, tooManyIndexProblems}
@@ -341,7 +404,7 @@ public abstract class AbstractIndexerTask extends PDOMWriter {
 			info= createFileInfo(key, null);
 		}
 		info.fIsUpdated= true;
-		info.fMacros= null;
+		info.clearCaches();
 	}
 
 	private FileContent createFileInfo(FileKey key, IIndexFile ifile) {
@@ -793,12 +856,6 @@ public abstract class AbstractIndexerTask extends PDOMWriter {
 					return null;
 				}
 			}
-			if (info.fMacros == null) {
-				info.fMacros= info.fIndexFile.getMacros();
-			}
-			if (info.fDirectives == null) {
-				info.fDirectives= info.fIndexFile.getUsingDirectives();
-			}
 			return info;
 		}
 		return null;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -172,11 +172,12 @@ public class PDOM extends PlatformObject implements IPDOM {
 	 *  79.0 - instantiation of values, bug 245027
 	 *  80.0 - support for specializations of partial specializations, bug 259872
 	 *  81.0 - change to c++ function types, bug 264479
+	 *  82.0 - offsets for using directives, bug 270806
 	 */
 	private static int version(int major, int minor) {
 		return major << 16 + minor;
 	}
-	public static final int MAJOR_VERSION = 81; 
+	public static final int MAJOR_VERSION = 82; 
 	public static final int MINOR_VERSION = 0;	// minor versions must be compatible	
 	
 	public static final int CURRENT_VERSION=       version(MAJOR_VERSION, MINOR_VERSION);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMFile.java

@@ -69,7 +69,7 @@ public class PDOMFile implements IIndexFragmentFile {
 	private static final int LINKAGE_ID= 20;
 	private static final int TIME_STAMP = 24;
 	private static final int SCANNER_CONFIG_HASH= 32;
-	private static final int FIRST_USING_DIRECTIVE= 36;
+	private static final int LAST_USING_DIRECTIVE= 36;
 	private static final int FIRST_MACRO_REFERENCE= 40;
 
 	private static final int RECORD_SIZE= 44;
@@ -623,12 +623,12 @@ public class PDOMFile implements IIndexFragmentFile {
 		setFirstIncludedBy(null);
 	}
 
-	public int getFirstUsingDirectiveRec() throws CoreException {
+	public int getLastUsingDirectiveRec() throws CoreException {
-		return fLinkage.getDB().getInt(record + FIRST_USING_DIRECTIVE);
+		return fLinkage.getDB().getInt(record + LAST_USING_DIRECTIVE);
 	}
 
 	public void setFirstUsingDirectiveRec(int rec) throws CoreException {
-		fLinkage.getDB().putInt(record + FIRST_USING_DIRECTIVE, rec);
+		fLinkage.getDB().putInt(record + LAST_USING_DIRECTIVE, rec);
 	}
 
 	/* (non-Javadoc)

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPLinkage.java

@@ -891,9 +891,9 @@ class PDOMCPPLinkage extends PDOMLinkage implements IIndexCPPBindingConstants {
 					}
 				}
 				if (doit) {
-					int rec= file.getFirstUsingDirectiveRec();
+					int rec= file.getLastUsingDirectiveRec();
 					PDOMCPPUsingDirective ud= new PDOMCPPUsingDirective(this, rec, containerNS,
-							pdomName.getBinding());
+							pdomName.getBinding(), pdomName.getFileLocation().getNodeOffset());
 					file.setFirstUsingDirectiveRec(ud.getRecord());
 				}
 			} catch (DOMException e) {
@@ -931,7 +931,7 @@ class PDOMCPPLinkage extends PDOMLinkage implements IIndexCPPBindingConstants {
 	 */
 	@Override
 	public ICPPUsingDirective[] getUsingDirectives(PDOMFile file) throws CoreException {
-		int rec= file.getFirstUsingDirectiveRec();
+		int rec= file.getLastUsingDirectiveRec();
 		if (rec == 0) {
 			return ICPPUsingDirective.EMPTY_ARRAY;
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPUsingDirective.java

@@ -35,7 +35,8 @@ public class PDOMCPPUsingDirective implements ICPPUsingDirective, IPDOMNode {
 	private static final int CONTAINER_NAMESPACE 	= 0;
 	private static final int NOMINATED_NAMESPACE    = 4;
 	private static final int PREV_DIRECTIVE_OF_FILE	= 8;
-	private static final int RECORD_SIZE 			= 12;
+	private static final int FILE_OFFSET	        = 12;
+	private static final int RECORD_SIZE 			= 16;
 
 	private final PDOMCPPLinkage fLinkage;
 	private final int fRecord;
@@ -45,7 +46,8 @@ public class PDOMCPPUsingDirective implements ICPPUsingDirective, IPDOMNode {
 		fRecord= record;
 	}
 
-	public PDOMCPPUsingDirective(PDOMCPPLinkage linkage, int prevRecInFile, PDOMCPPNamespace containerNS, PDOMBinding nominated) throws CoreException {
+	public PDOMCPPUsingDirective(PDOMCPPLinkage linkage, int prevRecInFile, PDOMCPPNamespace containerNS, 
+			PDOMBinding nominated, int fileOffset) throws CoreException {
 		final Database db= linkage.getDB();
 		final int containerRec= containerNS == null ? 0 : containerNS.getRecord();
 		final int nominatedRec= nominated.getRecord();
@@ -55,6 +57,7 @@ public class PDOMCPPUsingDirective implements ICPPUsingDirective, IPDOMNode {
 		db.putInt(fRecord + CONTAINER_NAMESPACE, containerRec);
 		db.putInt(fRecord + NOMINATED_NAMESPACE, nominatedRec);
 		db.putInt(fRecord + PREV_DIRECTIVE_OF_FILE, prevRecInFile);
+		db.putInt(fRecord + FILE_OFFSET, fileOffset);
 	}
 
 	/* (non-Javadoc)
@@ -97,8 +100,13 @@ public class PDOMCPPUsingDirective implements ICPPUsingDirective, IPDOMNode {
 	 * @see org.eclipse.cdt.core.dom.ast.cpp.ICPPUsingDirective#getPointOfDeclaration()
 	 */
 	public int getPointOfDeclaration() {
+		final Database db= fLinkage.getDB();
+		try {
+			return db.getInt(fRecord + FILE_OFFSET);
+		} catch (CoreException e) {
 			return 0;
 		}
+	}
 
 	public int getRecord() {
 		return fRecord;