Bug 535548 - Infinite recursion loading function whose return type refers to a parameter from the index

Change-Id: Ibff680983cc8ac0bd9d33d8f2247a9f0ca2fe2f3

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexCPPTemplateResolutionTest.java

@@ -3219,4 +3219,15 @@ public class IndexCPPTemplateResolutionTest extends IndexBindingResolutionTestBa
 	public void testClassCastException_533216() throws Exception {
 		checkBindings();
 	}
+	
+	//	template <int N>
+	//	struct array {};
+	//
+	//	template <class... Ts>
+	//	auto make_array(Ts... ts) -> array<sizeof...(ts)>;
+
+	//	auto x = make_array(2);
+	public void testRecursion_535548() throws Exception {
+		checkBindings();
+	}
 }

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java

@@ -22,7 +22,8 @@ import org.eclipse.core.runtime.CoreException;
  */
 public class ProblemFunctionType extends ProblemType implements ICPPFunctionType {
 	@SuppressWarnings("hiding")
-	public static final IType RECURSION_IN_LOOKUP = new ProblemFunctionType(BINDING_RECURSION_IN_LOOKUP);
+	public static final ICPPFunctionType RECURSION_IN_LOOKUP = new ProblemFunctionType(BINDING_RECURSION_IN_LOOKUP);
+	public static final ICPPFunctionType NOT_PERSISTED = new ProblemFunctionType(TYPE_NOT_PERSISTED);
 
 	public ProblemFunctionType(int id) {
 		super(id);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java

@@ -26,6 +26,7 @@ public class ProblemType implements IProblemType, ISerializableType {
 	public static final IType CANNOT_DEDUCE_DECLTYPE_AUTO_TYPE = new ProblemType(TYPE_CANNOT_DEDUCE_DECLTYPE_AUTO_TYPE);
 	public static final IType ENUMERATION_EXPECTED = new ProblemType(TYPE_ENUMERATION_EXPECTED);
 	public static final IType NO_NAME = new ProblemType(TYPE_NO_NAME);
+	public static final IType NOT_PERSISTED = new ProblemType(TYPE_NOT_PERSISTED);
 	public static final IType RECURSION_IN_LOOKUP = new ProblemType(BINDING_RECURSION_IN_LOOKUP);
 	public static final IType UNKNOWN_FOR_EXPRESSION = new ProblemType(TYPE_UNKNOWN_FOR_EXPRESSION);
 	public static final IType UNRESOLVED_NAME = new ProblemType(TYPE_UNRESOLVED_NAME);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java

@@ -167,7 +167,6 @@ import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPMethodSpecialization;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPMethodTemplateSpecialization;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPParameterPackType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPParameterSpecialization;
-import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPlaceholderType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerToMemberType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPTemplateDefinition;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java

@@ -17,6 +17,8 @@ package org.eclipse.cdt.internal.core.pdom.dom;
 
 import java.lang.ref.Reference;
 import java.lang.ref.SoftReference;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.eclipse.cdt.core.dom.IPDOMVisitor;
 import org.eclipse.cdt.core.dom.ast.IASTDeclSpecifier;
@@ -41,6 +43,7 @@ import org.eclipse.cdt.core.parser.util.CharArrayMap;
 import org.eclipse.cdt.internal.core.dom.parser.ASTInternal;
 import org.eclipse.cdt.internal.core.dom.parser.ITypeMarshalBuffer;
 import org.eclipse.cdt.internal.core.dom.parser.ProblemBinding;
+import org.eclipse.cdt.internal.core.dom.parser.ProblemType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPEvaluation;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPExecution;
 import org.eclipse.cdt.internal.core.index.IIndexBindingConstants;
@@ -77,6 +80,17 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 	private final PDOM fPDOM;
 	private final Database fDatabase;
 
+	/**
+	 * The set of types currently being loaded from the index on each thread, represented as record numbers.
+	 * This is used to guard against infinite recursion while loading types.
+	 */
+	private static final ThreadLocal<Set<Long>> fLoadTypeInProgress = new ThreadLocal<Set<Long>>() {
+		@Override
+		protected Set<Long> initialValue() {
+			return new HashSet<>();
+		}
+	};
+	
 	public PDOMLinkage(PDOM pdom, long record) {
 		super(null, record);
 		fPDOM= pdom;
@@ -550,6 +564,11 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 	}
 
 	public IType loadType(long offset) throws CoreException {
+		Set<Long> recursionProtectionSet = fLoadTypeInProgress.get();
+		if (!recursionProtectionSet.add(offset)) {
+			return ProblemType.NOT_PERSISTED;
+		}
+		try {
 			final Database db= getDB();
 			final byte firstByte= db.getByte(offset);
 			byte[] data= null;
@@ -567,6 +586,9 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 				break;
 			}
 			return new TypeMarshalBuffer(this, data).unmarshalType();
+		} finally {
+			recursionProtectionSet.remove(offset);
+		}
 	}
 
 	public void storeBinding(long offset, IBinding binding) throws CoreException {

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPFunction.java

@@ -333,10 +333,17 @@ class PDOMCPPFunction extends PDOMCPPBinding implements ICPPFunction, IPDOMOverl
 	public final ICPPFunctionType getType() {
 		if (fType == null) {
 			try {
-				fType = (ICPPFunctionType) getLinkage().loadType(record + FUNCTION_TYPE);
+				IType type = getLinkage().loadType(record + FUNCTION_TYPE);
+				if (type instanceof ICPPFunctionType) {
+					fType = (ICPPFunctionType) type;
+				} else {
+					// Something went wrong while loading the type and we didn't
+					// get a function type. Treat it similar to an exception.
+					fType = ProblemFunctionType.NOT_PERSISTED;
+				}
 			} catch (CoreException e) {
 				CCorePlugin.log(e);
-				fType = new ProblemFunctionType(ISemanticProblem.TYPE_NOT_PERSISTED);
+				fType = ProblemFunctionType.NOT_PERSISTED;
 			}
 		}
 		return fType;