Bug 495952 - Problem with SFINAE in trailing return type

Change-Id: Ida1d168803da5304c834b19fa83148921f69b2b6

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2TemplateTests.java

@@ -7818,6 +7818,43 @@ public class AST2TemplateTests extends AST2TestBase {
 		parseAndCheckBindings();
 	}
 
+	//	template<typename T>
+	//	T a();
+	//
+	//	template <class T>
+	//	struct A {};
+	//
+	//	template <class T>
+	//	A<T> b(T t);
+	//
+	//	template <class T, class U>
+	//	void c(U u);
+	//
+	//	template <class T, class U, class W>
+	//	decltype(c<T>(1)) d(W w, U u);
+	//
+	//	template <class T, class U>
+	//	auto d(U u, T t) -> decltype(d<typename A<T>::type>(u, t));
+	//
+	//	template <class T, class U>
+	//	auto e(U u, T t) -> decltype(d(b(u), t));
+	//
+	//	template <typename T, typename U = decltype(e(1, a<T>()))>
+	//	class B {};
+	//
+	//	template <typename T>
+	//	typename B<T>::type waldo(T p);
+	//
+	//	template <typename T>
+	//	int waldo(T p);
+	//
+	//	void test() {
+	//	  waldo(1);
+	//	}
+	public void testSfinaeInTrailingReturnType_495952() throws Exception {
+		parseAndCheckBindings();
+	}
+
 	//	template <typename>
 	//	struct M {
 	//	    template <typename... Args>

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java

@@ -21,6 +21,8 @@ import org.eclipse.core.runtime.CoreException;
  * Implementation of problem types.
  */
 public class ProblemFunctionType extends ProblemType implements ICPPFunctionType {
+	@SuppressWarnings("hiding")
+	public static final IType RECURSION_IN_LOOKUP = new ProblemFunctionType(BINDING_RECURSION_IN_LOOKUP);
 
 	public ProblemFunctionType(int id) {
 		super(id);

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java

@@ -12,7 +12,6 @@
 package org.eclipse.cdt.internal.core.dom.parser;
 
 import org.eclipse.cdt.core.dom.ast.IProblemType;
-import org.eclipse.cdt.core.dom.ast.ISemanticProblem;
 import org.eclipse.cdt.core.dom.ast.IType;
 import org.eclipse.cdt.internal.core.parser.ParserMessages;
 import org.eclipse.core.runtime.CoreException;
@@ -22,8 +21,9 @@ import org.eclipse.core.runtime.CoreException;
  */
 public class ProblemType implements IProblemType, ISerializableType {
 	public static final IType UNRESOLVED_NAME = new ProblemType(TYPE_UNRESOLVED_NAME);
-	public static final IType UNKNOWN_FOR_EXPRESSION = new ProblemType(ISemanticProblem.TYPE_UNKNOWN_FOR_EXPRESSION);
+	public static final IType UNKNOWN_FOR_EXPRESSION = new ProblemType(TYPE_UNKNOWN_FOR_EXPRESSION);
-	public static final IType ENUMERATION_EXPECTED = new ProblemType(ISemanticProblem.TYPE_ENUMERATION_EXPECTED);
+	public static final IType ENUMERATION_EXPECTED = new ProblemType(TYPE_ENUMERATION_EXPECTED);
+	public static final IType RECURSION_IN_LOOKUP = new ProblemType(BINDING_RECURSION_IN_LOOKUP);
 	
 	private final int fID;
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPEvaluation.java

@@ -58,9 +58,9 @@ public abstract class CPPEvaluation implements ICPPEvaluation {
 	}
 
 	protected static ICPPTemplateArgument[] instantiateArguments(ICPPTemplateArgument[] args,
-			InstantiationContext context) {
+			InstantiationContext context, boolean strict) {
 		try {
-			return CPPTemplates.instantiateArguments(args, context, false);
+			return CPPTemplates.instantiateArguments(args, context, strict);
 		} catch (DOMException e) {
 			CCorePlugin.log(e);
 		}

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java

@@ -22,7 +22,9 @@ import static org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.SemanticUti
 import static org.eclipse.cdt.internal.core.dom.parser.cpp.semantics.SemanticUtil.getNestedType;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.dom.ast.ASTTypeUtil;
@@ -121,6 +123,7 @@ import org.eclipse.cdt.internal.core.dom.parser.ASTQueries;
 import org.eclipse.cdt.internal.core.dom.parser.IASTInternalScope;
 import org.eclipse.cdt.internal.core.dom.parser.ITypeContainer;
 import org.eclipse.cdt.internal.core.dom.parser.ProblemBinding;
+import org.eclipse.cdt.internal.core.dom.parser.ProblemFunctionType;
 import org.eclipse.cdt.internal.core.dom.parser.ProblemType;
 import org.eclipse.cdt.internal.core.dom.parser.Value;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTName;
@@ -219,6 +222,13 @@ public class CPPTemplates {
 			return 0;
 		}
 	};
+	private static final ThreadLocal<Set<TypeInstantiationRequest>> instantiationsInProgress =
+			new ThreadLocal<Set<TypeInstantiationRequest>>() {
+		@Override
+		protected Set<TypeInstantiationRequest> initialValue() {
+			return new HashSet<>();
+		}
+	};
 
 	/**
 	 * Instantiates a class template with the given arguments. May return {@code null}.
@@ -1426,19 +1436,24 @@ public class CPPTemplates {
 	 * Instantiates the given type with the provided map and packОffset.
 	 * The context is used to replace templates with their specialization, where appropriate.
 	 */
-	public static IType instantiateType(IType type, InstantiationContext context) {
+	public static IType instantiateType(final IType type, InstantiationContext context) {
-		try {
 		if (context.getParameterMap() == null)
 			return type;
 
+		TypeInstantiationRequest instantiationRequest = new TypeInstantiationRequest(type, context);
+		if (!instantiationsInProgress.get().add(instantiationRequest)) {
+			System.out.println("Recursion in instantiation of type \"" + type + "\"");  //$NON-NLS-1$//$NON-NLS-2$ //XXX
+			return type instanceof ICPPFunctionType ?
+					ProblemFunctionType.RECURSION_IN_LOOKUP : ProblemType.RECURSION_IN_LOOKUP;
+		}
+
+		try {
 			if (type instanceof ICPPFunctionType) {
 				final ICPPFunctionType ft = (ICPPFunctionType) type;
-				IType ret = null;
-				IType[] params = null;
-				final IType r = ft.getReturnType();
-				ret = instantiateType(r, context);
 				IType[] ps = ft.getParameterTypes();
-				params = instantiateTypes(ps, context);
+				IType[] params = instantiateTypes(ps, context);
+				final IType r = ft.getReturnType();
+				IType ret = instantiateType(r, context);
 				if (ret == r && params == ps) {
 					return type;
 				}
@@ -1574,6 +1589,8 @@ public class CPPTemplates {
 			return type;
 		} catch (DOMException e) {
 			return e.getProblem();
+		} finally {
+			instantiationsInProgress.get().remove(instantiationRequest);
 		}
 	}
 

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/EvalFunctionSet.java

@@ -240,7 +240,7 @@ public class EvalFunctionSet extends CPPDependentEvaluation {
 		ICPPTemplateArgument[] originalArguments = fFunctionSet.getTemplateArguments();
 		ICPPTemplateArgument[] arguments = originalArguments;
 		if (originalArguments != null)
-			arguments = instantiateArguments(originalArguments, context);
+			arguments = instantiateArguments(originalArguments, context, true);
 
 		IBinding originalOwner = fFunctionSet.getOwner();
 		IBinding owner = originalOwner;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/EvalID.java

@@ -343,7 +343,7 @@ public class EvalID extends CPPDependentEvaluation {
 	public ICPPEvaluation instantiate(InstantiationContext context, int maxDepth) {
 		ICPPTemplateArgument[] templateArgs = fTemplateArgs;
 		if (templateArgs != null) {
-			templateArgs = instantiateArguments(templateArgs, context);
+			templateArgs = instantiateArguments(templateArgs, context, false);
 		}
 
 		ICPPEvaluation fieldOwner = fFieldOwner;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/TypeInstantiationRequest.java

@@ -0,0 +1,124 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Google, Inc and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * 	   Sergey Prigogin (Google) - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.cdt.internal.core.dom.parser.cpp.semantics;
+
+import java.util.Arrays;
+
+import org.eclipse.cdt.core.CCorePlugin;
+import org.eclipse.cdt.core.dom.ast.IASTNode;
+import org.eclipse.cdt.core.dom.ast.IType;
+import org.eclipse.cdt.core.dom.ast.cpp.ICPPTemplateArgument;
+import org.eclipse.cdt.core.dom.ast.cpp.ICPPTemplateParameterMap;
+import org.eclipse.cdt.core.dom.ast.cpp.ICPPTypeSpecialization;
+import org.eclipse.cdt.core.parser.util.CharArrayUtils;
+import org.eclipse.cdt.internal.core.dom.parser.cpp.InstantiationContext;
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * Used to track ongoing instantiations as a safeguard against infinite recursion.
+ */
+public class TypeInstantiationRequest {
+	private final IType type;
+	private final ICPPTemplateParameterMap parameterMap;
+	private final int packOffset;
+	private final ICPPTypeSpecialization contextTypeSpecialization;
+	private final IASTNode point;
+	private int hashCode;
+
+	public TypeInstantiationRequest(IType type, InstantiationContext context) {
+		this.type = type;
+		this.parameterMap = context.getParameterMap();
+		this.packOffset = context.getPackOffset();
+		this.contextTypeSpecialization = context.getContextTypeSpecialization();
+		this.point = context.getPoint();
+	}
+
+	@Override
+	public int hashCode() {
+		if (hashCode == 0) {
+			SignatureBuilder builder = new SignatureBuilder();
+			try {
+				builder.marshalType(type);
+				char[] signature = builder.getSignature();
+				hashCode = CharArrayUtils.hash(signature);
+			} catch (CoreException e) {
+				CCorePlugin.log(e);
+				hashCode = Integer.MIN_VALUE;
+			}
+		}
+		return hashCode;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (!obj.getClass().equals(getClass()))
+			return false;
+		TypeInstantiationRequest other = (TypeInstantiationRequest) obj;
+		if (!type.isSameType(other.type))
+			return false;
+		if (!equals(contextTypeSpecialization, other.contextTypeSpecialization))
+			return false;
+		if (!equals(parameterMap, other.parameterMap))
+			return false;
+		if (packOffset != other.packOffset)
+			return false;
+		if (point != other.point)
+			return false;
+		return true;
+	}
+
+	private boolean equals(IType type1, IType type2) {
+		if (type1 == type2)
+			return true;
+		if (type1 == null || type2 == null)
+			return false;
+		return type1.isSameType(type1);
+	}
+
+	private boolean equals(ICPPTemplateParameterMap map1, ICPPTemplateParameterMap map2) {
+		if (map1 == map2)
+			return true;
+		if (map1 == null || map2 == null)
+			return false;
+		Integer[] p1 = map1.getAllParameterPositions();
+		Integer[] p2 = map2.getAllParameterPositions();
+		if (!Arrays.equals(p1, p2))
+			return false;
+		for (Integer paramId : p1) {
+			ICPPTemplateArgument[] packExpansion1 = map1.getPackExpansion(paramId);
+			ICPPTemplateArgument[] packExpansion2 = map2.getPackExpansion(paramId);
+			if (packExpansion1 != null && packExpansion2 != null) {
+				if (packExpansion1.length != packExpansion2.length)
+					return false;
+				for (int i = 0; i < packExpansion1.length; i++) {
+					if (!equals(packExpansion1[i], packExpansion2[i]))
+						return false;
+				}
+			} else if (packExpansion1 == null && packExpansion2 == null) {
+				ICPPTemplateArgument arg1 = map1.getArgument(paramId);
+				ICPPTemplateArgument arg2 = map2.getArgument(paramId);
+				if (!equals(arg1, arg2))
+					return false;
+			} else {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	private boolean equals(ICPPTemplateArgument arg1, ICPPTemplateArgument arg2) {
+		if (arg1 == arg2)
+			return true;
+		if (arg1 == null || arg2 == null)
+			return false;
+		return arg1.isSameValue(arg2);
+	}
+}