Bug 314428: speedup build console - better handle duplicate markers

This changes the algorithm used to handle duplicate markers when build
is running.

Change-Id: I8d8b61edd80ae4da4c0e0eea3806b0efecb570e0

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/core/ExternalBuildRunner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2010, 2016 Wind River Systems and others.
+ * Copyright (c) 2010, 2017 Wind River Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -125,7 +125,14 @@ public class ExternalBuildRunner extends AbstractBuildRunner {
 				buildRunnerHelper.removeOldMarkers(project, new SubProgressMonitor(monitor, TICKS_DELETE_MARKERS, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK));
 
 				buildRunnerHelper.greeting(kind, cfgName, toolchainName, isSupported);
-				int state = buildRunnerHelper.build(new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK));
+				
+				int state;
+				epm.deferDeDuplication();
+				try {
+					state = buildRunnerHelper.build(new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK));
+				} finally {
+					epm.deDuplicate();
+				}
 				buildRunnerHelper.close();
 				buildRunnerHelper.goodbye();
 

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/core/InternalBuildRunner.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2010, 2013 Wind River Systems and others.
+ * Copyright (c) 2010, 2017 Wind River Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -124,21 +124,26 @@ public class InternalBuildRunner extends AbstractBuildRunner {
 			OutputStream stderr = buildRunnerHelper.getErrorStream();
 
 			int status;
-			if (dBuilder != null) {
+			epm.deferDeDuplication();
-				status = dBuilder.build(stdout, stderr, new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK));
+			try {
-			} else {
+				
-				status = ParallelBuilder.build(des, null, null, stdout, stderr, new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK), resumeOnErr, buildIncrementaly);
+				if (dBuilder != null) {
-				// Bug 403670:
+					status = dBuilder.build(stdout, stderr, new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK));
-				// Make sure the build configuration's rebuild status is updated with the result of
+				} else {
-				// this successful build.  In the non-parallel case this happens within dBuilder.build
+					status = ParallelBuilder.build(des, null, null, stdout, stderr, new SubProgressMonitor(monitor, TICKS_EXECUTE_COMMAND, SubProgressMonitor.PREPEND_MAIN_LABEL_TO_SUBTASK), resumeOnErr, buildIncrementaly);
-				// (the cBS is passed as an instance of IResourceRebuildStateContainer).
+					// Bug 403670:
-				if (status == ParallelBuilder.STATUS_OK)
+					// Make sure the build configuration's rebuild status is updated with the result of
-					cBS.setState(0);
+					// this successful build.  In the non-parallel case this happens within dBuilder.build
-				buildRunnerHelper.printLine(ManagedMakeMessages.getFormattedString("CommonBuilder.7", Integer.toString(ParallelBuilder.lastThreadsUsed))); //$NON-NLS-1$
+					// (the cBS is passed as an instance of IResourceRebuildStateContainer).
+					if (status == ParallelBuilder.STATUS_OK)
+						cBS.setState(0);
+					buildRunnerHelper.printLine(ManagedMakeMessages.getFormattedString("CommonBuilder.7", Integer.toString(ParallelBuilder.lastThreadsUsed))); //$NON-NLS-1$
+				}
+			} finally {
+				epm.deDuplicate();
 			}
-
+	
 			bsMngr.setProjectBuildState(project, pBS);
-
 			buildRunnerHelper.close();
 			buildRunnerHelper.goodbye();
 

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/ErrorParserManager.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- *  Copyright (c) 2005, 2016 IBM Corporation and others.
+ *  Copyright (c) 2005, 2017 IBM Corporation and others.
  *  All rights reserved. This program and the accompanying materials
  *  are made available under the terms of the Eclipse Public License v1.0
  *  which accompanies this distribution, and is available at
@@ -101,6 +101,8 @@ public class ErrorParserManager extends OutputStream implements IConsoleParser,
 	private URI cachedWorkingDirectory = null;
 	private IFile cachedFile = null;
 
+	private boolean deferDeDuplication = false;
+
 	private static boolean isCygwin = true;
 
 	/**
@@ -599,6 +601,7 @@ outer:
 		if ( ! ProblemMarkerFilterManager.getInstance().acceptMarker(problemMarkerInfo) )
 			return;
 		fErrors.add(problemMarkerInfo);
+		problemMarkerInfo.setDeferDeDuplication(deferDeDuplication);
 		fMarkerGenerator.addMarker(problemMarkerInfo);
 		if (problemMarkerInfo.severity == IMarkerGenerator.SEVERITY_ERROR_RESOURCE) {
 			hasErrors = true;
@@ -897,4 +900,29 @@ outer:
 			}
 		}
 	}
+
+	/**
+	 * Flag the marker generator to defer the de-duplication of error markers
+	 * until {@link #deDuplicate()} is called
+	 * 
+	 * @since 6.3
+	 */
+	public void deferDeDuplication() {
+		if (fMarkerGenerator instanceof ACBuilder) {
+			deferDeDuplication = true;
+		}
+	}
+
+	/**
+	 * De-duplicate error markers on resource that have had error markers added
+	 * since {@link #deferDeDuplication()} was called.
+	 * 
+	 * @since 6.3
+	 */
+	public void deDuplicate() {
+		if (deferDeDuplication) {
+			deferDeDuplication = false;
+			((ACBuilder) fMarkerGenerator).deDuplicate();
+		}
+	}
 }

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/ProblemMarkerInfo.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2015 Siemens AG and others.
+ * Copyright (c) 2006, 2017 Siemens AG and others.
  * All rights reserved. This content and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -50,6 +50,12 @@ public  class ProblemMarkerInfo {
 		private Map<String, String> attributes;
 		private String type;
 
+		/**
+		 * Flag marker for potential deferred de-duplication. See
+		 * {@link ACBuilder#deDuplicate()}
+		 */
+		private boolean deferDeDuplication;
+
 		/**
 		 * Create a new {@link ProblemMarkerInfo} object.
 		 * 
@@ -161,4 +167,26 @@ public  class ProblemMarkerInfo {
 		public void setType(String type){
 			this.type = type;
 		}
-}
+
+		/**
+		 * Flag marker for potential deferred de-duplication. See
+		 * {@link ACBuilder#deDuplicate()}
+		 * 
+		 * @return the deferDeDuplication
+		 * @since 6.3
+		 */
+		public boolean isDeferDeDuplication() {
+			return deferDeDuplication;
+		}
+
+		/**
+		 * Flag marker for potential deferred de-duplication. See
+		 * {@link ACBuilder#deDuplicate()}
+		 * 
+		 * @param deferDeDuplication the deferDeDuplication to set
+		 * @since 6.3
+		 */
+		public void setDeferDeDuplication(boolean deferDeDuplication) {
+			this.deferDeDuplication = deferDeDuplication;
+		}
+}

core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/resources/ACBuilder.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2016 QNX Software Systems and others.
+ * Copyright (c) 2000, 2017 QNX Software Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -9,13 +9,19 @@
  *     QNX Software Systems - Initial API and implementation
  *     IBM Corporation
  *     James Blackburn (Broadcom Corp.)
+ *     Jonah Graham (Kichwa Coders) - Bug 314428: New implementation for removing duplicate error markers
  *******************************************************************************/
 package org.eclipse.cdt.core.resources;
 
 
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.eclipse.cdt.core.CCorePlugin;
 import org.eclipse.cdt.core.CCorePreferenceConstants;
@@ -44,6 +50,7 @@ public abstract class ACBuilder extends IncrementalProjectBuilder implements IMa
 	protected static final boolean DEBUG_EVENTS = false;
 
 	private IProject currentProject;
+	private Set<IResource> resourcesToDeduplicate = new HashSet<>();
 
 	/**
 	 * Constructor for ACBuilder
@@ -54,7 +61,7 @@ public abstract class ACBuilder extends IncrementalProjectBuilder implements IMa
 
 	/**
 	 * Set the current project that this builder is running.
-	 * 
+	 *
 	 * @since 5.11
 	 */
 	protected void setCurrentProject(IProject project) {
@@ -63,7 +70,7 @@ public abstract class ACBuilder extends IncrementalProjectBuilder implements IMa
 
 	/**
 	 * Returns the current project that this builder is running.
-	 * 
+	 *
 	 * @return the project
 	 * @since 5.11
 	 */
@@ -80,6 +87,79 @@ public abstract class ACBuilder extends IncrementalProjectBuilder implements IMa
 		addMarker(problemMarkerInfo);
 	}
 
+	private static class MarkerWithInfo {
+		private static final String[] ATTRIBUTE_NAMES = new String[] { IMarker.LINE_NUMBER, IMarker.SEVERITY,
+				IMarker.MESSAGE, ICModelMarker.C_MODEL_MARKER_EXTERNAL_LOCATION, IMarker.SOURCE_ID };
+		private Object[] attributes;
+
+		MarkerWithInfo(IMarker marker) throws CoreException {
+			attributes = marker.getAttributes(ATTRIBUTE_NAMES);
+		}
+
+		@Override
+		public int hashCode() {
+			return Arrays.hashCode(attributes);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			MarkerWithInfo otherInfo = (MarkerWithInfo)obj;
+			return Arrays.equals(attributes, otherInfo.attributes);
+		}
+	}
+
+	/**
+	 * Remove duplicate error markers that may have been created by
+	 * {@link ACBuilder#addMarker(ProblemMarkerInfo)} with the
+	 * {@link ProblemMarkerInfo#isDeferDeDuplication()} flag set.
+	 * 
+	 * This method will also remove other duplicate
+	 * ICModelMarker.C_MODEL_PROBLEM_MARKER markers on the resources referred to
+	 * by ProblemMarkerInfo.
+	 * 
+	 * @since 6.3
+	 */
+	public void deDuplicate() {
+		/*
+		 * In practice it is actually faster to create all the markers and then
+		 * remove duplicates than try to search on each marker creation if the
+		 * marker already exists. This code is faster because it makes one pass
+		 * through the markers, instead of one pass for each new marker. As
+		 * getting attributes for makers is very expensive, only having to fetch
+		 * marker attributes once per marker speeds things up considerably.
+		 */
+		for (IResource resource : resourcesToDeduplicate) {
+			try {
+				IMarker[] markers = resource.findMarkers(ICModelMarker.C_MODEL_PROBLEM_MARKER, true,
+						IResource.DEPTH_ZERO);
+				List<IMarker> dups = new ArrayList<>(markers.length);
+				Set<MarkerWithInfo> unique = new HashSet<>(markers.length);
+				for (IMarker marker : markers) {
+					MarkerWithInfo info = new MarkerWithInfo(marker);
+					if (!unique.add(info)) {
+						dups.add(marker);
+					}
+				}
+				for (IMarker marker : dups) {
+					marker.delete();
+				}
+			} catch (CoreException e) {
+				CCorePlugin.log(e.getStatus());
+			}
+		}
+
+		resourcesToDeduplicate = new HashSet<>();
+	}
+
 	/**
 	 * Callback from Output Parser
 	 */
@@ -96,24 +176,28 @@ public abstract class ACBuilder extends IncrementalProjectBuilder implements IMa
 				externalLocation = problemMarkerInfo.externalPath.toOSString();
 			}
 
-			// Try to find matching markers and don't put in duplicates
+			if (!problemMarkerInfo.isDeferDeDuplication()) {
-			IMarker[] markers = markerResource.findMarkers(ICModelMarker.C_MODEL_PROBLEM_MARKER, true, IResource.DEPTH_ONE);
+				// Try to find matching markers and don't put in duplicates
-			for (IMarker m : markers) {
+				IMarker[] markers = markerResource.findMarkers(ICModelMarker.C_MODEL_PROBLEM_MARKER, true, IResource.DEPTH_ONE);
-				int line = m.getAttribute(IMarker.LINE_NUMBER, -1);
+				for (IMarker m : markers) {
-				int sev = m.getAttribute(IMarker.SEVERITY, -1);
+					int line = m.getAttribute(IMarker.LINE_NUMBER, -1);
-				String msg = (String) m.getAttribute(IMarker.MESSAGE);
+					int sev = m.getAttribute(IMarker.SEVERITY, -1);
-				if (line == problemMarkerInfo.lineNumber && sev == mapMarkerSeverity(problemMarkerInfo.severity) && msg.equals(problemMarkerInfo.description)) {
+					String msg = (String) m.getAttribute(IMarker.MESSAGE);
-					String extloc = (String) m.getAttribute(ICModelMarker.C_MODEL_MARKER_EXTERNAL_LOCATION);
+					if (line == problemMarkerInfo.lineNumber && sev == mapMarkerSeverity(problemMarkerInfo.severity) && msg.equals(problemMarkerInfo.description)) {
-					if (extloc == externalLocation || (extloc != null && extloc.equals(externalLocation))) {
+						String extloc = (String) m.getAttribute(ICModelMarker.C_MODEL_MARKER_EXTERNAL_LOCATION);
-						if (project == null || project.equals(markerResource.getProject())) {
+						if (extloc == externalLocation || (extloc != null && extloc.equals(externalLocation))) {
-							return;
+							if (project == null || project.equals(markerResource.getProject())) {
-						}
+								return;
-						String source = (String) m.getAttribute(IMarker.SOURCE_ID);
+							}
-						if (project.getName().equals(source)) {
+							String source = (String) m.getAttribute(IMarker.SOURCE_ID);
-							return;
+							if (project.getName().equals(source)) {
+								return;
+							}
 						}
 					}
 				}
+			} else {
+				resourcesToDeduplicate.add(markerResource);
 			}
 
 			String type = problemMarkerInfo.getType();