[202822] various synchronization issues with dstore spirit

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java

@@ -1657,7 +1657,7 @@ public final class DataElement implements IDataElement
 			_isUpdated = false;
 			_isExpanded = true;
 			_buffer = null;
-			//_nestedData = null;
+			//_nestedData = null; // commented out this null setting so that the children of this element can be properly deleted
 		}
 	}
 	

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java

@@ -14,6 +14,7 @@
  * Contributors:
  * Michael Berger (IBM) - 146326 fixed erroneously disconnected dstore elements.
  * Michael Berger (IBM) - 145799 added refresh() method with depth parameter.
+ * David McKnight (IBM) - 202822 findDeleted should not be synchronized
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -1657,7 +1658,11 @@ public final class DataStore
 	 */
 	public void disconnectObjects(DataElement from)
 	{
-		if (!isDoSpirit()) return;
+		if (!isDoSpirit())
+			{
+			System.out.println("not doing spirit");
+			return;
+			}
 		if (from != null)
 		{
 			for (int i = from.getNestedSize() - 1; i >= 0; i--)
@@ -2629,7 +2634,7 @@ public final class DataStore
 	 * @param root where to search from 
 	 * @return a list of elements
 	 */
-	public synchronized List findDeleted(DataElement root)
+	public List findDeleted(DataElement root)
 	{
 		return findDeleted(root, 10);
 	}
@@ -2641,10 +2646,12 @@ public final class DataStore
 	 * @param type the descriptor representing the type of the objects to search for 
 	 * @return a list of elements
 	 */
-	public synchronized List findDeleted(DataElement root, int depth)
+	public List findDeleted(DataElement root, int depth)
 	{
 		ArrayList results = new ArrayList();
-		synchronized (root)
+//		synchronized (root)
+		// synchronized can cause hang here..but may not be necessary anyway since
+		// we're not adding or removing anything here
 		{
 			if (root != null && root.getDataStore() == this)
 			{
@@ -3756,8 +3763,7 @@ public final class DataStore
 			for (int i = 0; i < toDelete.getNestedSize(); i++)
 			{
 				DataElement subDelete = toDelete.get(i);
-				if (subDelete != null && subDelete.getDataStore() == this/* && !subDelete.isDeleted()*/) // on server, spirited are considered deleted
+				if (subDelete != null && subDelete.getDataStore() == this && !subDelete.isDeleted()) 				{
-				{
 					deleteObjectHelper(toDelete, subDelete, depth);
 				}
 			}
@@ -4204,6 +4210,35 @@ public final class DataStore
 	{
 		referenceTag = tag;
 	}
-	
+	/*
+	public int printTree(String indent, DataElement root)
+	{
+		return printTree(indent, 0, root);
+	}
 
+	public int printTree(String indent, int number, DataElement root)
+	{
+		int total = number;
+		if (root != null)
+		{
+			total++;
+			boolean isSpirit = root.isSpirit();
+			boolean isDeleted = root.isDeleted();
+			String prefix = "DataElement";
+			if (isSpirit)
+				prefix += "<spirit>";
+			if (isDeleted)
+				prefix += "<deleted>";
+			
+			String msg = indent + prefix + "["+ total + "]("+root.getType()+", "+root.getName()+")";
+			System.out.println(msg);
+			for (int i = 0; i < root.getNestedSize(); i++)
+			{
+				DataElement currentElement = (DataElement) root.get(i);				
+				total = printTree(indent + " ", total, currentElement);
+			}
+		}
+		return total;
+	}
+*/
 }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java

@@ -13,6 +13,7 @@
  * 
  * Contributors:
  * {Name} (company) - description of contribution.
+ * David McKnight   (IBM)   [202822] should not be synchronizing on clean method
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -67,7 +68,7 @@ public abstract class UpdateHandler extends Handler
 		clean(object, 2);
 	}
 
-	protected synchronized void clean(DataElement object, int depth)
+	protected void clean(DataElement object, int depth)
 	{
 		if ((depth > 0) && (object != null) && object.getNestedSize() > 0)
 		{
@@ -78,7 +79,6 @@ public abstract class UpdateHandler extends Handler
 				DataElement child = (DataElement) deletedList.get(i);
 				if (child != null && child.isDeleted())
 				{
-					clean(child, depth - 1);
 					DataElement parent = child.getParent();
 					DataElementRemover.addToRemovedCount();
 					
@@ -91,15 +91,19 @@ public abstract class UpdateHandler extends Handler
 					
 					if (parent != null)
 					{
-						parent.removeNestedData(child);
+						synchronized (parent)
+						{
+							parent.removeNestedData(child);
+						}
 					}
-				  _dataStore.addToRecycled(child);
+				//  _dataStore.addToRecycled(child);
 				}
 			}
 
 			deletedList.clear();
 		}
-		
+		// delete objects under temproot
+		_dataStore.getTempRoot().removeNestedData();
 		
 	}
 

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java

@@ -13,12 +13,13 @@
  * 
  * Contributors:
  * {Name} (company) - description of contribution.
+ *  David McKnight  (IBM)  - [202822] don't need to remove children from map here
  *******************************************************************************/
 
 package org.eclipse.dstore.internal.core.util;
 
+import java.util.ArrayList;
 import java.util.LinkedList;
-import java.util.List;
 
 import org.eclipse.dstore.core.model.DataElement;
 import org.eclipse.dstore.core.model.DataStore;
@@ -39,8 +40,8 @@ public class DataElementRemover extends Handler
 	// that are older than _expiryTime milliseconds are removed.
 	public static final int DEFAULT_EXPIRY_TIME = 600; // in seconds
 	public static final int DEFAULT_INTERVAL_TIME = 60; // in seconds
-	private int _intervalTime = DEFAULT_INTERVAL_TIME * 100;
+	private int _intervalTime = DEFAULT_INTERVAL_TIME * 10;
-	private int _expiryTime = DEFAULT_EXPIRY_TIME * 100;
+	private int _expiryTime = DEFAULT_EXPIRY_TIME * 10;
 	public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME"; //$NON-NLS-1$
 	public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$
 	
@@ -96,9 +97,11 @@ public class DataElementRemover extends Handler
 	}
 
 	public synchronized void addToQueueForRemoval(DataElement element)
-	{
+	{		
+		System.out.println("dis:"+element.getName());
 		synchronized (_queue) 
 		{
+			
 			if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore())
 			{
 				QueueItem item = new QueueItem(element, System.currentTimeMillis());
@@ -116,6 +119,8 @@ public class DataElementRemover extends Handler
 	{
 		synchronized (_queue)
 		{
+			
+			System.out.println("spiriting");
 			_dataStore.memLog("           "); //$NON-NLS-1$
 			int disconnected = 0;
 			if (!_dataStore.isDoSpirit())
@@ -136,13 +141,20 @@ public class DataElementRemover extends Handler
 			
 			_dataStore.memLog("Size of queue: " + _queue.size()); //$NON-NLS-1$
 			
+			ArrayList toRefresh = new ArrayList();
 			while (_queue.size() > 0 && System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime)
 			{
 				DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement;
 				if (!toBeDisconnected.isSpirit()) 
 				{
 					toBeDisconnected.setSpirit(true);
-					_dataStore.refresh(toBeDisconnected);
+					toBeDisconnected.setUpdated(false);
+					DataElement parent = toBeDisconnected.getParent();
+					if (!toRefresh.contains(parent))
+					{
+						toRefresh.add(toBeDisconnected.getParent());
+					}
+						//_dataStore.refresh(toBeDisconnected);
 					disconnected++;
 					numDisconnected++;
 				}
@@ -152,6 +164,9 @@ public class DataElementRemover extends Handler
 				}
 				unmap(toBeDisconnected);
 			}
+	
+			_dataStore.refresh(toRefresh);
+			
 			_dataStore.memLog("Disconnected " + disconnected + " DataElements."); //$NON-NLS-1$ //$NON-NLS-2$
 			_dataStore.memLog("Elements created so far: " + numCreated); //$NON-NLS-1$
 			_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
@@ -162,15 +177,6 @@ public class DataElementRemover extends Handler
 	
 	private void unmap(DataElement element)
 	{	 
-		List children = element.getNestedData();
-		if (children != null)
-		{
-			for (int i = 0; i < children.size(); i++)
-			{
-				DataElement child = (DataElement)children.get(i);
-				unmap(child);
-			}
-		}
 		_dataStore.getHashMap().remove(element.getId());
 	}
 	

rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java

@@ -15,6 +15,7 @@
  * Martin Oberhuber (Wind River) - [175262] IHost.getSystemType() should return IRSESystemType 
  * Martin Oberhuber (Wind River) - [186640] Add IRSESystemType.testProperty()
  * Martin Oberhuber (Wind River) - [186128][refactoring] Move IProgressMonitor last in public base classes 
+ * David McKnight   (IBM)        - [202822] need to enable spiriting on the server side
  ********************************************************************************/
 
 package org.eclipse.rse.connectorservice.dstore;
@@ -85,6 +86,20 @@ import org.osgi.framework.Version;
  */
 public class DStoreConnectorService extends StandardConnectorService implements IDataStoreProvider
 {
+	private class StartSpiritThread extends Thread
+	{
+		private DataStore _dataStore;
+		public StartSpiritThread(DataStore dataStore)
+		{
+			_dataStore = dataStore;
+		}
+		
+		public void run()
+		{
+			if (_dataStore.isDoSpirit()) _dataStore.queryServerSpiritState();
+		}
+	}
+	
 	private ClientConnection clientConnection = null;
 	private ConnectionStatusListener _connectionStatusListener = null;
 	private IServerLauncher starter = null;
@@ -834,7 +849,9 @@ public class DStoreConnectorService extends StandardConnectorService implements
 				dataStore.showTicket(null);
 			}
 			
-	        if (dataStore.isDoSpirit()) dataStore.queryServerSpiritState();
+	      //  if (dataStore.isDoSpirit()) dataStore.queryServerSpiritState();
+			StartSpiritThread thread = new StartSpiritThread(dataStore);
+			thread.start();
 
 			// Fire comm event to signal state changed
 			fireCommunicationsEvent(CommunicationsEvent.AFTER_CONNECT);
@@ -869,6 +886,7 @@ public class DStoreConnectorService extends StandardConnectorService implements
 				boolean cacheRemoteClasses = store.getBoolean(IUniversalDStoreConstants.RESID_PREF_CACHE_REMOTE_CLASSES);
 				
 				dataStore.setPreference(RemoteClassLoader.CACHING_PREFERENCE, cacheRemoteClasses ? "true" : "false"); //$NON-NLS-1$  //$NON-NLS-2$
+				
 			}
 			else
 			{						

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/CommandMiner.java

@@ -13,6 +13,7 @@
  * 
  * Contributors:
  * David McKnight  (IBM)  - [191599] use specified encoding for shell
+ * David McKnight  (IBM)  - [202822] canceled output should be created before thread cleanup
  *******************************************************************************/
 
 package org.eclipse.rse.dstore.universal.miners;
@@ -265,7 +266,9 @@ public class CommandMiner extends Miner
 	{
 		CommandMinerThread theThread = (CommandMinerThread) _threads.get(status.getAttribute(DE.A_ID));
 		if (theThread != null)
-		{		    
+		{	
+			_dataStore.createObject(status, "stdout", "Command Cancelled by User Request"); //$NON-NLS-1$ //$NON-NLS-2$
+
 			theThread.stopThread();
 			theThread.sendExit();
 			
@@ -274,14 +277,12 @@ public class CommandMiner extends Miner
 			while (!done)
 				if ((!theThread.isAlive()) || (stopIn < System.currentTimeMillis()))
 					done = true;
-			_dataStore.createObject(status, "stdout", "Command Cancelled by User Request"); //$NON-NLS-1$ //$NON-NLS-2$
-			_dataStore.refresh(status);
 		}
 	}
 
 	public String getVersion()
 	{
-		return "6.4.0"; //$NON-NLS-1$
+		return "8.0.0"; //$NON-NLS-1$
 	}
 
 }

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/internal/dstore/universal/miners/command/CommandMinerThread.java

@@ -13,6 +13,7 @@
  * 
  * Contributors:
  * {Name} (company) - description of contribution.
+ *  David McKnight  (IBM)  - [202822] updating cleanup
  *******************************************************************************/
 
 package org.eclipse.rse.internal.dstore.universal.miners.command;
@@ -833,41 +834,23 @@ public class CommandMinerThread extends MinerThread
 	    }
 	}
 	
+
+	
 	public void cleanupThread()
 	{
-		// disconnecting all
-		_dataStore.disconnectObjects(_status);
 		
-
+		_isDone = true;
-		// clean up the associated environment
+		try
-		List projectEnvReference = _subject.getAssociated("inhabits"); //$NON-NLS-1$
-
-		if (projectEnvReference != null)
 		{
-			DataElement env = (DataElement)projectEnvReference.get(0);
+
-			DataElement envParent = env.getParent();			
-			_dataStore.deleteObject(envParent, env);
-			_dataStore.refresh(envParent);
-		}
-		_dataStore.disconnectObject(_subject); //bug 70420
 		
-		
+
-		refreshStatus();
 	    /*
 		if (_isShell)
 		{
 			sendInput("#exit");
 		}*/
-		
+
-		_isDone = true;
-		try
-		{
-			_status.setAttribute(DE.A_NAME, "done"); //$NON-NLS-1$
-			_dataStore.refresh(_status, true);
-			_stdOutputHandler.finish();
-			_stdErrorHandler.finish();
-			_stdInput.close();
-			_stdError.close();
 			if (_theProcess != null)
 			{
 				int exitcode;
@@ -880,7 +863,7 @@ public class CommandMinerThread extends MinerThread
 					else
 					{
 						exitcode = _theProcess.exitValue();
-						createObject("prompt", "> Shell Completed (exit code = " + exitcode + ")"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+						createObject("prompt", "> Shell Completed (exit code = " + exitcode + ")");
 					}
 				}
 				catch (IllegalThreadStateException e)
@@ -890,13 +873,41 @@ public class CommandMinerThread extends MinerThread
 				}
 				_theProcess = null;
 			}
+			
+
+			_stdOutputHandler.finish();
+			_stdErrorHandler.finish();
+			_stdInput.close();
+			_stdError.close();
+			
+			_status.setAttribute(DE.A_NAME, "done");
+			_dataStore.refresh(_status);
+	
+			System.out.println("cleaupThread");
+		
+		// disconnecting all
+	
+		_dataStore.disconnectObjects(_status);
+		
+
+		// clean up the associated environment
+		List projectEnvReference = _subject.getAssociated("inhabits");
+
+		if (projectEnvReference != null)
+		{
+			DataElement env = (DataElement)projectEnvReference.get(0);
+			DataElement envParent = env.getParent();			
+			_dataStore.deleteObject(envParent, env);
+			_dataStore.refresh(envParent);
+		}
+		_dataStore.disconnectObject(_subject); //bug 70420
 		}
 		catch (IOException e)
 		{
 			e.printStackTrace();
-		}
+		}				
 	}
-
+	
 	public void interpretLine(String line, boolean stdError)
 	{
 		// for prompting

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java

@@ -200,11 +200,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 			String hostEncoding, IProgressMonitor monitor)
 	{
 		BufferedInputStream bufInputStream = null;
-
-	
-		boolean isCancelled = false;
-
 	
+		boolean isCancelled = false;	
 	
 		try
 		{	

rse/plugins/org.eclipse.rse.subsystems.shells.core/src/org/eclipse/rse/subsystems/shells/core/model/RemoteCommandShell.java

@@ -127,7 +127,12 @@ public abstract class RemoteCommandShell implements IAdaptable, IRemoteCommandSh
 
 	public Object[] listOutput()
 	{
-		return _output.toArray();
+		Object[] array = new Object[_output.size()];
+		synchronized (_output)
+		{
+		  _output.toArray(array);
+		}
+		return array;
 	}
 
 	public int getIndexOf(Object output)

rse/plugins/org.eclipse.rse.subsystems.shells.dstore/src/org/eclipse/rse/internal/subsystems/shells/dstore/DStoreServiceCommandShell.java

@@ -13,12 +13,16 @@
  * 
  * Contributors:
  * {Name} (company) - description of contribution.
+ *  David McKnight  (IBM)  - [202822] cleanup output datalements after use
  *******************************************************************************/
 
 package org.eclipse.rse.internal.subsystems.shells.dstore;
 
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.dstore.core.model.DataElement;
+import org.eclipse.dstore.core.model.DataStore;
+import org.eclipse.dstore.extra.DomainEvent;
+import org.eclipse.dstore.extra.IDomainListener;
 import org.eclipse.rse.internal.services.dstore.shells.DStoreHostOutput;
 import org.eclipse.rse.internal.services.dstore.shells.DStoreHostShell;
 import org.eclipse.rse.internal.services.dstore.shells.DStoreShellOutputReader;
@@ -32,10 +36,79 @@ import org.eclipse.rse.subsystems.shells.core.model.RemoteOutput;
 import org.eclipse.rse.subsystems.shells.core.subsystems.IRemoteCmdSubSystem;
 import org.eclipse.rse.subsystems.shells.core.subsystems.IRemoteOutput;
 import org.eclipse.rse.subsystems.shells.core.subsystems.servicesubsystem.ServiceCommandShell;
+import org.eclipse.swt.widgets.Shell;
 
 public class DStoreServiceCommandShell extends ServiceCommandShell
 {
+	private class CleanUpSpirited extends Thread implements IDomainListener
+	{
+		private DataElement _status;
+		private DataStore _ds;
+		private String _name;
+		private boolean _done = false;
+		
+		public CleanUpSpirited(DataElement status, String name)
+		{
+			_status = status;
+			_ds = status.getDataStore();
+			_ds.getDomainNotifier().addDomainListener(this);
+			_name = name;
+		}
+		
+		public void domainChanged(DomainEvent e) 
+		{
+			deleteElements();
+		}
+		
+		public void run()
+		{			
+			while (!_done)
+			{
+				try
+				{
+					Thread.sleep(10000);
+				}
+				catch (Exception e)
+				{					
+				}
+				deleteElements();
+			}
+		}
+		
+		private void deleteElements()
+		{
+			if (_status.getNestedSize() > 0)
+			{
+				//synchronized (_status)
+				{
+					int ssize = _status.getNestedSize();
+				if (_status.get(ssize - 1).isSpirit())
+				{
+					System.out.println("deleting for "+ _name);
+					// delete
+					_ds.deleteObjects(_status);
+					_ds.refresh(_status);			
+					
+					_ds.getDomainNotifier().removeDomainListener(this);
+					_done = true;
+				}
+				}
+			}		
+		}
 
+		public Shell getShell() {
+			// TODO Auto-generated method stub
+			return null;
+		}
+
+		public boolean listeningTo(DomainEvent e) {
+			if (e.getParent() == _status)
+				return true;
+			return false;
+		}
+		
+	}
+	
 	public DStoreServiceCommandShell(IRemoteCmdSubSystem cmdSS, IHostShell hostShell)
 	{
 		super(cmdSS, hostShell);
@@ -158,4 +231,40 @@ public class DStoreServiceCommandShell extends ServiceCommandShell
 		return activeShell;
 	}
 	
+	public void removeOutput()
+	{			
+		DStoreHostShell shell = (DStoreHostShell)getHostShell();
+		DataElement status = shell.getStatus();
+		DataStore ds = status.getDataStore();
+
+		int ssize = status.getNestedSize();
+		if (status.get(ssize - 1).isSpirit() || !ds.isDoSpirit()) 
+		{
+			// objects can be deleted directly at this point since there will be no more updates from the server
+			ds.deleteObjects(status);
+			ds.refresh(status);
+		}
+		else
+		{
+			// cleanup later
+			// objects need to be deleted later since the server will still be sending spirited update
+			// if we don't defer this, then the deleted elements would get recreated when the spirits are updated
+			CleanUpSpirited cleanUp = new CleanUpSpirited(status, getId());					
+			cleanUp.start();
+		}
+
+		
+		// cleanup on host should be taking care of that
+		// ds.setObject(_commandElement);	
+		
+		synchronized(_output)
+		{
+			_output.clear();			
+		}
+		
+	//	noDE = ds.getHashMap().size();
+	//	System.out.println("DataElements:"+noDE);
+		//ds.printTree("2>", ds.getLogRoot());
+	}
+	
 }