applying patches for dstore spirit support

rse/plugins/org.eclipse.dstore.core/export.jardesc

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<jardesc>
+    <jar path="D:/servers/800/dstore_core.jar"/>
+    <options buildIfNeeded="true" compress="true" descriptionLocation="/org.eclipse.dstore.core/export.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="false" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
+    <refactoring deprecationInfo="true" structuralOnly="true"/>
+    <selectedProjects>
+        <project name="org.eclipse.dstore.core"/>
+    </selectedProjects>
+    <manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
+        <sealing sealJar="false">
+            <packagesToSeal/>
+            <packagesToUnSeal/>
+        </sealing>
+    </manifest>
+    <selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
+        <javaElement handleIdentifier="=org.eclipse.dstore.core/src"/>
+    </selectedElements>
+</jardesc>

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DE.java

@@ -105,10 +105,15 @@ public class DE
     public static final String P_VALUE           = "value";
 
     /*
-     * The <I>is reference?</I> property identifier of a <code>DataElement</code>.  
+     * The <I>is reference?</I> property identifier of a <code>DataElement</code>. Deprecated. Use P_REF_TYPE.  
      */
     public static final String P_ISREF           = "isRef";
     
+    /*
+     * The <I>is reference?</I> property identifier of a <code>DataElement</code>.  
+     */
+    public static final String P_REF_TYPE           = "refType";
+
     /*
      * The visibility property identifier of a <code>DataElement</code>.  
      */
@@ -207,10 +212,15 @@ public class DE
     public static final int    A_SOURCE_LOCATION     = 5;
 
     /*
-     * IsRef attribute index.  
+     * IsRef attribute index. Deprecated. Use A_REF_TYPE. 
      */
     public static final int    A_ISREF      = 6;
     
+    /*
+     * RefType attribute index.  
+     */
+    public static final int    A_REF_TYPE   = 6;
+
     /*
      * Visibility attribute index.  
      */

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java

@@ -19,6 +19,7 @@ package org.eclipse.dstore.core.model;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.eclipse.dstore.core.util.DataElementRemover;
 import org.eclipse.dstore.core.util.StringCompare;
 import org.eclipse.dstore.extra.internal.extra.DataElementActionFilter;
 import org.eclipse.dstore.extra.internal.extra.DesktopElement;
@@ -44,6 +45,7 @@ public final class DataElement implements IDataElement
 	private boolean _isExpanded = false;
 	private boolean _isUpdated = false;
 	private boolean _isPendingTransfer = false;
+	private boolean _isSpirit = false;
 	
 	private int _depth = 1;
 
@@ -65,6 +67,7 @@ public final class DataElement implements IDataElement
 	{
 		_dataStore = null;
 		_parent = null;
+		DataElementRemover.addToCreatedCount();
 	}
 
 	/**
@@ -76,7 +79,12 @@ public final class DataElement implements IDataElement
 	{
 		_dataStore = dataStore;
 		_parent = null;
+		DataElementRemover.addToCreatedCount();
+	}
 	
+	protected void finalize()
+	{
+		DataElementRemover.addToGCedCount();
 	}
 
 	/**
@@ -324,13 +332,13 @@ public final class DataElement implements IDataElement
 
 		String valueAttribute = getAttribute(DE.A_VALUE);
 
-		if (_depth == -1)
+		if (valueAttribute != null && valueAttribute.equals(DataStoreResources.DELETED))
 		{
 			return true;
 		}
-		else if (valueAttribute != null && valueAttribute.equals(DataStoreResources.DELETED))
+		
+		if (_isSpirit && !_dataStore.isVirtual())
 		{
-			_depth = -1;
 			return true; 
 		}
 		
@@ -726,6 +734,18 @@ public final class DataElement implements IDataElement
 		return _isPendingTransfer;
 	}
 	
+	/**
+	 * Indicates whether this element is a 'spirit' element. 
+	 * In a client datastore, this means that the element's counterpart on the
+	 * server is either also a spirit (and will be deleted soon) or has already
+	 * been deleted. In a server datastore, this means that the element is
+	 * to be deleted at the next opportunity in order to free memory.
+	 */
+	public boolean isSpirit()
+	{
+		return _isSpirit;
+	}
+	
 	/**
 	 * Sets an attribute of the element. 
 	 *
@@ -838,6 +858,18 @@ public final class DataElement implements IDataElement
 		_isPendingTransfer = flag;
 	}
 	
+	/**
+	 * Sets indication of whether this element is a 'spirit' element. 
+	 * In a client datastore, this means that the element's counterpart on the
+	 * server is either also a spirit (and will be deleted soon) or has already
+	 * been deleted. In a server datastore, this means that the element is
+	 * to be deleted at the next opportunity in order to free memory.
+	 */
+	public void setSpirit(boolean flag)
+	{
+		_isSpirit = flag;
+	}
+
 	/**
 	 * Sets the parent for this element. 
 	 *
@@ -1534,10 +1566,24 @@ public final class DataElement implements IDataElement
 		
 		
 
-		String isRef = getAttribute(DE.A_ISREF);
+		String refType = getAttribute(DE.A_REF_TYPE);
-		if (isRef != null && isRef.equals(DataStoreResources.TRUE))
+		if (refType != null)
+		{
+			if (refType.equals(DataStoreResources.TRUE) || refType.equals(DataStoreResources.REFERENCE))
 			{
 				_isReference = true;
+				_isSpirit = false;
+			}
+			else if (refType.equals(DataStoreResources.FALSE) || refType.equals(DataStoreResources.VALUE))
+			{
+				_isReference = false;
+				_isSpirit = false;
+			}
+			else if (refType.equals(DataStoreResources.SPIRIT))
+			{
+				_isReference = false;
+				_isSpirit = true;
+			}
 		}
 
 		String type = getAttribute(DE.A_TYPE);

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java

@@ -28,6 +28,7 @@ import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
@@ -42,6 +43,7 @@ import org.eclipse.dstore.core.java.ClassByteStreamHandlerRegistry;
 import org.eclipse.dstore.core.java.IClassByteStreamHandler;
 import org.eclipse.dstore.core.java.IRemoteClassInstance;
 import org.eclipse.dstore.core.java.RemoteClassLoader;
+import org.eclipse.dstore.core.util.DataElementRemover;
 import org.eclipse.dstore.core.util.ExternalLoader;
 import org.eclipse.dstore.core.util.StringCompare;
 import org.eclipse.dstore.core.util.XMLgenerator;
@@ -120,6 +122,12 @@ public final class DataStore
 	private RandomAccessFile _traceFile;
 	private boolean _tracingOn;
 	
+	private boolean _spiritModeOn = false;
+	private boolean _spiritCommandReceived = false;
+	private File _memLoggingFileHandle;
+	private RandomAccessFile _memLogFile;
+	private boolean _memLoggingOn;
+
 	private ArrayList _waitingStatuses = null;
 	
 	private String _userPreferencesDirectory = null;
@@ -128,6 +136,9 @@ public final class DataStore
 	private File _cacheJar;
 	public static final String REMOTE_CLASS_CACHE_JARFILE_NAME = "rmt_classloader_cache";
 	public static final String JARFILE_EXTENSION = ".jar";	
+	private DataElementRemover _deRemover;
+	public static final int SPIRIT_ON_INITIAL_SIZE = 1000;
+	private String referenceTag = null;
 	
 	private int _serverVersion;
 	private int _serverMinor;
@@ -147,8 +158,8 @@ public final class DataStore
 		_domainNotifier = null;
 		_isConnected = false;
 		_logTimes = false;
-		_initialSize = 100000;
+		setSpiritModeOnState();
-
+		_initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : 100000;
 		initialize();
 	}
 
@@ -166,8 +177,8 @@ public final class DataStore
 		_domainNotifier = null;
 		_isConnected = false;
 		_logTimes = false;
-		_initialSize = initialSize;
+		setSpiritModeOnState();
-
+		_initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : initialSize;
 		initialize();
 	}
 
@@ -187,8 +198,8 @@ public final class DataStore
 		_domainNotifier = domainNotifier;
 		_isConnected = true;
 		_logTimes = false;
-		_initialSize = 10000;
+		setSpiritModeOnState();
-
+		_initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : 10000;
 		initialize();
 		createRoot();
 	}
@@ -210,12 +221,21 @@ public final class DataStore
 		_domainNotifier = domainNotifier;
 		_isConnected = true;
 		_logTimes = false;
-		_initialSize = initialSize;
+		setSpiritModeOnState();
-
+		_initialSize = _spiritModeOn && !isVirtual() ? SPIRIT_ON_INITIAL_SIZE : initialSize;
 		initialize();
 		createRoot();
 	}
 
+	protected void setSpiritModeOnState()
+	{
+		if (isVirtual()) _spiritModeOn = true;
+		else
+		{
+			String doSpirit = System.getProperty("DSTORE_SPIRIT_ON");
+			_spiritModeOn = (doSpirit != null && doSpirit.equals("true"));
+		}
+	}
 	
 	public void setServerVersion(int version)
 	{
@@ -1626,6 +1646,28 @@ public final class DataStore
 		}
 	}
 	
+	/**
+	 * Disconnect all the elements contained in from
+	 *
+	 * @param from the element from which to disconnect objects
+	 */
+	public void disconnectObjects(DataElement from)
+	{
+		if (from != null)
+		{
+			for (int i = from.getNestedSize() - 1; i >= 0; i--)
+			{
+				DataElement disconnectee = from.get(i);
+				if (disconnectee != null)
+				{
+					disconnectObjectHelper(disconnectee, 5);
+				}
+			}
+
+			//			refresh(from);
+		}
+	}
+
 	/**
 	 * Deletes an element from another element
 	 *
@@ -1642,6 +1684,21 @@ public final class DataStore
 		}
 	}
 	
+	/**
+	 * Disconnects an element and makes it a "spirit"
+	 *
+	 * @param toDisconnect the element to disconnect  
+	 */
+	public void disconnectObject(DataElement toDisconnect)
+	{
+		if (toDisconnect != null)
+		{
+			disconnectObjectHelper(toDisconnect, 5);
+			// refresh(toDisconnect);
+			//			refresh(from);
+		}
+	}
+
 	/**
 	 * Replaces a deleted object
 	 */
@@ -2024,6 +2081,19 @@ public final class DataStore
 		return synchronizedCommand(cmd, _dummy);
 	}
 	
+	public boolean queryServerSpiritState()
+	{
+		DataElement spirittype = findObjectDescriptor(IDataStoreConstants.DATASTORE_SPIRIT_DESCRIPTOR);
+		if (spirittype == null) return false;
+		DataElement cmd = localDescriptorQuery(spirittype, IDataStoreConstants.C_START_SPIRIT, 2);
+		if (cmd == null) return false;
+		
+		DataElement status = synchronizedCommand(cmd, _dummy);
+		if ((status != null) && status.getName().equals(DataStoreResources.model_done))
+			return true;
+		else return false;
+	}
+	
 	public DataElement queryHostJVM()
 	{
 		DataElement cmd = findCommandDescriptor(DataStoreSchema.C_QUERY_JVM);
@@ -3361,11 +3431,9 @@ public final class DataStore
 		{
 			_tracingOn = false;
 		}
-		
-		String logDir = getUserPreferencesDirectory();
 		if (_tracingOn)
 		{
-			
+			String logDir = getUserPreferencesDirectory();
 			_traceFileHandle = new File(logDir, ".dstoreTrace");
 
 			try
@@ -3393,6 +3461,42 @@ public final class DataStore
 		registerLocalClassLoader(this.getClass().getClassLoader());
 	}
 	
+	public void startDataElementRemoverThread()
+	{
+		if (!isVirtual() && _deRemover == null)
+		{
+			String memLogging = System.getProperty("DSTORE_MEMLOGGING_ON");
+			_memLoggingOn = (memLogging != null && memLogging.equals("true"));
+			
+			if (_memLoggingOn)
+			{
+				String logDir = getUserPreferencesDirectory();
+				_memLoggingFileHandle = new File(logDir, ".dstoreMemLogging");
+	
+				try
+				{
+					_memLogFile = new RandomAccessFile(_memLoggingFileHandle, "rw");
+					startMemLogging();
+				}
+				catch (IOException e)
+				{
+				}
+			}
+			_deRemover = new DataElementRemover(this);
+			_deRemover.start();
+		}
+	}
+
+	public boolean isDoSpirit()
+	{
+		if (isVirtual()) return _spiritModeOn;
+		else return _spiritModeOn && _spiritCommandReceived;
+	}
+	
+	public void receiveStartSpiritCommand()
+	{
+		_spiritCommandReceived = true;
+	}
 	
 	public IByteStreamHandler getDefaultByteStreamHandler()
 	{
@@ -3627,6 +3731,23 @@ public final class DataStore
 		}
 	}
 
+	private void disconnectObjectHelper(DataElement toDisconnect, int depth)
+	{
+		if (depth > 0)
+		{
+			depth--;
+			_deRemover.addToQueueForRemoval(toDisconnect);
+			for (int i = 0; i < toDisconnect.getNestedSize(); i++)
+			{
+				DataElement subDisconnect = toDisconnect.get(i);
+				if (subDisconnect != null && subDisconnect.getDataStore() == this && !subDisconnect.isSpirit())
+				{
+					disconnectObjectHelper(subDisconnect, depth);
+				}
+			}
+		}
+	}
+	
 	private String makeIdUnique(String id)
 	{
 		
@@ -3694,6 +3815,28 @@ public final class DataStore
 		}
 	}
 	
+	public void startMemLogging()
+	{
+		if (_memLoggingOn && _memLogFile != null && _memLoggingFileHandle != null)
+		{
+			try
+			{
+				_memLogFile.seek(_memLoggingFileHandle.length());
+			}
+			catch (IOException e)
+			{
+			}
+
+			memLog("-----------------------------------------");
+			memLog("Start Memory Logging at " + System.currentTimeMillis());
+		}
+	}
+	
+	public void memLog(String str)
+	{
+		internalMemLog(str);
+	}
+
 	public void trace(String str)
 	{
 		internalTrace(str);
@@ -3740,6 +3883,22 @@ public final class DataStore
 		}
 	}
 	
+	private void internalMemLog(String message)
+	{
+		if (_memLoggingOn && _memLogFile != null && message != null)
+		{
+			try
+			{
+				_memLogFile.writeBytes((new Date()).toString() + ": ");
+				_memLogFile.writeBytes(message);
+				_memLogFile.writeBytes(System.getProperty("line.separator"));
+			}
+			catch (IOException e)
+			{
+			}
+		}
+	}
+
 	public void finish()
 	{
 		// dy: the call to flush deletes all the elements in the tree
@@ -3989,4 +4148,23 @@ public final class DataStore
 		}
 	}
 	
+	/**
+	 * @return what type of attribute tag is used on the peer DataStore to indicate whether dataelements
+	 * are references, values, or spirit elements. If the peer DataStore is an older one, this will return
+	 * "isRef", if its up-to-date, it will return "refType", and if the tag hasnt been determined yet, this method
+	 * will return null.
+	 */
+	public String getReferenceTag()
+	{
+		return referenceTag;
+	}
+
+	/**
+	 * Sets what type of attribute tag is used on the peer DataStore to indicate whether dataelements
+	 * are references, values, or spirit elements.
+	 */
+	public void setReferenceTag(String tag)
+	{
+		referenceTag = tag;
+	}
 }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreResources.java

@@ -109,4 +109,7 @@ public class DataStoreResources
 	public static String DELETED="deleted";
 	public static String KEEPALIVE_TYPE="KEEPALIVE";
 	public static String KEEPALIVECONFIRM_TYPE="CONFIRMKEEPALIVE";
+	public static String REFERENCE="reference";
+	public static String VALUE="value";
+	public static String SPIRIT="spirit";
 }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/Handler.java

@@ -27,7 +27,7 @@ public abstract class Handler extends Thread
 
 	protected int _waitIncrement;
 	protected DataStore _dataStore;
-	private boolean _keepRunning;
+	protected boolean _keepRunning;
 
 	/**
 	 * Constructor

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/IDataStoreConstants.java

@@ -25,4 +25,7 @@ public interface IDataStoreConstants
     public static final String UNKNOWN_PROBLEM = "unknown problem connecting to server";
     public static final String SERVER_FAILURE = "server failure: ";
     public static final String ATTEMPT_RECONNECT = "attempt reconnect";
+    
+    public static final String DATASTORE_SPIRIT_DESCRIPTOR = "datastore.spirit";
+    public static final String C_START_SPIRIT = "C_START_SPIRIT";
 }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java

@@ -20,7 +20,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.dstore.core.java.IRemoteClassInstance;
-
+import org.eclipse.dstore.core.util.DataElementRemover;
 
 /**
  * <p>
@@ -61,16 +61,12 @@ public abstract class UpdateHandler extends Handler
 	}
 
 	protected void clean(DataElement object)
-	{
-		if (_dataObjects.size() == 0)
 	{
 		clean(object, 2);
 	}
-	}
 
 	protected synchronized void clean(DataElement object, int depth)
 	{
-
 		if ((depth > 0) && (object != null) && object.getNestedSize() > 0)
 		{
 			List deletedList = _dataStore.findDeleted(object);
@@ -81,6 +77,7 @@ public abstract class UpdateHandler extends Handler
 				if (child != null && child.isDeleted())
 				{
 					DataElement parent = child.getParent();
+					if (child.isSpirit()) DataElementRemover.addToRemovedCount();
 					child.clear();
 					if (parent != null)
 					{

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/Server.java

@@ -37,6 +37,8 @@ public class Server
 	 */
 	public static void main(String[] args)
 	{
+		//Tell the Launcher that we are starting
+		System.err.println(ServerReturnCodes.RC_DSTORE_SERVER_MAGIC);
 
 		String jversion = System.getProperty("java.version");
 

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerCommandHandler.java

@@ -29,6 +29,7 @@ import org.eclipse.dstore.core.model.DataStore;
 import org.eclipse.dstore.core.model.DataStoreAttributes;
 import org.eclipse.dstore.core.model.DataStoreResources;
 import org.eclipse.dstore.core.model.DataStoreSchema;
+import org.eclipse.dstore.core.model.IDataStoreConstants;
 
 /**
  * The ServerCommandHandler is reponsible for maintaining
@@ -224,6 +225,7 @@ public class ServerCommandHandler extends CommandHandler
 					clientTicket.setAttribute(DE.A_VALUE,DataStoreResources.model_invalid);
 				}
 				_dataStore.update(clientTicket);
+				_dataStore.startDataElementRemoverThread();
 				status.setAttribute(DE.A_NAME,DataStoreResources.model_done);
 			}
 			else if (commandName.equals(DataStoreSchema.C_SET))
@@ -330,10 +332,13 @@ public class ServerCommandHandler extends CommandHandler
 				_dataStore.refresh(schemaRoot);
 				status.setAttribute(DE.A_NAME,DataStoreResources.model_done);
 			}
-			else if (_dataStore.validTicket())
+			else if (commandName.equals(IDataStoreConstants.C_START_SPIRIT))
+			{
+				_dataStore.receiveStartSpiritCommand();
+				status.setAttribute(DE.A_NAME, DataStoreResources.model_done);
+			}
+			else if (_dataStore.validTicket() && _minerLoader != null)
 			{
-			
-
 				if (status != null)
 				{
 					boolean failure = false;

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerLauncher.java

@@ -247,6 +247,8 @@ public class ServerLauncher extends Thread
 								+ timeout
 								+ " "
 								+ ticket
+								+ " " //$NON-NLS-1$
+								+ System.getProperty("java.home"); //$NON-NLS-1$
 								;
 
 						String[] authArray = { "sh", "-c", authString };
@@ -290,11 +292,20 @@ public class ServerLauncher extends Thread
 					}
 					else
 					{
+						//read over stuff coming from the login shell invocation
 						String status = _errReader.readLine();
-						_port = _errReader.readLine();
+						while (status!=null && !status.equals(ServerReturnCodes.RC_DSTORE_SERVER_MAGIC))
-
+						{
+							status = _errReader.readLine();
+						}
+						//now read the real server status
+						if (status != null)
+						{
+							status = _errReader.readLine();
+						}
 						if ((status != null) && status.equals(ServerReturnCodes.RC_SUCCESS))
 						{
+							_port = _errReader.readLine();
 							_errReader.readLine();
 							_writer.println(IDataStoreConstants.CONNECTED);
 							_writer.println(_port);
@@ -309,7 +320,7 @@ public class ServerLauncher extends Thread
 							{
 								status = new String(IDataStoreConstants.UNKNOWN_PROBLEM);
 							}
-
+							//TODO Make sure that the client doesnt try connecting forever
 							_writer.println(status);
 
 							_serverProcess.destroy();

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerReturnCodes.java

@@ -22,7 +22,7 @@ package org.eclipse.dstore.core.server;
  */
 public class ServerReturnCodes
 {
-
+	public static final String RC_DSTORE_SERVER_MAGIC = "Dstore Server Starting..."; //$NON-NLS-1$
 
 	public static final String RC_SUCCESS = "Server Started Successfully";
 

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/server/ServerSSLProperties.java

@@ -48,21 +48,21 @@ public class ServerSSLProperties implements ISSLProperties
 			ResourceBundle properties = ResourceBundle.getBundle("ssl");
 			if (properties != null)
 			{
-				_enableSSL = properties.getString(ENABLE_SSL).equals("true");
+				_enableSSL = properties.getString(ENABLE_SSL).trim().equals("true");
 				if (_enableSSL)
 				{
 					try
 					{
-						_daemonKeyStorePath = properties.getString(DAEMON_KEYSTORE_FILE);
+						_daemonKeyStorePath = properties.getString(DAEMON_KEYSTORE_FILE).trim();
-						_daemonKeyStorePassword = properties.getString(DAEMON_KEYSTORE_PASSWORD);
+						_daemonKeyStorePassword = properties.getString(DAEMON_KEYSTORE_PASSWORD).trim();
 					}
 					catch (Exception e)
 					{					
 					}
 					try
 					{
-						_serverKeyStorePath = properties.getString(SERVER_KEYSTORE_FILE);
+						_serverKeyStorePath = properties.getString(SERVER_KEYSTORE_FILE).trim();
-						_serverKeyStorePassword = properties.getString(SERVER_KEYSTORE_PASSWORD);			
+						_serverKeyStorePassword = properties.getString(SERVER_KEYSTORE_PASSWORD).trim();			
 					}
 					catch (Exception e)
 					{

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java

@@ -151,7 +151,7 @@ public class CommandGenerator
 
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 
-            if (dataObject.isUpdated())
+            if (dataObject.isUpdated() && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
             }
@@ -168,7 +168,7 @@ public class CommandGenerator
                     DataElement arg = (DataElement) arguments.get(i);
                     if (arg != null)
                     {
-                        if (!arg.isUpdated())
+                        if (!arg.isUpdated() || arg.isSpirit())
                         {
                             commandObject.addNestedData(arg, false);
                         }
@@ -208,7 +208,7 @@ public class CommandGenerator
             DataElement tempRoot = _dataStore.getTempRoot();
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 			clearDeleted(dataObject);
-            if (refArg || dataObject.isUpdated())
+            if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
             }
@@ -218,7 +218,7 @@ public class CommandGenerator
                 commandObject.addNestedData(dataObject, false);
             }
 
-            if (!arg.isUpdated())
+            if (!arg.isUpdated() || arg.isSpirit())
             {
                 commandObject.addNestedData(arg, false);
             }
@@ -254,7 +254,7 @@ public class CommandGenerator
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 
 			clearDeleted(dataObject);
-            if (refArg || dataObject.isUpdated())
+            if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_arguments);
             }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/DataElementRemover.java

@@ -0,0 +1,179 @@
+package org.eclipse.dstore.core.util;
+
+import java.util.LinkedList;
+
+import org.eclipse.dstore.core.model.DataElement;
+import org.eclipse.dstore.core.model.DataStore;
+import org.eclipse.dstore.core.model.Handler;
+import org.eclipse.dstore.core.model.IDataStoreConstants;
+
+public class DataElementRemover extends Handler 
+{
+	private LinkedList _queue;
+	private static int numRemoved = 0;
+	private static int numDisconnected = 0;
+	private static int numCreated = 0;
+	private static int numGCed = 0;
+	
+	// The following determine how DataElements are chosen to be removed once they
+	// are in the queue for removal. 	
+	// The queue is checked every _intervalTime milliseconds and all elements
+	// that are older than _expiryTime milliseconds are removed.
+	public static final int DEFAULT_EXPIRY_TIME = 600; // in seconds
+	public static final int DEFAULT_INTERVAL_TIME = 60; // in seconds
+	private int _intervalTime = DEFAULT_INTERVAL_TIME * 1000;
+	private int _expiryTime = DEFAULT_EXPIRY_TIME * 1000;
+	public static final String EXPIRY_TIME_PROPERTY_NAME = "SPIRIT_EXPIRY_TIME";
+	public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME";
+	
+	public DataElementRemover(DataStore dataStore)
+	{
+		super();
+		_dataStore = dataStore;
+		_queue = new LinkedList();
+		getTimes();
+		setWaitTime(_intervalTime);
+		DataElement spiritnode = _dataStore.createObjectDescriptor(_dataStore.getDescriptorRoot(), IDataStoreConstants.DATASTORE_SPIRIT_DESCRIPTOR);
+		_dataStore.createCommandDescriptor(spiritnode, "StartSpirit", "DataElementRemover", IDataStoreConstants.C_START_SPIRIT);
+		_dataStore.refresh(_dataStore.getDescriptorRoot());
+	}
+	
+	protected void getTimes()
+	{
+		try
+		{
+			String expiryTime = System.getProperty(EXPIRY_TIME_PROPERTY_NAME);
+			if (expiryTime != null && !expiryTime.equals("")) _expiryTime = Integer.parseInt(expiryTime) * 1000;			
+		}
+		catch (Exception e)
+		{
+			System.out.println("Invalid spirit expiry time property, using default.");
+			_expiryTime = DEFAULT_EXPIRY_TIME;
+		}
+		try
+		{
+			String intervalTime = System.getProperty(INTERVAL_TIME_PROPERTY_NAME);
+			if (intervalTime != null && !intervalTime.equals("")) _intervalTime = Integer.parseInt(intervalTime) * 1000;			
+		}
+		catch (Exception e)
+		{
+			System.out.println("Invalid spirit interval time property, using default.");
+			_intervalTime = DEFAULT_INTERVAL_TIME;
+		}
+	}
+	
+	public static void addToRemovedCount()
+	{
+		numRemoved++;
+	}
+	
+	public static void addToCreatedCount()
+	{
+		numCreated++;
+	}
+	
+	public static void addToGCedCount()
+	{
+		numGCed++;
+	}
+
+	public synchronized void addToQueueForRemoval(DataElement element)
+	{
+		synchronized (_queue) 
+		{
+			if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore())
+			{
+				QueueItem item = new QueueItem(element, System.currentTimeMillis());
+				_queue.add(item);
+			}
+		}
+	}
+	
+	public void handle()
+	{
+		clearQueue();
+	}
+	
+	public synchronized void clearQueue()
+	{
+		synchronized (_queue)
+		{
+			_dataStore.memLog("           ");
+			int disconnected = 0;
+			if (!_dataStore.isDoSpirit())
+			{
+				if (_queue.size() > 0) 
+				{
+					_dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false.");
+					_queue.clear();
+				}
+				_dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory());
+				_dataStore.memLog("Elements created so far: " + numCreated);
+				_dataStore.memLog("Elements disconnected so far: " + numDisconnected);
+				_dataStore.memLog("Spirit elements cleaned so far: " + numRemoved);
+				_dataStore.memLog("DataElements GCed so far: " + numGCed);
+				return;
+			}
+			_dataStore.memLog("Total heap size before disconnection: " + Runtime.getRuntime().totalMemory());
+			
+			_dataStore.memLog("Size of queue: " + _queue.size());
+			
+			while (_queue.size() > 0 && System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime)
+			{
+				DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement;
+				if (!toBeDisconnected.isSpirit()) 
+				{
+					toBeDisconnected.setSpirit(true);
+					_dataStore.refresh(toBeDisconnected);
+					disconnected++;
+					numDisconnected++;
+				}
+				else
+				{
+					//_dataStore.memLog(toBeDisconnected.toString());
+				}
+				_dataStore.getHashMap().remove(toBeDisconnected.getId());
+			}
+			_dataStore.memLog("Disconnected " + disconnected + " DataElements.");
+			_dataStore.memLog("Elements created so far: " + numCreated);
+			_dataStore.memLog("Elements disconnected so far: " + numDisconnected);
+			_dataStore.memLog("Spirit elements cleaned so far: " + numRemoved);
+			_dataStore.memLog("DataElements GCed so far: " + numGCed);
+		}
+	}
+	
+	protected class QueueItem
+	{
+		public DataElement dataElement;
+		public long timeStamp;
+		
+		public QueueItem(DataElement element, long stamp)
+		{
+			dataElement = element;
+			timeStamp = stamp;
+		}
+	}
+	
+	/**
+	 * Runs the handler loop in a thread.
+	 */
+	public void run()
+	{
+		while (_keepRunning)
+		{
+			try
+			{
+				Thread.sleep(_waitIncrement);
+				Thread.yield();
+			}
+			catch (InterruptedException e)
+			{
+				e.printStackTrace();
+				finish();
+				return;
+			}
+			handle();
+		}
+	}
+}
+

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLgenerator.java

@@ -17,8 +17,6 @@
 package org.eclipse.dstore.core.util;
 
 import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
 import java.io.PipedInputStream;
@@ -291,6 +289,27 @@ public class XMLgenerator
 
 	}
 	
+	private void addReferenceTypeAttribute(DataElement object)
+	{
+		if (object.isSpirit())
+		{
+			addAttribute(DE.P_REF_TYPE, DataStoreResources.SPIRIT);
+		}
+		else
+		{
+			if (object.isReference())
+			{
+				if (_dataStore.getReferenceTag() != null && _dataStore.getReferenceTag().equals(DE.P_REF_TYPE)) addAttribute(DE.P_REF_TYPE, DataStoreResources.REFERENCE);
+				else addAttribute(DE.P_REF_TYPE, DataStoreResources.TRUE);
+			}
+			else 
+			{
+				if (_dataStore.getReferenceTag() != null && _dataStore.getReferenceTag().equals(DE.P_REF_TYPE)) addAttribute(DE.P_REF_TYPE, DataStoreResources.VALUE);
+				else addAttribute(DE.P_REF_TYPE, DataStoreResources.FALSE);
+			}
+		}
+	}
+
 	private void addFile(byte[] bytes, int size, boolean binary)
 	{
 		if (_state == OPEN)
@@ -502,14 +521,7 @@ public class XMLgenerator
 			addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE));
 			addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION));
 
-			if (object.isReference())
+			addReferenceTypeAttribute(object);
-			{
-				addAttribute(DE.P_ISREF, DataStoreResources.TRUE);
-			}
-			else
-			{
-				addAttribute(DE.P_ISREF, DataStoreResources.FALSE);
-			}
 
 			addAttribute(DE.P_DEPTH, "" + size);
 			addFile(bytes, size, binary);
@@ -538,14 +550,8 @@ public class XMLgenerator
 			addAttribute(DE.P_VALUE, object.getAttribute(DE.A_VALUE));
 			addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE));
 			addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION));
-			if (object.isReference())
+
-			{
+			addReferenceTypeAttribute(object);
-				addAttribute(DE.P_ISREF, "true");
-			}
-			else
-			{
-				addAttribute(DE.P_ISREF, "false");
-			}
 
 			addAttribute(DE.P_DEPTH, "" + size);
 			addFile(bytes, size, true);
@@ -586,14 +592,7 @@ public class XMLgenerator
 					addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE));
 					addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION));
 
-					if (object.isReference())
+					addReferenceTypeAttribute(object);
-					{
-						addAttribute(DE.P_ISREF, "true");
-					}
-					else
-					{
-						addAttribute(DE.P_ISREF, "false");
-					}
 
 					addAttribute(DE.P_DEPTH, "" + object.depth());
 					addData(object.getBuffer());
@@ -627,14 +626,7 @@ public class XMLgenerator
 			addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE));
 			addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION));
 
-			if (object.isReference())
+			addReferenceTypeAttribute(object);
-			{
-				addAttribute(DE.P_ISREF, "true");
-			}
-			else
-			{
-				addAttribute(DE.P_ISREF, "false");
-			}
 			_state = BODY;
 			endTag(tagType);
 		}
@@ -654,14 +646,7 @@ public class XMLgenerator
 			addAttribute(DE.P_SOURCE, object.getAttribute(DE.A_SOURCE));
 			addAttribute(DE.P_SOURCE_LOCATION, object.getAttribute(DE.A_SOURCE_LOCATION));
 
-			if (object.isReference())
+			addReferenceTypeAttribute(object);
-			{
-				addAttribute(DE.P_ISREF, "true");
-			}
-			else
-			{
-				addAttribute(DE.P_ISREF, "false");
-			}
 			
 			try
 			{

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/XMLparser.java

@@ -474,6 +474,12 @@ public class XMLparser
 			{
 				String trimmedTag = xmlTag.trim();
 				
+				if (_dataStore.getReferenceTag() == null)
+				{
+					if (trimmedTag.indexOf(DE.P_ISREF + "=") > -1) _dataStore.setReferenceTag(DE.P_ISREF);
+					else if (trimmedTag.indexOf(DE.P_REF_TYPE + "=") > -1) _dataStore.setReferenceTag(DE.P_REF_TYPE);
+				}
+
 				if (!_tagStack.empty())
 				{
 					matchTag = (String) _tagStack.peek();
@@ -752,9 +758,11 @@ public class XMLparser
 				}			
 				else
 				{
-					String isRefStr = attributes[DE.A_ISREF];
+					String refType = attributes[DE.A_REF_TYPE];
+					boolean isSpirit = false;
+					if (refType != null) isSpirit = refType.equals(DataStoreResources.SPIRIT);
 					
-					if ((isRefStr != null) && isRefStr.equals("true"))
+					if ((refType != null) && (refType.equals(DataStoreResources.TRUE) || refType.equals(DataStoreResources.REFERENCE)))
 					{
 						// new reference
 						String origId = attributes[DE.A_NAME];
@@ -806,6 +814,15 @@ public class XMLparser
 							}
 							else
 							{
+								if (isSpirit)
+								{
+									if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE;
+									result.setSpirit(_dataStore.isVirtual());
+								}
+								else
+								{
+									result.setSpirit(false);
+								}
 								result.setAttributes(attributes);
 							}
 	
@@ -853,7 +870,24 @@ public class XMLparser
 						else
 						{
 							// new object
+							if (_dataStore.isVirtual() && parent != null) 
+							{
+								result = _dataStore.find(parent, DE.A_NAME, attributes[DE.A_NAME], 1);
+								if (result != null && result.getValue().equals(attributes[DE.A_VALUE]) && result.isSpirit())
+									_dataStore.deleteObject(parent, result);
+							}
+							if (isSpirit)
+							{
+								if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE;
 								result = _dataStore.createObject(parent, attributes);
+								result.setSpirit(_dataStore.isVirtual());
+							}
+							else
+							{
+								result = _dataStore.createObject(parent, attributes);
+								result.setSpirit(false);
+							}
+
 					
 						}
 

rse/plugins/org.eclipse.rse.connectorservice.dstore/src/org/eclipse/rse/connectorservice/dstore/DStoreConnectorService.java

@@ -768,6 +768,8 @@ public class DStoreConnectorService extends AbstractConnectorService implements
 				dataStore.showTicket(null);
 			}
 			
+	        if (dataStore.isDoSpirit()) dataStore.queryServerSpiritState();
+
 			// Fire comm event to signal state changed
 			fireCommunicationsEvent(CommunicationsEvent.AFTER_CONNECT);
 			

rse/plugins/org.eclipse.rse.eclipse.filesystem/src/org/eclipse/rse/eclipse/filesystem/RSEFileSystem.java

@@ -17,6 +17,7 @@
 package org.eclipse.rse.eclipse.filesystem;
 
 import java.net.URI;
+import java.rmi.RemoteException;
 import java.util.HashMap;
 
 import org.eclipse.core.filesystem.IFileStore;
@@ -27,6 +28,7 @@ import org.eclipse.rse.model.ISystemRegistry;
 import org.eclipse.rse.subsystems.files.core.model.RemoteFileUtility;
 import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFile;
 import org.eclipse.rse.subsystems.files.core.subsystems.IRemoteFileSubSystem;
+import org.eclipse.rse.subsystems.files.core.subsystems.RemoteFileEmpty;
 import org.eclipse.rse.ui.RSEUIPlugin;
 import org.eclipse.swt.widgets.Shell;
 import org.eclipse.ui.PlatformUI;
@@ -160,6 +162,7 @@ public class RSEFileSystem extends FileSystem
 		catch (Exception e) 
 		{
 			e.printStackTrace();
+			return FileStoreConversionUtility.convert(null, new RemoteFileEmpty());
 		}
 		return null;
 	}

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/CommandMinerThread.java

@@ -361,6 +361,7 @@ public class CommandMinerThread extends MinerThread
 
 		status.setAttribute(DE.A_NAME, "progress");
 		_dataStore.update(status);
+		_dataStore.disconnectObjects(status);
 		_stdOutputHandler = new OutputHandler(_stdInput, null, _isWindows || _isTTY, false, _isShell, this);
 		_stdOutputHandler.setWaitTime(10);
 		_stdOutputHandler.start();
@@ -1005,6 +1006,7 @@ public class CommandMinerThread extends MinerThread
 		}
 		
 		_dataStore.refresh(_status);
+		_dataStore.disconnectObjects(_status);
 	}
 
 	public void createPrompt(String line, String fileName)
@@ -1231,6 +1233,7 @@ public class CommandMinerThread extends MinerThread
 			    obj.setAttribute(DE.A_SOURCE, obj.getSource() + ':' + line.toString());				
 			}
 			_dataStore.refresh(_status);
+			_dataStore.disconnectObjects(_status);
 			return obj;
 		}
 		else

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/command/QueryPathThread.java

@@ -126,6 +126,7 @@ public class QueryPathThread extends Thread
 		}
 		status.setAttribute(DE.A_NAME, "done");
 		_dataStore.refresh(status);
+		_dataStore.disconnectObjects(status);
 	}
 
 	private void resolveCommandsInPath(File file, DataElement status)

rse/plugins/org.eclipse.rse.services.dstore/miners/org/eclipse/rse/dstore/universal/miners/filesystem/FileClassifier.java

@@ -302,6 +302,7 @@ public class FileClassifier extends Thread
                 }
             }
         }
+        _dataStore.disconnectObject(_subject);
     }
 
     /**