[dstore] fix issues with the spiriting mechanism and other memory

improvements (phase 1)

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataElement.java

@@ -19,6 +19,7 @@
  * David McKnight   (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
  * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
  * David McKnight   (IBM) - [391065] [dstore][regression] DataElement with "" type treated as deleted (when it's not)
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -28,7 +29,6 @@ import java.util.List;
 
 import org.eclipse.dstore.core.util.StringCompare;
 import org.eclipse.dstore.extra.IDataElement;
-import org.eclipse.dstore.internal.core.util.DataElementRemover;
 import org.eclipse.dstore.internal.extra.DataElementActionFilter;
 import org.eclipse.dstore.internal.extra.DesktopElement;
 import org.eclipse.dstore.internal.extra.PropertySource;
@@ -76,7 +76,6 @@ public final class DataElement implements IDataElement
 	{
 		_dataStore = null;
 		_parent = null;
-		DataElementRemover.addToCreatedCount();
 	}
 
 	/**
@@ -88,7 +87,6 @@ public final class DataElement implements IDataElement
 	{
 		_dataStore = dataStore;
 		_parent = null;
-		DataElementRemover.addToCreatedCount();
 	}
 	
 	/* Apparently having this method causes the GC to delay
@@ -304,6 +302,7 @@ public final class DataElement implements IDataElement
 	public void reInitAsTransient(String attributes[])
 	{
 		_attributes = attributes;
+		setAttribute(DE.A_ID, DataStoreResources.model_transient); 
 		
 		_isReference = false;
 		_isDescriptor = false;
@@ -886,14 +885,19 @@ public final class DataElement implements IDataElement
 	 */
 	public void setSpirit(boolean flag)
 	{
-		_isSpirit = flag;
+		if (flag && isDescriptor()){
-		String refType = getAttribute(DE.A_REF_TYPE);
+			// descriptors should not be spirited
-		if (refType != null){
+		}
-			if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) {
+		else {
-				setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT);
+			_isSpirit = flag;
-			}
+			String refType = getAttribute(DE.A_REF_TYPE);
-			else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back
+			if (refType != null){
-				setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE);
+				if (_isSpirit && !refType.equals(DataStoreResources.SPIRIT)) {
+					setAttribute(DE.A_REF_TYPE, DataStoreResources.SPIRIT);
+				}
+				else if (refType.equals(DataStoreResources.SPIRIT)){ // if it was a spirit, change it back
+					setAttribute(DE.A_REF_TYPE, DataStoreResources.VALUE);
+				}
 			}
 		}
 	}
@@ -1454,6 +1458,8 @@ public final class DataElement implements IDataElement
 			+ getSource()
 			+ "\n\tDepth:\t" //$NON-NLS-1$
 			+ _depth
+			+ "\n\tisSpirit:\t" //$NON-NLS-1$
+			+ _isSpirit
 			+ "\n\tDataStore:\t" //$NON-NLS-1$
 			+ _dataStore.getName()
 			+ "\n}\n"; //$NON-NLS-1$

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStore.java

@@ -45,6 +45,7 @@
  * David McKnight   (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
  * David McKnight   (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
  * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -199,7 +200,7 @@ public final class DataStore
 	 * 
 	 *  This needs to be updated for each major release.
 	 */
-	private String _RSE_version = "3.3.1"; //$NON-NLS-1$
+	private String _RSE_version = "3.4.2"; //$NON-NLS-1$
 	
 	/**
 	 * Creates a new <code>DataStore</code> instance
@@ -950,10 +951,6 @@ public final class DataStore
 			reference.reInit(parent, realObject, relationType);
 			parent.addNestedData(reference, false);
 
-			String sugId = reference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(sugId, reference);
-			}
 			refresh(parent);
 
 			return reference;
@@ -1005,11 +1002,6 @@ public final class DataStore
 
 			parent.addNestedData(reference, false);
 
-			String sugId = reference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(sugId, reference);
-			}
-			
 			if (doRefresh)
 			{
 			    refresh(parent);
@@ -1078,22 +1070,12 @@ public final class DataStore
 			toReference.reInit(parent, realObject, toRelation);
 
 			parent.addNestedData(toReference, false);
-
-			String toId = toReference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(toId, toReference);
-			}
 			// reference with "from" relationship
 			DataElement fromReference = createElement();
 
 			fromReference.reInit(realObject, parent, fromRelation);
 
 			realObject.addNestedData(fromReference, false);
-
-			String fromId = fromReference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(fromId, fromReference);
-			}
 			refresh(parent);
 
 
@@ -1128,12 +1110,6 @@ public final class DataStore
 			}
 
 			parent.addNestedData(toReference, false);
-
-			String toId = toReference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(toId, toReference);
-			}
-
 			// reference with "from" relationship
 			DataElement fromReference = createElement();
 
@@ -1149,10 +1125,6 @@ public final class DataStore
 
 			realObject.addNestedData(fromReference, false);
 
-			String fromId = fromReference.getId();
-			synchronized (_hashMap){
-				_hashMap.put(fromId, fromReference);
-			}
 			 refresh(parent);
 
 
@@ -1207,7 +1179,7 @@ public final class DataStore
 
 	public DataElement createTransientObject(String attributes[])
 	{
-		DataElement newObject = createElement();
+		DataElement newObject = new DataElement(this);
 
 		newObject.reInitAsTransient(attributes);
 		return newObject;
@@ -2537,7 +2509,7 @@ public final class DataStore
 	 */
 	public DataElement command(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject)
 	{
-		return command(commandDescriptor, arguments, dataObject, false);
+		return command(commandDescriptor, arguments, dataObject, true);
 	}
 
 	/**
@@ -2553,7 +2525,8 @@ public final class DataStore
 	{
 		if (_commandHandler != null)
 		{
-			return _commandHandler.command(commandDescriptor, arguments, dataObject, true, immediate);
+			// as per bug #396440, default is now to not use references
+			return _commandHandler.command(commandDescriptor, arguments, dataObject, false, immediate);
 		}
 		return null;
 	}
@@ -2584,7 +2557,8 @@ public final class DataStore
 	{
 		if (_commandHandler != null)
 		{
-			return _commandHandler.command(commandDescriptor, arg, dataObject, true, immediate);
+			// as per bug #396440, default is now to not use references
+			return _commandHandler.command(commandDescriptor, arg, dataObject, false, immediate);
 		}
 		return null;
 	}
@@ -2598,6 +2572,7 @@ public final class DataStore
 	 */
 	public DataElement command(DataElement commandDescriptor, DataElement dataObject)
 	{
+		// as per bug #396440, default is now to not use references
 		return command(commandDescriptor, dataObject, false);
 	}
 
@@ -2888,31 +2863,19 @@ public final class DataStore
 
 
 				List searchList = root.getNestedData();
-
+				if (searchList != null){
-				if (searchList != null)
+					for (int i = 0; i < searchList.size(); i++){
-				{
-					for (int i = 0; i < searchList.size(); i++)
-					{
 						DataElement child = (DataElement) searchList.get(i);
-						if (child != null)
+						if (child != null){
-						{
+							if (child.isDeleted() && !results.contains(child)){
-							synchronized (child)
+								results.add(child);
-							{
+								if (!child.isReference()){
-								if (child.isDeleted() && !results.contains(child))
+									if (depth > 0){
-								{
+										List sResults = findDeleted(child, depth - 1);
-									results.add(child);
+										for (int j = 0; j < sResults.size(); j++){
-									if (!child.isReference())
+											results.add(sResults.get(j));
-									{
-										if (depth > 0)
-										{
-											List sResults = findDeleted(child, depth - 1);
-											for (int j = 0; j < sResults.size(); j++)
-											{
-												results.add(sResults.get(j));
-											}
 										}
-
+									}									
-									}
 								}
 							}
 						}
@@ -3983,16 +3946,12 @@ public final class DataStore
 
 		int numRecycled = _recycled.size();
 
-		if (numRecycled > 1)
+		if (numRecycled > 0){
-		{
+			synchronized (_recycled){				
-			synchronized (_recycled)
+				if (numRecycled > _MAX_FREE){
-			{				
-				if (numRecycled > _MAX_FREE)
-				    {
 					int numRemoved = numRecycled - _MAX_FREE;
-					for (int i = 1; i <= numRemoved; i++)
+					for (int i = numRemoved - 1; i >=0; i--){
-					 {
+						_recycled.remove(i);						
-						_recycled.remove(numRemoved - i);						
 					 }
 				  }			
 				newObject = (DataElement) _recycled.remove((_recycled.size() - 1));
@@ -4076,16 +4035,16 @@ public final class DataStore
 
 	private void disconnectObjectHelper(DataElement toDisconnect, int depth)
 	{
-		if (depth > 0)
+		if (depth > 0){
-		{
 			depth--;
 			_deRemover.addToQueueForRemoval(toDisconnect);
-			for (int i = 0; i < toDisconnect.getNestedSize(); i++)
+			List nestedData = toDisconnect.getNestedData();
-			{
+			if (nestedData != null){
-				DataElement subDisconnect = toDisconnect.get(i);
+				for (int i = 0; i < nestedData.size(); i++){
-				if (subDisconnect != null && subDisconnect.getDataStore() == this && !subDisconnect.isSpirit())
+					DataElement subDisconnect = (DataElement)nestedData.get(i);
-				{
+					if (subDisconnect != null && !subDisconnect.isSpirit() && !subDisconnect.isDescriptor() && !subDisconnect.isReference()){
-					disconnectObjectHelper(subDisconnect, depth);
+						disconnectObjectHelper(subDisconnect, depth);
+					}
 				}
 			}
 		}
@@ -4105,18 +4064,8 @@ public final class DataStore
 		else
 		{
 			return generateId();
-			/*
-			String newId = String.valueOf(_random.nextInt());
-			while (_hashMap.containsKey(newId))
-			{
-				newId = String.valueOf(_random.nextInt());
-			}
-
-			return newId;
-			*/
 		}
 
-
 	}
 
 	private String generateId(DataElement parent, String type, String name)

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/DataStoreSchema.java

@@ -13,6 +13,7 @@
  * Contributors:
  * David McKnight   (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
  * David McKnight   (IBM)  - [385097] [dstore] DataStore spirit mechanism is not enabled
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  ********************************************************************************/
 
 package org.eclipse.dstore.core.model;
@@ -191,9 +192,6 @@ public class DataStoreSchema
 		// miner-specific descriptors are defined in the miners when they extend the schema
 
 		// these first elements are the most fundamental	  
-//		DataElement uiCmdD = _dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR);
-		_dataStore.createObject(schemaRoot, DE.T_UI_COMMAND_DESCRIPTOR, DE.T_UI_COMMAND_DESCRIPTOR);
-
 		_commandDescriptor = _dataStore.createCommandDescriptor(schemaRoot, DE.T_COMMAND_DESCRIPTOR);
 		_objectDescriptor = _dataStore.createObjectDescriptor(schemaRoot, DE.T_OBJECT_DESCRIPTOR);
 		_relationDescriptor = _dataStore.createRelationDescriptor(schemaRoot, DE.T_RELATION_DESCRIPTOR);

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/model/UpdateHandler.java

@@ -17,15 +17,16 @@
  * David McKnight   (IBM) - [385793] [dstore] DataStore spirit mechanism and other memory improvements needed
  * David McKnight   (IBM) - [389286] [dstore] element delete should not clear _attributes since elements get recycled
  * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.core.model;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 import org.eclipse.dstore.core.java.IRemoteClassInstance;
-import org.eclipse.dstore.internal.core.util.DataElementRemover;
 
 /**
  * <p>
@@ -75,49 +76,83 @@ public abstract class UpdateHandler extends Handler
 		clean(object, 2);
 	}
 
+	
+	private void unmap(DataElement element)
+	{	
+		if (element.isDeleted() || element.isSpirit()){
+			HashMap map = _dataStore.getHashMap();					
+			synchronized (map){
+				String id = element.getId();
+			//	_dataStore.memLog("unmapping " + id);
+				map.remove(id);
+				_dataStore.addToRecycled(element);
+			}
+		}
+	}
+	
 	protected void clean(DataElement object, int depth)
 	{
-		if ((depth > 0) && (object != null) && object.getNestedSize() > 0)
+		boolean isServer = !_dataStore.isVirtual();
-		{
+		if ((depth > 0) && object != null){
-			List deletedList = _dataStore.findDeleted(object);
+			if (object.isSpirit() || object.isDeleted()){
+				DataElement parent = object.getParent();
+				cleanChildren(object);
+				unmap(object);			
+				if (object.isSpirit() && isServer){						
+					// officially delete this now 
+					object.delete();								
+				}
+				if (isServer){
+					object.clear();
+				}
+				if (parent != null){
+					synchronized (parent){
+						parent.removeNestedData(object);
+					}
+				}
+			}
+			else if (object.getNestedSize() > 0){
+				cleanChildren(object);
+			}		
+		}
+		// remove objects under temproot
+		DataElement tmpRoot = _dataStore.getTempRoot();
+		if (tmpRoot != null){
+			tmpRoot.removeNestedData();
+		}
+	}
 
-			for (int i = 0; i < deletedList.size(); i++)
+	
-			{
+	/*
+	protected void clean(DataElement object, int depth)
+	{
+		boolean isServer = !_dataStore.isVirtual();
+		if ((depth > 0) && object != null){
+			List deletedList = _dataStore.findDeleted(object);
+			for (int i = 0; i < deletedList.size(); i++){
 				DataElement child = (DataElement) deletedList.get(i);
-				if (child != null && child.isDeleted())
+				if (child != null && child.isDeleted()){
-				{
 					DataElement parent = child.getParent();
-					DataElementRemover.addToRemovedCount();
 
 					cleanChildren(child); // clean the children
-
+					unmap(child);
-
+					if (child.isSpirit() && isServer){						
-					if (child.isSpirit())
-					{
 						// officially delete this now 
-						// will only happen on server since, on client, 
-						// the above call to isDeleted() returns false for spirited
 						child.delete();						
 					}
 
-					child.clear();
+					if (parent != null){
-					if (parent != null)
+						synchronized (parent){
-					{
-						synchronized (parent)
-						{
 							parent.removeNestedData(child);
 						}
 					}
-				//  _dataStore.addToRecycled(child);
 				}
 			}
 
 			deletedList.clear();
 		}
-		// delete objects under temproot
-		_dataStore.getTempRoot().removeNestedData();
-
 	}
+*/
 
 	/**
 	 * Recursively clean children for deletion
@@ -125,25 +160,26 @@ public abstract class UpdateHandler extends Handler
 	 */
 	protected void cleanChildren(DataElement parent)
 	{
+		boolean isServer = !_dataStore.isVirtual();
 		List nestedData = parent.getNestedData();
-		if (nestedData != null)
+		if (nestedData != null){
-		{
 			synchronized (nestedData){
 				for (int i = nestedData.size() - 1; i >= 0; i--){
 					DataElement child = (DataElement)nestedData.get(i);
-					cleanChildren(child);
+					if (child.isSpirit() || child.isDeleted()){
-	
+						cleanChildren(child);
-					if (child.isSpirit())
+						unmap(child);
-					{
+					
-						// officially delete this now
+						if (isServer){
-						child.delete();
+							// officially delete this now
+							child.delete();
+							child.clear();							
+						}
+						nestedData.remove(child);
 					}
-				
-					child.clear();
-					parent.removeNestedData(child);
 				}
 			}
-		}	
+		}
 	}
 
 	/**
@@ -172,51 +208,39 @@ public abstract class UpdateHandler extends Handler
 	 * @param object an object to get updated
 	 * @param immediate true indicates that this object should be first in the queue
 	 */
-	public void update(DataElement object, boolean immediate)
+	public void update(DataElement object, boolean immediate){
-	{
+		if (object != null){
-		synchronized (_dataObjects)
+			String type = object.getType();
-		{
+			boolean statusDone = false;
-			if (object != null){
+			boolean isStatus = DataStoreResources.model_status.equals(type);				
-				String type = object.getType();
+			if (isStatus){
-				boolean isStatus = DataStoreResources.model_status.equals(type);
+				statusDone = DataStoreResources.model_done.equals(object.getName()) || DataStoreResources.model_done.equals(object.getValue());
-			if (immediate)
-			{
-				_dataObjects.add(0, object);
-				// none of this immediate stuff - just put it at the beginning
-				//handle();
 			}
-			else 
+			synchronized (_dataObjects){
-			{				
+				if (immediate){
-				if (!_dataObjects.contains(object))
+					_dataObjects.add(0, object);
-				{
-						_dataObjects.add(object);
 				}
-				else
+				else {
-				{
+					if (!_dataObjects.contains(object)){
-
+						_dataObjects.add(object);
-					if (_dataStore != null && object != null && !object.isDeleted())
+					}
-					{
+					else {
-						if (isStatus)
+						if (_dataStore != null && object != null && !object.isDeleted()){
-						{
+							if (isStatus && statusDone){
-							if (object.getName().equals(DataStoreResources.model_done))
-							{
-								//DKM
 								// move to the back of the queue
 								// this is done so that if status that was already queued changed to done in between
 								// requests, and had not yet been transferred over comm layer, the completed status
 								// object does not come back to client (as "done") before the results of a query
 								_dataObjects.remove(object);
-								_dataObjects.add(object);
+								_dataObjects.add(object);											
-								
 							}
 						}
 					}
 				}
 			}
-				if (_dataStore != null && !_dataStore.isVirtual() && isStatus){
+			if (_dataStore != null && !_dataStore.isVirtual() && isStatus && statusDone){
-					_dataStore.disconnectObjects(object); // spirit the status
+				_dataStore.disconnectObject(object.getParent()); // spirit the command and its children
-				}
+			//	_dataStore.disconnectObject(object);
-			
 			}
 		}
 		notifyInput();

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/core/util/CommandGenerator.java

@@ -15,6 +15,7 @@
  * David McKnight   (IBM) - [225507][api][breaking] RSE dstore API leaks non-API types
  * David McKnight   (IBM) - [226561] [apidoc] Add API markup to RSE Javadocs where extend / implement is allowed
  * David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.core.util;
@@ -43,7 +44,6 @@ public class CommandGenerator
     private DataStore _dataStore = null;
     private DataElement _log = null;
 
-    static private int _id = 0;
 
 	/**
 	 * Constructor
@@ -143,7 +143,7 @@ public class CommandGenerator
 	 */
     public DataElement generateCommand(DataElement commandDescriptor, ArrayList arguments, DataElement dataObject, boolean refArg)
     {
-
+    	//refArg = false;
         DataElement commandObject = createCommand(commandDescriptor);
         if (commandObject != null)
         {
@@ -151,7 +151,7 @@ public class CommandGenerator
 
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 
-            if (dataObject.isUpdated() && !dataObject.isSpirit())
+            if (refArg && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
             }
@@ -175,7 +175,7 @@ public class CommandGenerator
                     DataElement arg = (DataElement) arguments.get(i);
                     if (arg != null)
                     {
-                        if (!arg.isUpdated() || arg.isSpirit())
+                        if (!arg.isUpdated() || arg.isSpirit() || !refArg)
                         {
                             commandObject.addNestedData(arg, false);
                         }
@@ -207,14 +207,14 @@ public class CommandGenerator
 	 */
     public DataElement generateCommand(DataElement commandDescriptor, DataElement arg, DataElement dataObject, boolean refArg)
     {
-        _id++;
+        //refArg = false;
 
         DataElement commandObject = createCommand(commandDescriptor);
         if (commandObject != null)
         {
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 			clearDeleted(dataObject);
-            if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
+            if (refArg && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_contents);
             }
@@ -230,7 +230,7 @@ public class CommandGenerator
                 commandObject.addNestedData(dataObject, false);
             }
 
-            if (!arg.isUpdated() || arg.isSpirit())
+            if (!arg.isUpdated() || arg.isSpirit() || !refArg)
             {
                 commandObject.addNestedData(arg, false);
             }
@@ -258,15 +258,14 @@ public class CommandGenerator
 	 */
     public DataElement generateCommand(DataElement commandDescriptor, DataElement dataObject, boolean refArg)
     {
-    	_id++;
+    	//refArg = false;
-
         DataElement commandObject = createCommand(commandDescriptor);
         if (commandObject != null)
         {
             commandObject.setAttribute(DE.A_VALUE, commandDescriptor.getName());
 
 			clearDeleted(dataObject);
-            if ((refArg || dataObject.isUpdated()) && !dataObject.isSpirit())
+            if (refArg && !dataObject.isSpirit())
             {
                 _dataStore.createReference(commandObject, dataObject,DataStoreResources.model_arguments);
             }

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/server/ServerCommandHandler.java

@@ -382,6 +382,7 @@ public class ServerCommandHandler extends CommandHandler
 				String property = dataObject.getName();
 				String value = dataObject.getValue();
 				_dataStore.setPreference(property, value);
+				status.setAttribute(DE.A_NAME,DataStoreResources.model_done);
 			}
 			else if (commandName.equals(DataStoreSchema.C_QUERY_INSTALL))
 			{

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/DataElementRemover.java

@@ -19,26 +19,41 @@
  *  David McKnight   (IBM) - [331922] [dstore] enable DataElement recycling
  *  David McKnight   (IBM) - [371401] [dstore][multithread] avoid use of static variables - causes memory leak after disconnect
  *  David McKnight   (IBM)  - [373507] [dstore][multithread] reduce heap memory on disconnect for server
- *  David McKnight   (IBM)  - [385097] [dstore] DataStore spirit mechanism is not enabled
+ *  David McKnight   (IBM) - [385097] [dstore] DataStore spirit mechanism is not enabled
  *  David McKnight   (IBM) - [390037] [dstore] Duplicated items in the System view
+ *  David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.internal.core.util;
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.LinkedList;
 
 import org.eclipse.dstore.core.model.DataElement;
 import org.eclipse.dstore.core.model.DataStore;
+import org.eclipse.dstore.core.model.DataStoreResources;
 import org.eclipse.dstore.core.model.Handler;
 
 
 public class DataElementRemover extends Handler 
-{
+{	
-	private LinkedList _queue;
+	protected class QueueItem
-	private static int numDisconnected = 0;
+	{
-	//private int numGCed = 0;
+		public DataElement dataElement;
+		public long timeStamp;
+		
+		public QueueItem(DataElement element, long stamp)
+		{
+			dataElement = element;
+			timeStamp = stamp;
+		}
+		
+		public boolean equals(QueueItem item){
+			return item.dataElement == dataElement;
+		}
+	}
+
+	private ArrayList _queue;
 	
 	// The following determine how DataElements are chosen to be removed once they
 	// are in the queue for removal. 	
@@ -52,12 +67,19 @@ public class DataElementRemover extends Handler
 	public static final String INTERVAL_TIME_PROPERTY_NAME = "SPIRIT_INTERVAL_TIME"; //$NON-NLS-1$
 	public MemoryManager _memoryManager;
 	
+	private int _lastLive = 0;
+	private int _lastFree = 0;
+	private long _lastMem = 0;
+	
+	private boolean DEBUG = false; // extra tracing of hashmap when on
+	private long _lastDumpTime = System.currentTimeMillis();
+	
 	public DataElementRemover(DataStore dataStore)
 	{
 		super();
 		_memoryManager = new MemoryManager(dataStore);
 		_dataStore = dataStore;
-		_queue = new LinkedList();
+		_queue = new ArrayList();
 		getTimes();
 		setWaitTime(_intervalTime);
 	}
@@ -84,41 +106,28 @@ public class DataElementRemover extends Handler
 		}
 	}
 	
-	public static void addToRemovedCount()
-	{
-		// not using this anymore - better to get this from DataStore
-	}
-	
-	public static void addToCreatedCount()
-	{
-		// not using this anymore - better to get this from DataStore
-	}
-	
-	public static void addToGCedCount()
-	{
-	}
 
 	
-	public synchronized void addToQueueForRemoval(DataElement element)
+	public synchronized void addToQueueForRemoval(DataElement element){
-	{
+		if(isMemoryThresholdExceeded()) {
-		synchronized (_queue) 
+			// do immediate clearing of queue since we're low on memory
-		{
+			clearQueue(true);
-			if(isMemoryThresholdExceeded()) {
+			return;
-				if(element.isSpirit()) {
+		}
-					unmap(element);
+		if (_dataStore.isDoSpirit() &&
+				!element.isReference() &&
+				!element.isSpirit() &&
+				!element.isDescriptor() && 
+				!element.isDeleted()){
+			//_dataStore.memLog("queuing " + element);
+			QueueItem item = new QueueItem(element, System.currentTimeMillis());			
+			if (!_queue.contains(item)){
+				synchronized (_queue){
+					_queue.add(item);				
 				}
-				
+				notifyInput();
-				// do immediate clearing of queue since we're low on memory
-				clearQueue(true);
-				return;
-			}
-			if (_dataStore.isDoSpirit() && _dataStore == element.getDataStore())
-			{
-				QueueItem item = new QueueItem(element, System.currentTimeMillis());
-				_queue.add(item);
 			}
 		}
-		notifyInput();
 	}
 
 	private boolean isMemoryThresholdExceeded(){
@@ -130,93 +139,79 @@ public class DataElementRemover extends Handler
 		clearQueue(false);
 	}
 	
-	public synchronized void clearQueue(boolean force)
+	private void logMemory(){
-	{
+		long mem = Runtime.getRuntime().totalMemory();
-		synchronized (_queue)
+		int liveElements = _dataStore.getNumElements();
-		{
+		int freeElements = _dataStore.getNumRecycled();
-			_dataStore.memLog("           "); //$NON-NLS-1$
-			int disconnected = 0;
-			if (!_dataStore.isDoSpirit())
-			{
-				if (_queue.size() > 0) 
-				{
-					_dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$
-					_queue.clear();
-				}
-				_dataStore.memLog("Total heap size: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$
-				_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
-				_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
-				_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
-
-
-				// no longer a helpful stat since we no longer use finalize
-				// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
-				return;
-			}
-			_dataStore.memLog("Total heap size before disconnection: " + Runtime.getRuntime().totalMemory()); //$NON-NLS-1$
-			
-			_dataStore.memLog("Size of queue: " + _queue.size()); //$NON-NLS-1$
-			
-			ArrayList toRefresh = new ArrayList();
-			while (_queue.size() > 0 && (force || System.currentTimeMillis() - ((QueueItem) _queue.getFirst()).timeStamp > _expiryTime))
-			{
-				DataElement toBeDisconnected = ((QueueItem) _queue.removeFirst()).dataElement;
-				if (!toBeDisconnected.isSpirit()) 
-				{
-					toBeDisconnected.setSpirit(true);
-					toBeDisconnected.setUpdated(false);
-					DataElement parent = toBeDisconnected.getParent();
-					if (!toRefresh.contains(parent))
-					{
-						//System.out.println("disconnect parent:"+parent.getName());
-						toRefresh.add(toBeDisconnected.getParent());
-					}
-						//_dataStore.refresh(toBeDisconnected);
-					disconnected++;
-					numDisconnected++;
-				}
-				else
-				{
-					//_dataStore.memLog(toBeDisconnected.toString());
-				}
-				unmap(toBeDisconnected);
-			}
-	
-			_dataStore.refresh(toRefresh);
-			
-			_dataStore.memLog("Disconnected " + disconnected + " DataElements."); //$NON-NLS-1$ //$NON-NLS-2$
-			_dataStore.memLog("Live elements: " + _dataStore.getNumElements()); //$NON-NLS-1$
-			_dataStore.memLog("Recycled elements: " + _dataStore.getNumRecycled()); //$NON-NLS-1$
-			_dataStore.memLog("Elements disconnected so far: " + numDisconnected); //$NON-NLS-1$
-			
-			// no longer a helpful stat since we no longer use finalize
-			// _dataStore.memLog("DataElements GCed so far: " + numGCed); //$NON-NLS-1$
-			System.gc();
-		}
-	}
-	
-	private void unmap(DataElement element)
-	{	 
-		HashMap map = _dataStore.getHashMap();					
-		synchronized (map){
-			map.remove(element.getId());
-			_dataStore.addToRecycled(element);
-		}
-	}
-	
-	
-	protected class QueueItem
-	{
-		public DataElement dataElement;
-		public long timeStamp;
 		
-		public QueueItem(DataElement element, long stamp)
+		if (mem != _lastMem || liveElements != _lastLive || freeElements != _lastFree){
-		{
+			_dataStore.memLog("                                        "); //$NON-NLS-1$
-			dataElement = element;
+			_dataStore.memLog("Total heap size: " + mem); //$NON-NLS-1$
-			timeStamp = stamp;
+			_dataStore.memLog("Number of live DataStore elements: " + liveElements);  //$NON-NLS-1$
+			_dataStore.memLog("Number of free DataStore elements: " + freeElements);  //$NON-NLS-1$
+			
+			_lastMem = mem;
+			_lastLive = liveElements;
+			_lastFree = freeElements;
 		}
 	}
 	
+	public synchronized void clearQueue(boolean force){
+		if (!_dataStore.isDoSpirit()){ // spiriting disabled
+			if (_queue.size() > 0) {
+				_dataStore.memLog("Clearing queue of size " + _queue.size() + ". DSTORE_SPIRIT_ON not set or set to false."); //$NON-NLS-1$ //$NON-NLS-2$
+				synchronized (_queue){
+					_queue.clear();		
+				}
+			}				
+			logMemory();				
+			return;
+		}
+		else { // spiriting enabled
+			if (_queue.size() > 0){
+				int queueSize = _queue.size();
+
+				ArrayList toRefresh = new ArrayList();
+				long currentTime = System.currentTimeMillis();
+				for (int i = queueSize - 1; i >= 0; i--){
+					QueueItem qitem = null;
+					synchronized (_queue){
+						qitem = (QueueItem)_queue.get(i);
+					}
+					long deltaTime = currentTime - qitem.timeStamp;
+					if (force || (deltaTime > _expiryTime)){
+						DataElement toBeDisconnected = qitem.dataElement;
+						toBeDisconnected.setSpirit(true);
+						toBeDisconnected.setUpdated(false);
+						DataElement parent = toBeDisconnected.getParent();
+						if (!toRefresh.contains(parent)){
+							toRefresh.add(parent);
+						}
+						synchronized (_queue){ // if spirited, dequeue
+							_queue.remove(i);
+						}
+					}
+				}
+				if (!toRefresh.isEmpty()){
+					// refresh parents of spirited items
+					_dataStore.refresh(toRefresh);
+					System.gc();
+				}
+				// print dump of elements on interval
+				if (DEBUG){
+					if (currentTime - _lastDumpTime > 100000){
+						_lastDumpTime = currentTime;
+						printHashmap();											
+					}
+				}
+				
+				logMemory();															
+			}
+		}
+	}
+	
+
+
 	/**
 	 * Runs the handler loop in a thread.
 	 */
@@ -238,5 +233,38 @@ public class DataElementRemover extends Handler
 			handle();
 		}
 	}
+	
+	// just used for tracing 
+	private void printHashmap(){
+		_dataStore.memLog("                                        "); //$NON-NLS-1$	
+		_dataStore.memLog("------------------------------Current Hashmap--------------------------------:"); //$NON-NLS-1$	
+		HashMap map = _dataStore.getHashMap();
+		synchronized (map){
+			DataElement[] elements = (DataElement[])map.values().toArray(new DataElement[map.size()]);
+			for (int i = 0; i < elements.length; i++){
+				DataElement element = elements[i];
+				if (!element.isDescriptor()){
+					String type = element.getType();
+					if (type.equals(DataStoreResources.model_abstracted_by) || 
+							type.equals(DataStoreResources.model_abstracts) ||
+							type.equals("Environment Variable") ||
+							type.equals("system.property")){
+						// schema and environment stuff
+					}
+					else {
+						if (type.equals(DataStoreResources.model_status)){
+							String value = element.getValue();
+							DataElement parent = element.getParent();
+							_dataStore.memLog("Command: " + parent.getName() + " is " + value);							
+						}		
+						else {
+							_dataStore.memLog(element.toString());
+						}
+					}
+				}
+			}
+		}		
+		_dataStore.memLog("-----------------------------------------------------------------------------:"); //$NON-NLS-1$	
+	}
 }
 

rse/plugins/org.eclipse.dstore.core/src/org/eclipse/dstore/internal/core/util/XMLparser.java

@@ -28,6 +28,7 @@
  * David McKnight   (IBM)  - [367449] [dstore] allow custom encoding for data transport layer
  * David McKnight     (IBM) - [378136][dstore] miner.finish is stuck
  * David McKnight   (IBM)   - [391966][dstore][performance] unnecessary call slows down large queries
+ * David McKnight   (IBM) - [396440] [dstore] fix issues with the spiriting mechanism and other memory improvements (phase 1)
  *******************************************************************************/
 
 package org.eclipse.dstore.internal.core.util;
@@ -881,6 +882,7 @@ public class XMLparser
 			if (attributes.length == DE.A_SIZE)
 			{
 				String type = attributes[DE.A_TYPE];
+				String id = attributes[DE.A_ID];
 				if (type.equals(DataStoreResources.KEEPALIVE_TYPE))
 				{
 					_isKeepAlive= true;
@@ -893,8 +895,7 @@ public class XMLparser
 				}
 				
 				else if (type.equals(DataStoreResources.DOCUMENT_TYPE))
-				{
+				{					
-					String id = attributes[DE.A_ID];
 					if (_dataStore.contains(id))
 					{
 						result = _dataStore.find(id);
@@ -905,7 +906,12 @@ public class XMLparser
 						result = _dataStore.createObject(null, attributes);
 					}
 				}
-
+				else if (id.equals(DataStoreResources.model_transient)){ // generic transient
+					result = _dataStore.createTransientObject(attributes);
+					if (parent != null){
+						parent.addNestedData(result, false);
+					}
+				}			
 				else if (_isFile || _isClass || _isSerialized || parent == null)
 				{
 					result = _dataStore.createTransientObject(attributes);
@@ -934,13 +940,6 @@ public class XMLparser
 					}
 					else
 					{
-						String id = attributes[DE.A_ID];
-						if (id == null)
-						{
-							handlePanic(new Exception(fullTag));
-							return null;
-						}
-	
 						if (_dataStore.contains(id))
 						{
 							result = _dataStore.find(id);
@@ -963,7 +962,7 @@ public class XMLparser
 								if (isSpirit)
 								{
 									if (!_dataStore.isVirtual()) attributes[DE.A_REF_TYPE] = DataStoreResources.VALUE;
-									result.setSpirit(_dataStore.isVirtual());
+									result.setSpirit(_dataStore.isVirtual()); // if this is the server, then the element is unspirited
 								}
 								else
 								{