Reducing the size of the index, bug 210392.

core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/pdom/tests/DBTest.java

@@ -64,14 +64,16 @@ public class DBTest extends BaseTestCase {
 		assertEquals(0, db.getVersion());
 
 		final int realsize = 42;
-		final int blocksize = (realsize / Database.MIN_SIZE + 1) * Database.MIN_SIZE;
+		final int deltas = (realsize+Database.BLOCK_HEADER_SIZE + Database.BLOCK_SIZE_DELTA - 1) / Database.BLOCK_SIZE_DELTA;
+		final int blocksize = deltas * Database.BLOCK_SIZE_DELTA;
+		final int freeDeltas= Database.CHUNK_SIZE/Database.BLOCK_SIZE_DELTA-deltas;
 		
 		int mem = db.malloc(realsize);
-		assertEquals(-blocksize, db.getInt(mem - Database.INT_SIZE));
+		assertEquals(-blocksize, db.getShort(mem - Database.BLOCK_HEADER_SIZE));
 		db.free(mem);
-		assertEquals(blocksize, db.getInt(mem - Database.INT_SIZE));
+		assertEquals(blocksize, db.getShort(mem - Database.BLOCK_HEADER_SIZE));
-		assertEquals(mem - Database.INT_SIZE, db.getInt((blocksize / Database.MIN_SIZE) * Database.INT_SIZE));
+		assertEquals(mem - Database.BLOCK_HEADER_SIZE, db.getInt((deltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
-		assertEquals(mem - Database.INT_SIZE + blocksize, db.getInt(((Database.CHUNK_SIZE - blocksize) / Database.MIN_SIZE) * Database.INT_SIZE));
+		assertEquals(mem - Database.BLOCK_HEADER_SIZE + blocksize, db.getInt((freeDeltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
 	}
 
 	public void testBug192437() throws IOException {
@@ -100,16 +102,18 @@ public class DBTest extends BaseTestCase {
 	
 	public void testFreeBlockLinking() throws Exception {
 		final int realsize = 42;
-		final int blocksize = (realsize / Database.MIN_SIZE + 1) * Database.MIN_SIZE;
+		final int deltas = (realsize+Database.BLOCK_HEADER_SIZE + Database.BLOCK_SIZE_DELTA - 1) / Database.BLOCK_SIZE_DELTA;
+		final int blocksize = deltas * Database.BLOCK_SIZE_DELTA;
+		final int freeDeltas= Database.MIN_BLOCK_DELTAS-deltas;
 
 		int mem1 = db.malloc(realsize);
 		int mem2 = db.malloc(realsize);
 		db.free(mem1);
 		db.free(mem2);
-		assertEquals(mem2 - Database.INT_SIZE, db.getInt((blocksize / Database.MIN_SIZE) * Database.INT_SIZE));
+		assertEquals(mem2 - Database.BLOCK_HEADER_SIZE, db.getInt((deltas-Database.MIN_BLOCK_DELTAS+1) * Database.INT_SIZE));
 		assertEquals(0, db.getInt(mem2));
-		assertEquals(mem1 - Database.INT_SIZE, db.getInt(mem2 + Database.INT_SIZE));
+		assertEquals(mem1 - Database.BLOCK_HEADER_SIZE, db.getInt(mem2 + Database.INT_SIZE));
-		assertEquals(mem2 - Database.INT_SIZE, db.getInt(mem1));
+		assertEquals(mem2 - Database.BLOCK_HEADER_SIZE, db.getInt(mem1));
 		assertEquals(0, db.getInt(mem1 + Database.INT_SIZE));
 	}
 	

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/PDOM.java

@@ -153,6 +153,7 @@ public class PDOM extends PlatformObject implements IPDOM {
 	 *  51 - modeling extern "C" (bug 191989)
 	 *  52 - files per linkage (bug 191989)
 	 *  53 - polymorphic method calls (bug 156691)
+	 *  54 - optimization of database size (bug 210392)
 	 */
 	
 	public static final int LINKAGES = Database.DATA_AREA;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Chunk.java

@@ -6,9 +6,9 @@
  * http://www.eclipse.org/legal/epl-v10.html
  *
  * Contributors:
- * QNX - Initial API and implementation
+ *    QNX - Initial API and implementation
- * Markus Schorn (Wind River Systems)
+ *    Markus Schorn (Wind River Systems)
- * IBM Corporation
+ *    IBM Corporation
  *******************************************************************************/
 package org.eclipse.cdt.internal.core.pdom.db;
 
@@ -95,6 +95,22 @@ final class Chunk {
 			((fBuffer[++idx] & 0xff) <<  0);
 	}
 
+	public void put3ByteUnsignedInt(final int offset, final int value) {
+		assert fLocked;
+		fDirty= true;
+		int idx= offset % Database.CHUNK_SIZE;
+		fBuffer[idx]= (byte)(value >> 16);
+		fBuffer[++idx]= (byte)(value >> 8);
+		fBuffer[++idx]= (byte)(value);
+	}
+	
+	public int get3ByteUnsignedInt(final int offset) {
+		int idx= offset % Database.CHUNK_SIZE;
+		return ((fBuffer[idx] & 0xff) << 16) |
+			((fBuffer[++idx] & 0xff) <<  8) |
+			((fBuffer[++idx] & 0xff) <<  0);
+	}
+
 	public void putShort(final int offset, final short value) {
 		assert fLocked;
 		fDirty= true;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/Database.java

@@ -42,12 +42,12 @@ import org.eclipse.core.runtime.Status;
  * offset            content
  * 	                 _____________________________
  * 0                | version number
- * INT_SIZE         | pointer to head of linked list of blocks of size MIN_SIZE
+ * INT_SIZE         | pointer to head of linked list of blocks of size MIN_BLOCK_DELTAS*BLOCK_SIZE_DELTA
  * ..               | ...
- * INT_SIZE * m (1) | pointer to head of linked list of blocks of size MIN_SIZE * m
+ * INT_SIZE * m (1) | pointer to head of linked list of blocks of size (m+MIN_BLOCK_DELTAS) * BLOCK_SIZE_DELTA 
  * DATA_AREA        | undefined (PDOM stores its own house-keeping data in this area) 
  * 
- * (1) where m <= (CHUNK_SIZE / MIN_SIZE)
+ * (1) where 2 <= m <= CHUNK_SIZE/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1
  * 
  * ===== block structure
  * 
@@ -59,7 +59,21 @@ import org.eclipse.core.runtime.Status;
  * 
  */
 public class Database {
+	// public for tests only, you shouldn't need these
+	public static final int INT_SIZE = 4;
+	public static final int CHUNK_SIZE = 1024 * 4;
+	public static final int BLOCK_HEADER_SIZE= 2;
+	public static final int BLOCK_SIZE_DELTA= 8;
+	public static final int MIN_BLOCK_DELTAS = 2;	// a block must at least be 2 + 2*4 bytes to link the free blocks.
+	public static final int MAX_BLOCK_DELTAS = CHUNK_SIZE/BLOCK_SIZE_DELTA;	
+	public static final int MAX_MALLOC_SIZE = MAX_BLOCK_DELTAS*BLOCK_SIZE_DELTA - BLOCK_HEADER_SIZE;  
 
+	public static final int VERSION_OFFSET = 0;
+	public static final int DATA_AREA = (CHUNK_SIZE / BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 2) * INT_SIZE;
+	
+	private static final int BLOCK_PREV_OFFSET = BLOCK_HEADER_SIZE;
+	private static final int BLOCK_NEXT_OFFSET = BLOCK_HEADER_SIZE + INT_SIZE;
+	
 	private final File fLocation;
 	private final RandomAccessFile fFile;
 	private boolean fExclusiveLock= false;	// necessary for any write operation
@@ -76,18 +90,6 @@ public class Database {
 	private long cacheHits;
 	private long cacheMisses;
 	
-	// public for tests only, you shouldn't need these
-	public static final int VERSION_OFFSET = 0;
-	public static final int CHUNK_SIZE = 1024 * 4;
-	public static final int MIN_SIZE = 16;
-	public static final int INT_SIZE = 4;
-	public static final int CHAR_SIZE = 2;
-	public static final int PREV_OFFSET = INT_SIZE;
-	public static final int NEXT_OFFSET = INT_SIZE * 2;
-	public static final int DATA_AREA = CHUNK_SIZE / MIN_SIZE * INT_SIZE + INT_SIZE;
-	
-	public static final int MAX_SIZE = CHUNK_SIZE - 4; // Room for overhead
-	
 	/**
 	 * Construct a new Database object, creating a backing file if necessary.
 	 * @param location the local file path for the database 
@@ -210,25 +212,25 @@ public class Database {
 	 * @param size
 	 * @return
 	 */ 
-	public int malloc(int size) throws CoreException {
+	public int malloc(final int datasize) throws CoreException {
 		assert fExclusiveLock;
-		if (size > MAX_SIZE)
+		if (datasize < 0 || datasize > MAX_MALLOC_SIZE)
 			// Too Big
 			throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0,
 					CCorePlugin.getResourceString("pdom.requestTooLarge"), new IllegalArgumentException())); //$NON-NLS-1$
 		
+		int needDeltas= (datasize + BLOCK_HEADER_SIZE + BLOCK_SIZE_DELTA - 1) / BLOCK_SIZE_DELTA;
+		if (needDeltas < MIN_BLOCK_DELTAS) {
+			needDeltas= MIN_BLOCK_DELTAS;
+		}
+
 		// Which block size
 		int freeblock = 0;
-		int blocksize;
+		int useDeltas;
-		int matchsize = 0;
+		for (useDeltas= needDeltas; useDeltas <= MAX_BLOCK_DELTAS; useDeltas++) {
-		for (blocksize = MIN_SIZE; blocksize <= CHUNK_SIZE; blocksize += MIN_SIZE) {
+			freeblock = getFirstBlock(useDeltas*BLOCK_SIZE_DELTA);
-			if (blocksize - INT_SIZE >= size) {
+			if (freeblock != 0)
-				if (matchsize == 0) // our real size
+				break;
-					matchsize = blocksize;
-				freeblock = getFirstBlock(blocksize);
-				if (freeblock != 0)
-					break;
-			}
 		}
 		
 		// get the block
@@ -236,26 +238,29 @@ public class Database {
 		if (freeblock == 0) {
 			// allocate a new chunk
 			freeblock= createNewChunk();
-			blocksize = CHUNK_SIZE;
+			useDeltas = MAX_BLOCK_DELTAS;
 			chunk = getChunk(freeblock);
 		} else {
 			chunk = getChunk(freeblock);
-			removeBlock(chunk, blocksize, freeblock);
+			removeBlock(chunk, useDeltas*BLOCK_SIZE_DELTA, freeblock);
 		}
  
-		if (blocksize != matchsize) {
+		final int unusedDeltas = useDeltas-needDeltas;
+		if (unusedDeltas >= MIN_BLOCK_DELTAS) {
 			// Add in the unused part of our block
-			addBlock(chunk, blocksize - matchsize, freeblock + matchsize);
+			addBlock(chunk, unusedDeltas*BLOCK_SIZE_DELTA, freeblock + needDeltas*BLOCK_SIZE_DELTA);
+			useDeltas= needDeltas;
 		}
 		
 		// Make our size negative to show in use
-		chunk.putInt(freeblock, - matchsize);
+		final int usedSize= useDeltas*BLOCK_SIZE_DELTA;
+		chunk.putShort(freeblock, (short) -usedSize);
 
 		// Clear out the block, lots of people are expecting this
-		chunk.clear(freeblock + 4, size);
+		chunk.clear(freeblock + BLOCK_HEADER_SIZE, usedSize-BLOCK_HEADER_SIZE);
 
-		malloced += matchsize;
+		malloced+= usedSize;
-		return freeblock + 4;
+		return freeblock + BLOCK_HEADER_SIZE;
 	}
 	
 	private int createNewChunk() throws CoreException {
@@ -276,38 +281,38 @@ public class Database {
 	
 	private int getFirstBlock(int blocksize) throws CoreException {
 		assert fLocked;
-		return fHeaderChunk.getInt((blocksize / MIN_SIZE) * INT_SIZE);
+		return fHeaderChunk.getInt((blocksize/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1) * INT_SIZE);
 	}
 	
 	private void setFirstBlock(int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
-		fHeaderChunk.putInt((blocksize / MIN_SIZE) * INT_SIZE, block);
+		fHeaderChunk.putInt((blocksize/BLOCK_SIZE_DELTA - MIN_BLOCK_DELTAS + 1) * INT_SIZE, block);
 	}
 	
 	private void removeBlock(Chunk chunk, int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
-		int prevblock = chunk.getInt(block + PREV_OFFSET);
+		int prevblock = chunk.getInt(block + BLOCK_PREV_OFFSET);
-		int nextblock = chunk.getInt(block + NEXT_OFFSET);
+		int nextblock = chunk.getInt(block + BLOCK_NEXT_OFFSET);
 		if (prevblock != 0)
-			putInt(prevblock + NEXT_OFFSET, nextblock);
+			putInt(prevblock + BLOCK_NEXT_OFFSET, nextblock);
 		else // we were the head
 			setFirstBlock(blocksize, nextblock);
 			
 		if (nextblock != 0)
-			putInt(nextblock + PREV_OFFSET, prevblock);
+			putInt(nextblock + BLOCK_PREV_OFFSET, prevblock);
 	}
 	
 	private void addBlock(Chunk chunk, int blocksize, int block) throws CoreException {
 		assert fExclusiveLock;
 		// Mark our size
-		chunk.putInt(block, blocksize);
+		chunk.putShort(block, (short) blocksize);
 
 		// Add us to the head of the list
 		int prevfirst = getFirstBlock(blocksize);
-		chunk.putInt(block + PREV_OFFSET, 0);
+		chunk.putInt(block + BLOCK_PREV_OFFSET, 0);
-		chunk.putInt(block + NEXT_OFFSET, prevfirst);
+		chunk.putInt(block + BLOCK_NEXT_OFFSET, prevfirst);
 		if (prevfirst != 0)
-			putInt(prevfirst + PREV_OFFSET, block);
+			putInt(prevfirst + BLOCK_PREV_OFFSET, block);
 		setFirstBlock(blocksize, block);
 	}
 	
@@ -319,9 +324,9 @@ public class Database {
 	public void free(int offset) throws CoreException {
 		assert fExclusiveLock;
 		// TODO - look for opportunities to merge blocks
-		int block = offset - 4;
+		int block = offset - BLOCK_HEADER_SIZE;
 		Chunk chunk = getChunk(block);
-		int blocksize = - chunk.getInt(block);
+		int blocksize = - chunk.getShort(block);
 		if (blocksize < 0)
 			// already freed
 			throw new CoreException(new Status(IStatus.ERROR, CCorePlugin.PLUGIN_ID, 0, "Already Freed", new Exception())); //$NON-NLS-1$
@@ -345,6 +350,14 @@ public class Database {
 		return getChunk(offset).getInt(offset);
 	}
 
+	public void put3ByteUnsignedInt(int offset, int value) throws CoreException {
+		getChunk(offset).put3ByteUnsignedInt(offset, value);
+	}
+	
+	public int get3ByteUnsignedInt(int offset) throws CoreException {
+		return getChunk(offset).get3ByteUnsignedInt(offset);
+	}
+	
 	public void putShort(int offset, short value) throws CoreException {
 		getChunk(offset).putShort(offset, value);
 	}
@@ -400,12 +413,12 @@ public class Database {
 		System.out.println("free'd: " + freed); //$NON-NLS-1$
 		System.out.println("wasted: " + (fChunks.length * CHUNK_SIZE - (malloced - freed))); //$NON-NLS-1$
 		System.out.println("Free blocks"); //$NON-NLS-1$
-		for (int bs = MIN_SIZE; bs <= CHUNK_SIZE; bs += MIN_SIZE) {
+		for (int bs = MIN_BLOCK_DELTAS*BLOCK_SIZE_DELTA; bs <= CHUNK_SIZE; bs += BLOCK_SIZE_DELTA) {
 			int count = 0;
 			int block = getFirstBlock(bs);
 			while (block != 0) {
 				++count;
-				block = getInt(block + NEXT_OFFSET);
+				block = getInt(block + BLOCK_NEXT_OFFSET);
 			}
 			if (count != 0)
 				System.out.println("Block size: " + bs + "=" + count); //$NON-NLS-1$ //$NON-NLS-2$

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/LongString.java

@@ -36,13 +36,13 @@ public class LongString implements IString {
 	private static final int NEXT1 = 4;
 	private static final int CHARS1 = 8;
 	
-	private static final int NUM_CHARS1 = (Database.MAX_SIZE - CHARS1) / 2;
+	private static final int NUM_CHARS1 = (Database.MAX_MALLOC_SIZE - CHARS1) / 2;
 	
 	// Additional fields of subsequent records
 	private static final int NEXTN = 0;
 	private static final int CHARSN = 4;
 	
-	private static final int NUM_CHARSN = (Database.MAX_SIZE - CHARSN) / 2;
+	private static final int NUM_CHARSN = (Database.MAX_MALLOC_SIZE - CHARSN) / 2;
 	
 	public LongString(Database db, int record) {
 		this.db = db;
@@ -55,7 +55,7 @@ public class LongString implements IString {
 
 	private int createString(int length, IWriter writer) throws CoreException {
 		// write the first record
-		int firstRecord = db.malloc(Database.MAX_SIZE);
+		int firstRecord = db.malloc(Database.MAX_MALLOC_SIZE);
 		int start = 0;
 		db.putInt(firstRecord, length);
 		writer.writeChars(start, NUM_CHARS1, firstRecord + CHARS1);
@@ -64,7 +64,7 @@ public class LongString implements IString {
 		int lastNext = firstRecord + NEXT1;
 		start += NUM_CHARS1;
 		while (length - start > NUM_CHARSN) {
-			int nextRecord = db.malloc(Database.MAX_SIZE);
+			int nextRecord = db.malloc(Database.MAX_MALLOC_SIZE);
 			db.putInt(lastNext, nextRecord);
 			writer.writeChars(start, NUM_CHARSN, nextRecord + CHARSN);
 			start += NUM_CHARSN;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/db/ShortString.java

@@ -30,7 +30,7 @@ public class ShortString implements IString {
 	private static final int LENGTH = 0;
 	private static final int CHARS = 4;
 	
-	public static final int MAX_LENGTH = (Database.MAX_SIZE - CHARS) / 2;
+	public static final int MAX_LENGTH = (Database.MAX_MALLOC_SIZE - CHARS) / 2;
 	
 	public ShortString(Database db, int offset) {
 		this.db = db;

core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMName.java

@@ -41,11 +41,11 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 	private static final int BINDING_REC_OFFSET  = 12;
 	private static final int BINDING_PREV_OFFSET = 16;
 	private static final int BINDING_NEXT_OFFSET = 20;
-	private static final int NODE_OFFSET_OFFSET  = 24;
+	private static final int NODE_OFFSET_OFFSET  = 24; // 3-byte unsigned int (sufficient for files <= 16mb)
-	private static final int NODE_LENGTH_OFFSET  = 28; 
+	private static final int NODE_LENGTH_OFFSET  = 27; // short (sufficient for names <= 32k)
-	private static final int FLAGS 				 = 32; 
+	private static final int FLAGS 				 = 29; 
 
-	private static final int RECORD_SIZE = 36;	// actual memory usage is the same from 28 - 44
+	private static final int RECORD_SIZE = 30;	// 30 yields a 32-byte block. (31 would trigger a 40-byte block)
 
 	public static final int IS_DECLARATION = 1;
 	public static final int IS_DEFINITION = 2;
@@ -71,7 +71,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 			flags = IS_REFERENCE;
 		
 		flags |= binding.getAdditionalNameFlags(flags, name);
-		db.putInt(record + FLAGS, flags);
+		db.putByte(record + FLAGS, (byte) flags);
 
 		// Hook us up to the binding
 		if (binding != null) {
@@ -97,8 +97,8 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 
 		// Record our location in the file
 		IASTFileLocation fileloc = name.getFileLocation();
-		db.putInt(record + NODE_OFFSET_OFFSET, fileloc.getNodeOffset());
+		db.put3ByteUnsignedInt(record + NODE_OFFSET_OFFSET, fileloc.getNodeOffset());
-		db.putInt(record + NODE_LENGTH_OFFSET, fileloc.getNodeLength());
+		db.putShort(record + NODE_LENGTH_OFFSET, (short) fileloc.getNodeLength());
 	}
 	
 	public PDOMName(PDOM pdom, int nameRecord) {
@@ -201,16 +201,16 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 	}
 	
 	private int getFlags(int mask) throws CoreException {
-		return pdom.getDB().getInt(record + FLAGS) & mask;
+		return pdom.getDB().getByte(record + FLAGS) & mask;
 	}
 
 	public void setIsBaseSpecifier(boolean val) throws CoreException {
-		int flags= pdom.getDB().getInt(record + FLAGS);
+		int flags= pdom.getDB().getByte(record + FLAGS) & 0xff;
 		if (val) 
 			flags |= IS_INHERITANCE_SPEC;
 		else
 			flags &= ~IS_INHERITANCE_SPEC;
-		pdom.getDB().putInt(record + FLAGS, flags);
+		pdom.getDB().putByte(record + FLAGS, (byte) flags);
 	}
 
 	public boolean isBaseSpecifier() throws CoreException {
@@ -288,7 +288,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 
 	public int getNodeLength() {
 		try {
-			return pdom.getDB().getInt(record + NODE_LENGTH_OFFSET);
+			return (pdom.getDB().getShort(record + NODE_LENGTH_OFFSET)) & 0xffff;
 		} catch (CoreException e) {
 			CCorePlugin.log(e);
 			return 0;
@@ -297,7 +297,7 @@ public final class PDOMName implements IIndexFragmentName, IASTFileLocation {
 
 	public int getNodeOffset() {
 		try {
-			return pdom.getDB().getInt(record + NODE_OFFSET_OFFSET);
+			return pdom.getDB().get3ByteUnsignedInt(record + NODE_OFFSET_OFFSET);
 		} catch (CoreException e) {
 			CCorePlugin.log(e);
 			return 0;