dos2unix

37 files changed, 8785 insertions, 8785 deletions

diff --git a/db/btree.cpp b/db/btree.cpp
index 8e2b982d079..2cbfe9811b4 100644
--- a/db/btree.cpp
+++ b/db/btree.cpp
@@ -1,901 +1,901 @@
-// btree.cpp

-

-#include "stdafx.h"

-#include "btree.h"

-#include "pdfile.h"

-

-/* it is easy to do custom sizes for a namespace - all the same for now */

-const int BucketSize = 8192;

-const int KeyMax = BucketSize / 10;

-

-int ninserts = 0;

-extern int otherTraceLevel;

-int split_debug = 0;

-int insert_debug = 0;

-DiskLoc maxDiskLoc(0x7fffffff, 0x7fffffff);

-DiskLoc minDiskLoc(0, 1);

-

-inline KeyNode::KeyNode(BucketBasics& bb, _KeyNode &k) : 

-  prevChildBucket(k.prevChildBucket), 

-  recordLoc(k.recordLoc), key(bb.data+k.keyDataOfs()) 

-{ }

-

-/* BucketBasics --------------------------------------------------- */

-

-inline void BucketBasics::setNotPacked() { flags &= ~Packed; }

-inline void BucketBasics::setPacked() { flags |= Packed; }

-

-void BucketBasics::_shape(int level, stringstream& ss) {

-	for( int i = 0; i < level; i++ ) ss << ' ';

-	ss << "*\n";

-	for( int i = 0; i < n; i++ ) 

-		if( !k(i).prevChildBucket.isNull() )

-			k(i).prevChildBucket.btree()->_shape(level+1,ss);

-	if( !nextChild.isNull() )

-		nextChild.btree()->_shape(level+1,ss);

-}

-

-int bt_fv=0;

-int bt_dmp=0;

-

-void BucketBasics::dumpTree(DiskLoc thisLoc) { 

-	bt_dmp=1;

-	fullValidate(thisLoc);

-	bt_dmp=0;

-}

-

-int BucketBasics::fullValidate(const DiskLoc& thisLoc) { 

-	assertValid(true);

-//	if( bt_fv==0 ) 

-//		return; 

-

-	if( bt_dmp ) {

-		cout << thisLoc.toString() << ' ';

-		((BtreeBucket *) this)->dump();

-	}

-

-	// keycount

-	int kc = 0;

-

-	for( int i = 0; i < n; i++ ) {

-		_KeyNode& kn = k(i);

-		if( kn.isUsed() ) kc++;

-		if( !kn.prevChildBucket.isNull() ) { 

-			DiskLoc left = kn.prevChildBucket;

-			BtreeBucket *b = left.btree();

-			wassert( b->parent == thisLoc );

-			kc += b->fullValidate(kn.prevChildBucket);

-		}

-	}

-	if( !nextChild.isNull() ) {

-		BtreeBucket *b = nextChild.btree();

-		wassert( b->parent == thisLoc );

-		kc += b->fullValidate(nextChild);

-	}

-

-	return kc;

-}

-

-int nDumped = 0;

-

-void BucketBasics::assertValid(bool force) { 

-	if( !debug && !force )

-		return;

-	wassert( n >= 0 && n < BucketSize );

-	wassert( emptySize >= 0 && emptySize < BucketSize );

-	wassert( topSize >= n && topSize <= BucketSize );

-	wassert( Size == BucketSize );

-	if( 1 ) {

-		// slow:

-		for( int i = 0; i < n-1; i++ ) {

-			JSObj k1 = keyNode(i).key;

-			JSObj k2 = keyNode(i+1).key;

-			int z = k1.woCompare(k2); //OK

-			if( z > 0 ) {

-				cout << "ERROR: btree key order corrupt.  Keys:" << endl;

-				if( ++nDumped < 5 ) {

-					for( int j = 0; j < n; j++ ) { 

-						cout << "  " << keyNode(j).key.toString() << endl;

-					}

-					((BtreeBucket *) this)->dump();

-				}

-				wassert(false);

-				break;

-			}

-			else if( z == 0 ) { 

-				wassert( k(i).recordLoc < k(i+1).recordLoc );

-			}

-		}

-	}

-	else {

-		//faster:

-		if( n > 1 ) {

-			JSObj k1 = keyNode(0).key;

-			JSObj k2 = keyNode(n-1).key;

-			int z = k1.woCompare(k2);

-			wassert( z <= 0 );

-		}

-	}

-}

-

-inline void BucketBasics::markUnused(int keypos) { 

-	assert( keypos >= 0 && keypos < n );

-	k(keypos).setUnused();

-}

-

-inline int BucketBasics::totalDataSize() const {

-	return Size - (data-(char*)this);

-}

-

-void BucketBasics::init(){

-	parent.Null(); nextChild.Null();

-	Size = BucketSize;

-	flags = Packed;

-	n = 0;

-	emptySize = totalDataSize(); topSize = 0;

-	reserved = 0;

-}

-

-/* we allocate space from the end of the buffer for data.

-   the keynodes grow from the front.

-*/

-inline int BucketBasics::_alloc(int bytes) {

-	topSize += bytes;

-	emptySize -= bytes;

-	int ofs = totalDataSize() - topSize;

-	assert( ofs > 0 );

-	return ofs;

-}

-

-void BucketBasics::_delKeyAtPos(int keypos) { 

-	assert( keypos >= 0 && keypos <= n );

-	assert( childForPos(keypos).isNull() );

-	n--;

-	assert( n > 0 || nextChild.isNull() );

-	for( int j = keypos; j < n; j++ )

-		k(j) = k(j+1);

-	emptySize += sizeof(_KeyNode);

-	setNotPacked();

-}

-

-/* add a key.  must be > all existing.  be careful to set next ptr right. */

-void BucketBasics::pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild) { 

-	int bytesNeeded = key.objsize() + sizeof(_KeyNode);

-	assert( bytesNeeded <= emptySize );

-	assert( n == 0 || keyNode(n-1).key.woCompare(key) <= 0 );

-	emptySize -= sizeof(_KeyNode);

-	_KeyNode& kn = k(n++);

-	kn.prevChildBucket = prevChild;

-	kn.recordLoc = recordLoc; 

-	kn.setKeyDataOfs( (short) _alloc(key.objsize()) );

-	char *p = dataAt(kn.keyDataOfs());

-	memcpy(p, key.objdata(), key.objsize());

-}

-

-bool BucketBasics::basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key) {

-	assert( keypos >= 0 && keypos <= n );

-	int bytesNeeded = key.objsize() + sizeof(_KeyNode);

-	if( bytesNeeded > emptySize ) { 

-		pack();

-		if( bytesNeeded > emptySize )

-			return false;

-	}

-	for( int j = n; j > keypos; j-- ) // make room

-		k(j) = k(j-1);

-	n++;

-	emptySize -= sizeof(_KeyNode);

-	_KeyNode& kn = k(keypos);

-	kn.prevChildBucket.Null();

-	kn.recordLoc = recordLoc;

-	kn.setKeyDataOfs((short) _alloc(key.objsize()) );

-	char *p = dataAt(kn.keyDataOfs());

-	memcpy(p, key.objdata(), key.objsize());

-	return true;

-}

-

-/* when we delete things we just leave empty space until the node is 

-   full and then we repack it.

-*/

-void BucketBasics::pack() { 

-	if( flags & Packed )

-		return;

-

-	int tdz = totalDataSize();

-	char temp[BucketSize];

-	int ofs = tdz;

-	topSize = 0;

-	for( int j = 0; j < n; j++ ) { 

-		short ofsold = k(j).keyDataOfs();

-		int sz = keyNode(j).key.objsize();

-		ofs -= sz; 

-		topSize += sz;

-		memcpy(temp+ofs, dataAt(ofsold), sz);

-		k(j).setKeyDataOfsSavingUse( ofs );

-	}

-	int dataUsed = tdz - ofs;

-	memcpy(data + ofs, temp + ofs, dataUsed);

-	emptySize = tdz - dataUsed - n * sizeof(_KeyNode);

-	assert( emptySize >= 0 );

-

-	setPacked();

-	assertValid();

-}

-

-inline void BucketBasics::truncateTo(int N) {

-	n = N;

-	setNotPacked();

-	pack();

-}

-

-/* - BtreeBucket --------------------------------------------------- */

-

-/* return largest key in the subtree. */

-void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey) {

-	DiskLoc loc = thisLoc;

-	while( 1 ) { 

-		BtreeBucket *b = loc.btree();

-//		b->dump();

-		if( !b->nextChild.isNull() ) { 

-			loc = b->nextChild;

-			continue;

-		}

-

-		assert(b->n>0);

-		largestLoc = loc;

-		largestKey = b->n-1;

-

-		break;

-	}

-} 

-

-/* pos: for existing keys k0...kn-1.

-   returns # it goes BEFORE.  so key[pos-1] < key < key[pos]

-   returns n if it goes after the last existing key.

-   note result might be Unused!

-*/

-bool BtreeBucket::find(JSObj& key, DiskLoc recordLoc, int& pos) { 

-	/* binary search for this key */

-	int l=0; int h=n-1;

-	while( l <= h ) { 

-		int m = (l+h)/2;

-		KeyNode M = keyNode(m);

-		int x = key.woCompare(M.key);

-		if( x == 0 ) 

-			x = recordLoc.compare(M.recordLoc);

-		if( x < 0 ) // key < M.key

-			h = m-1;

-		else if( x > 0 )

-			l = m+1;

-		else {

-			// found it.  however, if dup keys are here, be careful we might have

-			// found one in the middle.  we want find() to return the leftmost instance.

-/*

-			while( m >= 1 && keyNode(m-1).key.woEqual(key) )

-				m--;

-*/

-

-			pos = m;

-

-/*

-			DiskLoc ch = k(m).prevChildBucket;

-			if( !ch.isNull() ) {

-				// if dup keys, might be dups to the left.

-				DiskLoc largestLoc;

-				int largestKey;

-				ch.btree()->findLargestKey(ch, largestLoc, largestKey);

-				if( !largestLoc.isNull() ) { 

-					if( largestLoc.btree()->keyAt(largestKey).woEqual(key) )

-						return false;

-				}

-			}

-*/

-

-			return true;

-		}

-//?		x = key.woCompare(M.key);

-	}

-	// not found

-	pos = l;

-	if( pos != n ) {

-		JSObj keyatpos = keyNode(pos).key;

-		wassert( key.woCompare(keyatpos) <= 0 );

-		if( pos > 0 ) { 

-			wassert( keyNode(pos-1).key.woCompare(key) <= 0 );

-		}

-	}

-

-	return false;

-}

-

-void BtreeBucket::delBucket(const DiskLoc& thisLoc, IndexDetails& id) { 

-	assert( !isHead() );

-

-	BtreeBucket *p = parent.btree();

-	if( p->nextChild == thisLoc ) {

-		p->nextChild.Null();

-	}

-	else {

-		for( int i = 0; i < p->n; i++ ) {

-			if( p->k(i).prevChildBucket == thisLoc ) { 

-				p->k(i).prevChildBucket.Null();

-				goto found;

-			}

-		}

-		cout << "ERROR: can't find ref to deleted bucket.\n";

-		cout << "To delete:\n";

-		dump();

-		cout << "Parent:\n";

-		p->dump();

-		assert(false);

-	}

-found:

-	//defensive:

-	n = -1;

-	parent.Null();

-	theDataFileMgr.deleteRecord(id.indexNamespace().c_str(), thisLoc.rec(), thisLoc);

-}

-

-/* note: may delete the entire bucket!  this invalid upon return sometimes. */

-void BtreeBucket::delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p) { 

-	assert(n>0);

-	DiskLoc left = childForPos(p);

-

-	if( n == 1 ) {

-		if( left.isNull() && nextChild.isNull() ) { 

-			if( isHead() )

-				_delKeyAtPos(p); // we don't delete the top bucket ever

-			else

-				delBucket(thisLoc, id);

-			return;

-		}

-		markUnused(p);

-		return;

-	}

-

-	if( left.isNull() )

-		_delKeyAtPos(p);

-	else

-		markUnused(p);

-}

-

-int verbose = 0;

-int qqq = 0;

-

-bool BtreeBucket::unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc ) {

-	if( key.objsize() > KeyMax ) {

-		problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << ' ' << key.toString() << endl;

-		return false;

-	}

-

-	int pos;

-	bool found;

-	DiskLoc loc = locate(thisLoc, key, pos, found, recordLoc, 1);

-	if( found ) { 

-		loc.btree()->delKeyAtPos(loc, id, pos);

-		return true;

-	}

-	return false;

-}

-

-BtreeBucket* BtreeBucket::allocTemp() { 

-	BtreeBucket *b = (BtreeBucket*) malloc(BucketSize);

-	b->init();

-	return b;

-}

-

-inline void fix(const DiskLoc& thisLoc, const DiskLoc& child) { 

-	if( !child.isNull() ) {

-		if( insert_debug )

-			cout << "      " << child.toString() << ".parent=" << thisLoc.toString() << endl;

-		child.btree()->parent = thisLoc;

-	}

-}

-

-/* this sucks.  maybe get rid of parent ptrs. */

-void BtreeBucket::fixParentPtrs(const DiskLoc& thisLoc) { 

-	fix(thisLoc, nextChild);

-	for( int i = 0; i < n; i++ )

-		fix(thisLoc, k(i).prevChildBucket);

-}

-

-/* keypos - where to insert the key i3n range 0..n.  0=make leftmost, n=make rightmost.

-*/

-void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos, 

-							 DiskLoc recordLoc, JSObj& key,

-							 DiskLoc lchild, DiskLoc rchild, IndexDetails& idx) 

-{

-	if( insert_debug )

-		cout << "   " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' ' 

-		<< lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;

-

-	DiskLoc oldLoc = thisLoc;

-

-	if( basicInsert(keypos, recordLoc, key) ) {

-		_KeyNode& kn = k(keypos);

-		if( keypos+1 == n ) { // last key

-			if( nextChild != lchild ) {

-				cout << "ERROR nextChild != lchild" << endl;

-				cout << "  thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;

-				cout << "  keyPos: " << keypos << " n:" << n << endl;

-				cout << "  nextChild: " << nextChild.toString() << " lchild: " << lchild.toString() << endl;

-				cout << "  recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;

-				cout << "  key: " << key.toString() << endl;

-				dump();

-#if defined(_WIN32)

-				cout << "\n\nDUMPING FULL INDEX" << endl;

-				bt_dmp=1;

-				bt_fv=1;

-				idx.head.btree()->fullValidate(idx.head);

-#endif

-				assert(false);

-			}

-			kn.prevChildBucket = nextChild;

-			assert( kn.prevChildBucket == lchild );

-			nextChild = rchild;

-			if( !rchild.isNull() )

-				rchild.btree()->parent = thisLoc;

-		}

-		else {

-			k(keypos).prevChildBucket = lchild;

-			if( k(keypos+1).prevChildBucket != lchild ) { 

-				cout << "ERROR k(keypos+1).prevChildBucket != lchild" << endl;

-				cout << "  thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;

-				cout << "  keyPos: " << keypos << " n:" << n << endl;

-				cout << "  k(keypos+1).pcb: " << k(keypos+1).prevChildBucket.toString() << " lchild: " << lchild.toString() << endl;

-				cout << "  recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;

-				cout << "  key: " << key.toString() << endl;

-				dump();

-#if defined(_WIN32)

-				cout << "\n\nDUMPING FULL INDEX" << endl;

-				bt_dmp=1;

-				bt_fv=1;

-				idx.head.btree()->fullValidate(idx.head);

-#endif

-				assert(false);

-			}

-			k(keypos+1).prevChildBucket = rchild;

-			if( !rchild.isNull() )

-				rchild.btree()->parent = thisLoc;

-		}

-		return;

-	}

-

-	// split

-	if( split_debug )

-		cout << "    " << thisLoc.toString() << ".split" << endl;

-

-	int mid = n / 2;

-

-	/* on duplicate key, we need to ensure that they all end up on the RHS */

-	if( 0 ) {

-		assert(mid>0);

-		while( 1 ) {

-			KeyNode mn = keyNode(mid);

-			KeyNode left = keyNode(mid-1);

-			if( left.key < mn.key )

-				break;

-			mid--;

-			if( mid < 3 ) { 

-				problem() << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;

-				cout << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;

-				cout << "  ns:" << idx.indexNamespace() << endl;

-				cout << "  key:" << mn.key.toString() << endl;

-				break;

-			}

-		}

-	}

-

-	BtreeBucket *r = allocTemp();

-	DiskLoc rLoc;

-

-	if( split_debug )

-		cout << "     mid:" << mid << ' ' << keyNode(mid).key.toString() << " n:" << n << endl;

-	for( int i = mid+1; i < n; i++ ) {

-		KeyNode kn = keyNode(i);

-		if( i == keypos ) {

-			// slip in the new one

-			r->pushBack(recordLoc, key, kn.prevChildBucket);

-			r->pushBack(kn.recordLoc, kn.key, rchild);

-		}

-		else

-			r->pushBack(kn.recordLoc, kn.key, kn.prevChildBucket);

-	}

-	r->nextChild = nextChild;

-	r->assertValid();

-//r->dump();

-	rLoc = theDataFileMgr.insert(idx.indexNamespace().c_str(), r, r->Size, true);

-	if( split_debug )

-		cout << "     new rLoc:" << rLoc.toString() << endl;

-	free(r); r = 0;

-	rLoc.btree()->fixParentPtrs(rLoc);

-

-	{

-		KeyNode middle = keyNode(mid);

-		nextChild = middle.prevChildBucket; // middle key gets promoted, its children will be thisLoc (l) and rLoc (r)

-		if( split_debug ) {

-			//rLoc.btree()->dump();

-			cout << "    middle key:" << middle.key.toString() << endl;

-		}

-

-		// promote middle to a parent node

-		if( parent.isNull() ) { 

-			// make a new parent if we were the root

-			BtreeBucket *p = allocTemp();

-			p->pushBack(middle.recordLoc, middle.key, thisLoc);

-			p->nextChild = rLoc;

-			p->assertValid();

-			parent = idx.head = theDataFileMgr.insert(idx.indexNamespace().c_str(), p, p->Size, true);

-			if( split_debug )

-				cout << "    we were root, making new root:" << hex << parent.getOfs() << dec << endl;

-			free(p);

-			rLoc.btree()->parent = parent;

-		} 

-		else {

-			/* set this before calling _insert - if it splits it will do fixParent() logic and fix the value, 

-			   so we don't want to overwrite that if it happens.

-			*/

-			rLoc.btree()->parent = parent;

-			if( split_debug )

-				cout << "    promoting middle key " << middle.key.toString() << endl;

-			parent.btree()->_insert(parent, middle.recordLoc, middle.key, false, thisLoc, rLoc, idx);

-		}

-		BtreeBucket *br = rLoc.btree();

-//br->dump();

-

-//parent.btree()->dump();

-//idx.head.btree()->dump();

-

-	}

-

-	// mark on left that we no longer have anything from midpoint on.

-    bool highest = keypos == n;

-	truncateTo(mid);  // note this may trash middle.key!  thus we had to promote it before finishing up here.

-

-	// add our new key, there is room now

-	{

-		

-//dump();

-

-		if( keypos <= mid ) {

-//		if( keypos < mid ) {

-			if( split_debug )

-				cout << "  keypos<mid, insertHere() the new key" << endl;

-			insertHere(thisLoc, keypos, recordLoc, key, lchild, rchild, idx);

-//dump();

-		} else if( highest ) {

-			// else handled above already.

-			int kp = keypos-mid-1; assert(kp>=0);

-			rLoc.btree()->insertHere(rLoc, kp, recordLoc, key, lchild, rchild, idx);

-// set a bp here.

-//			if( !lchild.isNull() ) cout << lchild.btree()->parent.toString() << endl;

-//			if( !rchild.isNull() ) cout << rchild.btree()->parent.toString() << endl;

-		}

-	}

-

-	if( split_debug )

-		cout << "     split end " << hex << thisLoc.getOfs() << dec << endl;

-}

-

-DiskLoc BtreeBucket::addHead(IndexDetails& id) {

-	BtreeBucket *p = allocTemp();

-	DiskLoc loc = theDataFileMgr.insert(id.indexNamespace().c_str(), p, p->Size, true);

-	return loc;

-}

-

-DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) {

-	DiskLoc p = thisLoc;

-	while( !p.btree()->isHead() )

-		p = p.btree()->parent;

-	return p;

-}

-

-DiskLoc BtreeBucket::advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) {

-	if( keyOfs < 0 || keyOfs >= n ) {

-		cout << "ASSERT failure BtreeBucket::advance, caller: " << caller << endl;

-		cout << "  thisLoc: " << thisLoc.toString() << endl;

-		cout << "  keyOfs: " << keyOfs << " n:" << n << " direction: " << direction << endl;

-		cout << bucketSummary() << endl;

-		assert( keyOfs >= 0 && keyOfs < n );

-	}

-	int adj = direction < 0 ? 1 : 0;

-	int ko = keyOfs + direction;

-	DiskLoc nextDown = childForPos(ko+adj);

-	if( !nextDown.isNull() ) { 

-//		nextDown.btree()->dump();//TEMP:

-		while( 1 ) {

-			keyOfs = direction>0 ? 0 : nextDown.btree()->n - 1;

-			DiskLoc loc= nextDown.btree()->childForPos(keyOfs + adj);

-			if( loc.isNull() )

-				break;

-			nextDown = loc;

-		}

-		return nextDown;

-	}

-

-	if( ko < n && ko >= 0 ) {

-		keyOfs = ko;

-		return thisLoc;

-	}

-

-	// end of bucket.  traverse back up.

-	DiskLoc childLoc = thisLoc;

-	DiskLoc ancestor = parent;

-	while( 1 ) {

-		if( ancestor.isNull() )

-			break;

-		BtreeBucket *an = ancestor.btree();

-		for( int i = 0; i < an->n; i++ ) {

-			if( an->childForPos(i+adj) == childLoc ) {

-				keyOfs = i;

-				return ancestor;

-			}

-		}

-		assert( direction<0 || an->nextChild == childLoc );

-		// parent exhausted also, keep going up

-		childLoc = ancestor;

-		ancestor = an->parent;

-	}

-

-	return DiskLoc();

-}

-

-DiskLoc BtreeBucket::locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction) { 

-	int p;

-	found = find(key, recordLoc, p);

-	if( found ) {

-		pos = p;

-		return thisLoc;

-	}

-

-	DiskLoc child = childForPos(p);

-

-	if( !child.isNull() ) { 

-		DiskLoc l = child.btree()->locate(child, key, pos, found, recordLoc, direction);

-		if( !l.isNull() )

-			return l;

-	}

-

-	if( direction == -1 && p == n && n ) { 

-		p--;

-	}

-

-	pos = p;

-	return pos == n ? DiskLoc() /*theend*/ : thisLoc;

-}

-

-/* thisloc is the location of this bucket object.  you must pass that in. */

-int BtreeBucket::_insert(DiskLoc thisLoc, DiskLoc recordLoc, 

-						JSObj& key, bool dupsAllowed,

-						DiskLoc lChild, DiskLoc rChild, IndexDetails& idx) { 

-	if( key.objsize() > KeyMax ) { 

-		problem() << "ERROR: key too large len:" << key.objsize() << " max:" << KeyMax << ' ' << idx.indexNamespace() << endl;

-		return 2;

-	}

-	assert( key.objsize() > 0 );

-

-	int pos;

-	bool found = find(key, recordLoc, pos);

-	if( insert_debug ) {

-		cout << "  " << thisLoc.toString() << '.' << "_insert " << 

-			key.toString() << '/' << recordLoc.toString() <<

-			" l:" << lChild.toString() << " r:" << rChild.toString() << endl;

-		cout << "    found:" << found << " pos:" << pos << " n:" << n << endl;

-	}

-

-	if( found ) {

-		if( k(pos).isUnused() ) { 

-			cout << "an unused already occupying keyslot, write more code.\n";

-			cout << "  index may be corrupt (missing data) now.\n";

-		}

-

-		cout << "_insert(): key already exists in index\n";

-		cout << "  " << idx.indexNamespace().c_str() << " thisLoc:" << thisLoc.toString() << '\n';

-		cout << "  " << key.toString() << '\n';

-		cout << "  " << "recordLoc:" << recordLoc.toString() << " pos:" << pos << endl;

-		cout << "  old l r: " << childForPos(pos).toString() << ' ' << childForPos(pos+1).toString() << endl;

-		cout << "  new l r: " << lChild.toString() << ' ' << rChild.toString() << endl;

-		assert(false);

-

-		// on a dup key always insert on the right or else you will be broken.

-//		pos++;

-		// on a promotion, find the right point to update if dup keys.

-		/* not needed: we always insert right after the first key so we are ok with just pos++...

-		if( !rChild.isNull() ) { 

-			while( pos < n && k(pos).prevChildBucket != lchild ) { 

-				pos++;

-				cout << "looking for the right dup key" << endl;

-			}

-		}

-		*/

-	}

-

-	DEBUGGING cout << "TEMP: key: " << key.toString() << endl;

-	DiskLoc& child = getChild(pos);

-	if( insert_debug )

-		cout << "    getChild(" << pos << "): " << child.toString() << endl;

-	if( child.isNull() || !rChild.isNull() /* means an 'internal' insert */ ) { 

-		insertHere(thisLoc, pos, recordLoc, key, lChild, rChild, idx);

-		return 0;

-	}

-

-	return child.btree()->insert(child, recordLoc, key, dupsAllowed, idx, false);

-}

-

-void BtreeBucket::dump() { 

-	cout << "DUMP btreebucket: ";

-	cout << " parent:" << hex << parent.getOfs() << dec;

-	for( int i = 0; i < n; i++ ) {

-		cout << '\n';

-		KeyNode k = keyNode(i);

-		cout << '\t' << i << '\t' << k.key.toString() << "\tleft:" << hex << 

-			k.prevChildBucket.getOfs() << "\trec:" << k.recordLoc.getOfs() << dec;

-		if( this->k(i).isUnused() )

-			cout << " UNUSED";

-	}

-	cout << " right:" << hex << nextChild.getOfs() << dec << endl;

-}

-

-JSObj *music = 0;

-

-void tempMusic(DiskLoc thisLoc)

-{

-	BtreeCursor c(thisLoc, *music, 1, true);

-	while( c.ok() ) { 

-		KeyNode kn = c.currKeyNode();

-		if( !kn.key.woEqual(*music) )

-			break;

-		if( kn.recordLoc.getOfs() == 0x4c8d7c0 ) {

-			cout << "*** found it" << endl;

-				//				c.bucket.btree()->dump();

-			return;

-		}

-		c.advance();

-	}

-

-	cout << "*** NOT FOUND" << endl;

-}

-

-/* todo: meaning of return code unclear clean up */

-int BtreeBucket::insert(DiskLoc thisLoc, DiskLoc recordLoc, 

-						JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel) 

-{

-	if( toplevel ) {

-		if( key.objsize() > KeyMax ) { 

-			problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.toString() << '\n';

-			return 3;

-		}

-		++ninserts;

-		/*

-		if( ninserts % 1000 == 0 ) {

-			cout << "ninserts: " << ninserts << endl;

-			if( 0 && ninserts >= 127287 ) {

-				cout << "debug?" << endl;

-				split_debug = 1;

-			}

-		}

-		*/

-	}

-

-	bool chk = false;

-

-	int x = _insert(thisLoc, recordLoc, key, dupsAllowed, DiskLoc(), DiskLoc(), idx);

-	assertValid(); 

-

-/*	if( toplevel ) {

-		if( recordLoc.getOfs() == 0x4c8d7c0 ) {

-			if( key.toString() == "{ _searchIndex: \"music\" }" ) { 

-				tempMusic(thisLoc);

-			}

-		}

-	}

-*/

-	return x;

-}

-

-void BtreeBucket::shape(stringstream& ss) {

-	_shape(0, ss);

-}

-

-/* - BtreeCursor --------------------------------------------------- */

-

-BtreeCursor::BtreeCursor(DiskLoc head, JSObj& k, int _direction, bool sm) : 

-    direction(_direction), stopmiss(sm) 

-{

-//otherTraceLevel = 999;

-

-	bool found;

-	if( otherTraceLevel >= 12 ) {

-		if( otherTraceLevel >= 200 ) { 

-			cout << "::BtreeCursor() qtl>200.  validating entire index." << endl;

-			head.btree()->fullValidate(head);

-		}

-		else {

-			cout << "BTreeCursor(). dumping head bucket" << endl;

-			head.btree()->dump();

-		}

-	}

-	bucket = head.btree()->locate(head, k, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);

-	checkUnused();

-}

-

-int zzz = 0;

-

-/* skip unused keys. */

-void BtreeCursor::checkUnused() {

-	int u = 0;

-	while( 1 ) {

-		if( !ok() ) 

-			break;

-		BtreeBucket *b = bucket.btree();

-		_KeyNode& kn = b->k(keyOfs);

-		if( kn.isUsed() )

-			break;

-		bucket = b->advance(bucket, keyOfs, direction, "checkUnused");

-		u++;

-	}

-	if( u > 10 && ++zzz % 16 == 0 )

-		cout << "btree unused skipped:" << u << endl;

-}

-

-/*DiskLoc BtreeCursor::currLoc() {

-	assert( !bucket.isNull() );

-	_KeyNode& kn = bucket.btree()->k(keyOfs);

-	assert( kn.isUsed() );

-	return kn.recordLoc;

-}*/

-

-bool BtreeCursor::advance() { 

-	if( bucket.isNull() )

-		return false;

-	bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance");

-	checkUnused();

-	return !bucket.isNull();

-}

-

-void BtreeCursor::noteLocation() {

-	if( !eof() ) {

-		JSObj o = bucket.btree()->keyAt(keyOfs).copy();

-		keyAtKeyOfs = o;

-		locAtKeyOfs = bucket.btree()->k(keyOfs).recordLoc;

-	}

-}

-

-int clctr = 0;

-

-/* see if things moved around (deletes, splits, inserts) */

-void BtreeCursor::checkLocation() { 

-	try {

-		if( eof() ) 

-			return;

-		BtreeBucket *b = bucket.btree();

-		if( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) &&

-			b->k(keyOfs).recordLoc == locAtKeyOfs ) { 

-			if( !b->k(keyOfs).isUsed() )

-				checkUnused();

-			return;

-		}

-	}

-	catch( AssertionException ) { 

-		cout << "Caught exception in checkLocation(), that's maybe ok" << endl;

-	}

-

-	bool found;

-	DiskLoc bold = bucket;

-

-/* TODO: Switch to keep indexdetails and do idx.head! */

-	/* didn't find, check from the top */

-	DiskLoc head = bold.btree()->getHead(bold);

-	bucket = head.btree()->locate(head, keyAtKeyOfs, keyOfs, found, locAtKeyOfs, direction);

-	if( clctr++ % 128 == 0 )

-		cout << "  key seems to have moved in the index, refinding. found:" << found << endl;

-	if( found )

-		checkUnused();

-}

-

-/* ----------------------------------------------------------------------------- */

-

-struct BtreeUnitTest { 

-	BtreeUnitTest() { 

-		assert( minDiskLoc.compare(maxDiskLoc) < 0 );

-	}

-} btut;

+// btree.cpp
+
+#include "stdafx.h"
+#include "btree.h"
+#include "pdfile.h"
+
+/* it is easy to do custom sizes for a namespace - all the same for now */
+const int BucketSize = 8192;
+const int KeyMax = BucketSize / 10;
+
+int ninserts = 0;
+extern int otherTraceLevel;
+int split_debug = 0;
+int insert_debug = 0;
+DiskLoc maxDiskLoc(0x7fffffff, 0x7fffffff);
+DiskLoc minDiskLoc(0, 1);
+
+inline KeyNode::KeyNode(BucketBasics& bb, _KeyNode &k) : 
+  prevChildBucket(k.prevChildBucket), 
+  recordLoc(k.recordLoc), key(bb.data+k.keyDataOfs()) 
+{ }
+
+/* BucketBasics --------------------------------------------------- */
+
+inline void BucketBasics::setNotPacked() { flags &= ~Packed; }
+inline void BucketBasics::setPacked() { flags |= Packed; }
+
+void BucketBasics::_shape(int level, stringstream& ss) {
+	for( int i = 0; i < level; i++ ) ss << ' ';
+	ss << "*\n";
+	for( int i = 0; i < n; i++ ) 
+		if( !k(i).prevChildBucket.isNull() )
+			k(i).prevChildBucket.btree()->_shape(level+1,ss);
+	if( !nextChild.isNull() )
+		nextChild.btree()->_shape(level+1,ss);
+}
+
+int bt_fv=0;
+int bt_dmp=0;
+
+void BucketBasics::dumpTree(DiskLoc thisLoc) { 
+	bt_dmp=1;
+	fullValidate(thisLoc);
+	bt_dmp=0;
+}
+
+int BucketBasics::fullValidate(const DiskLoc& thisLoc) { 
+	assertValid(true);
+//	if( bt_fv==0 ) 
+//		return; 
+
+	if( bt_dmp ) {
+		cout << thisLoc.toString() << ' ';
+		((BtreeBucket *) this)->dump();
+	}
+
+	// keycount
+	int kc = 0;
+
+	for( int i = 0; i < n; i++ ) {
+		_KeyNode& kn = k(i);
+		if( kn.isUsed() ) kc++;
+		if( !kn.prevChildBucket.isNull() ) { 
+			DiskLoc left = kn.prevChildBucket;
+			BtreeBucket *b = left.btree();
+			wassert( b->parent == thisLoc );
+			kc += b->fullValidate(kn.prevChildBucket);
+		}
+	}
+	if( !nextChild.isNull() ) {
+		BtreeBucket *b = nextChild.btree();
+		wassert( b->parent == thisLoc );
+		kc += b->fullValidate(nextChild);
+	}
+
+	return kc;
+}
+
+int nDumped = 0;
+
+void BucketBasics::assertValid(bool force) { 
+	if( !debug && !force )
+		return;
+	wassert( n >= 0 && n < BucketSize );
+	wassert( emptySize >= 0 && emptySize < BucketSize );
+	wassert( topSize >= n && topSize <= BucketSize );
+	wassert( Size == BucketSize );
+	if( 1 ) {
+		// slow:
+		for( int i = 0; i < n-1; i++ ) {
+			JSObj k1 = keyNode(i).key;
+			JSObj k2 = keyNode(i+1).key;
+			int z = k1.woCompare(k2); //OK
+			if( z > 0 ) {
+				cout << "ERROR: btree key order corrupt.  Keys:" << endl;
+				if( ++nDumped < 5 ) {
+					for( int j = 0; j < n; j++ ) { 
+						cout << "  " << keyNode(j).key.toString() << endl;
+					}
+					((BtreeBucket *) this)->dump();
+				}
+				wassert(false);
+				break;
+			}
+			else if( z == 0 ) { 
+				wassert( k(i).recordLoc < k(i+1).recordLoc );
+			}
+		}
+	}
+	else {
+		//faster:
+		if( n > 1 ) {
+			JSObj k1 = keyNode(0).key;
+			JSObj k2 = keyNode(n-1).key;
+			int z = k1.woCompare(k2);
+			wassert( z <= 0 );
+		}
+	}
+}
+
+inline void BucketBasics::markUnused(int keypos) { 
+	assert( keypos >= 0 && keypos < n );
+	k(keypos).setUnused();
+}
+
+inline int BucketBasics::totalDataSize() const {
+	return Size - (data-(char*)this);
+}
+
+void BucketBasics::init(){
+	parent.Null(); nextChild.Null();
+	Size = BucketSize;
+	flags = Packed;
+	n = 0;
+	emptySize = totalDataSize(); topSize = 0;
+	reserved = 0;
+}
+
+/* we allocate space from the end of the buffer for data.
+   the keynodes grow from the front.
+*/
+inline int BucketBasics::_alloc(int bytes) {
+	topSize += bytes;
+	emptySize -= bytes;
+	int ofs = totalDataSize() - topSize;
+	assert( ofs > 0 );
+	return ofs;
+}
+
+void BucketBasics::_delKeyAtPos(int keypos) { 
+	assert( keypos >= 0 && keypos <= n );
+	assert( childForPos(keypos).isNull() );
+	n--;
+	assert( n > 0 || nextChild.isNull() );
+	for( int j = keypos; j < n; j++ )
+		k(j) = k(j+1);
+	emptySize += sizeof(_KeyNode);
+	setNotPacked();
+}
+
+/* add a key.  must be > all existing.  be careful to set next ptr right. */
+void BucketBasics::pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild) { 
+	int bytesNeeded = key.objsize() + sizeof(_KeyNode);
+	assert( bytesNeeded <= emptySize );
+	assert( n == 0 || keyNode(n-1).key.woCompare(key) <= 0 );
+	emptySize -= sizeof(_KeyNode);
+	_KeyNode& kn = k(n++);
+	kn.prevChildBucket = prevChild;
+	kn.recordLoc = recordLoc; 
+	kn.setKeyDataOfs( (short) _alloc(key.objsize()) );
+	char *p = dataAt(kn.keyDataOfs());
+	memcpy(p, key.objdata(), key.objsize());
+}
+
+bool BucketBasics::basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key) {
+	assert( keypos >= 0 && keypos <= n );
+	int bytesNeeded = key.objsize() + sizeof(_KeyNode);
+	if( bytesNeeded > emptySize ) { 
+		pack();
+		if( bytesNeeded > emptySize )
+			return false;
+	}
+	for( int j = n; j > keypos; j-- ) // make room
+		k(j) = k(j-1);
+	n++;
+	emptySize -= sizeof(_KeyNode);
+	_KeyNode& kn = k(keypos);
+	kn.prevChildBucket.Null();
+	kn.recordLoc = recordLoc;
+	kn.setKeyDataOfs((short) _alloc(key.objsize()) );
+	char *p = dataAt(kn.keyDataOfs());
+	memcpy(p, key.objdata(), key.objsize());
+	return true;
+}
+
+/* when we delete things we just leave empty space until the node is 
+   full and then we repack it.
+*/
+void BucketBasics::pack() { 
+	if( flags & Packed )
+		return;
+
+	int tdz = totalDataSize();
+	char temp[BucketSize];
+	int ofs = tdz;
+	topSize = 0;
+	for( int j = 0; j < n; j++ ) { 
+		short ofsold = k(j).keyDataOfs();
+		int sz = keyNode(j).key.objsize();
+		ofs -= sz; 
+		topSize += sz;
+		memcpy(temp+ofs, dataAt(ofsold), sz);
+		k(j).setKeyDataOfsSavingUse( ofs );
+	}
+	int dataUsed = tdz - ofs;
+	memcpy(data + ofs, temp + ofs, dataUsed);
+	emptySize = tdz - dataUsed - n * sizeof(_KeyNode);
+	assert( emptySize >= 0 );
+
+	setPacked();
+	assertValid();
+}
+
+inline void BucketBasics::truncateTo(int N) {
+	n = N;
+	setNotPacked();
+	pack();
+}
+
+/* - BtreeBucket --------------------------------------------------- */
+
+/* return largest key in the subtree. */
+void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey) {
+	DiskLoc loc = thisLoc;
+	while( 1 ) { 
+		BtreeBucket *b = loc.btree();
+//		b->dump();
+		if( !b->nextChild.isNull() ) { 
+			loc = b->nextChild;
+			continue;
+		}
+
+		assert(b->n>0);
+		largestLoc = loc;
+		largestKey = b->n-1;
+
+		break;
+	}
+} 
+
+/* pos: for existing keys k0...kn-1.
+   returns # it goes BEFORE.  so key[pos-1] < key < key[pos]
+   returns n if it goes after the last existing key.
+   note result might be Unused!
+*/
+bool BtreeBucket::find(JSObj& key, DiskLoc recordLoc, int& pos) { 
+	/* binary search for this key */
+	int l=0; int h=n-1;
+	while( l <= h ) { 
+		int m = (l+h)/2;
+		KeyNode M = keyNode(m);
+		int x = key.woCompare(M.key);
+		if( x == 0 ) 
+			x = recordLoc.compare(M.recordLoc);
+		if( x < 0 ) // key < M.key
+			h = m-1;
+		else if( x > 0 )
+			l = m+1;
+		else {
+			// found it.  however, if dup keys are here, be careful we might have
+			// found one in the middle.  we want find() to return the leftmost instance.
+/*
+			while( m >= 1 && keyNode(m-1).key.woEqual(key) )
+				m--;
+*/
+
+			pos = m;
+
+/*
+			DiskLoc ch = k(m).prevChildBucket;
+			if( !ch.isNull() ) {
+				// if dup keys, might be dups to the left.
+				DiskLoc largestLoc;
+				int largestKey;
+				ch.btree()->findLargestKey(ch, largestLoc, largestKey);
+				if( !largestLoc.isNull() ) { 
+					if( largestLoc.btree()->keyAt(largestKey).woEqual(key) )
+						return false;
+				}
+			}
+*/
+
+			return true;
+		}
+//?		x = key.woCompare(M.key);
+	}
+	// not found
+	pos = l;
+	if( pos != n ) {
+		JSObj keyatpos = keyNode(pos).key;
+		wassert( key.woCompare(keyatpos) <= 0 );
+		if( pos > 0 ) { 
+			wassert( keyNode(pos-1).key.woCompare(key) <= 0 );
+		}
+	}
+
+	return false;
+}
+
+void BtreeBucket::delBucket(const DiskLoc& thisLoc, IndexDetails& id) { 
+	assert( !isHead() );
+
+	BtreeBucket *p = parent.btree();
+	if( p->nextChild == thisLoc ) {
+		p->nextChild.Null();
+	}
+	else {
+		for( int i = 0; i < p->n; i++ ) {
+			if( p->k(i).prevChildBucket == thisLoc ) { 
+				p->k(i).prevChildBucket.Null();
+				goto found;
+			}
+		}
+		cout << "ERROR: can't find ref to deleted bucket.\n";
+		cout << "To delete:\n";
+		dump();
+		cout << "Parent:\n";
+		p->dump();
+		assert(false);
+	}
+found:
+	//defensive:
+	n = -1;
+	parent.Null();
+	theDataFileMgr.deleteRecord(id.indexNamespace().c_str(), thisLoc.rec(), thisLoc);
+}
+
+/* note: may delete the entire bucket!  this invalid upon return sometimes. */
+void BtreeBucket::delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p) { 
+	assert(n>0);
+	DiskLoc left = childForPos(p);
+
+	if( n == 1 ) {
+		if( left.isNull() && nextChild.isNull() ) { 
+			if( isHead() )
+				_delKeyAtPos(p); // we don't delete the top bucket ever
+			else
+				delBucket(thisLoc, id);
+			return;
+		}
+		markUnused(p);
+		return;
+	}
+
+	if( left.isNull() )
+		_delKeyAtPos(p);
+	else
+		markUnused(p);
+}
+
+int verbose = 0;
+int qqq = 0;
+
+bool BtreeBucket::unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc ) {
+	if( key.objsize() > KeyMax ) {
+		problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << ' ' << key.toString() << endl;
+		return false;
+	}
+
+	int pos;
+	bool found;
+	DiskLoc loc = locate(thisLoc, key, pos, found, recordLoc, 1);
+	if( found ) { 
+		loc.btree()->delKeyAtPos(loc, id, pos);
+		return true;
+	}
+	return false;
+}
+
+BtreeBucket* BtreeBucket::allocTemp() { 
+	BtreeBucket *b = (BtreeBucket*) malloc(BucketSize);
+	b->init();
+	return b;
+}
+
+inline void fix(const DiskLoc& thisLoc, const DiskLoc& child) { 
+	if( !child.isNull() ) {
+		if( insert_debug )
+			cout << "      " << child.toString() << ".parent=" << thisLoc.toString() << endl;
+		child.btree()->parent = thisLoc;
+	}
+}
+
+/* this sucks.  maybe get rid of parent ptrs. */
+void BtreeBucket::fixParentPtrs(const DiskLoc& thisLoc) { 
+	fix(thisLoc, nextChild);
+	for( int i = 0; i < n; i++ )
+		fix(thisLoc, k(i).prevChildBucket);
+}
+
+/* keypos - where to insert the key i3n range 0..n.  0=make leftmost, n=make rightmost.
+*/
+void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos, 
+							 DiskLoc recordLoc, JSObj& key,
+							 DiskLoc lchild, DiskLoc rchild, IndexDetails& idx) 
+{
+	if( insert_debug )
+		cout << "   " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' ' 
+		<< lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;
+
+	DiskLoc oldLoc = thisLoc;
+
+	if( basicInsert(keypos, recordLoc, key) ) {
+		_KeyNode& kn = k(keypos);
+		if( keypos+1 == n ) { // last key
+			if( nextChild != lchild ) {
+				cout << "ERROR nextChild != lchild" << endl;
+				cout << "  thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;
+				cout << "  keyPos: " << keypos << " n:" << n << endl;
+				cout << "  nextChild: " << nextChild.toString() << " lchild: " << lchild.toString() << endl;
+				cout << "  recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;
+				cout << "  key: " << key.toString() << endl;
+				dump();
+#if defined(_WIN32)
+				cout << "\n\nDUMPING FULL INDEX" << endl;
+				bt_dmp=1;
+				bt_fv=1;
+				idx.head.btree()->fullValidate(idx.head);
+#endif
+				assert(false);
+			}
+			kn.prevChildBucket = nextChild;
+			assert( kn.prevChildBucket == lchild );
+			nextChild = rchild;
+			if( !rchild.isNull() )
+				rchild.btree()->parent = thisLoc;
+		}
+		else {
+			k(keypos).prevChildBucket = lchild;
+			if( k(keypos+1).prevChildBucket != lchild ) { 
+				cout << "ERROR k(keypos+1).prevChildBucket != lchild" << endl;
+				cout << "  thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;
+				cout << "  keyPos: " << keypos << " n:" << n << endl;
+				cout << "  k(keypos+1).pcb: " << k(keypos+1).prevChildBucket.toString() << " lchild: " << lchild.toString() << endl;
+				cout << "  recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;
+				cout << "  key: " << key.toString() << endl;
+				dump();
+#if defined(_WIN32)
+				cout << "\n\nDUMPING FULL INDEX" << endl;
+				bt_dmp=1;
+				bt_fv=1;
+				idx.head.btree()->fullValidate(idx.head);
+#endif
+				assert(false);
+			}
+			k(keypos+1).prevChildBucket = rchild;
+			if( !rchild.isNull() )
+				rchild.btree()->parent = thisLoc;
+		}
+		return;
+	}
+
+	// split
+	if( split_debug )
+		cout << "    " << thisLoc.toString() << ".split" << endl;
+
+	int mid = n / 2;
+
+	/* on duplicate key, we need to ensure that they all end up on the RHS */
+	if( 0 ) {
+		assert(mid>0);
+		while( 1 ) {
+			KeyNode mn = keyNode(mid);
+			KeyNode left = keyNode(mid-1);
+			if( left.key < mn.key )
+				break;
+			mid--;
+			if( mid < 3 ) { 
+				problem() << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;
+				cout << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;
+				cout << "  ns:" << idx.indexNamespace() << endl;
+				cout << "  key:" << mn.key.toString() << endl;
+				break;
+			}
+		}
+	}
+
+	BtreeBucket *r = allocTemp();
+	DiskLoc rLoc;
+
+	if( split_debug )
+		cout << "     mid:" << mid << ' ' << keyNode(mid).key.toString() << " n:" << n << endl;
+	for( int i = mid+1; i < n; i++ ) {
+		KeyNode kn = keyNode(i);
+		if( i == keypos ) {
+			// slip in the new one
+			r->pushBack(recordLoc, key, kn.prevChildBucket);
+			r->pushBack(kn.recordLoc, kn.key, rchild);
+		}
+		else
+			r->pushBack(kn.recordLoc, kn.key, kn.prevChildBucket);
+	}
+	r->nextChild = nextChild;
+	r->assertValid();
+//r->dump();
+	rLoc = theDataFileMgr.insert(idx.indexNamespace().c_str(), r, r->Size, true);
+	if( split_debug )
+		cout << "     new rLoc:" << rLoc.toString() << endl;
+	free(r); r = 0;
+	rLoc.btree()->fixParentPtrs(rLoc);
+
+	{
+		KeyNode middle = keyNode(mid);
+		nextChild = middle.prevChildBucket; // middle key gets promoted, its children will be thisLoc (l) and rLoc (r)
+		if( split_debug ) {
+			//rLoc.btree()->dump();
+			cout << "    middle key:" << middle.key.toString() << endl;
+		}
+
+		// promote middle to a parent node
+		if( parent.isNull() ) { 
+			// make a new parent if we were the root
+			BtreeBucket *p = allocTemp();
+			p->pushBack(middle.recordLoc, middle.key, thisLoc);
+			p->nextChild = rLoc;
+			p->assertValid();
+			parent = idx.head = theDataFileMgr.insert(idx.indexNamespace().c_str(), p, p->Size, true);
+			if( split_debug )
+				cout << "    we were root, making new root:" << hex << parent.getOfs() << dec << endl;
+			free(p);
+			rLoc.btree()->parent = parent;
+		} 
+		else {
+			/* set this before calling _insert - if it splits it will do fixParent() logic and fix the value, 
+			   so we don't want to overwrite that if it happens.
+			*/
+			rLoc.btree()->parent = parent;
+			if( split_debug )
+				cout << "    promoting middle key " << middle.key.toString() << endl;
+			parent.btree()->_insert(parent, middle.recordLoc, middle.key, false, thisLoc, rLoc, idx);
+		}
+		BtreeBucket *br = rLoc.btree();
+//br->dump();
+
+//parent.btree()->dump();
+//idx.head.btree()->dump();
+
+	}
+
+	// mark on left that we no longer have anything from midpoint on.
+    bool highest = keypos == n;
+	truncateTo(mid);  // note this may trash middle.key!  thus we had to promote it before finishing up here.
+
+	// add our new key, there is room now
+	{
+		
+//dump();
+
+		if( keypos <= mid ) {
+//		if( keypos < mid ) {
+			if( split_debug )
+				cout << "  keypos<mid, insertHere() the new key" << endl;
+			insertHere(thisLoc, keypos, recordLoc, key, lchild, rchild, idx);
+//dump();
+		} else if( highest ) {
+			// else handled above already.
+			int kp = keypos-mid-1; assert(kp>=0);
+			rLoc.btree()->insertHere(rLoc, kp, recordLoc, key, lchild, rchild, idx);
+// set a bp here.
+//			if( !lchild.isNull() ) cout << lchild.btree()->parent.toString() << endl;
+//			if( !rchild.isNull() ) cout << rchild.btree()->parent.toString() << endl;
+		}
+	}
+
+	if( split_debug )
+		cout << "     split end " << hex << thisLoc.getOfs() << dec << endl;
+}
+
+DiskLoc BtreeBucket::addHead(IndexDetails& id) {
+	BtreeBucket *p = allocTemp();
+	DiskLoc loc = theDataFileMgr.insert(id.indexNamespace().c_str(), p, p->Size, true);
+	return loc;
+}
+
+DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) {
+	DiskLoc p = thisLoc;
+	while( !p.btree()->isHead() )
+		p = p.btree()->parent;
+	return p;
+}
+
+DiskLoc BtreeBucket::advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) {
+	if( keyOfs < 0 || keyOfs >= n ) {
+		cout << "ASSERT failure BtreeBucket::advance, caller: " << caller << endl;
+		cout << "  thisLoc: " << thisLoc.toString() << endl;
+		cout << "  keyOfs: " << keyOfs << " n:" << n << " direction: " << direction << endl;
+		cout << bucketSummary() << endl;
+		assert( keyOfs >= 0 && keyOfs < n );
+	}
+	int adj = direction < 0 ? 1 : 0;
+	int ko = keyOfs + direction;
+	DiskLoc nextDown = childForPos(ko+adj);
+	if( !nextDown.isNull() ) { 
+//		nextDown.btree()->dump();//TEMP:
+		while( 1 ) {
+			keyOfs = direction>0 ? 0 : nextDown.btree()->n - 1;
+			DiskLoc loc= nextDown.btree()->childForPos(keyOfs + adj);
+			if( loc.isNull() )
+				break;
+			nextDown = loc;
+		}
+		return nextDown;
+	}
+
+	if( ko < n && ko >= 0 ) {
+		keyOfs = ko;
+		return thisLoc;
+	}
+
+	// end of bucket.  traverse back up.
+	DiskLoc childLoc = thisLoc;
+	DiskLoc ancestor = parent;
+	while( 1 ) {
+		if( ancestor.isNull() )
+			break;
+		BtreeBucket *an = ancestor.btree();
+		for( int i = 0; i < an->n; i++ ) {
+			if( an->childForPos(i+adj) == childLoc ) {
+				keyOfs = i;
+				return ancestor;
+			}
+		}
+		assert( direction<0 || an->nextChild == childLoc );
+		// parent exhausted also, keep going up
+		childLoc = ancestor;
+		ancestor = an->parent;
+	}
+
+	return DiskLoc();
+}
+
+DiskLoc BtreeBucket::locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction) { 
+	int p;
+	found = find(key, recordLoc, p);
+	if( found ) {
+		pos = p;
+		return thisLoc;
+	}
+
+	DiskLoc child = childForPos(p);
+
+	if( !child.isNull() ) { 
+		DiskLoc l = child.btree()->locate(child, key, pos, found, recordLoc, direction);
+		if( !l.isNull() )
+			return l;
+	}
+
+	if( direction == -1 && p == n && n ) { 
+		p--;
+	}
+
+	pos = p;
+	return pos == n ? DiskLoc() /*theend*/ : thisLoc;
+}
+
+/* thisloc is the location of this bucket object.  you must pass that in. */
+int BtreeBucket::_insert(DiskLoc thisLoc, DiskLoc recordLoc, 
+						JSObj& key, bool dupsAllowed,
+						DiskLoc lChild, DiskLoc rChild, IndexDetails& idx) { 
+	if( key.objsize() > KeyMax ) { 
+		problem() << "ERROR: key too large len:" << key.objsize() << " max:" << KeyMax << ' ' << idx.indexNamespace() << endl;
+		return 2;
+	}
+	assert( key.objsize() > 0 );
+
+	int pos;
+	bool found = find(key, recordLoc, pos);
+	if( insert_debug ) {
+		cout << "  " << thisLoc.toString() << '.' << "_insert " << 
+			key.toString() << '/' << recordLoc.toString() <<
+			" l:" << lChild.toString() << " r:" << rChild.toString() << endl;
+		cout << "    found:" << found << " pos:" << pos << " n:" << n << endl;
+	}
+
+	if( found ) {
+		if( k(pos).isUnused() ) { 
+			cout << "an unused already occupying keyslot, write more code.\n";
+			cout << "  index may be corrupt (missing data) now.\n";
+		}
+
+		cout << "_insert(): key already exists in index\n";
+		cout << "  " << idx.indexNamespace().c_str() << " thisLoc:" << thisLoc.toString() << '\n';
+		cout << "  " << key.toString() << '\n';
+		cout << "  " << "recordLoc:" << recordLoc.toString() << " pos:" << pos << endl;
+		cout << "  old l r: " << childForPos(pos).toString() << ' ' << childForPos(pos+1).toString() << endl;
+		cout << "  new l r: " << lChild.toString() << ' ' << rChild.toString() << endl;
+		assert(false);
+
+		// on a dup key always insert on the right or else you will be broken.
+//		pos++;
+		// on a promotion, find the right point to update if dup keys.
+		/* not needed: we always insert right after the first key so we are ok with just pos++...
+		if( !rChild.isNull() ) { 
+			while( pos < n && k(pos).prevChildBucket != lchild ) { 
+				pos++;
+				cout << "looking for the right dup key" << endl;
+			}
+		}
+		*/
+	}
+
+	DEBUGGING cout << "TEMP: key: " << key.toString() << endl;
+	DiskLoc& child = getChild(pos);
+	if( insert_debug )
+		cout << "    getChild(" << pos << "): " << child.toString() << endl;
+	if( child.isNull() || !rChild.isNull() /* means an 'internal' insert */ ) { 
+		insertHere(thisLoc, pos, recordLoc, key, lChild, rChild, idx);
+		return 0;
+	}
+
+	return child.btree()->insert(child, recordLoc, key, dupsAllowed, idx, false);
+}
+
+void BtreeBucket::dump() { 
+	cout << "DUMP btreebucket: ";
+	cout << " parent:" << hex << parent.getOfs() << dec;
+	for( int i = 0; i < n; i++ ) {
+		cout << '\n';
+		KeyNode k = keyNode(i);
+		cout << '\t' << i << '\t' << k.key.toString() << "\tleft:" << hex << 
+			k.prevChildBucket.getOfs() << "\trec:" << k.recordLoc.getOfs() << dec;
+		if( this->k(i).isUnused() )
+			cout << " UNUSED";
+	}
+	cout << " right:" << hex << nextChild.getOfs() << dec << endl;
+}
+
+JSObj *music = 0;
+
+void tempMusic(DiskLoc thisLoc)
+{
+	BtreeCursor c(thisLoc, *music, 1, true);
+	while( c.ok() ) { 
+		KeyNode kn = c.currKeyNode();
+		if( !kn.key.woEqual(*music) )
+			break;
+		if( kn.recordLoc.getOfs() == 0x4c8d7c0 ) {
+			cout << "*** found it" << endl;
+				//				c.bucket.btree()->dump();
+			return;
+		}
+		c.advance();
+	}
+
+	cout << "*** NOT FOUND" << endl;
+}
+
+/* todo: meaning of return code unclear clean up */
+int BtreeBucket::insert(DiskLoc thisLoc, DiskLoc recordLoc, 
+						JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel) 
+{
+	if( toplevel ) {
+		if( key.objsize() > KeyMax ) { 
+			problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.toString() << '\n';
+			return 3;
+		}
+		++ninserts;
+		/*
+		if( ninserts % 1000 == 0 ) {
+			cout << "ninserts: " << ninserts << endl;
+			if( 0 && ninserts >= 127287 ) {
+				cout << "debug?" << endl;
+				split_debug = 1;
+			}
+		}
+		*/
+	}
+
+	bool chk = false;
+
+	int x = _insert(thisLoc, recordLoc, key, dupsAllowed, DiskLoc(), DiskLoc(), idx);
+	assertValid(); 
+
+/*	if( toplevel ) {
+		if( recordLoc.getOfs() == 0x4c8d7c0 ) {
+			if( key.toString() == "{ _searchIndex: \"music\" }" ) { 
+				tempMusic(thisLoc);
+			}
+		}
+	}
+*/
+	return x;
+}
+
+void BtreeBucket::shape(stringstream& ss) {
+	_shape(0, ss);
+}
+
+/* - BtreeCursor --------------------------------------------------- */
+
+BtreeCursor::BtreeCursor(DiskLoc head, JSObj& k, int _direction, bool sm) : 
+    direction(_direction), stopmiss(sm) 
+{
+//otherTraceLevel = 999;
+
+	bool found;
+	if( otherTraceLevel >= 12 ) {
+		if( otherTraceLevel >= 200 ) { 
+			cout << "::BtreeCursor() qtl>200.  validating entire index." << endl;
+			head.btree()->fullValidate(head);
+		}
+		else {
+			cout << "BTreeCursor(). dumping head bucket" << endl;
+			head.btree()->dump();
+		}
+	}
+	bucket = head.btree()->locate(head, k, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
+	checkUnused();
+}
+
+int zzz = 0;
+
+/* skip unused keys. */
+void BtreeCursor::checkUnused() {
+	int u = 0;
+	while( 1 ) {
+		if( !ok() ) 
+			break;
+		BtreeBucket *b = bucket.btree();
+		_KeyNode& kn = b->k(keyOfs);
+		if( kn.isUsed() )
+			break;
+		bucket = b->advance(bucket, keyOfs, direction, "checkUnused");
+		u++;
+	}
+	if( u > 10 && ++zzz % 16 == 0 )
+		cout << "btree unused skipped:" << u << endl;
+}
+
+/*DiskLoc BtreeCursor::currLoc() {
+	assert( !bucket.isNull() );
+	_KeyNode& kn = bucket.btree()->k(keyOfs);
+	assert( kn.isUsed() );
+	return kn.recordLoc;
+}*/
+
+bool BtreeCursor::advance() { 
+	if( bucket.isNull() )
+		return false;
+	bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance");
+	checkUnused();
+	return !bucket.isNull();
+}
+
+void BtreeCursor::noteLocation() {
+	if( !eof() ) {
+		JSObj o = bucket.btree()->keyAt(keyOfs).copy();
+		keyAtKeyOfs = o;
+		locAtKeyOfs = bucket.btree()->k(keyOfs).recordLoc;
+	}
+}
+
+int clctr = 0;
+
+/* see if things moved around (deletes, splits, inserts) */
+void BtreeCursor::checkLocation() { 
+	try {
+		if( eof() ) 
+			return;
+		BtreeBucket *b = bucket.btree();
+		if( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) &&
+			b->k(keyOfs).recordLoc == locAtKeyOfs ) { 
+			if( !b->k(keyOfs).isUsed() )
+				checkUnused();
+			return;
+		}
+	}
+	catch( AssertionException ) { 
+		cout << "Caught exception in checkLocation(), that's maybe ok" << endl;
+	}
+
+	bool found;
+	DiskLoc bold = bucket;
+
+/* TODO: Switch to keep indexdetails and do idx.head! */
+	/* didn't find, check from the top */
+	DiskLoc head = bold.btree()->getHead(bold);
+	bucket = head.btree()->locate(head, keyAtKeyOfs, keyOfs, found, locAtKeyOfs, direction);
+	if( clctr++ % 128 == 0 )
+		cout << "  key seems to have moved in the index, refinding. found:" << found << endl;
+	if( found )
+		checkUnused();
+}
+
+/* ----------------------------------------------------------------------------- */
+
+struct BtreeUnitTest { 
+	BtreeUnitTest() { 
+		assert( minDiskLoc.compare(maxDiskLoc) < 0 );
+	}
+} btut;
diff --git a/db/btree.h b/db/btree.h
index 407a954ca27..063196f67b4 100644
--- a/db/btree.h
+++ b/db/btree.h
@@ -1,188 +1,188 @@
-// btree.h

-

-#pragma once

-

-#include "../stdafx.h"

-#include "jsobj.h"

-#include "storage.h"

-#include "pdfile.h"

-

-#pragma pack(push)

-#pragma pack(1)

-

-struct _KeyNode {

-	DiskLoc prevChildBucket;

-	DiskLoc recordLoc;

-	short keyDataOfs() { return (short) _kdo; }

-	unsigned short _kdo;

-	void setKeyDataOfs(short s) { _kdo = s; assert(s>=0); }

-	void setKeyDataOfsSavingUse(short s) { _kdo = s; assert(s>=0); }

-	void setUnused() { recordLoc.GETOFS() |= 1; }

-//	void setUsed() { _kdo &= 0x7fff; }

-	int isUnused() { return (recordLoc.getOfs() & 1); }

-	int isUsed() { return !isUnused(); }

-};

-

-#pragma pack(pop)

-

-class BucketBasics;

-

-/* wrapper - this is our in memory representation of the key.  _KeyNode is the disk representation. */

-class KeyNode {

-public:

-	KeyNode(BucketBasics& bb, _KeyNode &k);

-	DiskLoc& prevChildBucket;

-	DiskLoc& recordLoc;

-	JSObj key;

-};

-

-#pragma pack(push)

-#pragma pack(1)

-

-/* this class is all about the storage management */

-class BucketBasics {

-	friend class KeyNode;

-public:

-	void dumpTree(DiskLoc thisLoc);

-	bool isHead() { return parent.isNull(); }

-	void assertValid(bool force = false);

-	int fullValidate(const DiskLoc& thisLoc); /* traverses everything */ 

-protected:

-	DiskLoc& getChild(int pos) { 

-		assert( pos >= 0 && pos <= n );

-		return pos == n ? nextChild : k(pos).prevChildBucket;

-	}

-	KeyNode keyNode(int i) { 

-		assert( i < n );

-		return KeyNode(*this, k(i));

-	}

-

-	char * dataAt(short ofs) { return data + ofs; }

-

-	void init(); // initialize a new node

-

-	/* returns false if node is full and must be split 

-	   keypos is where to insert -- inserted after that key #.  so keypos=0 is the leftmost one.

-	*/

-	bool basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key);

-	void pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild);

-	void _delKeyAtPos(int keypos); // low level version that doesn't deal with child ptrs. 

-

-	/* !Packed means there is deleted fragment space within the bucket.

-       We "repack" when we run out of space before considering the node

-	   to be full. 

-	   */

-	enum Flags { Packed=1 };

-

-	DiskLoc childForPos(int p) { 

-		return p == n ? nextChild : k(p).prevChildBucket;

-	}

-

-	int totalDataSize() const;

-	void pack(); void setNotPacked(); void setPacked();

-	int _alloc(int bytes);

-	void truncateTo(int N);

-	void markUnused(int keypos);

-public:

-	DiskLoc parent;

-

-	string bucketSummary() const {

-		stringstream ss;

-		ss << "  Bucket info:" << endl;

-		ss << "    n: " << n << endl;

-		ss << "    parent: " << parent.toString() << endl;

-		ss << "    nextChild: " << parent.toString() << endl;

-		ss << "    Size: " << Size << " flags:" << flags << endl;

-		ss << "    emptySize: " << emptySize << " topSize: " << topSize << endl;

-		return ss.str();

-	}

-

-protected:

-	void _shape(int level, stringstream&);

-	DiskLoc nextChild; // child bucket off and to the right of the highest key.

-	int Size; // total size of this btree node in bytes. constant.

-	int flags;

-	int emptySize; // size of the empty region

-	int topSize; // size of the data at the top of the bucket (keys are at the beginning or 'bottom')

-	int n; // # of keys so far.

-	int reserved;

-	_KeyNode& k(int i) { return ((_KeyNode*)data)[i]; }

-	char data[4];

-};

-

-class BtreeBucket : public BucketBasics { 

-	friend class BtreeCursor;

-public:

-	void dump();

-

-	static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */

-	int insert(DiskLoc thisLoc, DiskLoc recordLoc, 

-		JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel);

-

-	bool unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc);

-

-	/* locate may return an "unused" key that is just a marker.  so be careful.

-  	   looks for a key:recordloc pair.

-	*/

-	DiskLoc locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction=1);

-

-	/* advance one key position in the index: */

-	DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller);

-	DiskLoc getHead(const DiskLoc& thisLoc);

-

-	/* get tree shape */

-	void shape(stringstream&);

-private:

-	void fixParentPtrs(const DiskLoc& thisLoc);

-	void delBucket(const DiskLoc& thisLoc, IndexDetails&);

-	void delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p);

-	JSObj keyAt(int keyOfs) { return keyOfs >= n ? JSObj() : keyNode(keyOfs).key; }

-	static BtreeBucket* allocTemp(); /* caller must release with free() */

-	void insertHere(DiskLoc thisLoc, int keypos, 

-		DiskLoc recordLoc, JSObj& key,

-		DiskLoc lchild, DiskLoc rchild, IndexDetails&);

-	int _insert(DiskLoc thisLoc, DiskLoc recordLoc, 

-		JSObj& key, bool dupsAllowed,

-		DiskLoc lChild, DiskLoc rChild, IndexDetails&);

-	bool find(JSObj& key, DiskLoc recordLoc, int& pos);

-	static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey);

-};

-

-class BtreeCursor : public Cursor {

-	friend class BtreeBucket;

-public:

-	BtreeCursor(DiskLoc head, JSObj& startKey, int direction, bool stopmiss);

-	virtual bool ok() { return !bucket.isNull(); }

-	bool eof() { return !ok(); }

-	virtual bool advance();

-	virtual bool tempStopOnMiss() { return stopmiss; }

-	virtual void noteLocation(); // updates keyAtKeyOfs...

-	virtual void checkLocation();

-

-	_KeyNode& _currKeyNode() { 

-		assert( !bucket.isNull() );

-		_KeyNode& kn = bucket.btree()->k(keyOfs);

-		assert( kn.isUsed() );

-		return kn;

-	}

-	KeyNode currKeyNode() { 

-		assert( !bucket.isNull() );

-		return bucket.btree()->keyNode(keyOfs);

-	}

-

-	virtual DiskLoc currLoc() { return !bucket.isNull() ? _currKeyNode().recordLoc : DiskLoc(); }

-	virtual Record* _current() { return currLoc().rec(); }

-	virtual JSObj current() { return JSObj(_current()); }

-	virtual const char * toString() { return "BtreeCursor"; }

-

-private:

-	void checkUnused();

-	DiskLoc bucket;

-	int keyOfs;

-	int direction; // 1=fwd,-1=reverse

-	bool stopmiss;

-	JSObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call

-	DiskLoc locAtKeyOfs;

-};

-

-#pragma pack(pop)

+// btree.h
+
+#pragma once
+
+#include "../stdafx.h"
+#include "jsobj.h"
+#include "storage.h"
+#include "pdfile.h"
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct _KeyNode {
+	DiskLoc prevChildBucket;
+	DiskLoc recordLoc;
+	short keyDataOfs() { return (short) _kdo; }
+	unsigned short _kdo;
+	void setKeyDataOfs(short s) { _kdo = s; assert(s>=0); }
+	void setKeyDataOfsSavingUse(short s) { _kdo = s; assert(s>=0); }
+	void setUnused() { recordLoc.GETOFS() |= 1; }
+//	void setUsed() { _kdo &= 0x7fff; }
+	int isUnused() { return (recordLoc.getOfs() & 1); }
+	int isUsed() { return !isUnused(); }
+};
+
+#pragma pack(pop)
+
+class BucketBasics;
+
+/* wrapper - this is our in memory representation of the key.  _KeyNode is the disk representation. */
+class KeyNode {
+public:
+	KeyNode(BucketBasics& bb, _KeyNode &k);
+	DiskLoc& prevChildBucket;
+	DiskLoc& recordLoc;
+	JSObj key;
+};
+
+#pragma pack(push)
+#pragma pack(1)
+
+/* this class is all about the storage management */
+class BucketBasics {
+	friend class KeyNode;
+public:
+	void dumpTree(DiskLoc thisLoc);
+	bool isHead() { return parent.isNull(); }
+	void assertValid(bool force = false);
+	int fullValidate(const DiskLoc& thisLoc); /* traverses everything */ 
+protected:
+	DiskLoc& getChild(int pos) { 
+		assert( pos >= 0 && pos <= n );
+		return pos == n ? nextChild : k(pos).prevChildBucket;
+	}
+	KeyNode keyNode(int i) { 
+		assert( i < n );
+		return KeyNode(*this, k(i));
+	}
+
+	char * dataAt(short ofs) { return data + ofs; }
+
+	void init(); // initialize a new node
+
+	/* returns false if node is full and must be split 
+	   keypos is where to insert -- inserted after that key #.  so keypos=0 is the leftmost one.
+	*/
+	bool basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key);
+	void pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild);
+	void _delKeyAtPos(int keypos); // low level version that doesn't deal with child ptrs. 
+
+	/* !Packed means there is deleted fragment space within the bucket.
+       We "repack" when we run out of space before considering the node
+	   to be full. 
+	   */
+	enum Flags { Packed=1 };
+
+	DiskLoc childForPos(int p) { 
+		return p == n ? nextChild : k(p).prevChildBucket;
+	}
+
+	int totalDataSize() const;
+	void pack(); void setNotPacked(); void setPacked();
+	int _alloc(int bytes);
+	void truncateTo(int N);
+	void markUnused(int keypos);
+public:
+	DiskLoc parent;
+
+	string bucketSummary() const {
+		stringstream ss;
+		ss << "  Bucket info:" << endl;
+		ss << "    n: " << n << endl;
+		ss << "    parent: " << parent.toString() << endl;
+		ss << "    nextChild: " << parent.toString() << endl;
+		ss << "    Size: " << Size << " flags:" << flags << endl;
+		ss << "    emptySize: " << emptySize << " topSize: " << topSize << endl;
+		return ss.str();
+	}
+
+protected:
+	void _shape(int level, stringstream&);
+	DiskLoc nextChild; // child bucket off and to the right of the highest key.
+	int Size; // total size of this btree node in bytes. constant.
+	int flags;
+	int emptySize; // size of the empty region
+	int topSize; // size of the data at the top of the bucket (keys are at the beginning or 'bottom')
+	int n; // # of keys so far.
+	int reserved;
+	_KeyNode& k(int i) { return ((_KeyNode*)data)[i]; }
+	char data[4];
+};
+
+class BtreeBucket : public BucketBasics { 
+	friend class BtreeCursor;
+public:
+	void dump();
+
+	static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */
+	int insert(DiskLoc thisLoc, DiskLoc recordLoc, 
+		JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel);
+
+	bool unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc);
+
+	/* locate may return an "unused" key that is just a marker.  so be careful.
+  	   looks for a key:recordloc pair.
+	*/
+	DiskLoc locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction=1);
+
+	/* advance one key position in the index: */
+	DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller);
+	DiskLoc getHead(const DiskLoc& thisLoc);
+
+	/* get tree shape */
+	void shape(stringstream&);
+private:
+	void fixParentPtrs(const DiskLoc& thisLoc);
+	void delBucket(const DiskLoc& thisLoc, IndexDetails&);
+	void delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p);
+	JSObj keyAt(int keyOfs) { return keyOfs >= n ? JSObj() : keyNode(keyOfs).key; }
+	static BtreeBucket* allocTemp(); /* caller must release with free() */
+	void insertHere(DiskLoc thisLoc, int keypos, 
+		DiskLoc recordLoc, JSObj& key,
+		DiskLoc lchild, DiskLoc rchild, IndexDetails&);
+	int _insert(DiskLoc thisLoc, DiskLoc recordLoc, 
+		JSObj& key, bool dupsAllowed,
+		DiskLoc lChild, DiskLoc rChild, IndexDetails&);
+	bool find(JSObj& key, DiskLoc recordLoc, int& pos);
+	static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey);
+};
+
+class BtreeCursor : public Cursor {
+	friend class BtreeBucket;
+public:
+	BtreeCursor(DiskLoc head, JSObj& startKey, int direction, bool stopmiss);
+	virtual bool ok() { return !bucket.isNull(); }
+	bool eof() { return !ok(); }
+	virtual bool advance();
+	virtual bool tempStopOnMiss() { return stopmiss; }
+	virtual void noteLocation(); // updates keyAtKeyOfs...
+	virtual void checkLocation();
+
+	_KeyNode& _currKeyNode() { 
+		assert( !bucket.isNull() );
+		_KeyNode& kn = bucket.btree()->k(keyOfs);
+		assert( kn.isUsed() );
+		return kn;
+	}
+	KeyNode currKeyNode() { 
+		assert( !bucket.isNull() );
+		return bucket.btree()->keyNode(keyOfs);
+	}
+
+	virtual DiskLoc currLoc() { return !bucket.isNull() ? _currKeyNode().recordLoc : DiskLoc(); }
+	virtual Record* _current() { return currLoc().rec(); }
+	virtual JSObj current() { return JSObj(_current()); }
+	virtual const char * toString() { return "BtreeCursor"; }
+
+private:
+	void checkUnused();
+	DiskLoc bucket;
+	int keyOfs;
+	int direction; // 1=fwd,-1=reverse
+	bool stopmiss;
+	JSObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call
+	DiskLoc locAtKeyOfs;
+};
+
+#pragma pack(pop)
diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index c9d21d48aa8..d90762b90c5 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -1,214 +1,214 @@
-// clientcursor.cpp

-

-/* Cursor -- and its derived classes -- are our internal cursors.

-

-   ClientCursor is a wrapper that represents a cursorid from our client 

-   application's perspective.

-*/

-

-#include "stdafx.h"

-#include "query.h"

-#include "introspect.h"

-#include <time.h>

-

-/* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */

-

-typedef map<DiskLoc, ClientCursor*> DiskToCC;

-map<DiskLoc, ClientCursor*> byLocation;

-//HashTable<DiskLoc,ClientCursor*> byLocation(malloc(10000000), 10000000, "bylocation");

-

-CCMap clientCursors;

-

-class CursInspector : public SingleResultObjCursor { 

-	Cursor* clone() { 

-		return new CursInspector(); 

-	}

-//	Cursor* clone() { return new CursInspector(*this); }

-	void fill() { 

-		b.append("byLocation_size", byLocation.size());

-		b.append("clientCursors_size", clientCursors.size());

-

-

-

-		stringstream ss;

-		ss << '\n';

-		int x = 40;

-		DiskToCC::iterator it = byLocation.begin();

-		while( it != byLocation.end() ) {

-			DiskLoc dl = it->first;

-			ClientCursor *cc = it->second;

-			ss << dl.toString() << " -> \n";

-

-			while( cc ) {

-				ss << "    cid:" << cc->cursorid << ' ' << cc->ns << " pos:" << cc->pos << " LL:" << cc->lastLoc.toString();

-				try { 

-					setClient(cc->ns.c_str());

-					Record *r = dl.rec();

-					ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec << ' ' << cc->c->toString();

-					if( r->nextOfs >= 0 && r->nextOfs < 16 ) 

-						ss << " DELETED??? (!)";

-				}

-				catch(...) { 

-					ss << " EXCEPTION";

-				}

-				ss << "\n";

-				cc = cc->nextAtThisLocation;

-			}

-			if( --x <= 0 ) {

-				ss << "only first 40 shown\n" << endl;

-				break;

-			}

-			it++;

-		}

-		b.append("dump", ss.str().c_str());

-	}

-public:

-	CursInspector() { reg("intr.cursors"); }

-} _ciproto;

-

-/* must call this when a btree node is updated */

-void removedKey(const DiskLoc& btreeLoc, int keyPos) { 

-// TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

-}

-

-/* must call this on a delete so we clean up the cursors. */

-void aboutToDelete(const DiskLoc& dl) { 

-	DiskToCC::iterator it = byLocation.find(dl);

-//	cout << "atd:" << dl.toString() << endl;

-	if( it != byLocation.end() ) {

-		ClientCursor *cc = it->second;

-		byLocation.erase(it);

-

-		assert( cc != 0 );

-		int z = 0;

-		while( cc ) { 

-			z++;

-//			cout << "cc: " << cc->ns << endl;

-			ClientCursor *nxt = cc->nextAtThisLocation;

-			cc->nextAtThisLocation = 0; // updateLocation will manipulate linked list ptrs, so clean that up first.

-			cc->c->checkLocation();

-			cc->c->advance();

-			cc->lastLoc.Null(); // so updateLocation doesn't try to remove, just to be faster -- we handled that.

-			cc->updateLocation();

-			cc = nxt;

-		}

-//		cout << "z:" << z << endl;

-	}

-}

-

-void ClientCursor::cleanupByLocation(DiskLoc loc, long long cursorid) { 

-	if( loc.isNull() )

-		return;

-

-	DiskToCC::iterator it = byLocation.find(loc);

-	if( it != byLocation.end() ) {

-		ClientCursor *first = it->second;

-		ClientCursor *cc = first;

-		ClientCursor *prev = 0;

-

-		while( 1 ) {

-			if( cc == 0 ) 

-				break;

-			if( cc->cursorid == cursorid ) { 

-				// found one to remove.

-				if( prev == 0 ) { 

-					if( cc->nextAtThisLocation )

-						it->second = cc->nextAtThisLocation;

-					else

-						byLocation.erase(it);

-				}

-				else { 

-					prev->nextAtThisLocation = cc->nextAtThisLocation;

-				}

-				cc->nextAtThisLocation = 0;

-				return;

-			}

-			cc = cc->nextAtThisLocation;

-		}

-	}

-

-	// not found!

-	//cout << "Assertion failure - cleanupByLocation: not found " << cursorid << endl;

-}

-

-ClientCursor::~ClientCursor() {

-#if defined(_WIN32)

-	cout << "~clientcursor " << cursorid << endl;

-#endif

-	assert( pos != -2 );

-

-	cleanupByLocation(lastLoc, cursorid);

-

-	assert( pos != -2 );

-

-	// defensive

-	lastLoc.Null();

-	cursorid = -1; 

-	pos = -2;

-	nextAtThisLocation = 0;

-}

-

-// note this doesn't set lastLoc -- caller should.

-void ClientCursor::addToByLocation(DiskLoc cl) { 

-	assert( cursorid );

-

-	if( nextAtThisLocation ) { 

-		wassert( nextAtThisLocation == 0 );

-		return;

-	}

-

-	DiskToCC::iterator j = byLocation.find(cl);

-	nextAtThisLocation = j == byLocation.end() ? 0 : j->second;

-	byLocation[cl] = this;

-}

-

-void ClientCursor::updateLocation() {

-	assert( cursorid );

-

-	DiskLoc cl = c->currLoc();

-	//	cout<< "  TEMP: updateLocation last:" << lastLoc.toString() << " cl:" << cl.toString() << '\n';

-

-	if( !lastLoc.isNull() )

-		cleanupByLocation(lastLoc, cursorid);

-

-	if( !cl.isNull() )

-		addToByLocation(cl);

-

-	lastLoc = cl;

-	c->noteLocation();

-}

-

-/* report to us that a new clientcursor exists so we can track it. 

-   note you still must call updateLocation (which likely should be changed)

-*/

-void ClientCursor::add(ClientCursor* cc) {

-	clientCursors[cc->cursorid] = cc;

-}

-

-// todo: delete the ClientCursor.

-// todo: other map

-bool ClientCursor::erase(long long id) { 

-	CCMap::iterator it = clientCursors.find(id);

-	if( it != clientCursors.end() ) {

-		ClientCursor *cc = it->second;

-		it->second = 0; // defensive

-		clientCursors.erase(it);

-		delete cc; // destructor will fix byLocation map

-		return true;

-	}

-	return false;

-}

-

-long long allocCursorId() { 

-	long long x;

-	while( 1 ) {

-		x = (((long long)rand()) << 32);

-		x = x | (int) curTimeMillis() | 0x80000000; // OR to make sure not zero

-		if( clientCursors.count(x) == 0 )

-			break;

-	}

-#if defined(_WIN32)

-	cout << "alloccursorid " << x << endl;

-#endif

-	return x;

-}

+// clientcursor.cpp
+
+/* Cursor -- and its derived classes -- are our internal cursors.
+
+   ClientCursor is a wrapper that represents a cursorid from our client 
+   application's perspective.
+*/
+
+#include "stdafx.h"
+#include "query.h"
+#include "introspect.h"
+#include <time.h>
+
+/* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */
+
+typedef map<DiskLoc, ClientCursor*> DiskToCC;
+map<DiskLoc, ClientCursor*> byLocation;
+//HashTable<DiskLoc,ClientCursor*> byLocation(malloc(10000000), 10000000, "bylocation");
+
+CCMap clientCursors;
+
+class CursInspector : public SingleResultObjCursor { 
+	Cursor* clone() { 
+		return new CursInspector(); 
+	}
+//	Cursor* clone() { return new CursInspector(*this); }
+	void fill() { 
+		b.append("byLocation_size", byLocation.size());
+		b.append("clientCursors_size", clientCursors.size());
+
+
+
+		stringstream ss;
+		ss << '\n';
+		int x = 40;
+		DiskToCC::iterator it = byLocation.begin();
+		while( it != byLocation.end() ) {
+			DiskLoc dl = it->first;
+			ClientCursor *cc = it->second;
+			ss << dl.toString() << " -> \n";
+
+			while( cc ) {
+				ss << "    cid:" << cc->cursorid << ' ' << cc->ns << " pos:" << cc->pos << " LL:" << cc->lastLoc.toString();
+				try { 
+					setClient(cc->ns.c_str());
+					Record *r = dl.rec();
+					ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec << ' ' << cc->c->toString();
+					if( r->nextOfs >= 0 && r->nextOfs < 16 ) 
+						ss << " DELETED??? (!)";
+				}
+				catch(...) { 
+					ss << " EXCEPTION";
+				}
+				ss << "\n";
+				cc = cc->nextAtThisLocation;
+			}
+			if( --x <= 0 ) {
+				ss << "only first 40 shown\n" << endl;
+				break;
+			}
+			it++;
+		}
+		b.append("dump", ss.str().c_str());
+	}
+public:
+	CursInspector() { reg("intr.cursors"); }
+} _ciproto;
+
+/* must call this when a btree node is updated */
+void removedKey(const DiskLoc& btreeLoc, int keyPos) { 
+// TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+}
+
+/* must call this on a delete so we clean up the cursors. */
+void aboutToDelete(const DiskLoc& dl) { 
+	DiskToCC::iterator it = byLocation.find(dl);
+//	cout << "atd:" << dl.toString() << endl;
+	if( it != byLocation.end() ) {
+		ClientCursor *cc = it->second;
+		byLocation.erase(it);
+
+		assert( cc != 0 );
+		int z = 0;
+		while( cc ) { 
+			z++;
+//			cout << "cc: " << cc->ns << endl;
+			ClientCursor *nxt = cc->nextAtThisLocation;
+			cc->nextAtThisLocation = 0; // updateLocation will manipulate linked list ptrs, so clean that up first.
+			cc->c->checkLocation();
+			cc->c->advance();
+			cc->lastLoc.Null(); // so updateLocation doesn't try to remove, just to be faster -- we handled that.
+			cc->updateLocation();
+			cc = nxt;
+		}
+//		cout << "z:" << z << endl;
+	}
+}
+
+void ClientCursor::cleanupByLocation(DiskLoc loc, long long cursorid) { 
+	if( loc.isNull() )
+		return;
+
+	DiskToCC::iterator it = byLocation.find(loc);
+	if( it != byLocation.end() ) {
+		ClientCursor *first = it->second;
+		ClientCursor *cc = first;
+		ClientCursor *prev = 0;
+
+		while( 1 ) {
+			if( cc == 0 ) 
+				break;
+			if( cc->cursorid == cursorid ) { 
+				// found one to remove.
+				if( prev == 0 ) { 
+					if( cc->nextAtThisLocation )
+						it->second = cc->nextAtThisLocation;
+					else
+						byLocation.erase(it);
+				}
+				else { 
+					prev->nextAtThisLocation = cc->nextAtThisLocation;
+				}
+				cc->nextAtThisLocation = 0;
+				return;
+			}
+			cc = cc->nextAtThisLocation;
+		}
+	}
+
+	// not found!
+	//cout << "Assertion failure - cleanupByLocation: not found " << cursorid << endl;
+}
+
+ClientCursor::~ClientCursor() {
+#if defined(_WIN32)
+	cout << "~clientcursor " << cursorid << endl;
+#endif
+	assert( pos != -2 );
+
+	cleanupByLocation(lastLoc, cursorid);
+
+	assert( pos != -2 );
+
+	// defensive
+	lastLoc.Null();
+	cursorid = -1; 
+	pos = -2;
+	nextAtThisLocation = 0;
+}
+
+// note this doesn't set lastLoc -- caller should.
+void ClientCursor::addToByLocation(DiskLoc cl) { 
+	assert( cursorid );
+
+	if( nextAtThisLocation ) { 
+		wassert( nextAtThisLocation == 0 );
+		return;
+	}
+
+	DiskToCC::iterator j = byLocation.find(cl);
+	nextAtThisLocation = j == byLocation.end() ? 0 : j->second;
+	byLocation[cl] = this;
+}
+
+void ClientCursor::updateLocation() {
+	assert( cursorid );
+
+	DiskLoc cl = c->currLoc();
+	//	cout<< "  TEMP: updateLocation last:" << lastLoc.toString() << " cl:" << cl.toString() << '\n';
+
+	if( !lastLoc.isNull() )
+		cleanupByLocation(lastLoc, cursorid);
+
+	if( !cl.isNull() )
+		addToByLocation(cl);
+
+	lastLoc = cl;
+	c->noteLocation();
+}
+
+/* report to us that a new clientcursor exists so we can track it. 
+   note you still must call updateLocation (which likely should be changed)
+*/
+void ClientCursor::add(ClientCursor* cc) {
+	clientCursors[cc->cursorid] = cc;
+}
+
+// todo: delete the ClientCursor.
+// todo: other map
+bool ClientCursor::erase(long long id) { 
+	CCMap::iterator it = clientCursors.find(id);
+	if( it != clientCursors.end() ) {
+		ClientCursor *cc = it->second;
+		it->second = 0; // defensive
+		clientCursors.erase(it);
+		delete cc; // destructor will fix byLocation map
+		return true;
+	}
+	return false;
+}
+
+long long allocCursorId() { 
+	long long x;
+	while( 1 ) {
+		x = (((long long)rand()) << 32);
+		x = x | (int) curTimeMillis() | 0x80000000; // OR to make sure not zero
+		if( clientCursors.count(x) == 0 )
+			break;
+	}
+#if defined(_WIN32)
+	cout << "alloccursorid " << x << endl;
+#endif
+	return x;
+}
diff --git a/db/db.cpp b/db/db.cpp
index ebdedc8ce50..c50c5769420 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -1,757 +1,757 @@
-// db.cpp : Defines the entry point for the console application.

-//

-

-#include "stdafx.h"

-#include "db.h"

-#include "../grid/message.h"

-#include "../util/mmap.h"

-#include "../util/hashtab.h"

-#include "../util/goodies.h"

-#include "pdfile.h"

-#include "jsobj.h"

-#include "javajs.h"

-#include "query.h"

-#include "introspect.h"

-

-extern const char *dbpath;

-extern int curOp;

-

-boost::mutex dbMutex;

-

-struct MyStartupTests {

-	MyStartupTests() {

-		assert( sizeof(OID) == 12 );

-	}

-} mystartupdbcpp;

-

-/* example for

-	var zz = { x: 3, y: "foo", v: { z:5, arr: [1, 2] } }

-	zz.v.arr.prop = "zoo";

-*/

-

-void quicktest() { 

-	cout << "quicktest()\n";

-

-	MemoryMappedFile mmf;

-	char *m = (char *) mmf.map("/tmp/quicktest", 16384);

-	//	cout << "mmf reads: " << m << endl;

-	strcpy_s(m, 1000, "hello worldz");

-}

-

-void pdfileInit();

-

-class DbMessage {

-public:

-	DbMessage(Message& _m) : m(_m) {

-		theEnd = _m.data->_data + _m.data->dataLen();

-		int *r = (int *) _m.data->_data;

-		reserved = *r;

-		r++;

-		data = (const char *) r;

-		nextjsobj = data;

-	}

-

-	const char * getns() { return data; }

-	void getns(Namespace& ns) {

-		ns = data;

-	}

-

-	int pullInt() {

-		if( nextjsobj == data )

-			nextjsobj += strlen(data) + 1; // skip namespace

-		int i = *((int *)nextjsobj);

-		nextjsobj += 4;

-		return i;

-	}

-	long long pullInt64() {

-		if( nextjsobj == data )

-			nextjsobj += strlen(data) + 1; // skip namespace

-		long long i = *((long long *)nextjsobj);

-		nextjsobj += 8;

-		return i;

-	}

-

-	OID* getOID() {

-		return (OID *) (data + strlen(data) + 1); // skip namespace

-	}

-

-	void getQueryStuff(const char *&query, int& ntoreturn) {

-		int *i = (int *) (data + strlen(data) + 1);

-		ntoreturn = *i;

-		i++;

-		query = (const char *) i;

-	}

-

-	/* for insert and update msgs */

-	bool moreJSObjs() { return nextjsobj != 0; }

-	JSObj nextJsObj() {

-		if( nextjsobj == data )

-			nextjsobj += strlen(data) + 1; // skip namespace

-		JSObj js(nextjsobj);

-                assert( js.objsize() < ( theEnd - data ) );

-		if( js.objsize() <= 0 )

-			nextjsobj = null;

-		else {

-			nextjsobj += js.objsize();

-			if( nextjsobj >= theEnd )

-				nextjsobj = 0;

-		}

-		return js;

-	}

-

-private:

-	Message& m;

-	int reserved;

-	const char *data;

-	const char *nextjsobj;

-	const char *theEnd;

-};

-

-void killCursors(int n, long long *ids);

-void receivedKillCursors(Message& m) {

-	int *x = (int *) m.data->_data;

-	x++; // reserved

-	int n = *x++;

-	assert( n >= 1 );

-	if( n > 2000 ) { 

-		problem() << "Assertion failure, receivedKillCursors, n=" << n << endl;

-		cout << "Assertion failure, receivedKillCursors, n=" << n << endl;

-		assert( n < 30000 );

-	}

-	killCursors(n, (long long *) x);

-}

-

-void receivedUpdate(Message& m, stringstream& ss) {

-	DbMessage d(m);

-	const char *ns = d.getns();

-	assert(*ns);

-	setClient(ns);

-	if( client->profile )

-		ss << ns << ' ';

-	int flags = d.pullInt();

-	JSObj query = d.nextJsObj();

-

-	assert( d.moreJSObjs() );

-	assert( query.objsize() < m.data->dataLen() );

-	JSObj toupdate = d.nextJsObj();

-

-	assert( toupdate.objsize() < m.data->dataLen() );

-	assert( query.objsize() + toupdate.objsize() < m.data->dataLen() );

-	updateObjects(ns, toupdate, query, flags & 1, ss);

-}

-

-void receivedDelete(Message& m) {

-	DbMessage d(m);

-	const char *ns = d.getns();

-	assert(*ns);

-	setClient(ns);

-	int flags = d.pullInt();

-	assert( d.moreJSObjs() );

-	JSObj pattern = d.nextJsObj();

-	deleteObjects(ns, pattern, flags & 1);

-}

-

-void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {

-	DbMessage d(m);

-	const char *ns = d.getns();

-	setClient(ns);

-	int ntoskip = d.pullInt();

-	int ntoreturn = d.pullInt();

-	JSObj query = d.nextJsObj();

-	auto_ptr< set<string> > fields;

-	if( d.moreJSObjs() ) { 

-		fields = auto_ptr< set<string> >(new set<string>());

-		d.nextJsObj().getFieldNames(*fields);

-	}

-	QueryResult* msgdata;

-

-	try { 

-		msgdata = runQuery(ns, ntoskip, ntoreturn, query, fields, ss);

-	}

-	catch( AssertionException ) { 

-		ss << " exception ";

-		problem() << " Caught Assertion in runQuery " << ns << endl; 

-		cout << " Caught Assertion in runQuery, continuing" << endl; 

-		cout << "  ntoskip:" << ntoskip << " ntoreturn:" << ntoreturn << endl;

-		cout << "  ns:" << ns << endl;

-		cout << "  query:" << query.toString() << endl;

-		msgdata = (QueryResult*) malloc(sizeof(QueryResult));

-		QueryResult *qr = msgdata;

-		qr->_data[0] = 0;

-		qr->_data[1] = 0;

-		qr->_data[2] = 0;

-		qr->_data[3] = 0;

-		qr->len = sizeof(QueryResult);

-		qr->operation = opReply;

-		qr->cursorId = 0;

-		qr->startingFrom = 0;

-		qr->nReturned = 0;

-	}

-	Message resp;

-	resp.setData(msgdata, true); // transport will free

-	if( client ) { 

-		if( client->profile )

-			ss << " bytes:" << resp.data->dataLen();

-	}

-	else { 

-		cout << "ERROR: client is null; ns=" << ns << endl;

-	}

-	dbMsgPort.reply(m, resp);

-}

-

-void receivedGetMore(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {

-	DbMessage d(m);

-	const char *ns = d.getns();

-	ss << ns;

-	setClient(ns);

-	int ntoreturn = d.pullInt();

-	long long cursorid = d.pullInt64();

-	ss << " cid:" << cursorid;

-	ss << " ntoreturn:" << ntoreturn;

-	QueryResult* msgdata = getMore(ns, ntoreturn, cursorid);

-	Message resp;

-	resp.setData(msgdata, true);

-	ss << " bytes:" << resp.data->dataLen();

-	ss << " nreturned:" << msgdata->nReturned;

-	dbMsgPort.reply(m, resp);

-}

-

-void receivedInsert(Message& m, stringstream& ss) {

-//	cout << "GOT MSG id:" << m.data->id << endl;

-	DbMessage d(m);

-	while( d.moreJSObjs() ) {

-		JSObj js = d.nextJsObj();

-//		cout << "  temp dbinsert: got js object, size=" << js.objsize() << " ns:" << d.getns() << endl;

-		const char *ns = d.getns();

-		assert(*ns);

-		setClient(ns);

-		ss << ns;

-		theDataFileMgr.insert(ns, (void*) js.objdata(), js.objsize());

-	}

-}

-

-void testTheDb() {

-	stringstream ss;

-

-	setClient("sys.unittest.pdfile");

-

-	/* this is not validly formatted, if you query this namespace bad things will happen */

-	theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);

-	theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);

-

-	JSObj j1((const char *) &js1);

-	deleteObjects("sys.unittest.delete", j1, false);

-	theDataFileMgr.insert("sys.unittest.delete", &js1, sizeof(js1));

-	deleteObjects("sys.unittest.delete", j1, false);

-	updateObjects("sys.unittest.delete", j1, j1, true,ss);

-	updateObjects("sys.unittest.delete", j1, j1, false,ss);

-

-	auto_ptr<Cursor> c = theDataFileMgr.findAll("sys.unittest.pdfile");

-	while( c->ok() ) {

-		Record* r = c->_current();

-		c->advance();

-	}

-	cout << endl;

-

-	client = 0;

-}

-

-int port = DBPort;

-

-MessagingPort *grab = 0;

-void connThread();

-

-class OurListener : public Listener { 

-public:

-	OurListener(int p) : Listener(p) { }

-	virtual void accepted(MessagingPort *mp) {

-		assert( grab == 0 );

-		grab = mp;

-		boost::thread thr(connThread);

-		while( grab )

-			sleepmillis(1);

-	}

-};

-

-void listen(int port) { 

-	const char *Version = "db version: 110 2jun2008 embedded obj queries";

-	problem() << Version << endl;

-	cout << Version << endl;

-	pdfileInit();

-	testTheDb();

-	cout << curTimeMillis() % 10000 << " waiting for connections on port " << port << " ...\n" << endl;

-	OurListener l(port);

-	l.listen();

-}

-

-int ctr = 0;

-extern int callDepth;

-

-class JniMessagingPort : public AbstractMessagingPort { 

-public:

-	JniMessagingPort(Message& _container) : container(_container) { }

-	void reply(Message& received, Message& response) { 

-		container = response;

-	}

-	Message & container;

-};

-

-/* a call from java/js to the database locally. 

-

-     m - inbound message

-     out - outbound message, if there is any, will be set here.

-           if there is one, out.data will be non-null on return.

-		   The out.data buffer will automatically clean up when out

-		   goes out of scope (out.freeIt==true)

-

-   note we should already be in the mutex lock from connThread() at this point.

-*/

-void jniCallback(Message& m, Message& out)

-{

-	Client *clientOld = client;

-

-	JniMessagingPort jmp(out);

-	callDepth++;

-	int curOpOld = curOp;

-

-	try { 

-

-		stringstream ss;

-		char buf[64];

-		time_t_to_String(time(0), buf);

-		buf[20] = 0; // don't want the year

-		ss << buf << " dbjs ";

-

-		{

-			Timer t;

-

-			bool log = false;

-			curOp = m.data->operation;

-			if( m.data->operation == dbQuery ) { 

-				receivedQuery(jmp, m, ss);

-			}

-			else if( m.data->operation == dbInsert ) {

-				ss << "insert ";

-				receivedInsert(m, ss);

-			}

-			else if( m.data->operation == dbUpdate ) {

-				ss << "update ";

-				receivedUpdate(m, ss);

-			}

-			else if( m.data->operation == dbDelete ) {

-				ss << "remove ";

-				receivedDelete(m);

-			}

-			else if( m.data->operation == dbGetMore ) {

-				log = true;

-				ss << "getmore ";

-				receivedGetMore(jmp, m, ss);

-			}

-			else if( m.data->operation == dbKillCursors ) { 

-				try {

-					log = true;

-					ss << "killcursors ";

-					receivedKillCursors(m);

-				}

-				catch( AssertionException ) { 

-					problem() << "Caught Assertion in kill cursors, continuing" << endl; 

-					cout << "Caught Assertion in kill cursors, continuing" << endl; 

-					ss << " exception ";

-				}

-			}

-			else {

-				cout << "    jnicall: operation isn't supported: " << m.data->operation << endl;

-				assert(false);

-			}

-

-			int ms = t.millis();

-			log = log || ctr++ % 128 == 0;

-			if( log || ms > 100 ) {

-				ss << ' ' << t.millis() << "ms";

-				cout << ss.str().c_str() << endl;

-			}

-			if( client && client->profile >= 1 ) { 

-				if( client->profile >= 2 || ms >= 100 ) { 

-					// profile it

-					profile(ss.str().c_str()+20/*skip ts*/, ms);

-				}

-			}

-		}

-

-	}

-	catch( AssertionException ) { 

-		problem() << "Caught AssertionException in jniCall()" << endl;

-		cout << "Caught AssertionException in jniCall()" << endl;

-	}

-

-	curOp = curOpOld;

-	callDepth--;

-

-	if( client != clientOld ) { 

-		client = clientOld;

-		wassert(false);

-	}

-}

-

-void connThread()

-{

-	try { 

-

-	MessagingPort& dbMsgPort = *grab;

-	grab = 0;

-

-	Message m;

-	while( 1 ) { 

-		m.reset();

-		stringstream ss;

-

-		if( !dbMsgPort.recv(m) ) {

-			cout << "MessagingPort::recv(): returned false" << endl;

-			dbMsgPort.shutdown();

-			break;

-		}

-

-		char buf[64];

-		time_t_to_String(time(0), buf);

-		buf[20] = 0; // don't want the year

-		ss << buf;

-		//		ss << curTimeMillis() % 10000 << ' ';

-

-		{

-			lock lk(dbMutex);

-			Timer t;

-			client = 0;

-			curOp = 0;

-

-			bool log = false;

-			curOp = m.data->operation;

-			if( m.data->operation == dbMsg ) { 

-				ss << "msg ";

-				char *p = m.data->_data;

-				int len = strlen(p);

-				if( len > 400 ) 

-					cout << curTimeMillis() % 10000 << 

-					" long msg received, len:" << len << 

-					" ends with: " << p + len - 10 << endl;

-				bool end = strcmp("end", p) == 0;

-				Message resp;

-				resp.setData(opReply, "i am fine");

-				dbMsgPort.reply(m, resp);

-				if( end ) {

-					cout << curTimeMillis() % 10000 << "   end msg" << endl;

-					dbMsgPort.shutdown();

-					sleepmillis(500);

-					problem() << "exiting end msg" << endl;

-					exit(EXIT_SUCCESS);

-				}

-			}

-			else if( m.data->operation == dbQuery ) { 

-#if defined(_WIN32)

-				log = true;

-#endif

-				receivedQuery(dbMsgPort, m, ss);

-			}

-			else if( m.data->operation == dbInsert ) {

-				try { 

-					ss << "insert ";

-					receivedInsert(m, ss);

-				}

-				catch( AssertionException ) { 

-					problem() << " Caught Assertion insert, continuing" << endl; 

-					cout << "Caught Assertion, continuing" << endl; 

-					ss << " exception ";

-				}

-			}

-			else if( m.data->operation == dbUpdate ) {

-				try { 

-					ss << "update ";

-					receivedUpdate(m, ss);

-				}

-				catch( AssertionException ) { 

-					problem() << " Caught Assertion update, continuing" << endl; 

-					cout << "Caught Assertion update, continuing" << endl; 

-					ss << " exception ";

-				}

-			}

-			else if( m.data->operation == dbDelete ) {

-				try { 

-					ss << "remove ";

-					receivedDelete(m);

-				}

-				catch( AssertionException ) { 

-					problem() << " Caught Assertion receviedDelete, continuing" << endl; 

-					cout << "Caught Assertion receviedDelete, continuing" << endl; 

-					ss << " exception ";

-				}

-			}

-			else if( m.data->operation == dbGetMore ) {

-				log = true;

-				ss << "getmore ";

-				receivedGetMore(dbMsgPort, m, ss);

-			}

-			else if( m.data->operation == dbKillCursors ) { 

-				try {

-					log = true;

-					ss << "killcursors ";

-					receivedKillCursors(m);

-				}

-				catch( AssertionException ) { 

-					cout << "Caught Assertion in kill cursors, continuing" << endl; 

-					problem() << " Caught Assertion in kill cursors, continuing" << endl; 

-					ss << " exception ";

-				}

-			}

-			else {

-				cout << "    operation isn't supported: " << m.data->operation << endl;

-				assert(false);

-			}

-

-			int ms = t.millis();

-			log = log || ctr++ % 128 == 0;

-			if( log || ms > 100 ) {

-				ss << ' ' << t.millis() << "ms";

-				cout << ss.str().c_str() << endl;

-			}

-			if( client && client->profile >= 1 ) { 

-				if( client->profile >= 2 || ms >= 100 ) { 

-					// profile it

-					profile(ss.str().c_str()+20/*skip ts*/, ms);

-				}

-			}

-		}

-	}

-

-	}

-	catch( AssertionException ) { 

-		cout << "Caught AssertionException, terminating" << endl;

-		problem() << "Caught AssertionException, terminating" << endl;

-		exit(-7);

-	}

-}

-

-

-void msg(const char *m, const char *address, int port, int extras = 0) {

-

-    SockAddr db(address, port);

-

-//	SockAddr db("127.0.0.1", DBPort);

-//	SockAddr db("192.168.37.1", MessagingPort::DBPort);

-//	SockAddr db("10.0.21.60", MessagingPort::DBPort);

-//	SockAddr db("172.16.0.179", MessagingPort::DBPort);

-

-	MessagingPort p;

-	if( !p.connect(db) )

-		return;

-

-	for( int q = 0; q < 3; q++ ) {

-		Message send;

-		Message response;

-

-		send.setData( dbMsg , m);

-		int len = send.data->dataLen();

-

-		for( int i = 0; i < extras; i++ )

-			p.say(db, send);

-

-		Timer t;

-		bool ok = p.call(db, send, response);

-		double tm = t.micros() + 1;

-		cout << " ****ok. response.data:" << ok << " time:" << tm / 1000.0 << "ms " << 

-			((double) len) * 8 / 1000000 / (tm/1000000) << "Mbps" << endl;

-		if(  q+1 < 3 ) {

-			cout << "\t\tSLEEP 8 then sending again" << endl;

-			sleepsecs(8);

-		}

-	}

-

-	p.shutdown();

-}

-

-void msg(const char *m, int extras = 0) { 

-    msg(m, "127.0.0.1", DBPort, extras);

-}

-

-#if !defined(_WIN32)

-

-#include <signal.h>

-

-void pipeSigHandler( int signal ) {

-  psignal( signal, "Signal Received : ");

-}

-

-void segvhandler(int x) { 

-	cout << "got SIGSEGV " << x << ", terminating :-(" << endl;

-	problem() << "got SIGSEGV " << x << ", terminating :-(" << endl;

-	exit(-9);

+// db.cpp : Defines the entry point for the console application.
+//
+
+#include "stdafx.h"
+#include "db.h"
+#include "../grid/message.h"
+#include "../util/mmap.h"
+#include "../util/hashtab.h"
+#include "../util/goodies.h"
+#include "pdfile.h"
+#include "jsobj.h"
+#include "javajs.h"
+#include "query.h"
+#include "introspect.h"
+
+extern const char *dbpath;
+extern int curOp;
+
+boost::mutex dbMutex;
+
+struct MyStartupTests {
+	MyStartupTests() {
+		assert( sizeof(OID) == 12 );
+	}
+} mystartupdbcpp;
+
+/* example for
+	var zz = { x: 3, y: "foo", v: { z:5, arr: [1, 2] } }
+	zz.v.arr.prop = "zoo";
+*/
+
+void quicktest() { 
+	cout << "quicktest()\n";
+
+	MemoryMappedFile mmf;
+	char *m = (char *) mmf.map("/tmp/quicktest", 16384);
+	//	cout << "mmf reads: " << m << endl;
+	strcpy_s(m, 1000, "hello worldz");
 }
-

-void mysighandler(int x) { 

-	// [dm] not working.  why? 

-	// [gmj] because the VM catches sig 3 to do a thread dump

-	cout << "got kill or ctrl c signal " << x << ", terminating" << endl;

-	problem() << "got kill or ctrl c signal " << x << ", terminating" << endl;

-	exit(0);

-}

-

-void setupSignals() {

-	cout << "SETUPSIGNALS " << signal(SIGINT, mysighandler) << endl;

+
+void pdfileInit();
+
+class DbMessage {
+public:
+	DbMessage(Message& _m) : m(_m) {
+		theEnd = _m.data->_data + _m.data->dataLen();
+		int *r = (int *) _m.data->_data;
+		reserved = *r;
+		r++;
+		data = (const char *) r;
+		nextjsobj = data;
+	}
+
+	const char * getns() { return data; }
+	void getns(Namespace& ns) {
+		ns = data;
+	}
+
+	int pullInt() {
+		if( nextjsobj == data )
+			nextjsobj += strlen(data) + 1; // skip namespace
+		int i = *((int *)nextjsobj);
+		nextjsobj += 4;
+		return i;
+	}
+	long long pullInt64() {
+		if( nextjsobj == data )
+			nextjsobj += strlen(data) + 1; // skip namespace
+		long long i = *((long long *)nextjsobj);
+		nextjsobj += 8;
+		return i;
+	}
+
+	OID* getOID() {
+		return (OID *) (data + strlen(data) + 1); // skip namespace
+	}
+
+	void getQueryStuff(const char *&query, int& ntoreturn) {
+		int *i = (int *) (data + strlen(data) + 1);
+		ntoreturn = *i;
+		i++;
+		query = (const char *) i;
+	}
+
+	/* for insert and update msgs */
+	bool moreJSObjs() { return nextjsobj != 0; }
+	JSObj nextJsObj() {
+		if( nextjsobj == data )
+			nextjsobj += strlen(data) + 1; // skip namespace
+		JSObj js(nextjsobj);
+                assert( js.objsize() < ( theEnd - data ) );
+		if( js.objsize() <= 0 )
+			nextjsobj = null;
+		else {
+			nextjsobj += js.objsize();
+			if( nextjsobj >= theEnd )
+				nextjsobj = 0;
+		}
+		return js;
+	}
+
+private:
+	Message& m;
+	int reserved;
+	const char *data;
+	const char *nextjsobj;
+	const char *theEnd;
+};
+
+void killCursors(int n, long long *ids);
+void receivedKillCursors(Message& m) {
+	int *x = (int *) m.data->_data;
+	x++; // reserved
+	int n = *x++;
+	assert( n >= 1 );
+	if( n > 2000 ) { 
+		problem() << "Assertion failure, receivedKillCursors, n=" << n << endl;
+		cout << "Assertion failure, receivedKillCursors, n=" << n << endl;
+		assert( n < 30000 );
+	}
+	killCursors(n, (long long *) x);
+}
+
+void receivedUpdate(Message& m, stringstream& ss) {
+	DbMessage d(m);
+	const char *ns = d.getns();
+	assert(*ns);
+	setClient(ns);
+	if( client->profile )
+		ss << ns << ' ';
+	int flags = d.pullInt();
+	JSObj query = d.nextJsObj();
+
+	assert( d.moreJSObjs() );
+	assert( query.objsize() < m.data->dataLen() );
+	JSObj toupdate = d.nextJsObj();
+
+	assert( toupdate.objsize() < m.data->dataLen() );
+	assert( query.objsize() + toupdate.objsize() < m.data->dataLen() );
+	updateObjects(ns, toupdate, query, flags & 1, ss);
+}
+
+void receivedDelete(Message& m) {
+	DbMessage d(m);
+	const char *ns = d.getns();
+	assert(*ns);
+	setClient(ns);
+	int flags = d.pullInt();
+	assert( d.moreJSObjs() );
+	JSObj pattern = d.nextJsObj();
+	deleteObjects(ns, pattern, flags & 1);
+}
+
+void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
+	DbMessage d(m);
+	const char *ns = d.getns();
+	setClient(ns);
+	int ntoskip = d.pullInt();
+	int ntoreturn = d.pullInt();
+	JSObj query = d.nextJsObj();
+	auto_ptr< set<string> > fields;
+	if( d.moreJSObjs() ) { 
+		fields = auto_ptr< set<string> >(new set<string>());
+		d.nextJsObj().getFieldNames(*fields);
+	}
+	QueryResult* msgdata;
+
+	try { 
+		msgdata = runQuery(ns, ntoskip, ntoreturn, query, fields, ss);
+	}
+	catch( AssertionException ) { 
+		ss << " exception ";
+		problem() << " Caught Assertion in runQuery " << ns << endl; 
+		cout << " Caught Assertion in runQuery, continuing" << endl; 
+		cout << "  ntoskip:" << ntoskip << " ntoreturn:" << ntoreturn << endl;
+		cout << "  ns:" << ns << endl;
+		cout << "  query:" << query.toString() << endl;
+		msgdata = (QueryResult*) malloc(sizeof(QueryResult));
+		QueryResult *qr = msgdata;
+		qr->_data[0] = 0;
+		qr->_data[1] = 0;
+		qr->_data[2] = 0;
+		qr->_data[3] = 0;
+		qr->len = sizeof(QueryResult);
+		qr->operation = opReply;
+		qr->cursorId = 0;
+		qr->startingFrom = 0;
+		qr->nReturned = 0;
+	}
+	Message resp;
+	resp.setData(msgdata, true); // transport will free
+	if( client ) { 
+		if( client->profile )
+			ss << " bytes:" << resp.data->dataLen();
+	}
+	else { 
+		cout << "ERROR: client is null; ns=" << ns << endl;
+	}
+	dbMsgPort.reply(m, resp);
+}
+
+void receivedGetMore(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
+	DbMessage d(m);
+	const char *ns = d.getns();
+	ss << ns;
+	setClient(ns);
+	int ntoreturn = d.pullInt();
+	long long cursorid = d.pullInt64();
+	ss << " cid:" << cursorid;
+	ss << " ntoreturn:" << ntoreturn;
+	QueryResult* msgdata = getMore(ns, ntoreturn, cursorid);
+	Message resp;
+	resp.setData(msgdata, true);
+	ss << " bytes:" << resp.data->dataLen();
+	ss << " nreturned:" << msgdata->nReturned;
+	dbMsgPort.reply(m, resp);
+}
+
+void receivedInsert(Message& m, stringstream& ss) {
+//	cout << "GOT MSG id:" << m.data->id << endl;
+	DbMessage d(m);
+	while( d.moreJSObjs() ) {
+		JSObj js = d.nextJsObj();
+//		cout << "  temp dbinsert: got js object, size=" << js.objsize() << " ns:" << d.getns() << endl;
+		const char *ns = d.getns();
+		assert(*ns);
+		setClient(ns);
+		ss << ns;
+		theDataFileMgr.insert(ns, (void*) js.objdata(), js.objsize());
+	}
+}
+
+void testTheDb() {
+	stringstream ss;
+
+	setClient("sys.unittest.pdfile");
+
+	/* this is not validly formatted, if you query this namespace bad things will happen */
+	theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
+	theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
+
+	JSObj j1((const char *) &js1);
+	deleteObjects("sys.unittest.delete", j1, false);
+	theDataFileMgr.insert("sys.unittest.delete", &js1, sizeof(js1));
+	deleteObjects("sys.unittest.delete", j1, false);
+	updateObjects("sys.unittest.delete", j1, j1, true,ss);
+	updateObjects("sys.unittest.delete", j1, j1, false,ss);
+
+	auto_ptr<Cursor> c = theDataFileMgr.findAll("sys.unittest.pdfile");
+	while( c->ok() ) {
+		Record* r = c->_current();
+		c->advance();
+	}
+	cout << endl;
+
+	client = 0;
+}
+
+int port = DBPort;
+
+MessagingPort *grab = 0;
+void connThread();
+
+class OurListener : public Listener { 
+public:
+	OurListener(int p) : Listener(p) { }
+	virtual void accepted(MessagingPort *mp) {
+		assert( grab == 0 );
+		grab = mp;
+		boost::thread thr(connThread);
+		while( grab )
+			sleepmillis(1);
+	}
+};
+
+void listen(int port) { 
+	const char *Version = "db version: 110 2jun2008 embedded obj queries";
+	problem() << Version << endl;
+	cout << Version << endl;
+	pdfileInit();
+	testTheDb();
+	cout << curTimeMillis() % 10000 << " waiting for connections on port " << port << " ...\n" << endl;
+	OurListener l(port);
+	l.listen();
+}
+
+int ctr = 0;
+extern int callDepth;
+
+class JniMessagingPort : public AbstractMessagingPort { 
+public:
+	JniMessagingPort(Message& _container) : container(_container) { }
+	void reply(Message& received, Message& response) { 
+		container = response;
+	}
+	Message & container;
+};
+
+/* a call from java/js to the database locally. 
+
+     m - inbound message
+     out - outbound message, if there is any, will be set here.
+           if there is one, out.data will be non-null on return.
+		   The out.data buffer will automatically clean up when out
+		   goes out of scope (out.freeIt==true)
+
+   note we should already be in the mutex lock from connThread() at this point.
+*/
+void jniCallback(Message& m, Message& out)
+{
+	Client *clientOld = client;
+
+	JniMessagingPort jmp(out);
+	callDepth++;
+	int curOpOld = curOp;
+
+	try { 
+
+		stringstream ss;
+		char buf[64];
+		time_t_to_String(time(0), buf);
+		buf[20] = 0; // don't want the year
+		ss << buf << " dbjs ";
+
+		{
+			Timer t;
+
+			bool log = false;
+			curOp = m.data->operation;
+			if( m.data->operation == dbQuery ) { 
+				receivedQuery(jmp, m, ss);
+			}
+			else if( m.data->operation == dbInsert ) {
+				ss << "insert ";
+				receivedInsert(m, ss);
+			}
+			else if( m.data->operation == dbUpdate ) {
+				ss << "update ";
+				receivedUpdate(m, ss);
+			}
+			else if( m.data->operation == dbDelete ) {
+				ss << "remove ";
+				receivedDelete(m);
+			}
+			else if( m.data->operation == dbGetMore ) {
+				log = true;
+				ss << "getmore ";
+				receivedGetMore(jmp, m, ss);
+			}
+			else if( m.data->operation == dbKillCursors ) { 
+				try {
+					log = true;
+					ss << "killcursors ";
+					receivedKillCursors(m);
+				}
+				catch( AssertionException ) { 
+					problem() << "Caught Assertion in kill cursors, continuing" << endl; 
+					cout << "Caught Assertion in kill cursors, continuing" << endl; 
+					ss << " exception ";
+				}
+			}
+			else {
+				cout << "    jnicall: operation isn't supported: " << m.data->operation << endl;
+				assert(false);
+			}
+
+			int ms = t.millis();
+			log = log || ctr++ % 128 == 0;
+			if( log || ms > 100 ) {
+				ss << ' ' << t.millis() << "ms";
+				cout << ss.str().c_str() << endl;
+			}
+			if( client && client->profile >= 1 ) { 
+				if( client->profile >= 2 || ms >= 100 ) { 
+					// profile it
+					profile(ss.str().c_str()+20/*skip ts*/, ms);
+				}
+			}
+		}
+
+	}
+	catch( AssertionException ) { 
+		problem() << "Caught AssertionException in jniCall()" << endl;
+		cout << "Caught AssertionException in jniCall()" << endl;
+	}
+
+	curOp = curOpOld;
+	callDepth--;
+
+	if( client != clientOld ) { 
+		client = clientOld;
+		wassert(false);
+	}
+}
+
+void connThread()
+{
+	try { 
+
+	MessagingPort& dbMsgPort = *grab;
+	grab = 0;
+
+	Message m;
+	while( 1 ) { 
+		m.reset();
+		stringstream ss;
+
+		if( !dbMsgPort.recv(m) ) {
+			cout << "MessagingPort::recv(): returned false" << endl;
+			dbMsgPort.shutdown();
+			break;
+		}
+
+		char buf[64];
+		time_t_to_String(time(0), buf);
+		buf[20] = 0; // don't want the year
+		ss << buf;
+		//		ss << curTimeMillis() % 10000 << ' ';
+
+		{
+			lock lk(dbMutex);
+			Timer t;
+			client = 0;
+			curOp = 0;
+
+			bool log = false;
+			curOp = m.data->operation;
+			if( m.data->operation == dbMsg ) { 
+				ss << "msg ";
+				char *p = m.data->_data;
+				int len = strlen(p);
+				if( len > 400 ) 
+					cout << curTimeMillis() % 10000 << 
+					" long msg received, len:" << len << 
+					" ends with: " << p + len - 10 << endl;
+				bool end = strcmp("end", p) == 0;
+				Message resp;
+				resp.setData(opReply, "i am fine");
+				dbMsgPort.reply(m, resp);
+				if( end ) {
+					cout << curTimeMillis() % 10000 << "   end msg" << endl;
+					dbMsgPort.shutdown();
+					sleepmillis(500);
+					problem() << "exiting end msg" << endl;
+					exit(EXIT_SUCCESS);
+				}
+			}
+			else if( m.data->operation == dbQuery ) { 
+#if defined(_WIN32)
+				log = true;
+#endif
+				receivedQuery(dbMsgPort, m, ss);
+			}
+			else if( m.data->operation == dbInsert ) {
+				try { 
+					ss << "insert ";
+					receivedInsert(m, ss);
+				}
+				catch( AssertionException ) { 
+					problem() << " Caught Assertion insert, continuing" << endl; 
+					cout << "Caught Assertion, continuing" << endl; 
+					ss << " exception ";
+				}
+			}
+			else if( m.data->operation == dbUpdate ) {
+				try { 
+					ss << "update ";
+					receivedUpdate(m, ss);
+				}
+				catch( AssertionException ) { 
+					problem() << " Caught Assertion update, continuing" << endl; 
+					cout << "Caught Assertion update, continuing" << endl; 
+					ss << " exception ";
+				}
+			}
+			else if( m.data->operation == dbDelete ) {
+				try { 
+					ss << "remove ";
+					receivedDelete(m);
+				}
+				catch( AssertionException ) { 
+					problem() << " Caught Assertion receviedDelete, continuing" << endl; 
+					cout << "Caught Assertion receviedDelete, continuing" << endl; 
+					ss << " exception ";
+				}
+			}
+			else if( m.data->operation == dbGetMore ) {
+				log = true;
+				ss << "getmore ";
+				receivedGetMore(dbMsgPort, m, ss);
+			}
+			else if( m.data->operation == dbKillCursors ) { 
+				try {
+					log = true;
+					ss << "killcursors ";
+					receivedKillCursors(m);
+				}
+				catch( AssertionException ) { 
+					cout << "Caught Assertion in kill cursors, continuing" << endl; 
+					problem() << " Caught Assertion in kill cursors, continuing" << endl; 
+					ss << " exception ";
+				}
+			}
+			else {
+				cout << "    operation isn't supported: " << m.data->operation << endl;
+				assert(false);
+			}
+
+			int ms = t.millis();
+			log = log || ctr++ % 128 == 0;
+			if( log || ms > 100 ) {
+				ss << ' ' << t.millis() << "ms";
+				cout << ss.str().c_str() << endl;
+			}
+			if( client && client->profile >= 1 ) { 
+				if( client->profile >= 2 || ms >= 100 ) { 
+					// profile it
+					profile(ss.str().c_str()+20/*skip ts*/, ms);
+				}
+			}
+		}
+	}
+
+	}
+	catch( AssertionException ) { 
+		cout << "Caught AssertionException, terminating" << endl;
+		problem() << "Caught AssertionException, terminating" << endl;
+		exit(-7);
+	}
+}
+
+
+void msg(const char *m, const char *address, int port, int extras = 0) {
+
+    SockAddr db(address, port);
+
+//	SockAddr db("127.0.0.1", DBPort);
+//	SockAddr db("192.168.37.1", MessagingPort::DBPort);
+//	SockAddr db("10.0.21.60", MessagingPort::DBPort);
+//	SockAddr db("172.16.0.179", MessagingPort::DBPort);
+
+	MessagingPort p;
+	if( !p.connect(db) )
+		return;
+
+	for( int q = 0; q < 3; q++ ) {
+		Message send;
+		Message response;
+
+		send.setData( dbMsg , m);
+		int len = send.data->dataLen();
+
+		for( int i = 0; i < extras; i++ )
+			p.say(db, send);
+
+		Timer t;
+		bool ok = p.call(db, send, response);
+		double tm = t.micros() + 1;
+		cout << " ****ok. response.data:" << ok << " time:" << tm / 1000.0 << "ms " << 
+			((double) len) * 8 / 1000000 / (tm/1000000) << "Mbps" << endl;
+		if(  q+1 < 3 ) {
+			cout << "\t\tSLEEP 8 then sending again" << endl;
+			sleepsecs(8);
+		}
+	}
+
+	p.shutdown();
+}
+
+void msg(const char *m, int extras = 0) { 
+    msg(m, "127.0.0.1", DBPort, extras);
+}
+
+#if !defined(_WIN32)
+
+#include <signal.h>
+
+void pipeSigHandler( int signal ) {
+  psignal( signal, "Signal Received : ");
+}
+
+void segvhandler(int x) { 
+	cout << "got SIGSEGV " << x << ", terminating :-(" << endl;
+	problem() << "got SIGSEGV " << x << ", terminating :-(" << endl;
+	exit(-9);
+}
+
+void mysighandler(int x) { 
+	// [dm] not working.  why? 
+	// [gmj] because the VM catches sig 3 to do a thread dump
+	cout << "got kill or ctrl c signal " << x << ", terminating" << endl;
+	problem() << "got kill or ctrl c signal " << x << ", terminating" << endl;
+	exit(0);
+}
+
+void setupSignals() {
+	cout << "SETUPSIGNALS " << signal(SIGINT, mysighandler) << endl;
 //	assert( signal(SIGINT, mysighandler) != SIG_ERR );
 	assert( signal(SIGTERM, mysighandler) != SIG_ERR );
 	assert( signal(SIGQUIT, mysighandler) != SIG_ERR );
 	assert( signal(SIGSEGV, segvhandler) != SIG_ERR );
-}

-

-#else

-void setupSignals() {}

-#endif

-

-void initAndListen(int listenPort, const char *dbPath, const char *appserverLoc = null) { 

-

-	setupSignals();

-

-    /*

-     * ensure that the dbpath ends w/ '/' as that's key in preventing things like 

-     *    /data/dbadmin.ns

-     */

-    

-    if (dbPath && dbPath[strlen(dbPath)-1] != '/') {

-    	char *t = (char *) malloc(strlen(dbPath) + 2);

-    	

-    	strcpy(t, dbPath);

-    	strcat(t, "/");

-    	dbPath = t;

-    }

-

-    dbpath = dbPath;

-

-#if !defined(_WIN32)

-    pid_t pid = 0;

-    pid = getpid();

-#else

-	int pid=0;

-#endif

-    

-    cout << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;

-    problem() << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;

-

-    JavaJS = new JavaJSImpl(appserverLoc);

-    javajstest();

-

-    listen(listenPort);    

-}

-

-ofstream problems("dbproblems.log", ios_base::app | ios_base::out);

-

-int main(int argc, char* argv[], char *envp[] )

-{

-#if !defined(_WIN32)

-    signal(SIGPIPE, pipeSigHandler);

-#endif

-	srand(curTimeMillis());

-

-	if( argc >= 2 ) {

-	  if( strcmp(argv[1], "quicktest") == 0 ) {

-		  quicktest();

-		  return 0;

-	  }

-		if( strcmp(argv[1], "msg") == 0 ) {

-

-		    // msg(argc >= 3 ? argv[2] : "ping");

-

-		    const char *m = "ping";

-		    int thePort = DBPort;

-		    

-		    if (argc >= 3) { 

-		        m = argv[2];

-		        

-		        if (argc > 3) { 

-		            thePort = atoi(argv[3]);

-		        }

-		    }

-		    

-		    msg(m, "127.0.0.1", thePort);

-		    

-			goingAway = true;

-			return 0;

-		}

-		if( strcmp(argv[1], "msglots") == 0 ) {

-			msg(argc >= 3 ? argv[2] : "ping", 1000);

-			goingAway = true;

-			return 0;

-		}

-		if( strcmp(argv[1], "dev") == 0 ) { 

-			dbpath = "/home/dwight/db/";

-			cout << "dev mode: expect db files in " << dbpath << endl;

-			quicktest();

-			port++;

-			cout << "listening on port " << port << endl;

-			listen(port);

-			goingAway = true;

-			return 0;

-		}

-		if( strcmp(argv[1], "run") == 0 ) {

-		    

-		    initAndListen(port, dbpath);

-		    

-			goingAway = true;

-			return 0;

-		}

-		if( strcmp(argv[1], "longmsg") == 0 ) {

-			char buf[800000];

-			memset(buf, 'a', 799999);

-			buf[799999] = 0;

-			buf[799998] = 'b';

-			buf[0] = 'c';

-			msg(buf);

-			goingAway = true;

-			return 0;

-		}

-

-        /*

-         *  *** POST STANDARD SWITCH METHOD - if we don't satisfy, we switch to a 

-         *     slightly different mode where "run" is assumed and we can set values

-         */

-		

-        char *appsrvPath = null;

-		

-        for (int i = 1; i < argc; i++)  {

-    

-            char *s = argv[i];

-            

-            if (s && strcmp(s, "--port") == 0) { 

-                port = atoi(argv[++i]);

-            }

-            else if (s && strcmp(s, "--dbpath") == 0) { 

-            	dbpath = argv[++i];

-            }

-            else if (s && strcmp(s, "--appsrvpath") == 0) { 

-                appsrvPath = argv[++i];

-            }

-        }

-        

-        initAndListen(port, dbpath, appsrvPath);

-        

-        goingAway = true;

-        return 0;

-	}

-

-	cout << "usage:\n";

-	cout << "  quicktest         just check basic assertions and exit" << endl;

-	cout << "  msg [msg] [port]  send a request to the db server listening on port (or default)" << endl;

-	cout << "  msg end [port]    shut down db server listening on port (or default)" << endl;

-	cout << "  run               run db" << endl;

-	cout << "  dev               run in dev mode (diff db loc, diff port #)" << endl;

-	cout << "  longmsg           send a long test message to the db server" << endl;

-	cout << "  msglots           send a bunch of test messages, and then wait for answer on the last one" << endl;

-	cout << endl << "Alternate Usage :" << endl;

-	cout << " --port <portno>  --dbpath <root> --appsrvpath <root of appsrv>" << endl << endl;

-	

-	goingAway = true;

-	return 0;

-}

-

-//#if !defined(_WIN32)

-//int main( int argc, char *argv[], char *envp[] ) {

-//	return _tmain(argc, 0);

-//}

-//#endif

+}
+
+#else
+void setupSignals() {}
+#endif
+
+void initAndListen(int listenPort, const char *dbPath, const char *appserverLoc = null) { 
+
+	setupSignals();
+
+    /*
+     * ensure that the dbpath ends w/ '/' as that's key in preventing things like 
+     *    /data/dbadmin.ns
+     */
+    
+    if (dbPath && dbPath[strlen(dbPath)-1] != '/') {
+    	char *t = (char *) malloc(strlen(dbPath) + 2);
+    	
+    	strcpy(t, dbPath);
+    	strcat(t, "/");
+    	dbPath = t;
+    }
+
+    dbpath = dbPath;
+
+#if !defined(_WIN32)
+    pid_t pid = 0;
+    pid = getpid();
+#else
+	int pid=0;
+#endif
+    
+    cout << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;
+    problem() << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;
+
+    JavaJS = new JavaJSImpl(appserverLoc);
+    javajstest();
+
+    listen(listenPort);    
+}
+
+ofstream problems("dbproblems.log", ios_base::app | ios_base::out);
+
+int main(int argc, char* argv[], char *envp[] )
+{
+#if !defined(_WIN32)
+    signal(SIGPIPE, pipeSigHandler);
+#endif
+	srand(curTimeMillis());
+
+	if( argc >= 2 ) {
+	  if( strcmp(argv[1], "quicktest") == 0 ) {
+		  quicktest();
+		  return 0;
+	  }
+		if( strcmp(argv[1], "msg") == 0 ) {
+
+		    // msg(argc >= 3 ? argv[2] : "ping");
+
+		    const char *m = "ping";
+		    int thePort = DBPort;
+		    
+		    if (argc >= 3) { 
+		        m = argv[2];
+		        
+		        if (argc > 3) { 
+		            thePort = atoi(argv[3]);
+		        }
+		    }
+		    
+		    msg(m, "127.0.0.1", thePort);
+		    
+			goingAway = true;
+			return 0;
+		}
+		if( strcmp(argv[1], "msglots") == 0 ) {
+			msg(argc >= 3 ? argv[2] : "ping", 1000);
+			goingAway = true;
+			return 0;
+		}
+		if( strcmp(argv[1], "dev") == 0 ) { 
+			dbpath = "/home/dwight/db/";
+			cout << "dev mode: expect db files in " << dbpath << endl;
+			quicktest();
+			port++;
+			cout << "listening on port " << port << endl;
+			listen(port);
+			goingAway = true;
+			return 0;
+		}
+		if( strcmp(argv[1], "run") == 0 ) {
+		    
+		    initAndListen(port, dbpath);
+		    
+			goingAway = true;
+			return 0;
+		}
+		if( strcmp(argv[1], "longmsg") == 0 ) {
+			char buf[800000];
+			memset(buf, 'a', 799999);
+			buf[799999] = 0;
+			buf[799998] = 'b';
+			buf[0] = 'c';
+			msg(buf);
+			goingAway = true;
+			return 0;
+		}
+
+        /*
+         *  *** POST STANDARD SWITCH METHOD - if we don't satisfy, we switch to a 
+         *     slightly different mode where "run" is assumed and we can set values
+         */
+		
+        char *appsrvPath = null;
+		
+        for (int i = 1; i < argc; i++)  {
+    
+            char *s = argv[i];
+            
+            if (s && strcmp(s, "--port") == 0) { 
+                port = atoi(argv[++i]);
+            }
+            else if (s && strcmp(s, "--dbpath") == 0) { 
+            	dbpath = argv[++i];
+            }
+            else if (s && strcmp(s, "--appsrvpath") == 0) { 
+                appsrvPath = argv[++i];
+            }
+        }
+        
+        initAndListen(port, dbpath, appsrvPath);
+        
+        goingAway = true;
+        return 0;
+	}
+
+	cout << "usage:\n";
+	cout << "  quicktest         just check basic assertions and exit" << endl;
+	cout << "  msg [msg] [port]  send a request to the db server listening on port (or default)" << endl;
+	cout << "  msg end [port]    shut down db server listening on port (or default)" << endl;
+	cout << "  run               run db" << endl;
+	cout << "  dev               run in dev mode (diff db loc, diff port #)" << endl;
+	cout << "  longmsg           send a long test message to the db server" << endl;
+	cout << "  msglots           send a bunch of test messages, and then wait for answer on the last one" << endl;
+	cout << endl << "Alternate Usage :" << endl;
+	cout << " --port <portno>  --dbpath <root> --appsrvpath <root of appsrv>" << endl << endl;
+	
+	goingAway = true;
+	return 0;
+}
+
+//#if !defined(_WIN32)
+//int main( int argc, char *argv[], char *envp[] ) {
+//	return _tmain(argc, 0);
+//}
+//#endif
diff --git a/db/db.h b/db/db.h
index 11df40dca97..bf5f46e3c88 100644
--- a/db/db.h
+++ b/db/db.h
@@ -1,4 +1,4 @@
-#include "../stdafx.h"

-#include "../grid/message.h"

-

-void jniCallback(Message& m, Message& out);

+#include "../stdafx.h"
+#include "../grid/message.h"
+
+void jniCallback(Message& m, Message& out);
diff --git a/db/introspect.cpp b/db/introspect.cpp
index 95bbc73879a..34db42a143d 100644
--- a/db/introspect.cpp
+++ b/db/introspect.cpp
@@ -1,39 +1,39 @@
-// introspect.cpp 

-

-#include "stdafx.h"

-#include "introspect.h"

-#include "../util/builder.h"

-#include "../util/goodies.h"

-#include "pdfile.h"

-#include "jsobj.h"

-#include "pdfile.h"

-

-typedef map<string,Cursor*> StringToCursor;

-StringToCursor *specialNamespaces;

-

-auto_ptr<Cursor> getSpecialCursor(const char *ns) {

-	StringToCursor::iterator it = specialNamespaces->find(ns);

-	return auto_ptr<Cursor>

-		(it == specialNamespaces->end() ? 

-		0 : it->second->clone());

-}

-

-void SingleResultObjCursor::reg(const char *as) {

-	if( specialNamespaces == 0 )

-		specialNamespaces = new StringToCursor();

-	if( specialNamespaces->count(as) == 0 ) {

-		(*specialNamespaces)[as] = this;

-	}

-}

-

-void profile(const char *str,

-			 int millis)

-{

-	JSObjBuilder b;

-	b.appendDate("ts", jsTime());

-	b.append("info", str);

-	b.append("millis", (double) millis);

-	JSObj p = b.done();

-	theDataFileMgr.insert(client->profileName.c_str(), 

-		p.objdata(), p.objsize(), true);

-}

+// introspect.cpp 
+
+#include "stdafx.h"
+#include "introspect.h"
+#include "../util/builder.h"
+#include "../util/goodies.h"
+#include "pdfile.h"
+#include "jsobj.h"
+#include "pdfile.h"
+
+typedef map<string,Cursor*> StringToCursor;
+StringToCursor *specialNamespaces;
+
+auto_ptr<Cursor> getSpecialCursor(const char *ns) {
+	StringToCursor::iterator it = specialNamespaces->find(ns);
+	return auto_ptr<Cursor>
+		(it == specialNamespaces->end() ? 
+		0 : it->second->clone());
+}
+
+void SingleResultObjCursor::reg(const char *as) {
+	if( specialNamespaces == 0 )
+		specialNamespaces = new StringToCursor();
+	if( specialNamespaces->count(as) == 0 ) {
+		(*specialNamespaces)[as] = this;
+	}
+}
+
+void profile(const char *str,
+			 int millis)
+{
+	JSObjBuilder b;
+	b.appendDate("ts", jsTime());
+	b.append("info", str);
+	b.append("millis", (double) millis);
+	JSObj p = b.done();
+	theDataFileMgr.insert(client->profileName.c_str(), 
+		p.objdata(), p.objsize(), true);
+}
diff --git a/db/introspect.h b/db/introspect.h
index 10bcfd96925..1dd7f3c8ee4 100644
--- a/db/introspect.h
+++ b/db/introspect.h
@@ -1,46 +1,46 @@
-// introspect.h

-// system management stuff.

-

-#pragma once

-

-#include "../stdafx.h"

-#include "jsobj.h"

-#include "pdfile.h"

-

-auto_ptr<Cursor> getSpecialCursor(const char *ns);

-

-class SingleResultObjCursor : public Cursor {

-	int i;

-protected:

-	JSObjBuilder b;

-	void reg(const char *as); /* register as a certain namespace */

-public:

-	SingleResultObjCursor() { i = 0; }

-	virtual bool ok() { return i == 0; }

-	virtual Record* _current() { assert(false); return 0; }

-	virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }

-

-	virtual void fill() = 0;

-

-	virtual JSObj current() {

-		assert(i == 0);

-		fill();

-		return b.done();

-	}

-

-	virtual bool advance() {

-		i++;

-		return false;

-	}

-

-	virtual const char * toString() { return "SingleResultObjCursor"; }

-

-};

-

-/* --- profiling -------------------------------------------- 

-   do when client->profile is set

-*/

-

-void profile(const char *str,

-			 int millis);

-

+// introspect.h
+// system management stuff.
+
+#pragma once
+
+#include "../stdafx.h"
+#include "jsobj.h"
+#include "pdfile.h"
+
+auto_ptr<Cursor> getSpecialCursor(const char *ns);
+
+class SingleResultObjCursor : public Cursor {
+	int i;
+protected:
+	JSObjBuilder b;
+	void reg(const char *as); /* register as a certain namespace */
+public:
+	SingleResultObjCursor() { i = 0; }
+	virtual bool ok() { return i == 0; }
+	virtual Record* _current() { assert(false); return 0; }
+	virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }
+
+	virtual void fill() = 0;
+
+	virtual JSObj current() {
+		assert(i == 0);
+		fill();
+		return b.done();
+	}
+
+	virtual bool advance() {
+		i++;
+		return false;
+	}
+
+	virtual const char * toString() { return "SingleResultObjCursor"; }
+
+};
+
+/* --- profiling -------------------------------------------- 
+   do when client->profile is set
+*/
+
+void profile(const char *str,
+			 int millis);
+
diff --git a/db/javajs.cpp b/db/javajs.cpp
index 5ad2aa1f7ad..86a0a9a2ac1 100644
--- a/db/javajs.cpp
+++ b/db/javajs.cpp
@@ -24,27 +24,27 @@ using namespace boost::filesystem;
 
 using namespace std;
 
-#if defined(_WIN32)

-/* [dm] this being undefined without us adding it here means there is 

-        no tss cleanup on windows for boost lib?

-        we don't care for now esp on windows only

-

-		the boost source says:

-

-		  This function's sole purpose is to cause a link error in cases where

-		  automatic tss cleanup is not implemented by Boost.Threads as a

-		  reminder that user code is responsible for calling the necessary

-		  functions at the appropriate times (and for implementing an a

-		  tss_cleanup_implemented() function to eliminate the linker's

-		  missing symbol error).

-

-		  If Boost.Threads later implements automatic tss cleanup in cases

-		  where it currently doesn't (which is the plan), the duplicate

-		  symbol error will warn the user that their custom solution is no

-		  longer needed and can be removed. 

-*/

-extern "C" void tss_cleanup_implemented(void) {

-	//cout << "tss_cleanup_implemented called" << endl;

+#if defined(_WIN32)
+/* [dm] this being undefined without us adding it here means there is 
+        no tss cleanup on windows for boost lib?
+        we don't care for now esp on windows only
+
+		the boost source says:
+
+		  This function's sole purpose is to cause a link error in cases where
+		  automatic tss cleanup is not implemented by Boost.Threads as a
+		  reminder that user code is responsible for calling the necessary
+		  functions at the appropriate times (and for implementing an a
+		  tss_cleanup_implemented() function to eliminate the linker's
+		  missing symbol error).
+
+		  If Boost.Threads later implements automatic tss cleanup in cases
+		  where it currently doesn't (which is the plan), the duplicate
+		  symbol error will warn the user that their custom solution is no
+		  longer needed and can be removed. 
+*/
+extern "C" void tss_cleanup_implemented(void) {
+	//cout << "tss_cleanup_implemented called" << endl;
 }
 #endif
 
diff --git a/db/jsobj.cpp b/db/jsobj.cpp
index 26a9881fccf..e55de77d335 100644
--- a/db/jsobj.cpp
+++ b/db/jsobj.cpp
@@ -1,810 +1,810 @@
-// jsobj.cpp

-

-#include "stdafx.h"

-#include "jsobj.h"

-#include "../util/goodies.h"

-#include "javajs.h"

-

-#if defined(_WIN32)

-

-#include <hash_map>

-using namespace stdext;

-

-typedef const char * MyStr;

-struct less_str {

-   bool operator()(const MyStr & x, const MyStr & y) const {

-      if ( strcmp(x, y) > 0)

-         return true;

-

-      return false;

-   }

-};

-

-typedef hash_map<const char*, int, hash_compare<const char *, less_str> > strhashmap;

-

-#else

-

-#include <ext/hash_map>

-using namespace __gnu_cxx;

-

-typedef const char * MyStr;

-struct eq_str {

-   bool operator()(const MyStr & x, const MyStr & y) const {

-      if ( strcmp(x, y) == 0)

-         return true;

-

-      return false;

-   }

-};

-

-typedef hash_map<const char*, int, hash<const char *>, eq_str > strhashmap;

-

-#endif

-

-#include "minilex.h"

-

-MiniLex minilex;

-

-class Where { 

-public:

-	Where() { codeCopy = 0; }

-	~Where() {

-		JavaJS->scopeFree(scope);

-		delete codeCopy;

-		scope = 0; func = 0; codeCopy = 0;

-	}

-	jlong scope, func;

-	strhashmap fields;

-//	map<string,int> fields;

-	bool fullObject;

-	int nFields;

-	char *codeCopy;

-

-	void setFunc(const char *code) {

-		codeCopy = new char[strlen(code)+1];

-		strcpy(codeCopy,code);

-		func = JavaJS->functionCreate( code );

-		minilex.grabVariables(codeCopy, fields);

-		// if user references db, eg db.foo.save(obj), 

-		// we make sure we have the whole thing.

-		fullObject = fields.count("fullObject") +

-			fields.count("db") > 0;

-		nFields = fields.size();

-	}

-

-	void buildSubset(JSObj& src, JSObjBuilder& dst) { 

-		JSElemIter it(src);

-		int n = 0;

-		if( !it.more() ) return;

-		while( 1 ) {

-			Element e = it.next();

-			if( e.eoo() )

-				break;

-			if( //n == 0 && 

-				fields.find(e.fieldName()) != fields.end()

-				//fields.count(e.fieldName())

-				) {

-				dst.append(e);

-				if( ++n >= nFields )

-					break;

-			}

-		}

-	}

-};

-

-JSMatcher::~JSMatcher() { 

-	for( int i = 0; i < nBuilders; i++ )

-		delete builders[i];

-	delete where;

-}

-

-Element nullElement;

-

-string Element::toString() { 

-	stringstream s;

-	switch( type() ) {

-    case EOO:

-		return "EOO";

-    case Date:

-		s << fieldName() << ": Date(" << hex << date() << ')'; break;

-	case Number:

-		s << fieldName() << ": " << number(); break;

-	case Bool: 

-		s << fieldName() << ": " << boolean() ? "true" : "false"; break;

-	case Object:

-	case Array:

-		s << fieldName() << ": " << embeddedObject().toString(); break;

-	case Undefined:

-		s << fieldName() << ": undefined"; break;

-	case jstNULL:

-		s << fieldName() << ": null"; break;

-	case MaxKey:

-		s << fieldName() << ": MaxKey"; break;

-	case Code:

-		s << fieldName() << ": ";

-		if( valuestrsize() > 80 ) 

-			s << string(valuestr()).substr(0, 70) << "...";

-		else { 

-			s << valuestr();

-		}

-		break;

-	case String:

-		s << fieldName() << ": ";

-		if( valuestrsize() > 80 ) 

-			s << '"' << string(valuestr()).substr(0, 70) << "...\"";

-		else { 

-			s << '"' << valuestr() << '"';

-		}

-		break;

-	case jstOID: 

-          s << fieldName() << " : ObjId(";

-          s << hex << oid().a << hex << oid().b << ')';

-          break;

-    default:

-		s << fieldName() << ": ?type=" << type();

-		break;

-	}

-	return s.str();

-}

-

-int Element::size() {

-	if( totalSize >= 0 )

-		return totalSize;

-

-	int x = 1;

-	switch( type() ) {

-		case EOO:

-		case Undefined:

-		case jstNULL:

-		case MaxKey:

-			break;

-		case Bool:

-			x = 2;

-			break;

-		case Date:

-		case Number:

-			x = 9;

-			break;

-		case jstOID:

-			x = 13;

-			break;

-		case Code:

-		case String:

-			x = valuestrsize() + 4 + 1;

-			break;

-        case DBRef:

-          x = valuestrsize() + 4 + 12 + 1;

-          break;

-		case Object:

-		case Array:

-			x = objsize() + 1;

-			break;

-		case BinData:

-			x = valuestrsize() + 4 + 1 + 1/*subtype*/;

-			break;

-		case RegEx:

-			{

-				const char *p = value();

-				int len1 = strlen(p);

-				p = p + len1 + 1;

-				x = 1 + len1 + strlen(p) + 2;

-			}

-			break;

-		default:

-			cout << "Element: bad type " << (int) type() << endl;

-			assert(false);

-	}

-	totalSize =  x + fieldNameSize;

-

-	if( !eoo() ) { 

-		const char *next = data + totalSize;

-		if( *next < 0 || *next > JSTypeMax ) { 

-			// bad type.  

-			cout << "*********************************************\n";

-			cout << "Bad data or size in Element::size()" << endl;

-			cout << "bad type:" << (int) *next << endl;

-			cout << "totalsize:" << totalSize << " fieldnamesize:" << fieldNameSize << endl;

-			cout << "lastrec:" << endl;

-			dumpmemory(data, totalSize + 15);

-			assert(false);

-		}

-	}

-

-	return totalSize;

-}

-

-/* must be same type! */

-inline int compareElementValues(Element& l, Element& r) {

-	int f;

-	double x;

-	switch( l.type() ) {

-		case EOO:

-		case Undefined:

-		case jstNULL:

-		case MaxKey:

-			f = l.type() - r.type();

-			if( f<0 ) return -1;

-			return f==0 ? 0 : 1;

-		case Bool:

-			return *l.value() - *r.value();

-		case Date:

-			if( l.date() < r.date() )

-				return -1;

-			return l.date() == r.date() ? 0 : 1;

-		case Number:

-			x = l.number() - r.number();

-			if( x < 0 ) return -1;

-			return x == 0 ? 0 : 1;

-		case jstOID:

-			return memcmp(l.value(), r.value(), 12);

-		case Code:

-		case String:

-			/* todo: utf version */

-			return strcmp(l.valuestr(), r.valuestr());

-		case Object:

-		case Array:

-		case DBRef:

-			{

-				int lsz = l.valuesize();

-				int rsz = r.valuesize();

-				if( lsz - rsz != 0 ) return lsz - rsz;

-				return memcmp(l.value(), r.value(), lsz);

-			}

-		case BinData:

-		case RegEx:

-			cout << "compareElementValues: can't compare this type:" << (int) l.type() << endl;

-			assert(false);

-			break;

-		default:

-			cout << "compareElementValues: bad type " << (int) l.type() << endl;

-			assert(false);

-	}

-	return -1;

-}

-

-/* JSMatcher --------------------------------------*/

-

-// If the element is something like:

-//   a : { $gt : 3 }

-// we append

-//   a : 3

-// else we just append the element.

-//

-void appendElementHandlingGtLt(JSObjBuilder& b, Element& e) { 

-	if( e.type() == Object ) {

-		Element fe = e.embeddedObject().firstElement();

-		const char *fn = fe.fieldName();

-		if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 

-			b.appendAs(fe, e.fieldName());

-			return;

-		}

-	}

-	b.append(e);

-}

-

-int getGtLtOp(Element& e) { 

-	int op = JSMatcher::Equality;

-	if( e.type() != Object ) 

-		return op;

-

-	Element fe = e.embeddedObject().firstElement();

-	const char *fn = fe.fieldName();

-	if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 

-		if( fn[1] == 'g' ) { 

-			if( fn[3] == 0 ) op = JSMatcher::GT;

-			else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::GTE;

-		}

-		else if( fn[1] == 'l' ) { 

-			if( fn[3] == 0 ) op = JSMatcher::LT;

-			else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::LTE;

-		}

-	}

-	return op;

-}

-

-#include "pdfile.h"

-

-JSMatcher::JSMatcher(JSObj &_jsobj) : 

-   where(0), jsobj(_jsobj), nRegex(0)

-{

-	nBuilders = 0;

-

-	JSElemIter i(jsobj);

-	n = 0;

-	while( i.more() ) {

-		Element e = i.next();

-		if( e.eoo() )

-			break;

-

-		if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) { 

-			// $where: function()...

-			assert( where == 0 );

-			where = new Where();

-			const char *code = e.valuestr();

-			assert( JavaJS );

-			where->scope = JavaJS->scopeCreate();

-			JavaJS->scopeSetString(where->scope, "$client", client->name.c_str());

-			where->setFunc(code);

-			continue;

-		}

-

-		if( e.type() == RegEx ) {

-			if( nRegex >= 4 ) {

-				cout << "ERROR: too many regexes in query" << endl;

-			}

-			else {

-				pcrecpp::RE_Options options;

-				options.set_utf8(true);

-				const char *flags = e.regexFlags();

-				while( flags && *flags ) { 

-					if( *flags == 'i' )

-						options.set_caseless(true);

-					else if( *flags == 'm' )

-						options.set_multiline(true);

-					else if( *flags == 'x' )

-						options.set_extended(true);

-					flags++;

-				}

-				regexs[nRegex].re = new pcrecpp::RE(e.regex(), options);

-				regexs[nRegex].fieldName = e.fieldName();

-				nRegex++;

-			}

-			continue;

-		}

-

-		// greater than / less than...

-		// { a : { $gt: 3 } }

-		if( e.type() == Object ) {

-			Element fe = e.embeddedObject().firstElement();

-			const char *fn = fe.fieldName();

-			if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 

-				int op = Equality;

-				if( fn[1] == 'g' ) { 

-					if( fn[3] == 0 ) op = GT;

-					else if( fn[3] == 'e' && fn[4] == 0 ) op = GTE;

-				}

-				else if( fn[1] == 'l' ) { 

-					if( fn[3] == 0 ) op = LT;

-					else if( fn[3] == 'e' && fn[4] == 0 ) op = LTE;

-				}

-				if( op && nBuilders < 8) { 

-					JSObjBuilder *b = new JSObjBuilder();

-					builders[nBuilders++] = b;

-					b->appendAs(fe, e.fieldName());

-					toMatch.push_back( b->done().firstElement() );

-					compareOp.push_back(op);

-					n++;

-					continue;

-				}

-			}

-		}

-

-		{

-			toMatch.push_back(e);

-			compareOp.push_back(Equality);

-			n++;

-		}

-	}

-}

-

-inline int JSMatcher::valuesMatch(Element& l, Element& r, int op) { 

-	if( op == 0 ) 

-		return l.valuesEqual(r);

-

-	if( l.type() != r.type() )

-		return false;

-

-	int c = compareElementValues(l, r);

-	int z = 1 << (c+1); 

-	return (op & z);

-}

-

-/* return value

-   -1 mismatch

-    0 missing element

-    1 match

-*/

-int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj, int compareOp, bool *deep, bool isArr) { 

-	{

-		const char *p = strchr(fieldName, '.');

-		if( p ) { 

-			string left(fieldName, p-fieldName);

-

-			Element e = obj.getField(left.c_str());

-			if( e.eoo() )

-				return 0;

-			if( e.type() != Object && e.type() != Array )

-				return -1;

-

-			JSObj eo = e.embeddedObject();

-			return matchesDotted(p+1, toMatch, eo, compareOp, deep, e.type() == Array);

-		}

-	}

-

-	Element e = obj.getField(fieldName);

-

-	if( valuesMatch(e, toMatch, compareOp) ) {

-		return 1;

-	}

-	else if( e.type() == Array ) {

-		JSElemIter ai(e.embeddedObject());

-		while( ai.more() ) { 

-			Element z = ai.next();

-			if( valuesMatch( z, toMatch, compareOp) ) {

-				if( deep )

-					*deep = true;

-				return 1;

-			}

-		}

-	}

-	else if( isArr ) { 

-		JSElemIter ai(obj);

-		while( ai.more() ) { 

-			Element z = ai.next();

-			if( z.type() == Object ) {

-				JSObj eo = z.embeddedObject();

-				int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep);

-				if( cmp > 0 ) { 

-					if( deep ) *deep = true;

-					return 1;

-				}

-			}

-		}

-	}

-	else if( e.eoo() ) { 

-		return 0;

-	}

-	return -1;

-}

-

-/* deep means we looked into arrays for a match */

-bool JSMatcher::matches(JSObj& jsobj, bool *deep) {

-	if( deep ) 

-		*deep = false;

-

-	/* assuming there is usually only one thing to match.  if more this

-	could be slow sometimes. */

-

-	for( int r = 0; r < nRegex; r++ ) { 

-		RegexMatcher& rm = regexs[r];

-		Element e = jsobj.getFieldDotted(rm.fieldName);

-		if( e.eoo() )

-			return false;

-		{

-			char buf[64];

-			const char *p = buf;

-			if( e.type() == String )

-				p = e.valuestr();

-			else if( e.type() == Number ) { 

-				sprintf(buf, "%f", e.number());

-			}

-			else if( e.type() == Date ) { 

-				unsigned long long d = e.date();

-				time_t t = (d/1000);

-				time_t_to_String(t, buf);

-			}

-			else

-				return false;

-			if( !rm.re->PartialMatch(p) )

-				return false;

-		}

-	}

-

-	// check normal non-regex cases:

-	for( int i = 0; i < n; i++ ) {

-		Element& m = toMatch[i]; 

- 

-		int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep);

-

-		/* missing is ok iff we were looking for null */

-		if( cmp < 0 ) 

-			return false;

-		if( cmp == 0 && (m.type() != jstNULL && m.type() != Undefined ) )

-			return false;

-	}

-

-/*

-		Element e = jsobj.getFieldDotted(m.fieldName(), arrayElName);

-		if( !e.eoo() ) {

-			if( valuesMatch(e, m, compareOp[i]) ) {

-				goto ok;

-			}

-			else if( e.type() == Array ) {

-				JSElemIter ai(e.embeddedObject());

-				while( ai.more() ) { 

-					Element z = ai.next();

-					if( valuesMatch( z, m, compareOp[i]) ) {

-						if( deep )

-							*deep = true;

-						goto ok;

-					}

-				}

-			}

-			return false;

-		}

-*/

-

-		/* missing.  that is ok iff we were looking for null */

-//		if( m.type() == jstNULL || m.type() == Undefined )

-//			;

-//////		else

-//			return false;

-//ok:

-//		;

-//	}

-

-	if( where ) { 

-		if( where->func == 0 )

-			return false; // didn't compile

-		if( jsobj.objsize() < 200 || where->fullObject ) {

-			JavaJS->scopeSetObject(where->scope, "obj", &jsobj);

-		} else {

-			JSObjBuilder b;

-			where->buildSubset(jsobj, b);

-			JSObj temp = b.done();

-			JavaJS->scopeSetObject(where->scope, "obj", &temp);

-		}

-		if( JavaJS->invoke(where->scope, where->func) )

-			return false;

-		return JavaJS->scopeGetBoolean(where->scope, "return") != 0;

-	}

-

-	return true;

-}

-

-/* JSObj ------------------------------------------------------------*/

-

-string JSObj::toString() const { 

-	stringstream s;

-	s << "{ ";

-	JSElemIter i(*this);

-	Element e = i.next();

-	if( !e.eoo() )

-	while( 1 ) { 

-		s << e.toString();

-		e = i.next();

-		if( e.eoo() )

-			break;

-		s << ", ";

-	}

-	s << " }";

-	return s.str();

-}

-

-/* well ordered compare */

-int JSObj::woCompare(const JSObj& r) const { 

-        assert( _objdata );

-	if( isEmpty() )

-		return r.isEmpty() ? 0 : -1;

-	if( r.isEmpty() )

-		return 1;

-

-	JSElemIter i(*this);

-	JSElemIter j(r);

-	while( 1 ) { 

-		// so far, equal...

-

-		Element l = i.next();

-		Element r = j.next();

-

-		if( l == r ) {

-			if( l.eoo() )

-				return 0;

-			continue;

-		}

-

-		int x = (int) l.type() - (int) r.type();

-		if( x != 0 )

-			return x;

-		x = strcmp(l.fieldName(), r.fieldName());

-		if( x != 0 )

-			return x;

-		x = compareElementValues(l, r);

-		assert(x != 0);

-		return x;

-	}

-	return -1;

-} 

-

-/* return has eoo() true if no match 

-   supports "." notation to reach into embedded objects

-*/

-Element JSObj::getFieldDotted(const char *name) {

-	{

-		const char *p = strchr(name, '.');

-		if( p ) { 

-			string left(name, p-name);

-			JSObj sub = getObjectField(left.c_str());

-			return sub.isEmpty() ? nullElement : sub.getFieldDotted(p+1);

-		}

-	}

-

-	JSElemIter i(*this);

-	while( i.more() ) {

-		Element e = i.next();

-		if( e.eoo() )

-			break;

-		if( strcmp(e.fieldName(), name) == 0 )

-			return e;

-	}

-	return nullElement;

-}

-

-Element JSObj::getField(const char *name) {

-	JSElemIter i(*this);

-	while( i.more() ) {

-		Element e = i.next();

-		if( e.eoo() )

-			break;

-		if( strcmp(e.fieldName(), name) == 0 )

-			return e;

-	}

-	return nullElement;

-}

-

-/* makes a new JSObj with the fields specified in pattern.

-   fields returned in the order they appear in pattern.

-   if any field missing, you get back an empty object overall.

-

-   n^2 implementation bad if pattern and object have lots 

-   of fields - normally pattern doesn't so should be fine.

-*/

-JSObj JSObj::extractFields(JSObj pattern, JSObjBuilder& b) { 

-	JSElemIter i(pattern);

-	while( i.more() ) {

-		Element e = i.next();

-		if( e.eoo() )

-			break;

-		Element x = getField(e.fieldName());

-		if( x.eoo() )

-			return JSObj();

-		b.append(x);

-	}

-	return b.done();

-}

-

-const char * JSObj::getStringField(const char *name) { 

-	Element e = getField(name);

-	return e.type() == String ? e.valuestr() : 0;

-}

-

-JSObj JSObj::getObjectField(const char *name) { 

-	Element e = getField(name);

-	JSType t = e.type();

-	return t == Object || t == Array ? e.embeddedObject() : JSObj();

-}

-

-int JSObj::getFieldNames(set<string>& fields) {

-	int n = 0;

-	JSElemIter i(*this);

-	while( i.more() ) {

-		Element e = i.next();

-		if( e.eoo() )

-			break;

-		fields.insert(e.fieldName());

-		n++;

-	}

-	return n;

-}

-

-/* note: addFields always adds _id even if not specified 

-   returns n added not counting _id unless requested.

-*/

-int JSObj::addFields(JSObj& from, set<string>& fields) {

-	assert( _objdata == 0 ); /* partial implementation for now... */

-

-	JSObjBuilder b;

-

-	int N = fields.size();

-	int n = 0;

-	JSElemIter i(from);

-	bool gotId = false;

-	while( i.more() ) {

-		Element e = i.next();

-		const char *fname = e.fieldName();

-		if( fields.count(fname) ) {

-			b.append(e);

-			++n;

-			gotId = gotId || strcmp(fname, "_id")==0;

-			if( n == N && gotId )

-				break;

-		} else if( strcmp(fname, "_id")==0 ) {

-			b.append(e);

-			gotId = true;

-			if( n == N && gotId )

-				break;

-		}

-	}

-

-	if( n ) {

-		_objdata = b.decouple(_objsize);

-		iFree = true;

-	}

-

-	return n;

-}

-

-/*-- test things ----------------------------------------------------*/

-

-#pragma pack(push)

-#pragma pack(1)

-

-struct MaxKeyData { 

-	MaxKeyData() { totsize=7; maxkey=MaxKey; name=0; eoo=EOO; }

-	int totsize;

-	char maxkey;

-	char name;

-	char eoo;

-} maxkeydata;

-JSObj maxKey((const char *) &maxkeydata);

-

-struct JSObj0 {

-	JSObj0() { totsize = 5; eoo = EOO; }

-	int totsize;

-	char eoo;

-} js0;

-

-Element::Element() { 

-	data = &js0.eoo;

-	fieldNameSize = 0;

-	totalSize = -1;

-}

-

-struct JSObj1 js1;

-

-struct JSObj2 {

-	JSObj2() {

-		totsize=sizeof(JSObj2);

-		s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; 

-		strcpy_s(sval, 10, "123456789"); eoo = EOO;

-	}

-	unsigned totsize;

-	char s;

-	char sname[7];

-	unsigned slen;

-	char sval[10];

-	char eoo;

-} js2;

-

-struct JSUnitTest {

-	JSUnitTest() {

-		JSObj j1((const char *) &js1);

-		JSObj j2((const char *) &js2);

-		JSMatcher m(j2);

-		assert( m.matches(j1) );

-		js2.sval[0] = 'z';

-		assert( !m.matches(j1) );

-		JSMatcher n(j1);

-		assert( n.matches(j1) );

-		assert( !n.matches(j2) );

-

-		JSObj j0((const char *) &js0);

-		JSMatcher p(j0);

-		assert( p.matches(j1) );

-		assert( p.matches(j2) );

-	}

-} jsunittest;

-

-#pragma pack(pop)

-

-struct RXTest { 

-	RXTest() { 

-		/*

-		static const boost::regex e("(\\d{4}[- ]){3}\\d{4}");

-		static const boost::regex b(".....");

-		cout << "regex result: " << regex_match("hello", e) << endl;

-		cout << "regex result: " << regex_match("abcoo", b) << endl;

-		*/

-		pcrecpp::RE re1(")({a}h.*o");

-		pcrecpp::RE re("h.llo");

-		assert( re.FullMatch("hello") );

-		assert( !re1.FullMatch("hello") );

-

-

-		pcrecpp::RE_Options options;

-		options.set_utf8(true);

-		pcrecpp::RE part("dwi", options);

-		assert( part.PartialMatch("dwight") );

-	}

-} rxtest;

-

+// jsobj.cpp
+
+#include "stdafx.h"
+#include "jsobj.h"
+#include "../util/goodies.h"
+#include "javajs.h"
+
+#if defined(_WIN32)
+
+#include <hash_map>
+using namespace stdext;
+
+typedef const char * MyStr;
+struct less_str {
+   bool operator()(const MyStr & x, const MyStr & y) const {
+      if ( strcmp(x, y) > 0)
+         return true;
+
+      return false;
+   }
+};
+
+typedef hash_map<const char*, int, hash_compare<const char *, less_str> > strhashmap;
+
+#else
+
+#include <ext/hash_map>
+using namespace __gnu_cxx;
+
+typedef const char * MyStr;
+struct eq_str {
+   bool operator()(const MyStr & x, const MyStr & y) const {
+      if ( strcmp(x, y) == 0)
+         return true;
+
+      return false;
+   }
+};
+
+typedef hash_map<const char*, int, hash<const char *>, eq_str > strhashmap;
+
+#endif
+
+#include "minilex.h"
+
+MiniLex minilex;
+
+class Where { 
+public:
+	Where() { codeCopy = 0; }
+	~Where() {
+		JavaJS->scopeFree(scope);
+		delete codeCopy;
+		scope = 0; func = 0; codeCopy = 0;
+	}
+	jlong scope, func;
+	strhashmap fields;
+//	map<string,int> fields;
+	bool fullObject;
+	int nFields;
+	char *codeCopy;
+
+	void setFunc(const char *code) {
+		codeCopy = new char[strlen(code)+1];
+		strcpy(codeCopy,code);
+		func = JavaJS->functionCreate( code );
+		minilex.grabVariables(codeCopy, fields);
+		// if user references db, eg db.foo.save(obj), 
+		// we make sure we have the whole thing.
+		fullObject = fields.count("fullObject") +
+			fields.count("db") > 0;
+		nFields = fields.size();
+	}
+
+	void buildSubset(JSObj& src, JSObjBuilder& dst) { 
+		JSElemIter it(src);
+		int n = 0;
+		if( !it.more() ) return;
+		while( 1 ) {
+			Element e = it.next();
+			if( e.eoo() )
+				break;
+			if( //n == 0 && 
+				fields.find(e.fieldName()) != fields.end()
+				//fields.count(e.fieldName())
+				) {
+				dst.append(e);
+				if( ++n >= nFields )
+					break;
+			}
+		}
+	}
+};
+
+JSMatcher::~JSMatcher() { 
+	for( int i = 0; i < nBuilders; i++ )
+		delete builders[i];
+	delete where;
+}
+
+Element nullElement;
+
+string Element::toString() { 
+	stringstream s;
+	switch( type() ) {
+    case EOO:
+		return "EOO";
+    case Date:
+		s << fieldName() << ": Date(" << hex << date() << ')'; break;
+	case Number:
+		s << fieldName() << ": " << number(); break;
+	case Bool: 
+		s << fieldName() << ": " << boolean() ? "true" : "false"; break;
+	case Object:
+	case Array:
+		s << fieldName() << ": " << embeddedObject().toString(); break;
+	case Undefined:
+		s << fieldName() << ": undefined"; break;
+	case jstNULL:
+		s << fieldName() << ": null"; break;
+	case MaxKey:
+		s << fieldName() << ": MaxKey"; break;
+	case Code:
+		s << fieldName() << ": ";
+		if( valuestrsize() > 80 ) 
+			s << string(valuestr()).substr(0, 70) << "...";
+		else { 
+			s << valuestr();
+		}
+		break;
+	case String:
+		s << fieldName() << ": ";
+		if( valuestrsize() > 80 ) 
+			s << '"' << string(valuestr()).substr(0, 70) << "...\"";
+		else { 
+			s << '"' << valuestr() << '"';
+		}
+		break;
+	case jstOID: 
+          s << fieldName() << " : ObjId(";
+          s << hex << oid().a << hex << oid().b << ')';
+          break;
+    default:
+		s << fieldName() << ": ?type=" << type();
+		break;
+	}
+	return s.str();
+}
+
+int Element::size() {
+	if( totalSize >= 0 )
+		return totalSize;
+
+	int x = 1;
+	switch( type() ) {
+		case EOO:
+		case Undefined:
+		case jstNULL:
+		case MaxKey:
+			break;
+		case Bool:
+			x = 2;
+			break;
+		case Date:
+		case Number:
+			x = 9;
+			break;
+		case jstOID:
+			x = 13;
+			break;
+		case Code:
+		case String:
+			x = valuestrsize() + 4 + 1;
+			break;
+        case DBRef:
+          x = valuestrsize() + 4 + 12 + 1;
+          break;
+		case Object:
+		case Array:
+			x = objsize() + 1;
+			break;
+		case BinData:
+			x = valuestrsize() + 4 + 1 + 1/*subtype*/;
+			break;
+		case RegEx:
+			{
+				const char *p = value();
+				int len1 = strlen(p);
+				p = p + len1 + 1;
+				x = 1 + len1 + strlen(p) + 2;
+			}
+			break;
+		default:
+			cout << "Element: bad type " << (int) type() << endl;
+			assert(false);
+	}
+	totalSize =  x + fieldNameSize;
+
+	if( !eoo() ) { 
+		const char *next = data + totalSize;
+		if( *next < 0 || *next > JSTypeMax ) { 
+			// bad type.  
+			cout << "*********************************************\n";
+			cout << "Bad data or size in Element::size()" << endl;
+			cout << "bad type:" << (int) *next << endl;
+			cout << "totalsize:" << totalSize << " fieldnamesize:" << fieldNameSize << endl;
+			cout << "lastrec:" << endl;
+			dumpmemory(data, totalSize + 15);
+			assert(false);
+		}
+	}
+
+	return totalSize;
+}
+
+/* must be same type! */
+inline int compareElementValues(Element& l, Element& r) {
+	int f;
+	double x;
+	switch( l.type() ) {
+		case EOO:
+		case Undefined:
+		case jstNULL:
+		case MaxKey:
+			f = l.type() - r.type();
+			if( f<0 ) return -1;
+			return f==0 ? 0 : 1;
+		case Bool:
+			return *l.value() - *r.value();
+		case Date:
+			if( l.date() < r.date() )
+				return -1;
+			return l.date() == r.date() ? 0 : 1;
+		case Number:
+			x = l.number() - r.number();
+			if( x < 0 ) return -1;
+			return x == 0 ? 0 : 1;
+		case jstOID:
+			return memcmp(l.value(), r.value(), 12);
+		case Code:
+		case String:
+			/* todo: utf version */
+			return strcmp(l.valuestr(), r.valuestr());
+		case Object:
+		case Array:
+		case DBRef:
+			{
+				int lsz = l.valuesize();
+				int rsz = r.valuesize();
+				if( lsz - rsz != 0 ) return lsz - rsz;
+				return memcmp(l.value(), r.value(), lsz);
+			}
+		case BinData:
+		case RegEx:
+			cout << "compareElementValues: can't compare this type:" << (int) l.type() << endl;
+			assert(false);
+			break;
+		default:
+			cout << "compareElementValues: bad type " << (int) l.type() << endl;
+			assert(false);
+	}
+	return -1;
+}
+
+/* JSMatcher --------------------------------------*/
+
+// If the element is something like:
+//   a : { $gt : 3 }
+// we append
+//   a : 3
+// else we just append the element.
+//
+void appendElementHandlingGtLt(JSObjBuilder& b, Element& e) { 
+	if( e.type() == Object ) {
+		Element fe = e.embeddedObject().firstElement();
+		const char *fn = fe.fieldName();
+		if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 
+			b.appendAs(fe, e.fieldName());
+			return;
+		}
+	}
+	b.append(e);
+}
+
+int getGtLtOp(Element& e) { 
+	int op = JSMatcher::Equality;
+	if( e.type() != Object ) 
+		return op;
+
+	Element fe = e.embeddedObject().firstElement();
+	const char *fn = fe.fieldName();
+	if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 
+		if( fn[1] == 'g' ) { 
+			if( fn[3] == 0 ) op = JSMatcher::GT;
+			else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::GTE;
+		}
+		else if( fn[1] == 'l' ) { 
+			if( fn[3] == 0 ) op = JSMatcher::LT;
+			else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::LTE;
+		}
+	}
+	return op;
+}
+
+#include "pdfile.h"
+
+JSMatcher::JSMatcher(JSObj &_jsobj) : 
+   where(0), jsobj(_jsobj), nRegex(0)
+{
+	nBuilders = 0;
+
+	JSElemIter i(jsobj);
+	n = 0;
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+
+		if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) { 
+			// $where: function()...
+			assert( where == 0 );
+			where = new Where();
+			const char *code = e.valuestr();
+			assert( JavaJS );
+			where->scope = JavaJS->scopeCreate();
+			JavaJS->scopeSetString(where->scope, "$client", client->name.c_str());
+			where->setFunc(code);
+			continue;
+		}
+
+		if( e.type() == RegEx ) {
+			if( nRegex >= 4 ) {
+				cout << "ERROR: too many regexes in query" << endl;
+			}
+			else {
+				pcrecpp::RE_Options options;
+				options.set_utf8(true);
+				const char *flags = e.regexFlags();
+				while( flags && *flags ) { 
+					if( *flags == 'i' )
+						options.set_caseless(true);
+					else if( *flags == 'm' )
+						options.set_multiline(true);
+					else if( *flags == 'x' )
+						options.set_extended(true);
+					flags++;
+				}
+				regexs[nRegex].re = new pcrecpp::RE(e.regex(), options);
+				regexs[nRegex].fieldName = e.fieldName();
+				nRegex++;
+			}
+			continue;
+		}
+
+		// greater than / less than...
+		// { a : { $gt: 3 } }
+		if( e.type() == Object ) {
+			Element fe = e.embeddedObject().firstElement();
+			const char *fn = fe.fieldName();
+			if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { 
+				int op = Equality;
+				if( fn[1] == 'g' ) { 
+					if( fn[3] == 0 ) op = GT;
+					else if( fn[3] == 'e' && fn[4] == 0 ) op = GTE;
+				}
+				else if( fn[1] == 'l' ) { 
+					if( fn[3] == 0 ) op = LT;
+					else if( fn[3] == 'e' && fn[4] == 0 ) op = LTE;
+				}
+				if( op && nBuilders < 8) { 
+					JSObjBuilder *b = new JSObjBuilder();
+					builders[nBuilders++] = b;
+					b->appendAs(fe, e.fieldName());
+					toMatch.push_back( b->done().firstElement() );
+					compareOp.push_back(op);
+					n++;
+					continue;
+				}
+			}
+		}
+
+		{
+			toMatch.push_back(e);
+			compareOp.push_back(Equality);
+			n++;
+		}
+	}
+}
+
+inline int JSMatcher::valuesMatch(Element& l, Element& r, int op) { 
+	if( op == 0 ) 
+		return l.valuesEqual(r);
+
+	if( l.type() != r.type() )
+		return false;
+
+	int c = compareElementValues(l, r);
+	int z = 1 << (c+1); 
+	return (op & z);
+}
+
+/* return value
+   -1 mismatch
+    0 missing element
+    1 match
+*/
+int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj, int compareOp, bool *deep, bool isArr) { 
+	{
+		const char *p = strchr(fieldName, '.');
+		if( p ) { 
+			string left(fieldName, p-fieldName);
+
+			Element e = obj.getField(left.c_str());
+			if( e.eoo() )
+				return 0;
+			if( e.type() != Object && e.type() != Array )
+				return -1;
+
+			JSObj eo = e.embeddedObject();
+			return matchesDotted(p+1, toMatch, eo, compareOp, deep, e.type() == Array);
+		}
+	}
+
+	Element e = obj.getField(fieldName);
+
+	if( valuesMatch(e, toMatch, compareOp) ) {
+		return 1;
+	}
+	else if( e.type() == Array ) {
+		JSElemIter ai(e.embeddedObject());
+		while( ai.more() ) { 
+			Element z = ai.next();
+			if( valuesMatch( z, toMatch, compareOp) ) {
+				if( deep )
+					*deep = true;
+				return 1;
+			}
+		}
+	}
+	else if( isArr ) { 
+		JSElemIter ai(obj);
+		while( ai.more() ) { 
+			Element z = ai.next();
+			if( z.type() == Object ) {
+				JSObj eo = z.embeddedObject();
+				int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep);
+				if( cmp > 0 ) { 
+					if( deep ) *deep = true;
+					return 1;
+				}
+			}
+		}
+	}
+	else if( e.eoo() ) { 
+		return 0;
+	}
+	return -1;
+}
+
+/* deep means we looked into arrays for a match */
+bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
+	if( deep ) 
+		*deep = false;
+
+	/* assuming there is usually only one thing to match.  if more this
+	could be slow sometimes. */
+
+	for( int r = 0; r < nRegex; r++ ) { 
+		RegexMatcher& rm = regexs[r];
+		Element e = jsobj.getFieldDotted(rm.fieldName);
+		if( e.eoo() )
+			return false;
+		{
+			char buf[64];
+			const char *p = buf;
+			if( e.type() == String )
+				p = e.valuestr();
+			else if( e.type() == Number ) { 
+				sprintf(buf, "%f", e.number());
+			}
+			else if( e.type() == Date ) { 
+				unsigned long long d = e.date();
+				time_t t = (d/1000);
+				time_t_to_String(t, buf);
+			}
+			else
+				return false;
+			if( !rm.re->PartialMatch(p) )
+				return false;
+		}
+	}
+
+	// check normal non-regex cases:
+	for( int i = 0; i < n; i++ ) {
+		Element& m = toMatch[i]; 
+ 
+		int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep);
+
+		/* missing is ok iff we were looking for null */
+		if( cmp < 0 ) 
+			return false;
+		if( cmp == 0 && (m.type() != jstNULL && m.type() != Undefined ) )
+			return false;
+	}
+
+/*
+		Element e = jsobj.getFieldDotted(m.fieldName(), arrayElName);
+		if( !e.eoo() ) {
+			if( valuesMatch(e, m, compareOp[i]) ) {
+				goto ok;
+			}
+			else if( e.type() == Array ) {
+				JSElemIter ai(e.embeddedObject());
+				while( ai.more() ) { 
+					Element z = ai.next();
+					if( valuesMatch( z, m, compareOp[i]) ) {
+						if( deep )
+							*deep = true;
+						goto ok;
+					}
+				}
+			}
+			return false;
+		}
+*/
+
+		/* missing.  that is ok iff we were looking for null */
+//		if( m.type() == jstNULL || m.type() == Undefined )
+//			;
+//////		else
+//			return false;
+//ok:
+//		;
+//	}
+
+	if( where ) { 
+		if( where->func == 0 )
+			return false; // didn't compile
+		if( jsobj.objsize() < 200 || where->fullObject ) {
+			JavaJS->scopeSetObject(where->scope, "obj", &jsobj);
+		} else {
+			JSObjBuilder b;
+			where->buildSubset(jsobj, b);
+			JSObj temp = b.done();
+			JavaJS->scopeSetObject(where->scope, "obj", &temp);
+		}
+		if( JavaJS->invoke(where->scope, where->func) )
+			return false;
+		return JavaJS->scopeGetBoolean(where->scope, "return") != 0;
+	}
+
+	return true;
+}
+
+/* JSObj ------------------------------------------------------------*/
+
+string JSObj::toString() const { 
+	stringstream s;
+	s << "{ ";
+	JSElemIter i(*this);
+	Element e = i.next();
+	if( !e.eoo() )
+	while( 1 ) { 
+		s << e.toString();
+		e = i.next();
+		if( e.eoo() )
+			break;
+		s << ", ";
+	}
+	s << " }";
+	return s.str();
+}
+
+/* well ordered compare */
+int JSObj::woCompare(const JSObj& r) const { 
+        assert( _objdata );
+	if( isEmpty() )
+		return r.isEmpty() ? 0 : -1;
+	if( r.isEmpty() )
+		return 1;
+
+	JSElemIter i(*this);
+	JSElemIter j(r);
+	while( 1 ) { 
+		// so far, equal...
+
+		Element l = i.next();
+		Element r = j.next();
+
+		if( l == r ) {
+			if( l.eoo() )
+				return 0;
+			continue;
+		}
+
+		int x = (int) l.type() - (int) r.type();
+		if( x != 0 )
+			return x;
+		x = strcmp(l.fieldName(), r.fieldName());
+		if( x != 0 )
+			return x;
+		x = compareElementValues(l, r);
+		assert(x != 0);
+		return x;
+	}
+	return -1;
+} 
+
+/* return has eoo() true if no match 
+   supports "." notation to reach into embedded objects
+*/
+Element JSObj::getFieldDotted(const char *name) {
+	{
+		const char *p = strchr(name, '.');
+		if( p ) { 
+			string left(name, p-name);
+			JSObj sub = getObjectField(left.c_str());
+			return sub.isEmpty() ? nullElement : sub.getFieldDotted(p+1);
+		}
+	}
+
+	JSElemIter i(*this);
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+		if( strcmp(e.fieldName(), name) == 0 )
+			return e;
+	}
+	return nullElement;
+}
+
+Element JSObj::getField(const char *name) {
+	JSElemIter i(*this);
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+		if( strcmp(e.fieldName(), name) == 0 )
+			return e;
+	}
+	return nullElement;
+}
+
+/* makes a new JSObj with the fields specified in pattern.
+   fields returned in the order they appear in pattern.
+   if any field missing, you get back an empty object overall.
+
+   n^2 implementation bad if pattern and object have lots 
+   of fields - normally pattern doesn't so should be fine.
+*/
+JSObj JSObj::extractFields(JSObj pattern, JSObjBuilder& b) { 
+	JSElemIter i(pattern);
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+		Element x = getField(e.fieldName());
+		if( x.eoo() )
+			return JSObj();
+		b.append(x);
+	}
+	return b.done();
+}
+
+const char * JSObj::getStringField(const char *name) { 
+	Element e = getField(name);
+	return e.type() == String ? e.valuestr() : 0;
+}
+
+JSObj JSObj::getObjectField(const char *name) { 
+	Element e = getField(name);
+	JSType t = e.type();
+	return t == Object || t == Array ? e.embeddedObject() : JSObj();
+}
+
+int JSObj::getFieldNames(set<string>& fields) {
+	int n = 0;
+	JSElemIter i(*this);
+	while( i.more() ) {
+		Element e = i.next();
+		if( e.eoo() )
+			break;
+		fields.insert(e.fieldName());
+		n++;
+	}
+	return n;
+}
+
+/* note: addFields always adds _id even if not specified 
+   returns n added not counting _id unless requested.
+*/
+int JSObj::addFields(JSObj& from, set<string>& fields) {
+	assert( _objdata == 0 ); /* partial implementation for now... */
+
+	JSObjBuilder b;
+
+	int N = fields.size();
+	int n = 0;
+	JSElemIter i(from);
+	bool gotId = false;
+	while( i.more() ) {
+		Element e = i.next();
+		const char *fname = e.fieldName();
+		if( fields.count(fname) ) {
+			b.append(e);
+			++n;
+			gotId = gotId || strcmp(fname, "_id")==0;
+			if( n == N && gotId )
+				break;
+		} else if( strcmp(fname, "_id")==0 ) {
+			b.append(e);
+			gotId = true;
+			if( n == N && gotId )
+				break;
+		}
+	}
+
+	if( n ) {
+		_objdata = b.decouple(_objsize);
+		iFree = true;
+	}
+
+	return n;
+}
+
+/*-- test things ----------------------------------------------------*/
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct MaxKeyData { 
+	MaxKeyData() { totsize=7; maxkey=MaxKey; name=0; eoo=EOO; }
+	int totsize;
+	char maxkey;
+	char name;
+	char eoo;
+} maxkeydata;
+JSObj maxKey((const char *) &maxkeydata);
+
+struct JSObj0 {
+	JSObj0() { totsize = 5; eoo = EOO; }
+	int totsize;
+	char eoo;
+} js0;
+
+Element::Element() { 
+	data = &js0.eoo;
+	fieldNameSize = 0;
+	totalSize = -1;
+}
+
+struct JSObj1 js1;
+
+struct JSObj2 {
+	JSObj2() {
+		totsize=sizeof(JSObj2);
+		s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; 
+		strcpy_s(sval, 10, "123456789"); eoo = EOO;
+	}
+	unsigned totsize;
+	char s;
+	char sname[7];
+	unsigned slen;
+	char sval[10];
+	char eoo;
+} js2;
+
+struct JSUnitTest {
+	JSUnitTest() {
+		JSObj j1((const char *) &js1);
+		JSObj j2((const char *) &js2);
+		JSMatcher m(j2);
+		assert( m.matches(j1) );
+		js2.sval[0] = 'z';
+		assert( !m.matches(j1) );
+		JSMatcher n(j1);
+		assert( n.matches(j1) );
+		assert( !n.matches(j2) );
+
+		JSObj j0((const char *) &js0);
+		JSMatcher p(j0);
+		assert( p.matches(j1) );
+		assert( p.matches(j2) );
+	}
+} jsunittest;
+
+#pragma pack(pop)
+
+struct RXTest { 
+	RXTest() { 
+		/*
+		static const boost::regex e("(\\d{4}[- ]){3}\\d{4}");
+		static const boost::regex b(".....");
+		cout << "regex result: " << regex_match("hello", e) << endl;
+		cout << "regex result: " << regex_match("abcoo", b) << endl;
+		*/
+		pcrecpp::RE re1(")({a}h.*o");
+		pcrecpp::RE re("h.llo");
+		assert( re.FullMatch("hello") );
+		assert( !re1.FullMatch("hello") );
+
+
+		pcrecpp::RE_Options options;
+		options.set_utf8(true);
+		pcrecpp::RE part("dwi", options);
+		assert( part.PartialMatch("dwight") );
+	}
+} rxtest;
+
diff --git a/db/jsobj.h b/db/jsobj.h
index f441064a97e..faa4d9f159a 100644
--- a/db/jsobj.h
+++ b/db/jsobj.h
@@ -1,514 +1,514 @@
-// jsobj.h

-

-#pragma once

-

-#include "../stdafx.h"

-#include "../util/builder.h"

-//#include "javajs.h"

-

-#include <set>

-

-class JSObj;

-class Record;

-class JSObjBuilder;

-

-#pragma pack(push)

-#pragma pack(1)

-

-/* BinData = binary data types. 

-   EOO = end of object

-*/

-enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5, 

-              Undefined=6, jstOID=7, Bool=8, Date=9 , jstNULL=10, RegEx=11 ,

-              DBRef=12, Code=13, JSTypeMax=13, MaxKey=127 };

-

-/* subtypes of BinData.

-   bdtCustom and above are ones that the JS compiler understands, but are

-   opaque to the database.

-*/

-enum BinDataType { Function=1, ByteArray=2, bdtCustom=128 };

-

-/*	Object id's are optional for JSObjects.  

-	When present they should be the first object member added.

-*/

-struct OID { 

-	long long a;

-	unsigned b;

-	bool operator==(const OID& r) { return a==r.a&&b==r.b; }

-          void out(){ cout << hex << a << hex << b << endl; };

-};

-

-/* marshalled js object format:

-

-   <unsigned totalSize> {<byte JSType><cstring FieldName><Data>}* EOO

-      totalSize includes itself.

-

-   Data:

-     Bool: <byte>

-     EOO: nothing follows

-     Undefined: nothing follows

-     OID: an OID object

-     Number: <double>

-     String: <unsigned32 strsizewithnull><cstring>

-	 Date:   <8bytes>

-	 Regex:  <cstring regex><cstring options>

-     Object: a nested object, leading with its entire size, which terminates with EOO.

-     Array:  same as object

-     BinData:

-       <int len>

-       <byte subtype>

-       <byte[len] data>

-*/

-

-/* db operation message format 

-

-   unsigned opid;         // arbitary; will be echoed back

-   byte operation;

-   

-   dbInsert:

-      int reserved;

-      string collection;

-      a series of JSObjects terminated with a null object (i.e., just EOO)

-   dbUpdate: see query.h

-   dbDelete: see query.h

-   dbQuery: see query.h

-*/

-

-#pragma pack(pop)

-

-/* <type><fieldName    ><value> 

-   -------- size() ------------

-         -fieldNameSize-

-                        value()

-   type()

-*/

-class Element {

-	friend class JSElemIter;

-	friend class JSObj;

-public:

-	string toString();

-	JSType type() { return (JSType) *data; }

-	bool eoo() { return type() == EOO; }

-	int size();

-

-	// wrap this element up as a singleton object.

-	JSObj wrap();

-

-	const char * fieldName() { 

-		if( eoo() ) return ""; // no fieldname for it.

-		return data + 1; 

-	}

-

-	// raw data be careful:

-	const char * value() const { return (data + fieldNameSize + 1); }

-	int valuesize() { return size() - fieldNameSize - 1; } 

-

-	bool boolean() { return *value() ? true : false; }

-

-	unsigned long long date() { return *((unsigned long long*) value()); }

-	double& number() { return *((double *) value()); }

-	OID& oid() { return *((OID*) value()); }

-

-	// for strings

-	int valuestrsize() { 

-		return *((int *) value());

-	}

-

-	// for objects the size *includes* the size of the size field

-	int objsize() { 

-		return *((int *) value());

-	}

-

-	// for strings.  also gives you start of the real data for an embedded object

-	const char * valuestr() { return value() + 4; }

-

-	JSObj embeddedObject();

-

-	const char *regex() { assert(type() == RegEx); return value(); }

-	const char *regexFlags() { 

-		const char *p = regex();

-		return p + strlen(p) + 1;

-	}

-

-	/* like operator== but doesn't check the fieldname,

-	   just the value.

-	   */

-	bool valuesEqual(Element& r) { 

-		bool match= valuesize() == r.valuesize() && 

-			memcmp(value(),r.value(),valuesize()) == 0;

-		return match;

-		// todo: make "0" == 0.0, undefined==null

-	}

-

-	bool operator==(Element& r) { 

-		if( strcmp(fieldName(), r.fieldName()) != 0 )

-			return false;

-		return valuesEqual(r);

-/*

-		int sz = size();

-		return sz == r.size() && 

-			memcmp(data, r.data, sz) == 0;

-*/

-	}

-

-	const char * rawdata() { return data; }

-

-	Element();

-

-private:

-	Element(const char *d) : data(d) {

-		fieldNameSize = eoo() ? 0 : strlen(fieldName()) + 1;

-		totalSize = -1;

-	}

-	const char *data;

-	int fieldNameSize;

-	int totalSize;

-};

-

-class JSObj {

-	friend class JSElemIter;

-public:

-explicit

-	JSObj(const char *msgdata, bool ifree = false) : iFree(ifree) {

-		_objdata = msgdata;

-		_objsize = *((int*) _objdata);

-	}

-	JSObj(Record *r);

-	JSObj() : _objsize(0), _objdata(0), iFree(false) { }

-

-	~JSObj() { if( iFree ) { free((void*)_objdata); _objdata=0; } }

-

-	void iWillFree() { 

-		assert(!iFree); iFree = true; 

-	}

-

-	string toString() const;

-	/* note: addFields always adds _id even if not specified */

-	int addFields(JSObj& from, set<string>& fields); /* returns n added */

-	int getFieldNames(set<string>& fields);

-

-	/* return has eoo() true if no match 

-	   supports "." notation to reach into embedded objects

-	*/

-	Element getFieldDotted(const char *name); 

-

-	Element getField(const char *name); /* return has eoo() true if no match */

-

-	const char * getStringField(const char *name);

-	JSObj getObjectField(const char *name);

-

-	/* makes a new JSObj with the fields specified in pattern.

-       fields returned in the order they appear in pattern.

-	   if any field missing, you get back an empty object overall.

-	   */

-	JSObj extractFields(JSObj pattern, JSObjBuilder& b);

-

-	const char *objdata() const { return _objdata; }

-	int objsize() const { return _objsize; } // includes the embedded size field

-	bool isEmpty() const { return objsize() <= 5; }

-

-	/* this is broken if elements aren't in the same order. */

-	bool operator<(const JSObj& r) const { return woCompare(r) < 0; }

-

-	/* -1: l<r. 0:l==r. 1:l>r 

-	   wo='well ordered'.  fields must be in same order in each object.

-	*/

-	int woCompare(const JSObj& r) const;

-	bool woEqual(const JSObj& r) const { 

-		return _objsize==r._objsize && memcmp(_objdata,r._objdata,_objsize)==0;

-	}

-	bool operator==(const JSObj& r) const { 

-		return this->woEqual(r);*this == r; 

-	}

-

-	Element firstElement() { 

-		return Element(objdata() + 4);

-	}

-	Element findElement(const char *name);

-

-	OID* getOID() {

-		Element e = firstElement();

-		if( e.type() != jstOID )

-			return 0;

-		return &e.oid();

-	}

-

-	JSObj(const JSObj& r) { 

-		_objsize = r._objsize;

-		_objdata = r._objdata;

-		iFree = r.iFree;

-		if( iFree ) { 

-			((JSObj&)r)._objdata = 0; 

-			((JSObj&)r).iFree = false; 

-		}

-

-                assert( _objsize == 0 || _objdata );

-

-	}

-	JSObj& operator=(JSObj& r) {

-		if( iFree ) free((void*)_objdata); 

-		_objsize = r._objsize;

-		_objdata = r._objdata;

-		iFree = r.iFree;

-		/* kind of like auto_ptrs here.  note we leave objsize as it was 

-		   so you'll get notified if you try to use the object, instead of just

-		   thinking it is empty.

-		   */

-		if( iFree ) { r._objdata = 0; r.iFree = false; }

-		return *this;

-	}

-

-	/* makes a copy of the object.  Normally, a jsobj points to data "owned" 

-	   by something else.  this is a useful way to get your own copy of the buffer 

-	   data (which is freed when the new jsobj destructs).

-	   */

-	JSObj copy() const;

-

-	int hash() const {

-		unsigned x = 0;

-		const char *p = _objdata;

-		for( int i = 0; i < _objsize; i++ )

-			x = x * 131 + p[i];

-		return (x & 0x7fffffff) | 0x8000000; // must be > 0

-	}

-

-	bool iFree;

-private:

-	int _objsize;

-	const char *_objdata;

-};

-

-class JSObjBuilder { 

-public:

-	JSObjBuilder() { b.skip(4); /*leave room for size field*/ }

-

-	/* add all the fields from the object specified to this object */

-	void appendElements(JSObj x);

-

-	void append(Element& e) { b.append((void*) e.rawdata(), e.size()); }

-

-	/* append an element but with a new name */

-	void appendAs(Element& e, const char *as) { 

-		b.append((char) e.type());

-		b.append(as);

-		b.append((void *) e.value(), e.valuesize());

-	}

-

-	/* add a subobject as a member */

-	void append(const char *fieldName, JSObj subObj) { 

-		b.append((char) Object);

-		b.append(fieldName);

-		b.append((void *) subObj.objdata(), subObj.objsize());

-	}

-

-	void appendBool(const char *fieldName, int val) { 

-		b.append((char) Bool);

-		b.append(fieldName);

-		b.append((char) (val?1:0));

-	}

-	void append(const char *fieldName, double n) { 

-		b.append((char) Number);

-		b.append(fieldName);

-		b.append(n);

-	}

-	void appendDate(const char *fieldName, unsigned long long dt) { 

-		b.append((char) Date);

-		b.append(fieldName);

-		b.append(dt);

-	}

-	void append(const char *fieldName, const char *str) {

-		b.append((char) String);

-		b.append(fieldName);

-		b.append((int) strlen(str)+1);

-		b.append(str);

-	}

-

-	JSObj doneAndDecouple() { 

-		int l;

-		return JSObj(decouple(l), true);

-	}

-

-	/* this version, jsobjbuilder still frees the jsobj

-	when the builder goes out of scope.  use it this way

-	by default, that's simplest.

-	*/

-	JSObj done() { 

-		return JSObj(_done());

-	}

-

-	/* assume ownership of the buffer - you must then free it (with free()) */

-	char* decouple(int& l) {

-		char *x = _done();

-                assert( x );

-		l = b.len();

-		b.decouple();

-		return x;

-	}

-	void decouple() { b.decouple(); } // post done() call version.  be sure jsobj frees...

-

-private:

-	char* _done() { 

-		b.append((char) EOO);

-		char *data = b.buf();

-		*((int*)data) = b.len();

-		return data;

-	}

-

-	BufBuilder b;

-};

-

-class JSElemIter {

-public:

-	JSElemIter(const JSObj& jso) {

-		pos = jso.objdata() + 4;

-		theend = jso.objdata() + jso.objsize();

-	}

-	bool more() { return pos < theend; }

-	Element next() {

-		Element e(pos);

-		pos += e.size();

-		return e;

-	}

-private:

-	const char *pos;

-	const char *theend;

-};

-

-#include <pcrecpp.h> 

-

-class RegexMatcher { 

-public:

-	const char *fieldName;

-	pcrecpp::RE *re;

-	RegexMatcher() { re = 0; }

-	~RegexMatcher() { delete re; }

-};

-

-// SQL where clause equivalent

-class Where; 

-

-/* For when a js object is a query pattern. 

-

-   e.g.

-       db.foo.find( { a : 3 } );

-

-   { a : 3 } is the pattern object.

-

-   GT/LT:

-   { a : { $gt

-

-*/

-class JSMatcher { 

-	int matchesDotted(

-		const char *fieldName, 

-		Element& toMatch, JSObj& obj, 

-		int compareOp, bool *deep, bool isArr = false);

-

-public:

-	enum { 

-		Equality = 0,

-		LT = 0x1,

-		LTE = 0x3,

-		GTE = 0x6,

-		GT = 0x4 

-	};

-

-	static int opDirection(int op) { 

-		return op <= LTE ? -1 : 1;

-	}

-

-	JSMatcher(JSObj& pattern);

-

-	~JSMatcher();

-

-	/* deep means we looked into arrays for a match */

-	bool matches(JSObj& j, bool *deep = 0);

-

-	int getN() { return n; }

-

-private:

-	int valuesMatch(Element& l, Element& r, int op);

-

-	Where *where;

-	JSObj& jsobj;

-	vector<Element> toMatch;

-	vector<int> compareOp;

-	int n;

-

-	RegexMatcher regexs[4];

-	int nRegex;

-

-	// so we delete the mem when we're done:

-	JSObjBuilder *builders[8];

-	int nBuilders;

-};

-

-extern JSObj maxKey;

-

-/*- just for testing -- */

-

-#pragma pack(push)

-#pragma pack(1)

-struct JSObj1 {

-	JSObj1() {

-		totsize=sizeof(JSObj1); 

-		n = Number; strcpy_s(nname, 5, "abcd"); N = 3.1;

-		s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; 

-		strcpy_s(sval, 10, "123456789"); eoo = EOO;

-	}

-	unsigned totsize;

-

-	char n;

-	char nname[5];

-	double N;

-

-	char s;

-	char sname[7];

-	unsigned slen;

-	char sval[10];

-

-	char eoo;

-};

-#pragma pack(pop)

-extern JSObj1 js1;

-

-inline JSObj Element::embeddedObject() { 

-	assert( type()==Object || type()==Array ); 

-	return JSObj(value()); 

-}

-

-inline JSObj JSObj::copy() const { 

-	if( _objsize == 0 )

-		return *this;

-	char *p = (char*) malloc(_objsize);

-	memcpy(p, _objdata, _objsize);

-	return JSObj(p, true);

-}

-

-// wrap this element up as a singleton object.

-inline JSObj Element::wrap() { 

-	JSObjBuilder b;

-	b.append(*this);

-	return b.doneAndDecouple();

-}

-

-inline Element JSObj::findElement(const char *name) { 

-	JSElemIter it(*this);

-	while( it.more() ) {

-		Element e = it.next();

-		if( strcmp(name, e.fieldName()) == 0 ) 

-			return e;

-	}

-	return Element();

-}

-

-/* add all the fields from the object specified to this object */

-inline void JSObjBuilder::appendElements(JSObj x) { 

-	JSElemIter it(x);

-	while( it.more() ) {

-		Element e = it.next();

-		if( e.eoo() ) break;

-		append(e);

-	}

-}

-

+// jsobj.h
+
+#pragma once
+
+#include "../stdafx.h"
+#include "../util/builder.h"
+//#include "javajs.h"
+
+#include <set>
+
+class JSObj;
+class Record;
+class JSObjBuilder;
+
+#pragma pack(push)
+#pragma pack(1)
+
+/* BinData = binary data types. 
+   EOO = end of object
+*/
+enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5, 
+              Undefined=6, jstOID=7, Bool=8, Date=9 , jstNULL=10, RegEx=11 ,
+              DBRef=12, Code=13, JSTypeMax=13, MaxKey=127 };
+
+/* subtypes of BinData.
+   bdtCustom and above are ones that the JS compiler understands, but are
+   opaque to the database.
+*/
+enum BinDataType { Function=1, ByteArray=2, bdtCustom=128 };
+
+/*	Object id's are optional for JSObjects.  
+	When present they should be the first object member added.
+*/
+struct OID { 
+	long long a;
+	unsigned b;
+	bool operator==(const OID& r) { return a==r.a&&b==r.b; }
+          void out(){ cout << hex << a << hex << b << endl; };
+};
+
+/* marshalled js object format:
+
+   <unsigned totalSize> {<byte JSType><cstring FieldName><Data>}* EOO
+      totalSize includes itself.
+
+   Data:
+     Bool: <byte>
+     EOO: nothing follows
+     Undefined: nothing follows
+     OID: an OID object
+     Number: <double>
+     String: <unsigned32 strsizewithnull><cstring>
+	 Date:   <8bytes>
+	 Regex:  <cstring regex><cstring options>
+     Object: a nested object, leading with its entire size, which terminates with EOO.
+     Array:  same as object
+     BinData:
+       <int len>
+       <byte subtype>
+       <byte[len] data>
+*/
+
+/* db operation message format 
+
+   unsigned opid;         // arbitary; will be echoed back
+   byte operation;
+   
+   dbInsert:
+      int reserved;
+      string collection;
+      a series of JSObjects terminated with a null object (i.e., just EOO)
+   dbUpdate: see query.h
+   dbDelete: see query.h
+   dbQuery: see query.h
+*/
+
+#pragma pack(pop)
+
+/* <type><fieldName    ><value> 
+   -------- size() ------------
+         -fieldNameSize-
+                        value()
+   type()
+*/
+class Element {
+	friend class JSElemIter;
+	friend class JSObj;
+public:
+	string toString();
+	JSType type() { return (JSType) *data; }
+	bool eoo() { return type() == EOO; }
+	int size();
+
+	// wrap this element up as a singleton object.
+	JSObj wrap();
+
+	const char * fieldName() { 
+		if( eoo() ) return ""; // no fieldname for it.
+		return data + 1; 
+	}
+
+	// raw data be careful:
+	const char * value() const { return (data + fieldNameSize + 1); }
+	int valuesize() { return size() - fieldNameSize - 1; } 
+
+	bool boolean() { return *value() ? true : false; }
+
+	unsigned long long date() { return *((unsigned long long*) value()); }
+	double& number() { return *((double *) value()); }
+	OID& oid() { return *((OID*) value()); }
+
+	// for strings
+	int valuestrsize() { 
+		return *((int *) value());
+	}
+
+	// for objects the size *includes* the size of the size field
+	int objsize() { 
+		return *((int *) value());
+	}
+
+	// for strings.  also gives you start of the real data for an embedded object
+	const char * valuestr() { return value() + 4; }
+
+	JSObj embeddedObject();
+
+	const char *regex() { assert(type() == RegEx); return value(); }
+	const char *regexFlags() { 
+		const char *p = regex();
+		return p + strlen(p) + 1;
+	}
+
+	/* like operator== but doesn't check the fieldname,
+	   just the value.
+	   */
+	bool valuesEqual(Element& r) { 
+		bool match= valuesize() == r.valuesize() && 
+			memcmp(value(),r.value(),valuesize()) == 0;
+		return match;
+		// todo: make "0" == 0.0, undefined==null
+	}
+
+	bool operator==(Element& r) { 
+		if( strcmp(fieldName(), r.fieldName()) != 0 )
+			return false;
+		return valuesEqual(r);
+/*
+		int sz = size();
+		return sz == r.size() && 
+			memcmp(data, r.data, sz) == 0;
+*/
+	}
+
+	const char * rawdata() { return data; }
+
+	Element();
+
+private:
+	Element(const char *d) : data(d) {
+		fieldNameSize = eoo() ? 0 : strlen(fieldName()) + 1;
+		totalSize = -1;
+	}
+	const char *data;
+	int fieldNameSize;
+	int totalSize;
+};
+
+class JSObj {
+	friend class JSElemIter;
+public:
+explicit
+	JSObj(const char *msgdata, bool ifree = false) : iFree(ifree) {
+		_objdata = msgdata;
+		_objsize = *((int*) _objdata);
+	}
+	JSObj(Record *r);
+	JSObj() : _objsize(0), _objdata(0), iFree(false) { }
+
+	~JSObj() { if( iFree ) { free((void*)_objdata); _objdata=0; } }
+
+	void iWillFree() { 
+		assert(!iFree); iFree = true; 
+	}
+
+	string toString() const;
+	/* note: addFields always adds _id even if not specified */
+	int addFields(JSObj& from, set<string>& fields); /* returns n added */
+	int getFieldNames(set<string>& fields);
+
+	/* return has eoo() true if no match 
+	   supports "." notation to reach into embedded objects
+	*/
+	Element getFieldDotted(const char *name); 
+
+	Element getField(const char *name); /* return has eoo() true if no match */
+
+	const char * getStringField(const char *name);
+	JSObj getObjectField(const char *name);
+
+	/* makes a new JSObj with the fields specified in pattern.
+       fields returned in the order they appear in pattern.
+	   if any field missing, you get back an empty object overall.
+	   */
+	JSObj extractFields(JSObj pattern, JSObjBuilder& b);
+
+	const char *objdata() const { return _objdata; }
+	int objsize() const { return _objsize; } // includes the embedded size field
+	bool isEmpty() const { return objsize() <= 5; }
+
+	/* this is broken if elements aren't in the same order. */
+	bool operator<(const JSObj& r) const { return woCompare(r) < 0; }
+
+	/* -1: l<r. 0:l==r. 1:l>r 
+	   wo='well ordered'.  fields must be in same order in each object.
+	*/
+	int woCompare(const JSObj& r) const;
+	bool woEqual(const JSObj& r) const { 
+		return _objsize==r._objsize && memcmp(_objdata,r._objdata,_objsize)==0;
+	}
+	bool operator==(const JSObj& r) const { 
+		return this->woEqual(r);*this == r; 
+	}
+
+	Element firstElement() { 
+		return Element(objdata() + 4);
+	}
+	Element findElement(const char *name);
+
+	OID* getOID() {
+		Element e = firstElement();
+		if( e.type() != jstOID )
+			return 0;
+		return &e.oid();
+	}
+
+	JSObj(const JSObj& r) { 
+		_objsize = r._objsize;
+		_objdata = r._objdata;
+		iFree = r.iFree;
+		if( iFree ) { 
+			((JSObj&)r)._objdata = 0; 
+			((JSObj&)r).iFree = false; 
+		}
+
+                assert( _objsize == 0 || _objdata );
+
+	}
+	JSObj& operator=(JSObj& r) {
+		if( iFree ) free((void*)_objdata); 
+		_objsize = r._objsize;
+		_objdata = r._objdata;
+		iFree = r.iFree;
+		/* kind of like auto_ptrs here.  note we leave objsize as it was 
+		   so you'll get notified if you try to use the object, instead of just
+		   thinking it is empty.
+		   */
+		if( iFree ) { r._objdata = 0; r.iFree = false; }
+		return *this;
+	}
+
+	/* makes a copy of the object.  Normally, a jsobj points to data "owned" 
+	   by something else.  this is a useful way to get your own copy of the buffer 
+	   data (which is freed when the new jsobj destructs).
+	   */
+	JSObj copy() const;
+
+	int hash() const {
+		unsigned x = 0;
+		const char *p = _objdata;
+		for( int i = 0; i < _objsize; i++ )
+			x = x * 131 + p[i];
+		return (x & 0x7fffffff) | 0x8000000; // must be > 0
+	}
+
+	bool iFree;
+private:
+	int _objsize;
+	const char *_objdata;
+};
+
+class JSObjBuilder { 
+public:
+	JSObjBuilder() { b.skip(4); /*leave room for size field*/ }
+
+	/* add all the fields from the object specified to this object */
+	void appendElements(JSObj x);
+
+	void append(Element& e) { b.append((void*) e.rawdata(), e.size()); }
+
+	/* append an element but with a new name */
+	void appendAs(Element& e, const char *as) { 
+		b.append((char) e.type());
+		b.append(as);
+		b.append((void *) e.value(), e.valuesize());
+	}
+
+	/* add a subobject as a member */
+	void append(const char *fieldName, JSObj subObj) { 
+		b.append((char) Object);
+		b.append(fieldName);
+		b.append((void *) subObj.objdata(), subObj.objsize());
+	}
+
+	void appendBool(const char *fieldName, int val) { 
+		b.append((char) Bool);
+		b.append(fieldName);
+		b.append((char) (val?1:0));
+	}
+	void append(const char *fieldName, double n) { 
+		b.append((char) Number);
+		b.append(fieldName);
+		b.append(n);
+	}
+	void appendDate(const char *fieldName, unsigned long long dt) { 
+		b.append((char) Date);
+		b.append(fieldName);
+		b.append(dt);
+	}
+	void append(const char *fieldName, const char *str) {
+		b.append((char) String);
+		b.append(fieldName);
+		b.append((int) strlen(str)+1);
+		b.append(str);
+	}
+
+	JSObj doneAndDecouple() { 
+		int l;
+		return JSObj(decouple(l), true);
+	}
+
+	/* this version, jsobjbuilder still frees the jsobj
+	when the builder goes out of scope.  use it this way
+	by default, that's simplest.
+	*/
+	JSObj done() { 
+		return JSObj(_done());
+	}
+
+	/* assume ownership of the buffer - you must then free it (with free()) */
+	char* decouple(int& l) {
+		char *x = _done();
+                assert( x );
+		l = b.len();
+		b.decouple();
+		return x;
+	}
+	void decouple() { b.decouple(); } // post done() call version.  be sure jsobj frees...
+
+private:
+	char* _done() { 
+		b.append((char) EOO);
+		char *data = b.buf();
+		*((int*)data) = b.len();
+		return data;
+	}
+
+	BufBuilder b;
+};
+
+class JSElemIter {
+public:
+	JSElemIter(const JSObj& jso) {
+		pos = jso.objdata() + 4;
+		theend = jso.objdata() + jso.objsize();
+	}
+	bool more() { return pos < theend; }
+	Element next() {
+		Element e(pos);
+		pos += e.size();
+		return e;
+	}
+private:
+	const char *pos;
+	const char *theend;
+};
+
+#include <pcrecpp.h> 
+
+class RegexMatcher { 
+public:
+	const char *fieldName;
+	pcrecpp::RE *re;
+	RegexMatcher() { re = 0; }
+	~RegexMatcher() { delete re; }
+};
+
+// SQL where clause equivalent
+class Where; 
+
+/* For when a js object is a query pattern. 
+
+   e.g.
+       db.foo.find( { a : 3 } );
+
+   { a : 3 } is the pattern object.
+
+   GT/LT:
+   { a : { $gt
+
+*/
+class JSMatcher { 
+	int matchesDotted(
+		const char *fieldName, 
+		Element& toMatch, JSObj& obj, 
+		int compareOp, bool *deep, bool isArr = false);
+
+public:
+	enum { 
+		Equality = 0,
+		LT = 0x1,
+		LTE = 0x3,
+		GTE = 0x6,
+		GT = 0x4 
+	};
+
+	static int opDirection(int op) { 
+		return op <= LTE ? -1 : 1;
+	}
+
+	JSMatcher(JSObj& pattern);
+
+	~JSMatcher();
+
+	/* deep means we looked into arrays for a match */
+	bool matches(JSObj& j, bool *deep = 0);
+
+	int getN() { return n; }
+
+private:
+	int valuesMatch(Element& l, Element& r, int op);
+
+	Where *where;
+	JSObj& jsobj;
+	vector<Element> toMatch;
+	vector<int> compareOp;
+	int n;
+
+	RegexMatcher regexs[4];
+	int nRegex;
+
+	// so we delete the mem when we're done:
+	JSObjBuilder *builders[8];
+	int nBuilders;
+};
+
+extern JSObj maxKey;
+
+/*- just for testing -- */
+
+#pragma pack(push)
+#pragma pack(1)
+struct JSObj1 {
+	JSObj1() {
+		totsize=sizeof(JSObj1); 
+		n = Number; strcpy_s(nname, 5, "abcd"); N = 3.1;
+		s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; 
+		strcpy_s(sval, 10, "123456789"); eoo = EOO;
+	}
+	unsigned totsize;
+
+	char n;
+	char nname[5];
+	double N;
+
+	char s;
+	char sname[7];
+	unsigned slen;
+	char sval[10];
+
+	char eoo;
+};
+#pragma pack(pop)
+extern JSObj1 js1;
+
+inline JSObj Element::embeddedObject() { 
+	assert( type()==Object || type()==Array ); 
+	return JSObj(value()); 
+}
+
+inline JSObj JSObj::copy() const { 
+	if( _objsize == 0 )
+		return *this;
+	char *p = (char*) malloc(_objsize);
+	memcpy(p, _objdata, _objsize);
+	return JSObj(p, true);
+}
+
+// wrap this element up as a singleton object.
+inline JSObj Element::wrap() { 
+	JSObjBuilder b;
+	b.append(*this);
+	return b.doneAndDecouple();
+}
+
+inline Element JSObj::findElement(const char *name) { 
+	JSElemIter it(*this);
+	while( it.more() ) {
+		Element e = it.next();
+		if( strcmp(name, e.fieldName()) == 0 ) 
+			return e;
+	}
+	return Element();
+}
+
+/* add all the fields from the object specified to this object */
+inline void JSObjBuilder::appendElements(JSObj x) { 
+	JSElemIter it(x);
+	while( it.more() ) {
+		Element e = it.next();
+		if( e.eoo() ) break;
+		append(e);
+	}
+}
+
diff --git a/db/minilex.h b/db/minilex.h
index 64c98fb9ff8..b63228f4822 100644
--- a/db/minilex.h
+++ b/db/minilex.h
@@ -1,97 +1,97 @@
-// minilex.h

-// mini js lexical analyzer.  idea is to be dumb and fast.

-

-struct MiniLex {

-	strhashmap reserved;

-	bool ic[256]; // ic=Identifier Character

-	bool starter[256];

-

-	// dm: very dumb about comments and escaped quotes -- but we are faster then at least, 

-	// albeit returning too much (which is ok for jsbobj current usage).

-	void grabVariables(char *code /*modified and must stay in scope*/, strhashmap& vars) { 

-		char *p = code;

-		char last = 0;

-		while( *p ) { 

-			if( starter[*p] ) { 

-				char *q = p+1;

-				while( *q && ic[*q] ) q++;

-				const char *identifier = p;

-				bool done = *q == 0;

-				*q = 0;

-				if( !reserved.count(identifier) ) {

-					// we try to be smart about 'obj' but have to be careful as obj.obj 

-					// can happen; this is so that nFields is right for simplistic where cases

-					// so we can stop scanning in jsobj when we find the field of interest.

-					if( strcmp(identifier,"obj")==0 && p>code && p[-1] != '.' )

-						;

-					else

-						vars[identifier] = 1;

-				}

-				if( done )

-					break;

-				p = q + 1;

-				continue;

-			}

-

-			if( *p == '\'' ) {

-				p++;

-				while( *p && *p != '\'' ) p++;

-			}

-			else if( *p == '"' ) {

-				p++;

-				while( *p && *p != '"' ) p++;

-			}

-			p++;

-		}

-	}

-

-	MiniLex() { 

-		strhashmap atest;

-		atest["foo"] = 3;

-		assert( atest.count("bar") == 0 );

-		assert( atest.count("foo") == 1 );

-		assert( atest["foo"] == 3 );

-

-		for( int i = 0; i < 256; i++ ) { 

-			ic[i] = starter[i] = false;

-		}

-		for( int i = 'a'; i <= 'z'; i++ )

-			ic[i] = starter[i] = true;

-		for( int i = 'A'; i <= 'Z'; i++ )

-			ic[i] = starter[i] = true;

-		for( int i = '0'; i <= '9'; i++ )

-			ic[i] = true;

-		for( int i = 128; i < 256; i++ )

-			ic[i] = starter[i] = true;

-		ic['$'] = starter['$'] = true;

-		ic['_'] = starter['_'] = true;

-

-		reserved["break"] = true;

-		reserved["case"] = true;

-		reserved["catch"] = true;

-		reserved["continue"] = true;

-		reserved["default"] = true;

-		reserved["delete"] = true;

-		reserved["do"] = true;

-		reserved["else"] = true;

-		reserved["finally"] = true;

-		reserved["for"] = true;

-		reserved["function"] = true;

-		reserved["if"] = true;

-		reserved["in"] = true;

-		reserved["instanceof"] = true;

-		reserved["new"] = true;

-		reserved["return"] = true;

-		reserved["switch"] = true;

-		reserved["this"] = true;

-		reserved["throw"] = true;

-		reserved["try"] = true;

-		reserved["typeof"] = true;

-		reserved["var"] = true;

-		reserved["void"] = true;

-		reserved["while"] = true;

-		reserved["with "] = true;

-	}

-};

-

-

+// minilex.h
+// mini js lexical analyzer.  idea is to be dumb and fast.
+
+struct MiniLex {
+	strhashmap reserved;
+	bool ic[256]; // ic=Identifier Character
+	bool starter[256];
+
+	// dm: very dumb about comments and escaped quotes -- but we are faster then at least, 
+	// albeit returning too much (which is ok for jsbobj current usage).
+	void grabVariables(char *code /*modified and must stay in scope*/, strhashmap& vars) { 
+		char *p = code;
+		char last = 0;
+		while( *p ) { 
+			if( starter[*p] ) { 
+				char *q = p+1;
+				while( *q && ic[*q] ) q++;
+				const char *identifier = p;
+				bool done = *q == 0;
+				*q = 0;
+				if( !reserved.count(identifier) ) {
+					// we try to be smart about 'obj' but have to be careful as obj.obj 
+					// can happen; this is so that nFields is right for simplistic where cases
+					// so we can stop scanning in jsobj when we find the field of interest.
+					if( strcmp(identifier,"obj")==0 && p>code && p[-1] != '.' )
+						;
+					else
+						vars[identifier] = 1;
+				}
+				if( done )
+					break;
+				p = q + 1;
+				continue;
+			}
+
+			if( *p == '\'' ) {
+				p++;
+				while( *p && *p != '\'' ) p++;
+			}
+			else if( *p == '"' ) {
+				p++;
+				while( *p && *p != '"' ) p++;
+			}
+			p++;
+		}
+	}
+
+	MiniLex() { 
+		strhashmap atest;
+		atest["foo"] = 3;
+		assert( atest.count("bar") == 0 );
+		assert( atest.count("foo") == 1 );
+		assert( atest["foo"] == 3 );
+
+		for( int i = 0; i < 256; i++ ) { 
+			ic[i] = starter[i] = false;
+		}
+		for( int i = 'a'; i <= 'z'; i++ )
+			ic[i] = starter[i] = true;
+		for( int i = 'A'; i <= 'Z'; i++ )
+			ic[i] = starter[i] = true;
+		for( int i = '0'; i <= '9'; i++ )
+			ic[i] = true;
+		for( int i = 128; i < 256; i++ )
+			ic[i] = starter[i] = true;
+		ic['$'] = starter['$'] = true;
+		ic['_'] = starter['_'] = true;
+
+		reserved["break"] = true;
+		reserved["case"] = true;
+		reserved["catch"] = true;
+		reserved["continue"] = true;
+		reserved["default"] = true;
+		reserved["delete"] = true;
+		reserved["do"] = true;
+		reserved["else"] = true;
+		reserved["finally"] = true;
+		reserved["for"] = true;
+		reserved["function"] = true;
+		reserved["if"] = true;
+		reserved["in"] = true;
+		reserved["instanceof"] = true;
+		reserved["new"] = true;
+		reserved["return"] = true;
+		reserved["switch"] = true;
+		reserved["this"] = true;
+		reserved["throw"] = true;
+		reserved["try"] = true;
+		reserved["typeof"] = true;
+		reserved["var"] = true;
+		reserved["void"] = true;
+		reserved["while"] = true;
+		reserved["with "] = true;
+	}
+};
+
+
diff --git a/db/namespace.h b/db/namespace.h
index ed9613d58f4..6caefc633f3 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -1,225 +1,225 @@
-// namespace.h

-

-#pragma once

-

-#include "../util/hashtab.h"

-#include "../util/mmap.h"

-

-class Cursor;

-

-#pragma pack(push)

-#pragma pack(1)

-

-class Namespace {

-public:

-	Namespace(const char *ns) { 

-		*this = ns;

-	}

-	Namespace& operator=(const char *ns) { 

-		memset(buf, 0, 128); /* this is just to keep stuff clean in the files for easy dumping and reading */

-		strcpy_s(buf, 128, ns); return *this; 

-	}

-

-	void kill() { 

-		buf[0] = 0x7f;

-	}

-

-	bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; }

-	int hash() const {

-		unsigned x = 0;

-		const char *p = buf;

-		while( *p ) { 

-			x = x * 131 + *p;

-			p++;

-		}

-		return (x & 0x7fffffff) | 0x8000000; // must be > 0

-	}

-

-	char buf[128];

-};

-

-const int Buckets = 19;

-const int MaxBucket = 18;

-const int MaxIndexes = 10;

-

-class IndexDetails { 

-public:

-	DiskLoc head; /* btree head */

-	/* index info object. 

-	  { name:"nameofindex", ns:"parentnsname", key: {keypattobject} } 

-	*/

-	DiskLoc info; 

-

-	/* pull out the relevant key objects from obj, so we

-       can index them.  Note that the set is multiple elements 

-	   only when it's a "multikey" array.

-       keys will be left empty if key not found in the object.

-	*/

-	void getKeysFromObject(JSObj& obj, set<JSObj>& keys);

-

-    // returns name of this index's storage area

-	// client.table.$index

-	string indexNamespace() { 

-		JSObj io = info.obj();

-		string s;

-		s.reserve(128);

-		s = io.getStringField("ns");

-		assert( !s.empty() );

-		s += ".$";

-		s += io.getStringField("name"); 

-		return s;

-	}

-};

-

-extern int bucketSizes[];

-

-class NamespaceDetails {

-public:

-	NamespaceDetails() { 

-		datasize = nrecords = 0;

-		lastExtentSize = 0;

-		nIndexes = 0;

-		capped = 0;

-		max = 0x7fffffff;

-		memset(reserved, 0, sizeof(reserved));

-	} 

-	DiskLoc firstExtent;

-	DiskLoc lastExtent;

-	DiskLoc deletedList[Buckets];

-	long long datasize;

-	long long nrecords;

-	int lastExtentSize;

-	int nIndexes;

-	IndexDetails indexes[MaxIndexes];

-	int capped;

-	int max; // max # of objects for a capped table.

-	char reserved[256-16-4-4-8*MaxIndexes-8-8];

-

-	//returns offset in indexes[]

-	int findIndexByName(const char *name) { 

-		for( int i = 0; i < nIndexes; i++ ) {

-			if( strcmp(indexes[i].info.obj().getStringField("name"),name) == 0 )

-				return i;

-		}

-		return -1;

-	}

-

-	/* return which "deleted bucket" for this size object */

-	static int bucket(int n) { 

-		for( int i = 0; i < Buckets; i++ )

-			if( bucketSizes[i] > n )

-				return i;

-		return Buckets-1;

-	}

-

-	DiskLoc alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc);

-	void addDeletedRec(DeletedRecord *d, DiskLoc dloc);

-	void dumpDeleted(set<DiskLoc> *extents = 0);

-private:

-	DiskLoc __stdAlloc(int len);

-	DiskLoc _alloc(const char *ns, int len);

-	void compact();

-};

-

-#pragma pack(pop)

-

-class NamespaceIndex {

-	friend class NamespaceCursor;

-public:

-	NamespaceIndex() { }

-

-	void init(const char *dir, const char *client) { 

-		string path = dir;

-		path += client;

-		path += ".ns";

-		const int LEN = 16 * 1024 * 1024;

-		void *p = f.map(path.c_str(), LEN);

-		if( p == 0 ) { 

-			problem() << "couldn't open namespace.idx " << path.c_str() << " terminating" << endl;

-			exit(-3);

-		}

-		ht = new HashTable<Namespace,NamespaceDetails>(p, LEN, "namespace index");

-	}

-

-	void add(const char *ns, DiskLoc& loc) { 

-		Namespace n(ns);

-		NamespaceDetails details;

-		details.lastExtent = details.firstExtent = loc;

-		ht->put(n, details);

-	}

-

-	NamespaceDetails* details(const char *ns) { 

-		Namespace n(ns);

-		return ht->get(n); 

-	}

-

-	void kill(const char *ns) {

-		Namespace n(ns);

-		ht->kill(n); 

-	}

-

-	bool find(const char *ns, DiskLoc& loc) { 

-		NamespaceDetails *l = details(ns);

-		if( l ) {

-			loc = l->firstExtent;

-			return true;

-		}

-		return false;

-	}

-

-private:

-	MemoryMappedFile f;

-	HashTable<Namespace,NamespaceDetails> *ht;

-};

-

-extern const char *dbpath;

-

-/*

-class NamespaceIndexMgr { 

-public:

-	NamespaceIndexMgr() { }

-	NamespaceIndex* get(const char *client) { 

-		map<string,NamespaceIndex*>::iterator it = m.find(client);

-		if( it != m.end() )

-			return it->second;

-		NamespaceIndex *ni = new NamespaceIndex();

-		ni->init(dbpath, client);

-		m[client] = ni;

-		return ni;

-	}

-private:

-	map<string,NamespaceIndex*> m;

-};

-

-extern NamespaceIndexMgr namespaceIndexMgr;

-*/

-

-// "client.a.b.c" -> "client"

-inline void nsToClient(const char *ns, char *client) { 

-	const char *p = ns;

-	char *q = client;

-	while( *p != '.' ) { 

-		if( *p == 0 ) { 

-			assert(false);

-			*client = 0;

-			return;

-		}

-		*q++ = *p++;

-	}

-	*q = 0;

-	assert(q-client<256);

-}

-

-/*

-inline NamespaceIndex* nsindex(const char *ns) { 

-	char client[256];

-	nsToClient(ns, client);

-	return namespaceIndexMgr.get(client);

-}

-

-inline NamespaceDetails* nsdetails(const char *ns) { 

-	return nsindex(ns)->details(ns);

-}

-*/

-

-//auto_ptr<Cursor> makeNamespaceCursor();

+// namespace.h
+
+#pragma once
+
+#include "../util/hashtab.h"
+#include "../util/mmap.h"
+
+class Cursor;
+
+#pragma pack(push)
+#pragma pack(1)
+
+class Namespace {
+public:
+	Namespace(const char *ns) { 
+		*this = ns;
+	}
+	Namespace& operator=(const char *ns) { 
+		memset(buf, 0, 128); /* this is just to keep stuff clean in the files for easy dumping and reading */
+		strcpy_s(buf, 128, ns); return *this; 
+	}
+
+	void kill() { 
+		buf[0] = 0x7f;
+	}
+
+	bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; }
+	int hash() const {
+		unsigned x = 0;
+		const char *p = buf;
+		while( *p ) { 
+			x = x * 131 + *p;
+			p++;
+		}
+		return (x & 0x7fffffff) | 0x8000000; // must be > 0
+	}
+
+	char buf[128];
+};
+
+const int Buckets = 19;
+const int MaxBucket = 18;
+const int MaxIndexes = 10;
+
+class IndexDetails { 
+public:
+	DiskLoc head; /* btree head */
+	/* index info object. 
+	  { name:"nameofindex", ns:"parentnsname", key: {keypattobject} } 
+	*/
+	DiskLoc info; 
+
+	/* pull out the relevant key objects from obj, so we
+       can index them.  Note that the set is multiple elements 
+	   only when it's a "multikey" array.
+       keys will be left empty if key not found in the object.
+	*/
+	void getKeysFromObject(JSObj& obj, set<JSObj>& keys);
+
+    // returns name of this index's storage area
+	// client.table.$index
+	string indexNamespace() { 
+		JSObj io = info.obj();
+		string s;
+		s.reserve(128);
+		s = io.getStringField("ns");
+		assert( !s.empty() );
+		s += ".$";
+		s += io.getStringField("name"); 
+		return s;
+	}
+};
+
+extern int bucketSizes[];
+
+class NamespaceDetails {
+public:
+	NamespaceDetails() { 
+		datasize = nrecords = 0;
+		lastExtentSize = 0;
+		nIndexes = 0;
+		capped = 0;
+		max = 0x7fffffff;
+		memset(reserved, 0, sizeof(reserved));
+	} 
+	DiskLoc firstExtent;
+	DiskLoc lastExtent;
+	DiskLoc deletedList[Buckets];
+	long long datasize;
+	long long nrecords;
+	int lastExtentSize;
+	int nIndexes;
+	IndexDetails indexes[MaxIndexes];
+	int capped;
+	int max; // max # of objects for a capped table.
+	char reserved[256-16-4-4-8*MaxIndexes-8-8];
+
+	//returns offset in indexes[]
+	int findIndexByName(const char *name) { 
+		for( int i = 0; i < nIndexes; i++ ) {
+			if( strcmp(indexes[i].info.obj().getStringField("name"),name) == 0 )
+				return i;
+		}
+		return -1;
+	}
+
+	/* return which "deleted bucket" for this size object */
+	static int bucket(int n) { 
+		for( int i = 0; i < Buckets; i++ )
+			if( bucketSizes[i] > n )
+				return i;
+		return Buckets-1;
+	}
+
+	DiskLoc alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc);
+	void addDeletedRec(DeletedRecord *d, DiskLoc dloc);
+	void dumpDeleted(set<DiskLoc> *extents = 0);
+private:
+	DiskLoc __stdAlloc(int len);
+	DiskLoc _alloc(const char *ns, int len);
+	void compact();
+};
+
+#pragma pack(pop)
+
+class NamespaceIndex {
+	friend class NamespaceCursor;
+public:
+	NamespaceIndex() { }
+
+	void init(const char *dir, const char *client) { 
+		string path = dir;
+		path += client;
+		path += ".ns";
+		const int LEN = 16 * 1024 * 1024;
+		void *p = f.map(path.c_str(), LEN);
+		if( p == 0 ) { 
+			problem() << "couldn't open namespace.idx " << path.c_str() << " terminating" << endl;
+			exit(-3);
+		}
+		ht = new HashTable<Namespace,NamespaceDetails>(p, LEN, "namespace index");
+	}
+
+	void add(const char *ns, DiskLoc& loc) { 
+		Namespace n(ns);
+		NamespaceDetails details;
+		details.lastExtent = details.firstExtent = loc;
+		ht->put(n, details);
+	}
+
+	NamespaceDetails* details(const char *ns) { 
+		Namespace n(ns);
+		return ht->get(n); 
+	}
+
+	void kill(const char *ns) {
+		Namespace n(ns);
+		ht->kill(n); 
+	}
+
+	bool find(const char *ns, DiskLoc& loc) { 
+		NamespaceDetails *l = details(ns);
+		if( l ) {
+			loc = l->firstExtent;
+			return true;
+		}
+		return false;
+	}
+
+private:
+	MemoryMappedFile f;
+	HashTable<Namespace,NamespaceDetails> *ht;
+};
+
+extern const char *dbpath;
+
+/*
+class NamespaceIndexMgr { 
+public:
+	NamespaceIndexMgr() { }
+	NamespaceIndex* get(const char *client) { 
+		map<string,NamespaceIndex*>::iterator it = m.find(client);
+		if( it != m.end() )
+			return it->second;
+		NamespaceIndex *ni = new NamespaceIndex();
+		ni->init(dbpath, client);
+		m[client] = ni;
+		return ni;
+	}
+private:
+	map<string,NamespaceIndex*> m;
+};
+
+extern NamespaceIndexMgr namespaceIndexMgr;
+*/
+
+// "client.a.b.c" -> "client"
+inline void nsToClient(const char *ns, char *client) { 
+	const char *p = ns;
+	char *q = client;
+	while( *p != '.' ) { 
+		if( *p == 0 ) { 
+			assert(false);
+			*client = 0;
+			return;
+		}
+		*q++ = *p++;
+	}
+	*q = 0;
+	assert(q-client<256);
+}
+
+/*
+inline NamespaceIndex* nsindex(const char *ns) { 
+	char client[256];
+	nsToClient(ns, client);
+	return namespaceIndexMgr.get(client);
+}
+
+inline NamespaceDetails* nsdetails(const char *ns) { 
+	return nsindex(ns)->details(ns);
+}
+*/
+
+//auto_ptr<Cursor> makeNamespaceCursor();
diff --git a/db/objwrappers.h b/db/objwrappers.h
index afd3a5b479d..8868e9e7d4e 100644
--- a/db/objwrappers.h
+++ b/db/objwrappers.h
@@ -1,9 +1,9 @@
-// objwrappers.h

-

-#pragma once

-

-#include "../stdafx.h"

-#include "jsobj.h"

-

-

-

+// objwrappers.h
+
+#pragma once
+
+#include "../stdafx.h"
+#include "jsobj.h"
+
+
+
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
index c063d91b1c1..ddf74adf615 100644
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -1,969 +1,969 @@
-// pdfile.cpp

-

-/* 

-todo: 

-_ manage deleted records.  bucket?

-_ use deleted on inserts!

-_ quantize allocations

-_ table scans must be sequential, not next/prev pointers

-_ regex support

-*/

-

-#include "stdafx.h"

-#include "pdfile.h"

-#include "db.h"

-#include "../util/mmap.h"

-#include "../util/hashtab.h"

-#include "objwrappers.h"

-#include "btree.h"

-#include <algorithm>

-#include <list>

-

-const char *dbpath = "/data/db/";

-

-DataFileMgr theDataFileMgr;

-map<string,Client*> clients;

-Client *client;

-const char *curNs = "";

-int MAGIC = 0x1000;

-int curOp = -2;

-int callDepth = 0;

-

-extern int otherTraceLevel;

-

-void sayDbContext(const char *errmsg) { 

-	if( errmsg ) { 

-		cout << errmsg << '\n';

-		problem() << errmsg << endl;

-	}

-	cout << " client: " << (client ? client->name.c_str() : "null");

-	cout << " op:" << curOp << ' ' << callDepth << endl;

-	if( client )

-		cout << " ns: " << curNs << endl;

-}

-

-JSObj::JSObj(Record *r) { 

-	_objdata = r->data;

-	_objsize = *((int*) _objdata);

-	if( _objsize > r->netLength() ) { 

-		cout << "About to assert fail _objsize <= r->netLength()" << endl;

-		cout << " _objsize: " << _objsize << endl;

-		cout << " netLength(): " << r->netLength() << endl;

-		cout << " extentOfs: " << r->extentOfs << endl;

-		cout << " nextOfs: " << r->nextOfs << endl;

-		cout << " prevOfs: " << r->prevOfs << endl;

-		assert( _objsize <= r->netLength() );

-	}

-	iFree = false;

-}

-

-/*---------------------------------------------------------------------*/ 

-

-int bucketSizes[] = { 

-	32, 64, 128, 256, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000,

-	0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000,

-	0x400000, 0x800000

-};

-

-//NamespaceIndexMgr namespaceIndexMgr;

-

-void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) { 

-	DEBUGGING cout << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl;

-	int b = bucket(d->lengthWithHeaders);

-	DiskLoc& list = deletedList[b];

-	DiskLoc oldHead = list;

-	list = dloc;

-	d->nextDeleted = oldHead;

-}

-

-/* lenToAlloc is WITH header 

-*/

-DiskLoc NamespaceDetails::alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc) {

-	lenToAlloc = (lenToAlloc + 3) & 0xfffffffc;

-	DiskLoc loc = _alloc(ns, lenToAlloc);

-	if( loc.isNull() )

-		return loc;

-

-	DeletedRecord *r = loc.drec();

-

-	/* note we want to grab from the front so our next pointers on disk tend

-	to go in a forward direction which is important for performance. */

-	int regionlen = r->lengthWithHeaders;

-	extentLoc.set(loc.a(), r->extentOfs);

-

-	DEBUGGING cout << "TEMP: alloc() returns " << loc.toString() << ' ' << ns << " lentoalloc:" << lenToAlloc << " ext:" << extentLoc.toString() << endl;

-

-	int left = regionlen - lenToAlloc;

-	if( left < 24 || (left < (lenToAlloc >> 3) && capped == 0) ) {

-		// you get the whole thing.

-		return loc;

-	}

-

-	/* split off some for further use. */

-	r->lengthWithHeaders = lenToAlloc;

-	DiskLoc newDelLoc = loc;

-	newDelLoc.inc(lenToAlloc);

-	DeletedRecord *newDel = newDelLoc.drec();

-	newDel->extentOfs = r->extentOfs;

-	newDel->lengthWithHeaders = left;

-	newDel->nextDeleted.Null();

-

-	addDeletedRec(newDel, newDelLoc);

-

-	return loc;

-}

-

-/* for non-capped collections.

-   returned item is out of the deleted list upon return 

-*/

-DiskLoc NamespaceDetails::__stdAlloc(int len) {

-	DiskLoc *prev;

-	DiskLoc *bestprev = 0;

-	DiskLoc bestmatch;

-	int bestmatchlen = 0x7fffffff;

-	int b = bucket(len);

-	DiskLoc cur = deletedList[b]; prev = &deletedList[b];

-	int extra = 5; // look for a better fit, a little.

-	int chain = 0;

-	while( 1 ) { 

-		{

-			int a = cur.a();

-			if( a < -1 || a >= 100000 ) { 

-				problem() << "Assertion failure - a() out of range in _alloc() " << a << endl;

-				cout << "Assertion failure - a() out of range in _alloc() " << a << '\n';

-				sayDbContext();

-				if( cur == *prev )

-					prev->Null();

-				cur.Null();

-			}

-		}

-		if( cur.isNull() ) { 

-			// move to next bucket.  if we were doing "extra", just break

-			if( bestmatchlen < 0x7fffffff )

-				break;

-			b++;

-			if( b > MaxBucket ) {

-				// out of space. alloc a new extent.

-				return DiskLoc();

-			}

-			cur = deletedList[b]; prev = &deletedList[b];

-			continue;

-		}

-		DeletedRecord *r = cur.drec();

-		if( r->lengthWithHeaders >= len && 

-			r->lengthWithHeaders < bestmatchlen ) {

-				bestmatchlen = r->lengthWithHeaders;

-				bestmatch = cur;

-				bestprev = prev;

-		}

-		if( bestmatchlen < 0x7fffffff && --extra <= 0 )

-			break;

-		if( ++chain > 30 && b < MaxBucket ) {

-			// too slow, force move to next bucket to grab a big chunk

-			b++;

-			chain = 0;

-			cur.Null();

-		}

-		else {

-			cur = r->nextDeleted; prev = &r->nextDeleted;

-		}

-	}

-

-	/* unlink ourself from the deleted list */

-	*bestprev = bestmatch.drec()->nextDeleted;

-

-	return bestmatch;

-}

-

-void NamespaceDetails::dumpDeleted(set<DiskLoc> *extents) { 

-//	cout << "DUMP deleted chains" << endl;

-	for( int i = 0; i < Buckets; i++ ) { 

-//		cout << "  bucket " << i << endl;

-		DiskLoc dl = deletedList[i];

-		while( !dl.isNull() ) { 

-			DeletedRecord *r = dl.drec();

-			DiskLoc extLoc(dl.a(), r->extentOfs);

-			if( extents == 0 || extents->count(extLoc) <= 0 ) {

-				cout << "  bucket " << i << endl;

-				cout << "   " << dl.toString() << " ext:" << extLoc.toString();

-				if( extents && extents->count(extLoc) <= 0 )

-					cout << '?';

-				cout << " len:" << r->lengthWithHeaders << endl;

-			}

-			dl = r->nextDeleted;

-		}

-	}

-//	cout << endl;

-}

-

-/* combine adjacent deleted records

-

-   this is O(n^2) but we call it for capped tables where typically n==1 or 2! 

-   (or 3...there will be a little unused sliver at the end of the extent.)

-*/

-void NamespaceDetails::compact() { 

-	assert(capped);

-	list<DiskLoc> drecs;

-

-	for( int i = 0; i < Buckets; i++ ) { 

-		DiskLoc dl = deletedList[i];

-		deletedList[i].Null();

-		while( !dl.isNull() ) { 

-			DeletedRecord *r = dl.drec();

-			drecs.push_back(dl);

-			dl = r->nextDeleted;

-		}

-	}

-

-	drecs.sort();

-

-	list<DiskLoc>::iterator j = drecs.begin(); 

-	assert( j != drecs.end() );

-	DiskLoc a = *j;

-	while( 1 ) {

-		j++;

-		if( j == drecs.end() ) {

-			DEBUGGING cout << "TEMP: compact adddelrec\n";

-			addDeletedRec(a.drec(), a);

-			break;

-		}

-		DiskLoc b = *j;

-		while( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) { 

-			// a & b are adjacent.  merge.

-			a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders;

-			j++;

-			if( j == drecs.end() ) {

-				DEBUGGING cout << "temp: compact adddelrec2\n";

-				addDeletedRec(a.drec(), a);

-				return;

-			}

-			b = *j;

-		}

-		DEBUGGING cout << "temp: compact adddelrec3\n";

-		addDeletedRec(a.drec(), a);

-		a = b;

-	}

-}

-

-DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {

-	if( !capped )

-		return __stdAlloc(len);

-

-	assert( len < 400000000 );

-	int passes = 0;

-	DiskLoc loc;

-

-	// delete records until we have room and the max # objects limit achieved.

-	while( 1 ) {

-		if( nrecords < max ) { 

-			loc = __stdAlloc(len);

-			if( !loc.isNull() )

-				break;

-		}

-

-		DiskLoc fr = firstExtent.ext()->firstRecord;

-		if( fr.isNull() ) { 

-			cout << "couldn't make room for new record in capped ns\n";

-			cout << " ns:" << ns;

-			cout << "\n len: " << len << endl;

-			assert(false);

-			return DiskLoc();

-		}

-

-		theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true);

-		compact();

-		assert( ++passes < 5000 );

-	}

-

-	return loc;

-}

-

-/*

-class NamespaceCursor : public Cursor {

-public:

-	virtual bool ok() { return i >= 0; }

-	virtual Record* _current() { assert(false); return 0; }

-	virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }

-

-	virtual JSObj current() {

-		NamespaceDetails &d = namespaceIndex.ht->nodes[i].value;

-		JSObjBuilder b;

-		b.append("name", namespaceIndex.ht->nodes[i].k.buf);

-		return b.done();

-	}

-

-	virtual bool advance() {

-		while( 1 ) {

-			i++;

-			if( i >= namespaceIndex.ht->n )

-				break;

-			if( namespaceIndex.ht->nodes[i].inUse() )

-				return true;

-		}

-		i = -1000000;

-		return false;

-	}

-

-	NamespaceCursor() { 

-		i = -1;

-		advance();

-	}

-private:

-	int i;

-};

-

-auto_ptr<Cursor> makeNamespaceCursor() {

-	return auto_ptr<Cursor>(new NamespaceCursor());

-}*/

-

-void newNamespace(const char *ns) {

-	cout << "New namespace: " << ns << endl;

-	if( strstr(ns, "system.namespaces") == 0 ) {

-		JSObjBuilder b;

-		b.append("name", ns);

-		JSObj j = b.done();

-		char client[256];

-		nsToClient(ns, client);

-		string s = client;

-		s += ".system.namespaces";

-		theDataFileMgr.insert(s.c_str(), j.objdata(), j.objsize(), true);

-	}

-}

-

-int initialExtentSize(int len) { 

-	long long sz = len * 16;

-	if( len < 1000 ) sz = len * 64;

-	if( sz > 1000000000 )

-		sz = 1000000000;

-	int z = ((int)sz) & 0xffffff00;

-	assert( z > len );

-	cout << "initialExtentSize(" << len << ") returns " << z << endl;

-	return z;

-}

-

-// { ..., capped: true, size: ..., max: ... }

-bool userCreateNS(const char *ns, JSObj& j) { 

-	if( nsdetails(ns) )

-		return false;

-

-	cout << j.toString() << endl;

-

-	newNamespace(ns);

-

-	int ies = initialExtentSize(128);

-	Element e = j.findElement("size");

-	if( e.type() == Number ) {

-		ies = (int) e.number();

-		ies += 256;

-		ies &= 0xffffff00;

-		if( ies > 1024 * 1024 * 1024 + 256 ) return false;

-	}

-

-	client->newestFile()->newExtent(ns, ies);

-	NamespaceDetails *d = nsdetails(ns);

-	assert(d);

-

-	e = j.findElement("capped");

-	if( e.type() == Bool && e.boolean() ) {

-		d->capped = 1;

-		e = j.findElement("max");

-		if( e.type() == Number ) { 

-			int mx = (int) e.number();

-			if( mx > 0 )

-				d->max = mx;

-		}

-	}

-

-	return true;

-}

-

-/*---------------------------------------------------------------------*/ 

-

-void PhysicalDataFile::open(int fn, const char *filename) {

-	int length;

-

-	if( fn <= 4 ) {

-		length = (64*1024*1024) << fn;

-		if( strstr(filename, "alleyinsider") && length < 1024 * 1024 * 1024 )

-			length = 1024 * 1024 * 1024;

-	} else

-		length = 0x7ff00000;

-

-	assert( length >= 64*1024*1024 && length % 4096 == 0 );

-

-	assert(fn == fileNo);

-	header = (PDFHeader *) mmf.map(filename, length);

-	assert(header);

-	header->init(fileNo, length);

-}

-

-/* prev - previous extent for this namespace.  null=this is the first one. */

-Extent* PhysicalDataFile::newExtent(const char *ns, int approxSize, int loops) {

-	assert( approxSize >= 0 && approxSize <= 0x7ff00000 );

-

-	int ExtentSize = approxSize <= header->unusedLength ? approxSize : header->unusedLength;

-	DiskLoc loc;

-	if( ExtentSize <= 0 ) {

-		if( loops > 8 ) { 

-			assert(false);

-			return 0;

-		}

-		cout << "INFO: newExtent(): file full, adding a new file " << ns << endl;

-		return client->addAFile()->newExtent(ns, approxSize, loops+1);

-	}

-	int offset = header->unused.getOfs();

-	header->unused.setOfs( fileNo, offset + ExtentSize );

-	header->unusedLength -= ExtentSize;

-	loc.setOfs(fileNo, offset);

-	Extent *e = _getExtent(loc);

-	DiskLoc emptyLoc = e->init(ns, ExtentSize, fileNo, offset);

-

-	DiskLoc oldExtentLoc;

-	NamespaceIndex *ni = nsindex(ns);

-	NamespaceDetails *details = ni->details(ns);

-	if( details ) { 

-		assert( !details->firstExtent.isNull() );

-		e->xprev = details->lastExtent;

-		details->lastExtent.ext()->xnext = loc;

-		details->lastExtent = loc;

-	}

-	else {

-		ni->add(ns, loc);

-		details = ni->details(ns);

-	}

-

-	details->lastExtentSize = approxSize;

-	DEBUGGING cout << "temp: newextent adddelrec " << ns << endl;

-	details->addDeletedRec(emptyLoc.drec(), emptyLoc);

-

-	cout << "new extent size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset << dec;

-	cout << " emptyLoc:" << hex << emptyLoc.getOfs() << dec;

-	cout << ' ' << ns << endl;

-	return e;

-}

-

-/*---------------------------------------------------------------------*/ 

-

-/* assumes already zeroed -- insufficient for block 'reuse' perhaps */

-DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset) { 

-	magic = 0x41424344;

-	myLoc.setOfs(_fileNo, _offset);

-	xnext.Null(); xprev.Null();

-	ns = nsname;

-	length = _length;

-	firstRecord.Null(); lastRecord.Null();

-

-	DiskLoc emptyLoc = myLoc;

-	emptyLoc.inc( (extentData-(char*)this) );

-

-	DeletedRecord *empty1 = (DeletedRecord *) extentData;

-	DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);

-	assert( empty == empty1 );

-	empty->lengthWithHeaders = _length - (extentData - (char *) this);

-	empty->extentOfs = myLoc.getOfs();

-	return emptyLoc;

-}

-

-/*

-Record* Extent::newRecord(int len) {

-	if( firstEmptyRegion.isNull() )

-		return 0;

-

-	assert(len > 0);

-	int newRecSize = len + Record::HeaderSize;

-	DiskLoc newRecordLoc = firstEmptyRegion;

-	Record *r = getRecord(newRecordLoc);

-	int left = r->netLength() - len;

-	if( left < 0 ) {

-	//

-		firstEmptyRegion.Null();

-		return 0;

-	}

-

-	DiskLoc nextEmpty = r->next.getNextEmpty(firstEmptyRegion);

-	r->lengthWithHeaders = newRecSize;

-	r->next.markAsFirstOrLastInExtent(this); // we're now last in the extent

-	if( !lastRecord.isNull() ) {

-		assert(getRecord(lastRecord)->next.lastInExtent()); // it was the last one

-		getRecord(lastRecord)->next.set(newRecordLoc); // until now

-		r->prev.set(lastRecord);

-	}

-	else {

-		r->prev.markAsFirstOrLastInExtent(this); // we are the first in the extent

-		assert( firstRecord.isNull() );

-		firstRecord = newRecordLoc;

-	}

-	lastRecord = newRecordLoc;

-

-	if( left < Record::HeaderSize + 32 ) { 

-		firstEmptyRegion.Null();

-	}

-	else {

-		firstEmptyRegion.inc(newRecSize);

-		Record *empty = getRecord(firstEmptyRegion);

-		empty->next.set(nextEmpty); // not for empty records, unless in-use records, next and prev can be null.

-		empty->prev.Null();

-		empty->lengthWithHeaders = left;

-	}

-

-	return r;

-}

-*/

-

-/*---------------------------------------------------------------------*/ 

-

-auto_ptr<Cursor> DataFileMgr::findAll(const char *ns) {

-	DiskLoc loc;

-	bool found = nsindex(ns)->find(ns, loc);

-	if( !found ) {

-		//		cout << "info: findAll() namespace does not exist: " << ns << endl;

-		return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));

-	}

-

-	Extent *e = getExtent(loc);

-

-	DEBUGGING {

-		cout << "temp: listing extents for " << ns << endl;

-		DiskLoc tmp = loc;

-		set<DiskLoc> extents;

-

-		while( 1 ) { 

-			Extent *f = getExtent(tmp);

-			cout << "extent: " << tmp.toString() << endl;

-			extents.insert(tmp);

-			tmp = f->xnext;

-			if( tmp.isNull() )

-				break;

-			f = f->getNextExtent();

-		}

-

-		cout << endl;

-		nsdetails(ns)->dumpDeleted(&extents);

-	}

-

-	while( e->firstRecord.isNull() && !e->xnext.isNull() ) {

-	    OCCASIONALLY cout << "info DFM::findAll(): extent " << loc.toString() << " was empty, skipping ahead" << endl;

-		// find a nonempty extent

-		// it might be nice to free the whole extent here!  but have to clean up free recs then.

-		e = e->getNextExtent();

-	}

-	return auto_ptr<Cursor>(new BasicCursor( e->firstRecord ));

-}

-

-/* get a table scan cursor, but can be forward or reverse direction */

-auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order) {

-	Element el = order.findElement("$natural");

-	if( el.type() != Number || el.number() >= 0 )

-		return DataFileMgr::findAll(ns);

-

-	// "reverse natural order"		// "reverse natural order"

-	NamespaceDetails *d = nsdetails(ns);

-	if( !d )

-		return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));

-	Extent *e = d->lastExtent.ext();

-	while( e->lastRecord.isNull() && !e->xprev.isNull() ) {

-		OCCASIONALLY cout << "  findTableScan: extent empty, skipping ahead" << endl;

-		e = e->getPrevExtent();

-	}

-	return auto_ptr<Cursor>(new ReverseCursor( e->lastRecord ));

-}

-

-void aboutToDelete(const DiskLoc& dl);

-

-/* pull out the relevant key objects from obj, so we

-   can index them.  Note that the set is multiple elements 

-   only when it's a "multikey" array.

-   keys will be left empty if key not found in the object.

-*/

-void IndexDetails::getKeysFromObject(JSObj& obj, set<JSObj>& keys) { 

-	JSObj keyPattern = info.obj().getObjectField("key");

-	JSObjBuilder b;

-	JSObj key = obj.extractFields(keyPattern, b);

-	if( key.isEmpty() )

-		return;

-	Element f = key.firstElement();

-	if( f.type() != Array ) {

-		b.decouple();

-		key.iWillFree();

-		keys.insert(key);

-		return;

-	}

-	JSObj arr = f.embeddedObject();

-//	cout << arr.toString() << endl;

-	JSElemIter i(arr);

-	while( i.more() ) { 

-		Element e = i.next();

-		if( e.eoo() ) break;

-		JSObjBuilder b;

-

-		b.appendAs(e, f.fieldName());

-		JSObj o = b.doneAndDecouple();

-                assert( o.objdata() );

-		keys.insert(o);

-	}

-}

-

-int nUnindexes = 0;

-

-void _unindexRecord(const char *ns, IndexDetails& id, JSObj& obj, const DiskLoc& dl) { 

-	set<JSObj> keys;

-	id.getKeysFromObject(obj, keys);

-	for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {

-		JSObj j = *i;

-//		cout << "UNINDEX: j:" << j.toString() << " head:" << id.head.toString() << dl.toString() << endl;

-		if( otherTraceLevel >= 5 ) {

-			cout << "_unindexRecord() " << obj.toString();

-			cout << "\n  unindex:" << j.toString() << endl;

-		}

-		nUnindexes++;

-		bool ok = false;

-		try {

-			ok = id.head.btree()->unindex(id.head, id, j, dl);

-		}

-		catch(AssertionException) { 

-			cout << " caught assertion _unindexRecord " << id.indexNamespace() << '\n';

-			problem() << "Assertion failure: _unindex failed " << id.indexNamespace() << endl;

-			cout << "Assertion failure: _unindex failed" << '\n';

-			cout << "  obj:" << obj.toString() << '\n';

-			cout << "  key:" << j.toString() << '\n';

-			cout << "  dl:" << dl.toString() << endl;

-			sayDbContext();

-		}

-

-		if( !ok ) { 

-			cout << "unindex failed (key too big?) " << id.indexNamespace() << '\n';

-		}

-	}

-}

-

-void  unindexRecord(const char *ns, NamespaceDetails *d, Record *todelete, const DiskLoc& dl) {

-	if( d->nIndexes == 0 ) return;

-	JSObj obj(todelete);

-	for( int i = 0; i < d->nIndexes; i++ ) { 

-		_unindexRecord(ns, d->indexes[i], obj, dl);

-	}

-}

-

-void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK) 

-{

-	int tempextofs = todelete->extentOfs;

-

-	NamespaceDetails* d = nsdetails(ns);

-	if( d->capped && !cappedOK ) { 

-		cout << "failing remove on a capped ns " << ns << endl;

-		return;

-	}

-

-	/* check if any cursors point to us.  if so, advance them. */

-	aboutToDelete(dl);

-

-	unindexRecord(ns, d, todelete, dl);

-

-	/* remove ourself from the record next/prev chain */

-	{

-		if( todelete->prevOfs != DiskLoc::NullOfs )

-			todelete->getPrev(dl).rec()->nextOfs = todelete->nextOfs;

-		if( todelete->nextOfs != DiskLoc::NullOfs )

-			todelete->getNext(dl).rec()->prevOfs = todelete->prevOfs;

-	}

-

-	/* remove ourself from extent pointers */

-	{

-		Extent *e = todelete->myExtent(dl);

-		if( e->firstRecord == dl ) {

-			if( todelete->nextOfs == DiskLoc::NullOfs )

-				e->firstRecord.Null();

-			else

-				e->firstRecord.setOfs(dl.a(), todelete->nextOfs);

-		}

-		if( e->lastRecord == dl ) {

-			if( todelete->prevOfs == DiskLoc::NullOfs )

-				e->lastRecord.Null();

-			else

-				e->lastRecord.setOfs(dl.a(), todelete->prevOfs);

-		}

-	}

-

-	/* add to the free list */

-	{

-		d->nrecords--;

-		d->datasize -= todelete->netLength();

-///		DEBUGGING << "temp: dddelrec deleterecord " << ns << endl;

-//		if( todelete->extentOfs == 0xaca500 ) { 

-//			cout << "break\n";

-//		}

-/*

-TEMP: add deleted rec 0:aca5b0 aca500

-temp: adddelrec deleterecord admin.blog.posts

-TEMP: add deleted rec 0:b9e750 b6a500

-temp: adddelrec deleterecord admin.blog.posts

-*/

-

-		d->addDeletedRec((DeletedRecord*)todelete, dl);

-	}

-}

-

-void setDifference(set<JSObj>& l, set<JSObj>& r, vector<JSObj*> &diff) { 

-	set<JSObj>::iterator i = l.begin();

-	set<JSObj>::iterator j = r.begin();

-	while( 1 ) { 

-		if( i == l.end() )

-			break;

-		while( j != r.end() && *j < *i )

-			j++;

-		if( j == r.end() || !i->woEqual(*j) ) {

-			const JSObj *jo = &*i;

-			diff.push_back( (JSObj *) jo );

-		}

-		i++;

-	}

-}

-

-/** Note: as written so far, if the object shrinks a lot, we don't free up space. */

-void DataFileMgr::update(

-		const char *ns,

-		Record *toupdate, const DiskLoc& dl,

-		const char *buf, int len, stringstream& ss) 

-{

-	NamespaceDetails *d = nsdetails(ns);

-

-	if( toupdate->netLength() < len ) {

-		if( d && d->capped ) { 

-			ss << " failing a growing update on a capped ns " << ns << endl;

-			return;

-		}

-

-		// doesn't fit.

-		if( client->profile )

-			ss << " moved ";

-		deleteRecord(ns, toupdate, dl);

-		insert(ns, buf, len);

-		return;

-	}

-

-	/* has any index keys changed? */

-	{

-		NamespaceDetails *d = nsdetails(ns);

-		if( d->nIndexes ) {

-			JSObj newObj(buf);

-			JSObj oldObj = dl.obj();

-			for( int i = 0; i < d->nIndexes; i++ ) {

-				IndexDetails& idx = d->indexes[i];

-				JSObj idxKey = idx.info.obj().getObjectField("key");

-

-				set<JSObj> oldkeys;

-				set<JSObj> newkeys;

-				idx.getKeysFromObject(oldObj, oldkeys);

-				idx.getKeysFromObject(newObj, newkeys);

-				vector<JSObj*> removed;

-				setDifference(oldkeys, newkeys, removed);

-				string idxns = idx.indexNamespace();

-				for( unsigned i = 0; i < removed.size(); i++ ) {

-					try {  

-						idx.head.btree()->unindex(idx.head, idx, *removed[i], dl);

-					}

-					catch(AssertionException) { 

-						ss << " exception update unindex ";

-						cout << " caught assertion update unindex " << idxns.c_str() << '\n';

-						problem() << " caught assertion update unindex " << idxns.c_str() << endl;

-					}

-				}

-				vector<JSObj*> added;

-				setDifference(newkeys, oldkeys, added);

-				assert( !dl.isNull() );

-				for( unsigned i = 0; i < added.size(); i++ ) { 

-					try {

-						idx.head.btree()->insert(

-							idx.head, 

-							dl, *added[i], false, idx, true);

-					}

-					catch(AssertionException) {

-						ss << " exception update index "; 

-						cout << " caught assertion update index " << idxns.c_str() << '\n';

-						problem() << " caught assertion update index " << idxns.c_str() << endl;

-					}

-				}

-				if( client->profile )

-					ss << "<br>" << added.size() << " key updates ";

-

-			}

-		}

-	}

-

-	//	update in place

-	memcpy(toupdate->data, buf, len);

-}

-

-int followupExtentSize(int len, int lastExtentLen) {

-	int x = initialExtentSize(len);

-	int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.2);

-	int sz = y > x ? y : x;

-	sz = ((int)sz) & 0xffffff00;

-	assert( sz > len );

-	return sz;

-}

-

-int deb=0;

-

-/* add keys to indexes for a new record */

-void  _indexRecord(IndexDetails& idx, JSObj& obj, DiskLoc newRecordLoc) { 

-

-	set<JSObj> keys;

-	idx.getKeysFromObject(obj, keys);

-	for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {

-		assert( !newRecordLoc.isNull() );

-		try {

-//			DEBUGGING << "temp index: " << newRecordLoc.toString() << obj.toString() << endl;

-			idx.head.btree()->insert(idx.head, newRecordLoc,

-				(JSObj&) *i, false, idx, true);

-		}

-		catch(AssertionException) { 

-			cout << " caught assertion _indexRecord " << idx.indexNamespace() << '\n';

-			problem() << " caught assertion _indexRecord " << idx.indexNamespace() << endl;

-		}

-	}

-}

-

-/* note there are faster ways to build an index in bulk, that can be 

-   done eventually */

-void addExistingToIndex(const char *ns, IndexDetails& idx) {

-	cout << "Adding all existing records for " << ns << " to new index" << endl;

-	int n = 0;

-	auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);

-	while( c->ok() ) {

-		JSObj js = c->current();

-		_indexRecord(idx, js, c->currLoc());

-		c->advance();

-		n++;

-	};

-	cout << "  indexing complete for " << n << " records" << endl;

-}

-

-/* add keys to indexes for a new record */

-void  indexRecord(NamespaceDetails *d, const void *buf, int len, DiskLoc newRecordLoc) { 

-	JSObj obj((const char *)buf);

-	for( int i = 0; i < d->nIndexes; i++ ) { 

-		_indexRecord(d->indexes[i], obj, newRecordLoc);

-	}

-}

-

-DiskLoc DataFileMgr::insert(const char *ns, const void *buf, int len, bool god) {

-	bool addIndex = false;

-	const char *sys = strstr(ns, "system.");

-	if( sys ) { 

-		if( sys == ns ) {

-			cout << "ERROR: attempt to insert for invalid client 'system': " << ns << endl;

-			return DiskLoc();

-		}

-		if( strstr(ns, ".system.") ) { 

-			if( strstr(ns, ".system.indexes") )

-				addIndex = true;

-			else if( !god ) { 

-				cout << "ERROR: attempt to insert in system namespace " << ns << endl;

-				return DiskLoc();

-			}

-		}

-	}

-

-	NamespaceDetails *d = nsdetails(ns);

-	if( d == 0 ) {

-		newNamespace(ns);

-		client->newestFile()->newExtent(ns, initialExtentSize(len));

-		d = nsdetails(ns);

-	}

-

-	NamespaceDetails *tableToIndex = 0;

-

-	const char *tabletoidxns = 0;

-	if( addIndex ) { 

-		JSObj io((const char *) buf);

-		const char *name = io.getStringField("name"); // name of the index

-		tabletoidxns = io.getStringField("ns");  // table it indexes

-		JSObj key = io.getObjectField("key");

-		if( name == 0 || *name == 0 || tabletoidxns == 0 || key.isEmpty() || key.objsize() > 2048 ) { 

-			cout << "user warning: bad add index attempt name:" << (name?name:"") << " ns:" << 

-				(tabletoidxns?tabletoidxns:"") << endl;

-			return DiskLoc();

-		}

-		tableToIndex = nsdetails(tabletoidxns);

-		if( tableToIndex == 0 ) {

-			cout << "user warning: bad add index attempt, no such table(ns):" << tabletoidxns << endl;

-			return DiskLoc();

-		}

-		if( tableToIndex->nIndexes >= MaxIndexes ) { 

-			cout << "user warning: bad add index attempt, too many indexes for:" << tabletoidxns << endl;

-			return DiskLoc();

-		}

-		if( tableToIndex->findIndexByName(name) >= 0 ) { 

-			//cout << "INFO: index:" << name << " already exists for:" << tabletoidxns << endl;

-			return DiskLoc();

-		}

-		//indexFullNS = tabletoidxns; 

-		//indexFullNS += ".$";

-		//indexFullNS += name; // client.table.$index -- note this doesn't contain jsobjs, it contains BtreeBuckets.

-	}

-

-	DiskLoc extentLoc;

-	int lenWHdr = len + Record::HeaderSize;

-	DiskLoc loc = d->alloc(ns, lenWHdr, extentLoc);

-	if( loc.isNull() ) {

-		// out of space

-		if( d->capped == 0 ) { // size capped doesn't grow

-			cout << "allocating new extent for " << ns << endl;

-			client->newestFile()->newExtent(ns, followupExtentSize(len, d->lastExtentSize));

-			loc = d->alloc(ns, lenWHdr, extentLoc);

-		}

-		if( loc.isNull() ) { 

-			cout << "out of space in datafile. capped:" << d->capped << endl;

-			assert(d->capped);

-			return DiskLoc();

-		}

-	}

-

-	Record *r = loc.rec();

-	assert( r->lengthWithHeaders >= lenWHdr );

-	memcpy(r->data, buf, len);

-	Extent *e = r->myExtent(loc);

-	if( e->lastRecord.isNull() ) { 

-		e->firstRecord = e->lastRecord = loc;

-		r->prevOfs = r->nextOfs = DiskLoc::NullOfs;

-	}

-	else {

-		Record *oldlast = e->lastRecord.rec();

-		r->prevOfs = e->lastRecord.getOfs();

-		r->nextOfs = DiskLoc::NullOfs;

-		oldlast->nextOfs = loc.getOfs();

-		e->lastRecord = loc;

-	}

-

-	d->nrecords++;

-	d->datasize += r->netLength();

-

-	if( tableToIndex ) { 

-		IndexDetails& idxinfo = tableToIndex->indexes[tableToIndex->nIndexes];

-		idxinfo.info = loc;

-		idxinfo.head = BtreeBucket::addHead(idxinfo);

-		tableToIndex->nIndexes++; 

-		/* todo: index existing records here */

-		addExistingToIndex(tabletoidxns, idxinfo);

-	}

-

-	/* add this record to our indexes */

-	if( d->nIndexes )

-		indexRecord(d, buf, len, loc);

-

-//	cout << "   inserted at loc:" << hex << loc.getOfs() << " lenwhdr:" << hex << lenWHdr << dec << ' ' << ns << endl;

-	return loc;

-}

-

-void DataFileMgr::init(const char *dir) {

-/*	string path = dir;

-	path += "temp.dat";

-	temp.open(path.c_str(), 64 * 1024 * 1024);

-*/

-}

-

-void pdfileInit() {

-//	namespaceIndex.init(dbpath);

-	theDataFileMgr.init(dbpath);

-}

+// pdfile.cpp
+
+/* 
+todo: 
+_ manage deleted records.  bucket?
+_ use deleted on inserts!
+_ quantize allocations
+_ table scans must be sequential, not next/prev pointers
+_ regex support
+*/
+
+#include "stdafx.h"
+#include "pdfile.h"
+#include "db.h"
+#include "../util/mmap.h"
+#include "../util/hashtab.h"
+#include "objwrappers.h"
+#include "btree.h"
+#include <algorithm>
+#include <list>
+
+const char *dbpath = "/data/db/";
+
+DataFileMgr theDataFileMgr;
+map<string,Client*> clients;
+Client *client;
+const char *curNs = "";
+int MAGIC = 0x1000;
+int curOp = -2;
+int callDepth = 0;
+
+extern int otherTraceLevel;
+
+void sayDbContext(const char *errmsg) { 
+	if( errmsg ) { 
+		cout << errmsg << '\n';
+		problem() << errmsg << endl;
+	}
+	cout << " client: " << (client ? client->name.c_str() : "null");
+	cout << " op:" << curOp << ' ' << callDepth << endl;
+	if( client )
+		cout << " ns: " << curNs << endl;
+}
+
+JSObj::JSObj(Record *r) { 
+	_objdata = r->data;
+	_objsize = *((int*) _objdata);
+	if( _objsize > r->netLength() ) { 
+		cout << "About to assert fail _objsize <= r->netLength()" << endl;
+		cout << " _objsize: " << _objsize << endl;
+		cout << " netLength(): " << r->netLength() << endl;
+		cout << " extentOfs: " << r->extentOfs << endl;
+		cout << " nextOfs: " << r->nextOfs << endl;
+		cout << " prevOfs: " << r->prevOfs << endl;
+		assert( _objsize <= r->netLength() );
+	}
+	iFree = false;
+}
+
+/*---------------------------------------------------------------------*/ 
+
+int bucketSizes[] = { 
+	32, 64, 128, 256, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000,
+	0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000,
+	0x400000, 0x800000
+};
+
+//NamespaceIndexMgr namespaceIndexMgr;
+
+void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) { 
+	DEBUGGING cout << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl;
+	int b = bucket(d->lengthWithHeaders);
+	DiskLoc& list = deletedList[b];
+	DiskLoc oldHead = list;
+	list = dloc;
+	d->nextDeleted = oldHead;
+}
+
+/* lenToAlloc is WITH header 
+*/
+DiskLoc NamespaceDetails::alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc) {
+	lenToAlloc = (lenToAlloc + 3) & 0xfffffffc;
+	DiskLoc loc = _alloc(ns, lenToAlloc);
+	if( loc.isNull() )
+		return loc;
+
+	DeletedRecord *r = loc.drec();
+
+	/* note we want to grab from the front so our next pointers on disk tend
+	to go in a forward direction which is important for performance. */
+	int regionlen = r->lengthWithHeaders;
+	extentLoc.set(loc.a(), r->extentOfs);
+
+	DEBUGGING cout << "TEMP: alloc() returns " << loc.toString() << ' ' << ns << " lentoalloc:" << lenToAlloc << " ext:" << extentLoc.toString() << endl;
+
+	int left = regionlen - lenToAlloc;
+	if( left < 24 || (left < (lenToAlloc >> 3) && capped == 0) ) {
+		// you get the whole thing.
+		return loc;
+	}
+
+	/* split off some for further use. */
+	r->lengthWithHeaders = lenToAlloc;
+	DiskLoc newDelLoc = loc;
+	newDelLoc.inc(lenToAlloc);
+	DeletedRecord *newDel = newDelLoc.drec();
+	newDel->extentOfs = r->extentOfs;
+	newDel->lengthWithHeaders = left;
+	newDel->nextDeleted.Null();
+
+	addDeletedRec(newDel, newDelLoc);
+
+	return loc;
+}
+
+/* for non-capped collections.
+   returned item is out of the deleted list upon return 
+*/
+DiskLoc NamespaceDetails::__stdAlloc(int len) {
+	DiskLoc *prev;
+	DiskLoc *bestprev = 0;
+	DiskLoc bestmatch;
+	int bestmatchlen = 0x7fffffff;
+	int b = bucket(len);
+	DiskLoc cur = deletedList[b]; prev = &deletedList[b];
+	int extra = 5; // look for a better fit, a little.
+	int chain = 0;
+	while( 1 ) { 
+		{
+			int a = cur.a();
+			if( a < -1 || a >= 100000 ) { 
+				problem() << "Assertion failure - a() out of range in _alloc() " << a << endl;
+				cout << "Assertion failure - a() out of range in _alloc() " << a << '\n';
+				sayDbContext();
+				if( cur == *prev )
+					prev->Null();
+				cur.Null();
+			}
+		}
+		if( cur.isNull() ) { 
+			// move to next bucket.  if we were doing "extra", just break
+			if( bestmatchlen < 0x7fffffff )
+				break;
+			b++;
+			if( b > MaxBucket ) {
+				// out of space. alloc a new extent.
+				return DiskLoc();
+			}
+			cur = deletedList[b]; prev = &deletedList[b];
+			continue;
+		}
+		DeletedRecord *r = cur.drec();
+		if( r->lengthWithHeaders >= len && 
+			r->lengthWithHeaders < bestmatchlen ) {
+				bestmatchlen = r->lengthWithHeaders;
+				bestmatch = cur;
+				bestprev = prev;
+		}
+		if( bestmatchlen < 0x7fffffff && --extra <= 0 )
+			break;
+		if( ++chain > 30 && b < MaxBucket ) {
+			// too slow, force move to next bucket to grab a big chunk
+			b++;
+			chain = 0;
+			cur.Null();
+		}
+		else {
+			cur = r->nextDeleted; prev = &r->nextDeleted;
+		}
+	}
+
+	/* unlink ourself from the deleted list */
+	*bestprev = bestmatch.drec()->nextDeleted;
+
+	return bestmatch;
+}
+
+void NamespaceDetails::dumpDeleted(set<DiskLoc> *extents) { 
+//	cout << "DUMP deleted chains" << endl;
+	for( int i = 0; i < Buckets; i++ ) { 
+//		cout << "  bucket " << i << endl;
+		DiskLoc dl = deletedList[i];
+		while( !dl.isNull() ) { 
+			DeletedRecord *r = dl.drec();
+			DiskLoc extLoc(dl.a(), r->extentOfs);
+			if( extents == 0 || extents->count(extLoc) <= 0 ) {
+				cout << "  bucket " << i << endl;
+				cout << "   " << dl.toString() << " ext:" << extLoc.toString();
+				if( extents && extents->count(extLoc) <= 0 )
+					cout << '?';
+				cout << " len:" << r->lengthWithHeaders << endl;
+			}
+			dl = r->nextDeleted;
+		}
+	}
+//	cout << endl;
+}
+
+/* combine adjacent deleted records
+
+   this is O(n^2) but we call it for capped tables where typically n==1 or 2! 
+   (or 3...there will be a little unused sliver at the end of the extent.)
+*/
+void NamespaceDetails::compact() { 
+	assert(capped);
+	list<DiskLoc> drecs;
+
+	for( int i = 0; i < Buckets; i++ ) { 
+		DiskLoc dl = deletedList[i];
+		deletedList[i].Null();
+		while( !dl.isNull() ) { 
+			DeletedRecord *r = dl.drec();
+			drecs.push_back(dl);
+			dl = r->nextDeleted;
+		}
+	}
+
+	drecs.sort();
+
+	list<DiskLoc>::iterator j = drecs.begin(); 
+	assert( j != drecs.end() );
+	DiskLoc a = *j;
+	while( 1 ) {
+		j++;
+		if( j == drecs.end() ) {
+			DEBUGGING cout << "TEMP: compact adddelrec\n";
+			addDeletedRec(a.drec(), a);
+			break;
+		}
+		DiskLoc b = *j;
+		while( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) { 
+			// a & b are adjacent.  merge.
+			a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders;
+			j++;
+			if( j == drecs.end() ) {
+				DEBUGGING cout << "temp: compact adddelrec2\n";
+				addDeletedRec(a.drec(), a);
+				return;
+			}
+			b = *j;
+		}
+		DEBUGGING cout << "temp: compact adddelrec3\n";
+		addDeletedRec(a.drec(), a);
+		a = b;
+	}
+}
+
+DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
+	if( !capped )
+		return __stdAlloc(len);
+
+	assert( len < 400000000 );
+	int passes = 0;
+	DiskLoc loc;
+
+	// delete records until we have room and the max # objects limit achieved.
+	while( 1 ) {
+		if( nrecords < max ) { 
+			loc = __stdAlloc(len);
+			if( !loc.isNull() )
+				break;
+		}
+
+		DiskLoc fr = firstExtent.ext()->firstRecord;
+		if( fr.isNull() ) { 
+			cout << "couldn't make room for new record in capped ns\n";
+			cout << " ns:" << ns;
+			cout << "\n len: " << len << endl;
+			assert(false);
+			return DiskLoc();
+		}
+
+		theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true);
+		compact();
+		assert( ++passes < 5000 );
+	}
+
+	return loc;
+}
+
+/*
+class NamespaceCursor : public Cursor {
+public:
+	virtual bool ok() { return i >= 0; }
+	virtual Record* _current() { assert(false); return 0; }
+	virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }
+
+	virtual JSObj current() {
+		NamespaceDetails &d = namespaceIndex.ht->nodes[i].value;
+		JSObjBuilder b;
+		b.append("name", namespaceIndex.ht->nodes[i].k.buf);
+		return b.done();
+	}
+
+	virtual bool advance() {
+		while( 1 ) {
+			i++;
+			if( i >= namespaceIndex.ht->n )
+				break;
+			if( namespaceIndex.ht->nodes[i].inUse() )
+				return true;
+		}
+		i = -1000000;
+		return false;
+	}
+
+	NamespaceCursor() { 
+		i = -1;
+		advance();
+	}
+private:
+	int i;
+};
+
+auto_ptr<Cursor> makeNamespaceCursor() {
+	return auto_ptr<Cursor>(new NamespaceCursor());
+}*/
+
+void newNamespace(const char *ns) {
+	cout << "New namespace: " << ns << endl;
+	if( strstr(ns, "system.namespaces") == 0 ) {
+		JSObjBuilder b;
+		b.append("name", ns);
+		JSObj j = b.done();
+		char client[256];
+		nsToClient(ns, client);
+		string s = client;
+		s += ".system.namespaces";
+		theDataFileMgr.insert(s.c_str(), j.objdata(), j.objsize(), true);
+	}
+}
+
+int initialExtentSize(int len) { 
+	long long sz = len * 16;
+	if( len < 1000 ) sz = len * 64;
+	if( sz > 1000000000 )
+		sz = 1000000000;
+	int z = ((int)sz) & 0xffffff00;
+	assert( z > len );
+	cout << "initialExtentSize(" << len << ") returns " << z << endl;
+	return z;
+}
+
+// { ..., capped: true, size: ..., max: ... }
+bool userCreateNS(const char *ns, JSObj& j) { 
+	if( nsdetails(ns) )
+		return false;
+
+	cout << j.toString() << endl;
+
+	newNamespace(ns);
+
+	int ies = initialExtentSize(128);
+	Element e = j.findElement("size");
+	if( e.type() == Number ) {
+		ies = (int) e.number();
+		ies += 256;
+		ies &= 0xffffff00;
+		if( ies > 1024 * 1024 * 1024 + 256 ) return false;
+	}
+
+	client->newestFile()->newExtent(ns, ies);
+	NamespaceDetails *d = nsdetails(ns);
+	assert(d);
+
+	e = j.findElement("capped");
+	if( e.type() == Bool && e.boolean() ) {
+		d->capped = 1;
+		e = j.findElement("max");
+		if( e.type() == Number ) { 
+			int mx = (int) e.number();
+			if( mx > 0 )
+				d->max = mx;
+		}
+	}
+
+	return true;
+}
+
+/*---------------------------------------------------------------------*/ 
+
+void PhysicalDataFile::open(int fn, const char *filename) {
+	int length;
+
+	if( fn <= 4 ) {
+		length = (64*1024*1024) << fn;
+		if( strstr(filename, "alleyinsider") && length < 1024 * 1024 * 1024 )
+			length = 1024 * 1024 * 1024;
+	} else
+		length = 0x7ff00000;
+
+	assert( length >= 64*1024*1024 && length % 4096 == 0 );
+
+	assert(fn == fileNo);
+	header = (PDFHeader *) mmf.map(filename, length);
+	assert(header);
+	header->init(fileNo, length);
+}
+
+/* prev - previous extent for this namespace.  null=this is the first one. */
+Extent* PhysicalDataFile::newExtent(const char *ns, int approxSize, int loops) {
+	assert( approxSize >= 0 && approxSize <= 0x7ff00000 );
+
+	int ExtentSize = approxSize <= header->unusedLength ? approxSize : header->unusedLength;
+	DiskLoc loc;
+	if( ExtentSize <= 0 ) {
+		if( loops > 8 ) { 
+			assert(false);
+			return 0;
+		}
+		cout << "INFO: newExtent(): file full, adding a new file " << ns << endl;
+		return client->addAFile()->newExtent(ns, approxSize, loops+1);
+	}
+	int offset = header->unused.getOfs();
+	header->unused.setOfs( fileNo, offset + ExtentSize );
+	header->unusedLength -= ExtentSize;
+	loc.setOfs(fileNo, offset);
+	Extent *e = _getExtent(loc);
+	DiskLoc emptyLoc = e->init(ns, ExtentSize, fileNo, offset);
+
+	DiskLoc oldExtentLoc;
+	NamespaceIndex *ni = nsindex(ns);
+	NamespaceDetails *details = ni->details(ns);
+	if( details ) { 
+		assert( !details->firstExtent.isNull() );
+		e->xprev = details->lastExtent;
+		details->lastExtent.ext()->xnext = loc;
+		details->lastExtent = loc;
+	}
+	else {
+		ni->add(ns, loc);
+		details = ni->details(ns);
+	}
+
+	details->lastExtentSize = approxSize;
+	DEBUGGING cout << "temp: newextent adddelrec " << ns << endl;
+	details->addDeletedRec(emptyLoc.drec(), emptyLoc);
+
+	cout << "new extent size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset << dec;
+	cout << " emptyLoc:" << hex << emptyLoc.getOfs() << dec;
+	cout << ' ' << ns << endl;
+	return e;
+}
+
+/*---------------------------------------------------------------------*/ 
+
+/* assumes already zeroed -- insufficient for block 'reuse' perhaps */
+DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset) { 
+	magic = 0x41424344;
+	myLoc.setOfs(_fileNo, _offset);
+	xnext.Null(); xprev.Null();
+	ns = nsname;
+	length = _length;
+	firstRecord.Null(); lastRecord.Null();
+
+	DiskLoc emptyLoc = myLoc;
+	emptyLoc.inc( (extentData-(char*)this) );
+
+	DeletedRecord *empty1 = (DeletedRecord *) extentData;
+	DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);
+	assert( empty == empty1 );
+	empty->lengthWithHeaders = _length - (extentData - (char *) this);
+	empty->extentOfs = myLoc.getOfs();
+	return emptyLoc;
+}
+
+/*
+Record* Extent::newRecord(int len) {
+	if( firstEmptyRegion.isNull() )
+		return 0;
+
+	assert(len > 0);
+	int newRecSize = len + Record::HeaderSize;
+	DiskLoc newRecordLoc = firstEmptyRegion;
+	Record *r = getRecord(newRecordLoc);
+	int left = r->netLength() - len;
+	if( left < 0 ) {
+	//
+		firstEmptyRegion.Null();
+		return 0;
+	}
+
+	DiskLoc nextEmpty = r->next.getNextEmpty(firstEmptyRegion);
+	r->lengthWithHeaders = newRecSize;
+	r->next.markAsFirstOrLastInExtent(this); // we're now last in the extent
+	if( !lastRecord.isNull() ) {
+		assert(getRecord(lastRecord)->next.lastInExtent()); // it was the last one
+		getRecord(lastRecord)->next.set(newRecordLoc); // until now
+		r->prev.set(lastRecord);
+	}
+	else {
+		r->prev.markAsFirstOrLastInExtent(this); // we are the first in the extent
+		assert( firstRecord.isNull() );
+		firstRecord = newRecordLoc;
+	}
+	lastRecord = newRecordLoc;
+
+	if( left < Record::HeaderSize + 32 ) { 
+		firstEmptyRegion.Null();
+	}
+	else {
+		firstEmptyRegion.inc(newRecSize);
+		Record *empty = getRecord(firstEmptyRegion);
+		empty->next.set(nextEmpty); // not for empty records, unless in-use records, next and prev can be null.
+		empty->prev.Null();
+		empty->lengthWithHeaders = left;
+	}
+
+	return r;
+}
+*/
+
+/*---------------------------------------------------------------------*/ 
+
+auto_ptr<Cursor> DataFileMgr::findAll(const char *ns) {
+	DiskLoc loc;
+	bool found = nsindex(ns)->find(ns, loc);
+	if( !found ) {
+		//		cout << "info: findAll() namespace does not exist: " << ns << endl;
+		return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
+	}
+
+	Extent *e = getExtent(loc);
+
+	DEBUGGING {
+		cout << "temp: listing extents for " << ns << endl;
+		DiskLoc tmp = loc;
+		set<DiskLoc> extents;
+
+		while( 1 ) { 
+			Extent *f = getExtent(tmp);
+			cout << "extent: " << tmp.toString() << endl;
+			extents.insert(tmp);
+			tmp = f->xnext;
+			if( tmp.isNull() )
+				break;
+			f = f->getNextExtent();
+		}
+
+		cout << endl;
+		nsdetails(ns)->dumpDeleted(&extents);
+	}
+
+	while( e->firstRecord.isNull() && !e->xnext.isNull() ) {
+	    OCCASIONALLY cout << "info DFM::findAll(): extent " << loc.toString() << " was empty, skipping ahead" << endl;
+		// find a nonempty extent
+		// it might be nice to free the whole extent here!  but have to clean up free recs then.
+		e = e->getNextExtent();
+	}
+	return auto_ptr<Cursor>(new BasicCursor( e->firstRecord ));
+}
+
+/* get a table scan cursor, but can be forward or reverse direction */
+auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order) {
+	Element el = order.findElement("$natural");
+	if( el.type() != Number || el.number() >= 0 )
+		return DataFileMgr::findAll(ns);
+
+	// "reverse natural order"		// "reverse natural order"
+	NamespaceDetails *d = nsdetails(ns);
+	if( !d )
+		return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
+	Extent *e = d->lastExtent.ext();
+	while( e->lastRecord.isNull() && !e->xprev.isNull() ) {
+		OCCASIONALLY cout << "  findTableScan: extent empty, skipping ahead" << endl;
+		e = e->getPrevExtent();
+	}
+	return auto_ptr<Cursor>(new ReverseCursor( e->lastRecord ));
+}
+
+void aboutToDelete(const DiskLoc& dl);
+
+/* pull out the relevant key objects from obj, so we
+   can index them.  Note that the set is multiple elements 
+   only when it's a "multikey" array.
+   keys will be left empty if key not found in the object.
+*/
+void IndexDetails::getKeysFromObject(JSObj& obj, set<JSObj>& keys) { 
+	JSObj keyPattern = info.obj().getObjectField("key");
+	JSObjBuilder b;
+	JSObj key = obj.extractFields(keyPattern, b);
+	if( key.isEmpty() )
+		return;
+	Element f = key.firstElement();
+	if( f.type() != Array ) {
+		b.decouple();
+		key.iWillFree();
+		keys.insert(key);
+		return;
+	}
+	JSObj arr = f.embeddedObject();
+//	cout << arr.toString() << endl;
+	JSElemIter i(arr);
+	while( i.more() ) { 
+		Element e = i.next();
+		if( e.eoo() ) break;
+		JSObjBuilder b;
+
+		b.appendAs(e, f.fieldName());
+		JSObj o = b.doneAndDecouple();
+                assert( o.objdata() );
+		keys.insert(o);
+	}
+}
+
+int nUnindexes = 0;
+
+void _unindexRecord(const char *ns, IndexDetails& id, JSObj& obj, const DiskLoc& dl) { 
+	set<JSObj> keys;
+	id.getKeysFromObject(obj, keys);
+	for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {
+		JSObj j = *i;
+//		cout << "UNINDEX: j:" << j.toString() << " head:" << id.head.toString() << dl.toString() << endl;
+		if( otherTraceLevel >= 5 ) {
+			cout << "_unindexRecord() " << obj.toString();
+			cout << "\n  unindex:" << j.toString() << endl;
+		}
+		nUnindexes++;
+		bool ok = false;
+		try {
+			ok = id.head.btree()->unindex(id.head, id, j, dl);
+		}
+		catch(AssertionException) { 
+			cout << " caught assertion _unindexRecord " << id.indexNamespace() << '\n';
+			problem() << "Assertion failure: _unindex failed " << id.indexNamespace() << endl;
+			cout << "Assertion failure: _unindex failed" << '\n';
+			cout << "  obj:" << obj.toString() << '\n';
+			cout << "  key:" << j.toString() << '\n';
+			cout << "  dl:" << dl.toString() << endl;
+			sayDbContext();
+		}
+
+		if( !ok ) { 
+			cout << "unindex failed (key too big?) " << id.indexNamespace() << '\n';
+		}
+	}
+}
+
+void  unindexRecord(const char *ns, NamespaceDetails *d, Record *todelete, const DiskLoc& dl) {
+	if( d->nIndexes == 0 ) return;
+	JSObj obj(todelete);
+	for( int i = 0; i < d->nIndexes; i++ ) { 
+		_unindexRecord(ns, d->indexes[i], obj, dl);
+	}
+}
+
+void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK) 
+{
+	int tempextofs = todelete->extentOfs;
+
+	NamespaceDetails* d = nsdetails(ns);
+	if( d->capped && !cappedOK ) { 
+		cout << "failing remove on a capped ns " << ns << endl;
+		return;
+	}
+
+	/* check if any cursors point to us.  if so, advance them. */
+	aboutToDelete(dl);
+
+	unindexRecord(ns, d, todelete, dl);
+
+	/* remove ourself from the record next/prev chain */
+	{
+		if( todelete->prevOfs != DiskLoc::NullOfs )
+			todelete->getPrev(dl).rec()->nextOfs = todelete->nextOfs;
+		if( todelete->nextOfs != DiskLoc::NullOfs )
+			todelete->getNext(dl).rec()->prevOfs = todelete->prevOfs;
+	}
+
+	/* remove ourself from extent pointers */
+	{
+		Extent *e = todelete->myExtent(dl);
+		if( e->firstRecord == dl ) {
+			if( todelete->nextOfs == DiskLoc::NullOfs )
+				e->firstRecord.Null();
+			else
+				e->firstRecord.setOfs(dl.a(), todelete->nextOfs);
+		}
+		if( e->lastRecord == dl ) {
+			if( todelete->prevOfs == DiskLoc::NullOfs )
+				e->lastRecord.Null();
+			else
+				e->lastRecord.setOfs(dl.a(), todelete->prevOfs);
+		}
+	}
+
+	/* add to the free list */
+	{
+		d->nrecords--;
+		d->datasize -= todelete->netLength();
+///		DEBUGGING << "temp: dddelrec deleterecord " << ns << endl;
+//		if( todelete->extentOfs == 0xaca500 ) { 
+//			cout << "break\n";
+//		}
+/*
+TEMP: add deleted rec 0:aca5b0 aca500
+temp: adddelrec deleterecord admin.blog.posts
+TEMP: add deleted rec 0:b9e750 b6a500
+temp: adddelrec deleterecord admin.blog.posts
+*/
+
+		d->addDeletedRec((DeletedRecord*)todelete, dl);
+	}
+}
+
+void setDifference(set<JSObj>& l, set<JSObj>& r, vector<JSObj*> &diff) { 
+	set<JSObj>::iterator i = l.begin();
+	set<JSObj>::iterator j = r.begin();
+	while( 1 ) { 
+		if( i == l.end() )
+			break;
+		while( j != r.end() && *j < *i )
+			j++;
+		if( j == r.end() || !i->woEqual(*j) ) {
+			const JSObj *jo = &*i;
+			diff.push_back( (JSObj *) jo );
+		}
+		i++;
+	}
+}
+
+/** Note: as written so far, if the object shrinks a lot, we don't free up space. */
+void DataFileMgr::update(
+		const char *ns,
+		Record *toupdate, const DiskLoc& dl,
+		const char *buf, int len, stringstream& ss) 
+{
+	NamespaceDetails *d = nsdetails(ns);
+
+	if( toupdate->netLength() < len ) {
+		if( d && d->capped ) { 
+			ss << " failing a growing update on a capped ns " << ns << endl;
+			return;
+		}
+
+		// doesn't fit.
+		if( client->profile )
+			ss << " moved ";
+		deleteRecord(ns, toupdate, dl);
+		insert(ns, buf, len);
+		return;
+	}
+
+	/* has any index keys changed? */
+	{
+		NamespaceDetails *d = nsdetails(ns);
+		if( d->nIndexes ) {
+			JSObj newObj(buf);
+			JSObj oldObj = dl.obj();
+			for( int i = 0; i < d->nIndexes; i++ ) {
+				IndexDetails& idx = d->indexes[i];
+				JSObj idxKey = idx.info.obj().getObjectField("key");
+
+				set<JSObj> oldkeys;
+				set<JSObj> newkeys;
+				idx.getKeysFromObject(oldObj, oldkeys);
+				idx.getKeysFromObject(newObj, newkeys);
+				vector<JSObj*> removed;
+				setDifference(oldkeys, newkeys, removed);
+				string idxns = idx.indexNamespace();
+				for( unsigned i = 0; i < removed.size(); i++ ) {
+					try {  
+						idx.head.btree()->unindex(idx.head, idx, *removed[i], dl);
+					}
+					catch(AssertionException) { 
+						ss << " exception update unindex ";
+						cout << " caught assertion update unindex " << idxns.c_str() << '\n';
+						problem() << " caught assertion update unindex " << idxns.c_str() << endl;
+					}
+				}
+				vector<JSObj*> added;
+				setDifference(newkeys, oldkeys, added);
+				assert( !dl.isNull() );
+				for( unsigned i = 0; i < added.size(); i++ ) { 
+					try {
+						idx.head.btree()->insert(
+							idx.head, 
+							dl, *added[i], false, idx, true);
+					}
+					catch(AssertionException) {
+						ss << " exception update index "; 
+						cout << " caught assertion update index " << idxns.c_str() << '\n';
+						problem() << " caught assertion update index " << idxns.c_str() << endl;
+					}
+				}
+				if( client->profile )
+					ss << "<br>" << added.size() << " key updates ";
+
+			}
+		}
+	}
+
+	//	update in place
+	memcpy(toupdate->data, buf, len);
+}
+
+int followupExtentSize(int len, int lastExtentLen) {
+	int x = initialExtentSize(len);
+	int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.2);
+	int sz = y > x ? y : x;
+	sz = ((int)sz) & 0xffffff00;
+	assert( sz > len );
+	return sz;
+}
+
+int deb=0;
+
+/* add keys to indexes for a new record */
+void  _indexRecord(IndexDetails& idx, JSObj& obj, DiskLoc newRecordLoc) { 
+
+	set<JSObj> keys;
+	idx.getKeysFromObject(obj, keys);
+	for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {
+		assert( !newRecordLoc.isNull() );
+		try {
+//			DEBUGGING << "temp index: " << newRecordLoc.toString() << obj.toString() << endl;
+			idx.head.btree()->insert(idx.head, newRecordLoc,
+				(JSObj&) *i, false, idx, true);
+		}
+		catch(AssertionException) { 
+			cout << " caught assertion _indexRecord " << idx.indexNamespace() << '\n';
+			problem() << " caught assertion _indexRecord " << idx.indexNamespace() << endl;
+		}
+	}
+}
+
+/* note there are faster ways to build an index in bulk, that can be 
+   done eventually */
+void addExistingToIndex(const char *ns, IndexDetails& idx) {
+	cout << "Adding all existing records for " << ns << " to new index" << endl;
+	int n = 0;
+	auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+	while( c->ok() ) {
+		JSObj js = c->current();
+		_indexRecord(idx, js, c->currLoc());
+		c->advance();
+		n++;
+	};
+	cout << "  indexing complete for " << n << " records" << endl;
+}
+
+/* add keys to indexes for a new record */
+void  indexRecord(NamespaceDetails *d, const void *buf, int len, DiskLoc newRecordLoc) { 
+	JSObj obj((const char *)buf);
+	for( int i = 0; i < d->nIndexes; i++ ) { 
+		_indexRecord(d->indexes[i], obj, newRecordLoc);
+	}
+}
+
+DiskLoc DataFileMgr::insert(const char *ns, const void *buf, int len, bool god) {
+	bool addIndex = false;
+	const char *sys = strstr(ns, "system.");
+	if( sys ) { 
+		if( sys == ns ) {
+			cout << "ERROR: attempt to insert for invalid client 'system': " << ns << endl;
+			return DiskLoc();
+		}
+		if( strstr(ns, ".system.") ) { 
+			if( strstr(ns, ".system.indexes") )
+				addIndex = true;
+			else if( !god ) { 
+				cout << "ERROR: attempt to insert in system namespace " << ns << endl;
+				return DiskLoc();
+			}
+		}
+	}
+
+	NamespaceDetails *d = nsdetails(ns);
+	if( d == 0 ) {
+		newNamespace(ns);
+		client->newestFile()->newExtent(ns, initialExtentSize(len));
+		d = nsdetails(ns);
+	}
+
+	NamespaceDetails *tableToIndex = 0;
+
+	const char *tabletoidxns = 0;
+	if( addIndex ) { 
+		JSObj io((const char *) buf);
+		const char *name = io.getStringField("name"); // name of the index
+		tabletoidxns = io.getStringField("ns");  // table it indexes
+		JSObj key = io.getObjectField("key");
+		if( name == 0 || *name == 0 || tabletoidxns == 0 || key.isEmpty() || key.objsize() > 2048 ) { 
+			cout << "user warning: bad add index attempt name:" << (name?name:"") << " ns:" << 
+				(tabletoidxns?tabletoidxns:"") << endl;
+			return DiskLoc();
+		}
+		tableToIndex = nsdetails(tabletoidxns);
+		if( tableToIndex == 0 ) {
+			cout << "user warning: bad add index attempt, no such table(ns):" << tabletoidxns << endl;
+			return DiskLoc();
+		}
+		if( tableToIndex->nIndexes >= MaxIndexes ) { 
+			cout << "user warning: bad add index attempt, too many indexes for:" << tabletoidxns << endl;
+			return DiskLoc();
+		}
+		if( tableToIndex->findIndexByName(name) >= 0 ) { 
+			//cout << "INFO: index:" << name << " already exists for:" << tabletoidxns << endl;
+			return DiskLoc();
+		}
+		//indexFullNS = tabletoidxns; 
+		//indexFullNS += ".$";
+		//indexFullNS += name; // client.table.$index -- note this doesn't contain jsobjs, it contains BtreeBuckets.
+	}
+
+	DiskLoc extentLoc;
+	int lenWHdr = len + Record::HeaderSize;
+	DiskLoc loc = d->alloc(ns, lenWHdr, extentLoc);
+	if( loc.isNull() ) {
+		// out of space
+		if( d->capped == 0 ) { // size capped doesn't grow
+			cout << "allocating new extent for " << ns << endl;
+			client->newestFile()->newExtent(ns, followupExtentSize(len, d->lastExtentSize));
+			loc = d->alloc(ns, lenWHdr, extentLoc);
+		}
+		if( loc.isNull() ) { 
+			cout << "out of space in datafile. capped:" << d->capped << endl;
+			assert(d->capped);
+			return DiskLoc();
+		}
+	}
+
+	Record *r = loc.rec();
+	assert( r->lengthWithHeaders >= lenWHdr );
+	memcpy(r->data, buf, len);
+	Extent *e = r->myExtent(loc);
+	if( e->lastRecord.isNull() ) { 
+		e->firstRecord = e->lastRecord = loc;
+		r->prevOfs = r->nextOfs = DiskLoc::NullOfs;
+	}
+	else {
+		Record *oldlast = e->lastRecord.rec();
+		r->prevOfs = e->lastRecord.getOfs();
+		r->nextOfs = DiskLoc::NullOfs;
+		oldlast->nextOfs = loc.getOfs();
+		e->lastRecord = loc;
+	}
+
+	d->nrecords++;
+	d->datasize += r->netLength();
+
+	if( tableToIndex ) { 
+		IndexDetails& idxinfo = tableToIndex->indexes[tableToIndex->nIndexes];
+		idxinfo.info = loc;
+		idxinfo.head = BtreeBucket::addHead(idxinfo);
+		tableToIndex->nIndexes++; 
+		/* todo: index existing records here */
+		addExistingToIndex(tabletoidxns, idxinfo);
+	}
+
+	/* add this record to our indexes */
+	if( d->nIndexes )
+		indexRecord(d, buf, len, loc);
+
+//	cout << "   inserted at loc:" << hex << loc.getOfs() << " lenwhdr:" << hex << lenWHdr << dec << ' ' << ns << endl;
+	return loc;
+}
+
+void DataFileMgr::init(const char *dir) {
+/*	string path = dir;
+	path += "temp.dat";
+	temp.open(path.c_str(), 64 * 1024 * 1024);
+*/
+}
+
+void pdfileInit() {
+//	namespaceIndex.init(dbpath);
+	theDataFileMgr.init(dbpath);
+}
diff --git a/db/pdfile.h b/db/pdfile.h
index bb94089d917..f88f11d704f 100644
--- a/db/pdfile.h
+++ b/db/pdfile.h
@@ -1,456 +1,456 @@
-/* pdfile.h

-

-   Files:

-     client.ns - namespace index

-     client.1  - data files

-     client.2 

-     ...

-*/

-

-#pragma once

-

-// see version, versionMinor, below.

-const int VERSION = 4;

-

-#include "../stdafx.h"

-#include "../util/mmap.h"

-#include "storage.h"

-#include "jsobj.h"

-#include "namespace.h"

-

-class PDFHeader;

-class Extent;

-class Record;

-class Cursor;

-

-/*---------------------------------------------------------------------*/ 

-

-class PDFHeader;

-class PhysicalDataFile {

-	friend class DataFileMgr;

-	friend class BasicCursor;

-public:

-	PhysicalDataFile(int fn) : fileNo(fn) { }

-	void open(int fileNo, const char *filename);

-

-

-	Extent* newExtent(const char *ns, int approxSize, int loops = 0);

-private:

-	Extent* getExtent(DiskLoc loc);

-	Extent* _getExtent(DiskLoc loc);

-	Record* recordAt(DiskLoc dl);

-

-	MemoryMappedFile mmf;

-	PDFHeader *header;

-int __unUsEd;

-//	int length;

-	int fileNo;

-};

-

-class DataFileMgr {

-	friend class BasicCursor;

-public:

-	void init(const char *);

-

-	void update(

-		const char *ns,

-		Record *toupdate, const DiskLoc& dl,

-		const char *buf, int len, stringstream& profiling);

-	DiskLoc insert(const char *ns, const void *buf, int len, bool god = false);

-	void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false);

-	static auto_ptr<Cursor> findAll(const char *ns);

-

-	static Extent* getExtent(const DiskLoc& dl);

-	static Record* getRecord(const DiskLoc& dl);

-private:

-	vector<PhysicalDataFile *> files;

-};

-

-extern DataFileMgr theDataFileMgr;

-

-#pragma pack(push)

-#pragma pack(1)

-

-class DeletedRecord {

-public:

-	int lengthWithHeaders;

-	int extentOfs;

-	DiskLoc nextDeleted;

-};

-

-class Record {

-public:

-	enum { HeaderSize = 16 };

-	int lengthWithHeaders;

-	int extentOfs;

-	int nextOfs;

-	int prevOfs;

-	char data[4];

-	int netLength() { return lengthWithHeaders - HeaderSize; }

-	//void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }

-

-	/* use this when a record is deleted. basically a union with next/prev fields */

-	DeletedRecord& asDeleted() { return *((DeletedRecord*) this); }

-

-	Extent* myExtent(const DiskLoc& myLoc) { 

-		return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs));

-	}

-	/* get the next record in the namespace, traversing extents as necessary */

-	DiskLoc getNext(const DiskLoc& myLoc);

-	DiskLoc getPrev(const DiskLoc& myLoc);

-};

-

-/* extents are regions where all the records within the region 

-   belong to the same namespace.

-*/

-class Extent {

-public:

-	unsigned magic;

-	DiskLoc myLoc;

-	DiskLoc xnext, xprev; /* next/prev extent for this namespace */

-	Namespace ns; /* which namespace this extent is for.  this is just for troubleshooting really */

-	int length;   /* size of the extent, including these fields */

-	//	DiskLoc firstEmptyRegion;

-	DiskLoc firstRecord, lastRecord;

-	char extentData[4];

-

-	bool validates() { 

-		return !(firstRecord.isNull() ^ lastRecord.isNull()) &&

-			length >= 0 && !myLoc.isNull();

-	}

-

-	void dump(iostream& s) { 

-		s << "    loc:" << myLoc.toString() << " xnext:" << xnext.toString() << " xprev:" << xprev.toString() << '\n';

-		s << "    ns:" << ns.buf << '\n';

-		s << "    size:" << length << " firstRecord:" << firstRecord.toString() << " lastRecord:" << lastRecord.toString() << '\n';

-	}

-

-	/* assumes already zeroed -- insufficient for block 'reuse' perhaps 

-	Returns a DeletedRecord location which is the data in the extent ready for us.

-	Caller will need to add that to the freelist structure in namespacedetail.

-	*/

-	DiskLoc init(const char *nsname, int _length, int _fileNo, int _offset);

-

-	void assertOk() { assert(magic == 0x41424344); }

-

-	Record* newRecord(int len);

-

-	Record* getRecord(DiskLoc dl) {

-		assert( !dl.isNull() );

-		assert( dl.sameFile(myLoc) );

-		int x = dl.getOfs() - myLoc.getOfs();

-		assert( x > 0 );

-		return (Record *) (((char *) this) + x);

-	}

-

-	Extent* getNextExtent() { return xnext.isNull() ? 0 : DataFileMgr::getExtent(xnext); }

-	Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }

-};

-

-/*

-      ----------------------

-      Header

-      ----------------------

-      Extent (for a particular namespace)

-        Record

-        ...

-        Record (some chained for unused space)

-      ----------------------

-      more Extents...

-      ----------------------

-*/

-

-/* data file header */

-class PDFHeader {

-public:

-	int version;

-	int versionMinor;

-	int fileLength;

-	DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */

-	int unusedLength;

-	char reserved[8192 - 4*4 - 8];

-

-	char data[4];

-

-	static int headerSize() { return sizeof(PDFHeader) - 4; }

-

-	bool uninitialized() { if( version == 0 ) return true; assert(version == VERSION); return false; }

-

-	Record* getRecord(DiskLoc dl) {

-		int ofs = dl.getOfs();

-		assert( ofs >= headerSize() );

-		return (Record*) (((char *) this) + ofs);

-	}

-

-	void init(int fileno, int filelength) {

-		if( uninitialized() ) {

-			assert(filelength > 32768 );

-			assert( headerSize() == 8192 );

-			fileLength = filelength;

-			version = VERSION;

-			versionMinor = 0;

-			unused.setOfs( fileno, headerSize() );

-			assert( (data-(char*)this) == headerSize() );

-			unusedLength = fileLength - headerSize() - 16;

-			memcpy(data+unusedLength, "      \nthe end\n", 16); 

-		}

-	}

-};

-

-#pragma pack(pop)

-

-inline Extent* PhysicalDataFile::_getExtent(DiskLoc loc) {

-	loc.assertOk();

-	Extent *e = (Extent *) (((char *)header) + loc.getOfs());

-	return e;

-}

-

-inline Extent* PhysicalDataFile::getExtent(DiskLoc loc) {

-	Extent *e = _getExtent(loc);

-	e->assertOk();

-	return e;

-}

-

-class Cursor {

-public:

-	virtual bool ok() = 0;

-	bool eof() { return !ok(); }

-	virtual Record* _current() = 0;

-	virtual JSObj current() = 0;

-	virtual DiskLoc currLoc() = 0;

-	virtual bool advance() = 0; /*true=ok*/

-

-	/* optional to implement.  if implemented, means 'this' is a prototype */

-	virtual Cursor* clone() { return 0; }

-

-	virtual bool tempStopOnMiss() { return false; }

-

-	/* called after every query block is iterated -- i.e. between getMore() blocks

-	   so you can note where we are, if necessary.

-	   */

-	virtual void noteLocation() { } 

-

-	/* called before query getmore block is iterated */

-	virtual void checkLocation() { } 

-

-	virtual const char * toString() { return "abstract?"; }

-

-	/* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */

-	set<DiskLoc> dups;

-	bool dup(DiskLoc loc) {

-		/* to save mem only call this when there is risk of dups (e.g. when 'deep'/multikey) */

-		if( dups.count(loc) > 0 )

-			return true;

-		dups.insert(loc);

-		return false;

-	}

-};

-

-class BasicCursor : public Cursor {

-public:

-	bool ok() { return !curr.isNull(); }

-	Record* _current() {

-		assert( ok() );

-		return curr.rec();

-	}

-	JSObj current() { 

-		Record *r = _current();

-		JSObj j(r);

-		return j;

-	}

-	virtual DiskLoc currLoc() { return curr; }

-

-	bool advance() { 

-		if( eof() )

-			return false;

-		Record *r = _current();

-		curr = r->getNext(curr);

-		return ok();

-	}

-

-	BasicCursor(DiskLoc dl) : curr(dl) { }

-	BasicCursor() { }

-	virtual const char * toString() { return "BasicCursor"; }

-

-	DiskLoc curr;

-};

-

-class ReverseCursor : public BasicCursor {

-public:

-	bool advance() { 

-		if( eof() )

-			return false;

-		Record *r = _current();

-		curr = r->getPrev(curr);

-		return ok();

-	}

-

-	ReverseCursor(DiskLoc dl) : BasicCursor(dl) { }

-	ReverseCursor() { }

-	virtual const char * toString() { return "ReverseCursor"; }

-};

-

-inline Record* PhysicalDataFile::recordAt(DiskLoc dl) { return header->getRecord(dl); }

-

-void sayDbContext(const char *msg = 0);

-

-inline DiskLoc Record::getNext(const DiskLoc& myLoc) {

-	if( nextOfs != DiskLoc::NullOfs ) {

-		/* defensive */

-		if( nextOfs >= 0 && nextOfs < 10 ) { 

-			sayDbContext("Assertion failure - Record::getNext() referencing a deleted record?");

-			return DiskLoc();

-		}

-

-		return DiskLoc(myLoc.a(), nextOfs);

-	}

-	Extent *e = myExtent(myLoc);

-	while( 1 ) {

-		if( e->xnext.isNull() )

-			return DiskLoc(); // end of table.

-		e = e->xnext.ext();

-		if( !e->firstRecord.isNull() ) 

-			break;

-		// entire extent could be empty, keep looking

-	}

-	return e->firstRecord;

-}

-inline DiskLoc Record::getPrev(const DiskLoc& myLoc) {

-	if( prevOfs != DiskLoc::NullOfs )

-		return DiskLoc(myLoc.a(), prevOfs);

-	Extent *e = myExtent(myLoc);

-	if( e->xprev.isNull() )

-		return DiskLoc();

-	return e->xprev.ext()->lastRecord;

-}

-

-inline Record* DiskLoc::rec() const {

-	return DataFileMgr::getRecord(*this);

-}

-inline JSObj DiskLoc::obj() const {

-	return JSObj(rec());

-}

-inline DeletedRecord* DiskLoc::drec() const {

-	assert( fileNo != -1 );

-	return (DeletedRecord*) rec();

-}

-inline Extent* DiskLoc::ext() const {

-	return DataFileMgr::getExtent(*this);

-}

-

-inline BtreeBucket* DiskLoc::btree() const { 

-	assert( fileNo != -1 );

-	return (BtreeBucket*) rec()->data;

-}

-

-inline Bucket* DiskLoc::bucket() const { 

-	assert( fileNo != -1 );

-	return (Bucket*) rec()->data;

-}

-

-/*---------------------------------------------------------------------*/ 

-

-// customer, or rather a customer's database -- i guess down the line

-// there might be more than one for a cust, we'll see.

-class Client { 

-public:

-	Client(const char *nm) : name(nm) { 

-		namespaceIndex = new NamespaceIndex();

-		namespaceIndex->init(dbpath, nm);

-		profile = 0;

-		profileName = name + ".system.profile";

-	} 

-

-	PhysicalDataFile* getFile(int n) { 

-		assert(this);

-

-		if( n < 0 || n >= 10000 ) {

-			cout << "getFile(): n=" << n << endl;

-			assert( n >= 0 && n < 10000 );

-		}

-#if defined(_DEBUG)

-		if( n > 100 )

-			cout << "getFile(): n=" << n << "?" << endl;

-#endif

-		while( n >= (int) files.size() )

-			files.push_back(0);

-		PhysicalDataFile* p = files[n];

-		if( p == 0 ) {

-			p = new PhysicalDataFile(n);

-			files[n] = p;

-			stringstream out;

-			out << dbpath << name << '.' << n;

-			p->open(n, out.str().c_str());

-		}

-		return p;

-	}

-

-	PhysicalDataFile* addAFile() {

-		int n = (int) files.size();

-		return getFile(n);

-	}

-

-	PhysicalDataFile* newestFile() { 

-		int n = (int) files.size();

-		if( n > 0 ) n--;

-		return getFile(n);

-	}

-

-	vector<PhysicalDataFile*> files;

-	string name; // "alleyinsider"

-	NamespaceIndex *namespaceIndex;

-	int profile; // 0=off.

-	string profileName; // "alleyinsider.system.profile"

-};

-

-// tempish...move to TLS or pass all the way down as a parm

-extern map<string,Client*> clients;

-extern Client *client;

-extern const char *curNs;

-inline void setClient(const char *ns) { 

-	char cl[256];

-	curNs = ns;

-	nsToClient(ns, cl);

-	map<string,Client*>::iterator it = clients.find(cl);

-	if( it != clients.end() ) {

-		client = it->second;

-		return;

-	}

-	Client *c = new Client(cl);

-	clients[cl] = c;

-	client = c;

-}

-

-inline NamespaceIndex* nsindex(const char *ns) { 

-#if defined(_DEBUG)

-	char buf[256];

-	nsToClient(ns, buf);

-	if( client->name != buf ) { 

-		cout << "ERROR: attempt to write to wrong database client\n";

-		cout << " ns:" << ns << '\n';

-		cout << " client->name:" << client->name << endl;

-		assert( client->name == buf );

-	}

-#endif

-	return client->namespaceIndex;

-}

-

-inline NamespaceDetails* nsdetails(const char *ns) { 

-	return nsindex(ns)->details(ns);

-}

-

-inline PhysicalDataFile& DiskLoc::pdf() const { 

-	assert( fileNo != -1 );

-	return *client->getFile(fileNo);

-}

-

-inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) {

-	assert( dl.a() != -1 );

-	return client->getFile(dl.a())->getExtent(dl);

-}

-

-inline Record* DataFileMgr::getRecord(const DiskLoc& dl) {

-	assert( dl.a() != -1 );

-	return client->getFile(dl.a())->recordAt(dl);

-}

-

+/* pdfile.h
+
+   Files:
+     client.ns - namespace index
+     client.1  - data files
+     client.2 
+     ...
+*/
+
+#pragma once
+
+// see version, versionMinor, below.
+const int VERSION = 4;
+
+#include "../stdafx.h"
+#include "../util/mmap.h"
+#include "storage.h"
+#include "jsobj.h"
+#include "namespace.h"
+
+class PDFHeader;
+class Extent;
+class Record;
+class Cursor;
+
+/*---------------------------------------------------------------------*/ 
+
+class PDFHeader;
+class PhysicalDataFile {
+	friend class DataFileMgr;
+	friend class BasicCursor;
+public:
+	PhysicalDataFile(int fn) : fileNo(fn) { }
+	void open(int fileNo, const char *filename);
+
+
+	Extent* newExtent(const char *ns, int approxSize, int loops = 0);
+private:
+	Extent* getExtent(DiskLoc loc);
+	Extent* _getExtent(DiskLoc loc);
+	Record* recordAt(DiskLoc dl);
+
+	MemoryMappedFile mmf;
+	PDFHeader *header;
+int __unUsEd;
+//	int length;
+	int fileNo;
+};
+
+class DataFileMgr {
+	friend class BasicCursor;
+public:
+	void init(const char *);
+
+	void update(
+		const char *ns,
+		Record *toupdate, const DiskLoc& dl,
+		const char *buf, int len, stringstream& profiling);
+	DiskLoc insert(const char *ns, const void *buf, int len, bool god = false);
+	void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false);
+	static auto_ptr<Cursor> findAll(const char *ns);
+
+	static Extent* getExtent(const DiskLoc& dl);
+	static Record* getRecord(const DiskLoc& dl);
+private:
+	vector<PhysicalDataFile *> files;
+};
+
+extern DataFileMgr theDataFileMgr;
+
+#pragma pack(push)
+#pragma pack(1)
+
+class DeletedRecord {
+public:
+	int lengthWithHeaders;
+	int extentOfs;
+	DiskLoc nextDeleted;
+};
+
+class Record {
+public:
+	enum { HeaderSize = 16 };
+	int lengthWithHeaders;
+	int extentOfs;
+	int nextOfs;
+	int prevOfs;
+	char data[4];
+	int netLength() { return lengthWithHeaders - HeaderSize; }
+	//void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }
+
+	/* use this when a record is deleted. basically a union with next/prev fields */
+	DeletedRecord& asDeleted() { return *((DeletedRecord*) this); }
+
+	Extent* myExtent(const DiskLoc& myLoc) { 
+		return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs));
+	}
+	/* get the next record in the namespace, traversing extents as necessary */
+	DiskLoc getNext(const DiskLoc& myLoc);
+	DiskLoc getPrev(const DiskLoc& myLoc);
+};
+
+/* extents are regions where all the records within the region 
+   belong to the same namespace.
+*/
+class Extent {
+public:
+	unsigned magic;
+	DiskLoc myLoc;
+	DiskLoc xnext, xprev; /* next/prev extent for this namespace */
+	Namespace ns; /* which namespace this extent is for.  this is just for troubleshooting really */
+	int length;   /* size of the extent, including these fields */
+	//	DiskLoc firstEmptyRegion;
+	DiskLoc firstRecord, lastRecord;
+	char extentData[4];
+
+	bool validates() { 
+		return !(firstRecord.isNull() ^ lastRecord.isNull()) &&
+			length >= 0 && !myLoc.isNull();
+	}
+
+	void dump(iostream& s) { 
+		s << "    loc:" << myLoc.toString() << " xnext:" << xnext.toString() << " xprev:" << xprev.toString() << '\n';
+		s << "    ns:" << ns.buf << '\n';
+		s << "    size:" << length << " firstRecord:" << firstRecord.toString() << " lastRecord:" << lastRecord.toString() << '\n';
+	}
+
+	/* assumes already zeroed -- insufficient for block 'reuse' perhaps 
+	Returns a DeletedRecord location which is the data in the extent ready for us.
+	Caller will need to add that to the freelist structure in namespacedetail.
+	*/
+	DiskLoc init(const char *nsname, int _length, int _fileNo, int _offset);
+
+	void assertOk() { assert(magic == 0x41424344); }
+
+	Record* newRecord(int len);
+
+	Record* getRecord(DiskLoc dl) {
+		assert( !dl.isNull() );
+		assert( dl.sameFile(myLoc) );
+		int x = dl.getOfs() - myLoc.getOfs();
+		assert( x > 0 );
+		return (Record *) (((char *) this) + x);
+	}
+
+	Extent* getNextExtent() { return xnext.isNull() ? 0 : DataFileMgr::getExtent(xnext); }
+	Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }
+};
+
+/*
+      ----------------------
+      Header
+      ----------------------
+      Extent (for a particular namespace)
+        Record
+        ...
+        Record (some chained for unused space)
+      ----------------------
+      more Extents...
+      ----------------------
+*/
+
+/* data file header */
+class PDFHeader {
+public:
+	int version;
+	int versionMinor;
+	int fileLength;
+	DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */
+	int unusedLength;
+	char reserved[8192 - 4*4 - 8];
+
+	char data[4];
+
+	static int headerSize() { return sizeof(PDFHeader) - 4; }
+
+	bool uninitialized() { if( version == 0 ) return true; assert(version == VERSION); return false; }
+
+	Record* getRecord(DiskLoc dl) {
+		int ofs = dl.getOfs();
+		assert( ofs >= headerSize() );
+		return (Record*) (((char *) this) + ofs);
+	}
+
+	void init(int fileno, int filelength) {
+		if( uninitialized() ) {
+			assert(filelength > 32768 );
+			assert( headerSize() == 8192 );
+			fileLength = filelength;
+			version = VERSION;
+			versionMinor = 0;
+			unused.setOfs( fileno, headerSize() );
+			assert( (data-(char*)this) == headerSize() );
+			unusedLength = fileLength - headerSize() - 16;
+			memcpy(data+unusedLength, "      \nthe end\n", 16); 
+		}
+	}
+};
+
+#pragma pack(pop)
+
+inline Extent* PhysicalDataFile::_getExtent(DiskLoc loc) {
+	loc.assertOk();
+	Extent *e = (Extent *) (((char *)header) + loc.getOfs());
+	return e;
+}
+
+inline Extent* PhysicalDataFile::getExtent(DiskLoc loc) {
+	Extent *e = _getExtent(loc);
+	e->assertOk();
+	return e;
+}
+
+class Cursor {
+public:
+	virtual bool ok() = 0;
+	bool eof() { return !ok(); }
+	virtual Record* _current() = 0;
+	virtual JSObj current() = 0;
+	virtual DiskLoc currLoc() = 0;
+	virtual bool advance() = 0; /*true=ok*/
+
+	/* optional to implement.  if implemented, means 'this' is a prototype */
+	virtual Cursor* clone() { return 0; }
+
+	virtual bool tempStopOnMiss() { return false; }
+
+	/* called after every query block is iterated -- i.e. between getMore() blocks
+	   so you can note where we are, if necessary.
+	   */
+	virtual void noteLocation() { } 
+
+	/* called before query getmore block is iterated */
+	virtual void checkLocation() { } 
+
+	virtual const char * toString() { return "abstract?"; }
+
+	/* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */
+	set<DiskLoc> dups;
+	bool dup(DiskLoc loc) {
+		/* to save mem only call this when there is risk of dups (e.g. when 'deep'/multikey) */
+		if( dups.count(loc) > 0 )
+			return true;
+		dups.insert(loc);
+		return false;
+	}
+};
+
+class BasicCursor : public Cursor {
+public:
+	bool ok() { return !curr.isNull(); }
+	Record* _current() {
+		assert( ok() );
+		return curr.rec();
+	}
+	JSObj current() { 
+		Record *r = _current();
+		JSObj j(r);
+		return j;
+	}
+	virtual DiskLoc currLoc() { return curr; }
+
+	bool advance() { 
+		if( eof() )
+			return false;
+		Record *r = _current();
+		curr = r->getNext(curr);
+		return ok();
+	}
+
+	BasicCursor(DiskLoc dl) : curr(dl) { }
+	BasicCursor() { }
+	virtual const char * toString() { return "BasicCursor"; }
+
+	DiskLoc curr;
+};
+
+class ReverseCursor : public BasicCursor {
+public:
+	bool advance() { 
+		if( eof() )
+			return false;
+		Record *r = _current();
+		curr = r->getPrev(curr);
+		return ok();
+	}
+
+	ReverseCursor(DiskLoc dl) : BasicCursor(dl) { }
+	ReverseCursor() { }
+	virtual const char * toString() { return "ReverseCursor"; }
+};
+
+inline Record* PhysicalDataFile::recordAt(DiskLoc dl) { return header->getRecord(dl); }
+
+void sayDbContext(const char *msg = 0);
+
+inline DiskLoc Record::getNext(const DiskLoc& myLoc) {
+	if( nextOfs != DiskLoc::NullOfs ) {
+		/* defensive */
+		if( nextOfs >= 0 && nextOfs < 10 ) { 
+			sayDbContext("Assertion failure - Record::getNext() referencing a deleted record?");
+			return DiskLoc();
+		}
+
+		return DiskLoc(myLoc.a(), nextOfs);
+	}
+	Extent *e = myExtent(myLoc);
+	while( 1 ) {
+		if( e->xnext.isNull() )
+			return DiskLoc(); // end of table.
+		e = e->xnext.ext();
+		if( !e->firstRecord.isNull() ) 
+			break;
+		// entire extent could be empty, keep looking
+	}
+	return e->firstRecord;
+}
+inline DiskLoc Record::getPrev(const DiskLoc& myLoc) {
+	if( prevOfs != DiskLoc::NullOfs )
+		return DiskLoc(myLoc.a(), prevOfs);
+	Extent *e = myExtent(myLoc);
+	if( e->xprev.isNull() )
+		return DiskLoc();
+	return e->xprev.ext()->lastRecord;
+}
+
+inline Record* DiskLoc::rec() const {
+	return DataFileMgr::getRecord(*this);
+}
+inline JSObj DiskLoc::obj() const {
+	return JSObj(rec());
+}
+inline DeletedRecord* DiskLoc::drec() const {
+	assert( fileNo != -1 );
+	return (DeletedRecord*) rec();
+}
+inline Extent* DiskLoc::ext() const {
+	return DataFileMgr::getExtent(*this);
+}
+
+inline BtreeBucket* DiskLoc::btree() const { 
+	assert( fileNo != -1 );
+	return (BtreeBucket*) rec()->data;
+}
+
+inline Bucket* DiskLoc::bucket() const { 
+	assert( fileNo != -1 );
+	return (Bucket*) rec()->data;
+}
+
+/*---------------------------------------------------------------------*/ 
+
+// customer, or rather a customer's database -- i guess down the line
+// there might be more than one for a cust, we'll see.
+class Client { 
+public:
+	Client(const char *nm) : name(nm) { 
+		namespaceIndex = new NamespaceIndex();
+		namespaceIndex->init(dbpath, nm);
+		profile = 0;
+		profileName = name + ".system.profile";
+	} 
+
+	PhysicalDataFile* getFile(int n) { 
+		assert(this);
+
+		if( n < 0 || n >= 10000 ) {
+			cout << "getFile(): n=" << n << endl;
+			assert( n >= 0 && n < 10000 );
+		}
+#if defined(_DEBUG)
+		if( n > 100 )
+			cout << "getFile(): n=" << n << "?" << endl;
+#endif
+		while( n >= (int) files.size() )
+			files.push_back(0);
+		PhysicalDataFile* p = files[n];
+		if( p == 0 ) {
+			p = new PhysicalDataFile(n);
+			files[n] = p;
+			stringstream out;
+			out << dbpath << name << '.' << n;
+			p->open(n, out.str().c_str());
+		}
+		return p;
+	}
+
+	PhysicalDataFile* addAFile() {
+		int n = (int) files.size();
+		return getFile(n);
+	}
+
+	PhysicalDataFile* newestFile() { 
+		int n = (int) files.size();
+		if( n > 0 ) n--;
+		return getFile(n);
+	}
+
+	vector<PhysicalDataFile*> files;
+	string name; // "alleyinsider"
+	NamespaceIndex *namespaceIndex;
+	int profile; // 0=off.
+	string profileName; // "alleyinsider.system.profile"
+};
+
+// tempish...move to TLS or pass all the way down as a parm
+extern map<string,Client*> clients;
+extern Client *client;
+extern const char *curNs;
+inline void setClient(const char *ns) { 
+	char cl[256];
+	curNs = ns;
+	nsToClient(ns, cl);
+	map<string,Client*>::iterator it = clients.find(cl);
+	if( it != clients.end() ) {
+		client = it->second;
+		return;
+	}
+	Client *c = new Client(cl);
+	clients[cl] = c;
+	client = c;
+}
+
+inline NamespaceIndex* nsindex(const char *ns) { 
+#if defined(_DEBUG)
+	char buf[256];
+	nsToClient(ns, buf);
+	if( client->name != buf ) { 
+		cout << "ERROR: attempt to write to wrong database client\n";
+		cout << " ns:" << ns << '\n';
+		cout << " client->name:" << client->name << endl;
+		assert( client->name == buf );
+	}
+#endif
+	return client->namespaceIndex;
+}
+
+inline NamespaceDetails* nsdetails(const char *ns) { 
+	return nsindex(ns)->details(ns);
+}
+
+inline PhysicalDataFile& DiskLoc::pdf() const { 
+	assert( fileNo != -1 );
+	return *client->getFile(fileNo);
+}
+
+inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) {
+	assert( dl.a() != -1 );
+	return client->getFile(dl.a())->getExtent(dl);
+}
+
+inline Record* DataFileMgr::getRecord(const DiskLoc& dl) {
+	assert( dl.a() != -1 );
+	return client->getFile(dl.a())->recordAt(dl);
+}
+
diff --git a/db/query.cpp b/db/query.cpp
index 80f4bbf316c..bd19ea653dd 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -1,854 +1,854 @@
-// query.cpp

-

-#include "stdafx.h"

-#include "query.h"

-#include "pdfile.h"

-#include "jsobj.h"

-#include "../util/builder.h"

-#include <time.h>

-#include "introspect.h"

-#include "btree.h"

-#include "../util/lruishmap.h"

-#include "javajs.h"

-

-//ns->query->DiskLoc

-LRUishMap<JSObj,DiskLoc,5> lrutest(123);

-

-int nextCursorId = 1;

-

-#pragma pack(push)

-#pragma pack(1)

-struct EmptyObject {

-	EmptyObject() { len = 5; jstype = EOO; }

-	int len;

-	char jstype;

-} emptyObject;

-#pragma pack(pop)

-

-JSObj emptyObj((char *) &emptyObject);

-

-int getGtLtOp(Element& e);

-void appendElementHandlingGtLt(JSObjBuilder& b, Element& e);

-

-/* todo: _ cache query plans 

-         _ use index on partial match with the query

-

-   parameters

-     query - the query, e.g., { name: 'joe' }

-     order - order by spec, e.g., { name: 1 } 1=ASC, -1=DESC

-

-*/

-auto_ptr<Cursor> getIndexCursor(const char *ns, JSObj& query, JSObj& order) { 

-	NamespaceDetails *d = nsdetails(ns);

-	if( d == 0 ) return auto_ptr<Cursor>();

-

-	// queryFields, e.g. { 'name' }

-	set<string> queryFields;

-	query.getFieldNames(queryFields);

-

-	if( !order.isEmpty() ) {

-		set<string> orderFields;

-		order.getFieldNames(orderFields);

-		// order by

-		for(int i = 0; i < d->nIndexes; i++ ) { 

-			JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }

-			assert( strcmp(ns, idxInfo.getStringField("ns")) == 0 );

-			JSObj idxKey = idxInfo.getObjectField("key");

-			set<string> keyFields;

-			idxKey.getFieldNames(keyFields);

-			if( keyFields == orderFields ) {

-				bool reverse = 

-					order.firstElement().type() == Number && 

-					order.firstElement().number() < 0;

-				JSObjBuilder b;

-				/* todo: start with the right key, not just beginning of index, when query is also

-				         specified!

-				*/

-				return auto_ptr<Cursor>(new BtreeCursor(d->indexes[i].head, reverse ? maxKey : emptyObj, reverse ? -1 : 1, false));

-			}

-		}

-	}

-

-	// regular query without order by

-	for(int i = 0; i < d->nIndexes; i++ ) { 

-		JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }

-		JSObj idxKey = idxInfo.getObjectField("key");

-		set<string> keyFields;

-		idxKey.getFieldNames(keyFields);

-		if( keyFields == queryFields ) {

-			JSObjBuilder b;

-			JSObj q = query.extractFields(idxKey, b);

-			/* regexp: only supported if form is /^text/ */

-			JSObjBuilder b2;

-			JSElemIter it(q);

-			bool first = true;

-			while( it.more() ) {

-				Element e = it.next();

-				if( e.eoo() )

-					break;

-

-				// GT/LT

-				if( e.type() == Object ) { 

-					int op = getGtLtOp(e);

-					if( op ) { 

-						if( !first || !it.next().eoo() ) {

-							// compound keys with GT/LT not supported yet via index.

-							goto fail;

-						}

-						int direction = - JSMatcher::opDirection(op);

-						return auto_ptr<Cursor>( new BtreeCursor(

-							d->indexes[i].head, 

-							direction == 1 ? emptyObj : maxKey, 

-							direction, 

-							true) );

-					}

-				}

-

-				first = false;

-				if( e.type() == RegEx ) { 

-					if( *e.regexFlags() )

-						goto fail;

-					const char *re = e.regex();

-					const char *p = re;

-					if( *p++ != '^' ) goto fail;

-					while( *p ) {

-						if( *p == ' ' || (*p>='0'&&*p<='9') || (*p>='@'&&*p<='Z') || (*p>='a'&&*p<='z') )

-							;

-						else

-							goto fail;

-						p++;

-					}

-					if( it.more() && !it.next().eoo() ) // we must be the last part of the key (for now until we are smarter)

-						goto fail;

-					// ok!

-                    b2.append(e.fieldName(), re+1);

-					break;

-				}

-				else {

-					b2.append(e);

-					//appendElementHandlingGtLt(b2, e);

-				}

-			}

-			JSObj q2 = b2.done();

-			return auto_ptr<Cursor>( 

-				new BtreeCursor(d->indexes[i].head, q2, 1, true));

-		}

-	}

-

-fail:

-	return auto_ptr<Cursor>();

-}

-

-void deleteObjects(const char *ns, JSObj pattern, bool justOne) {

-//	cout << "TEMP delete ns:" << ns << " queryobjsize:" << 

-//		pattern.objsize() << endl;

-

-	if( strstr(ns, ".system.") ) {

-		if( strstr(ns, ".system.namespaces") ){ 

-			cout << "WARNING: delete on system namespace " << ns << endl;

-		}

-		else if( strstr(ns, ".system.indexes") ) {

-			cout << "WARNING: delete on system namespace " << ns << endl;

-		}

-		else { 

-			cout << "ERROR: attempt to delete in system namespace " << ns << endl;

-			return;

-		}

-	}

-

-	JSMatcher matcher(pattern);

-	JSObj order;

-	auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);

-	if( c.get() == 0 )

-		c = theDataFileMgr.findAll(ns);

-

-	Cursor &tempDebug = *c;

-	int temp = 0;

-	int tempd = 0;

-

-DiskLoc _tempDelLoc;

-

-	while( c->ok() ) {

-		temp++;

-

-		Record *r = c->_current();

-		DiskLoc rloc = c->currLoc();

-		c->advance(); // must advance before deleting as the next ptr will die

-		JSObj js(r);

-		//cout << "TEMP: " << js.toString() << endl;

-		bool deep;

-		if( !matcher.matches(js, &deep) ) {

-			if( c->tempStopOnMiss() )

-				break;

-		}

-		else { 

-			assert( !deep || !c->dup(rloc) ); // can't be a dup, we deleted it!

-//			cout << "  found match to delete" << endl;

-			if( !justOne )

-				c->noteLocation();

-_tempDelLoc = rloc;

-			theDataFileMgr.deleteRecord(ns, r, rloc);

-			tempd = temp;

-			if( justOne )

-				return;

-			c->checkLocation();

-		}

-	}

-}

-

-struct Mod { 

-	const char *fieldName;

-	double n;

-};

-

-void applyMods(vector<Mod>& mods, JSObj obj) { 

-	for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ ) { 

-		Mod& m = *i;

-		Element e = obj.findElement(m.fieldName);

-		if( e.type() == Number ) {

-			e.number() += m.n;

-		}

-	}

-}

-

-/* get special operations like $inc 

-   { $inc: { a:1, b:1 } }

-*/

-void getMods(vector<Mod>& mods, JSObj from) { 

-	JSElemIter it(from);

-	while( it.more() ) {

-		Element e = it.next();

-		if( strcmp(e.fieldName(), "$inc") == 0 && e.type() == Object ) {

-			JSObj j = e.embeddedObject();

-			JSElemIter jt(j);

-			while( jt.more() ) { 

-				Element f = jt.next();

-				if( f.eoo() )

-					break;

-				Mod m;

-				m.fieldName = f.fieldName();

-				if( f.type() == Number ) {

-					m.n = f.number();

-					mods.push_back(m);

-				} 

-			}

-		}

-	}

-}

-

-/*

-todo:

- smart requery find record immediately

-*/

-void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss) {

-//cout << "TEMP BAD";

-//lrutest.find(updateobj);

-

-	int profile = client->profile;

-

-	//	cout << "update ns:" << ns << " objsize:" << updateobj.objsize() << " queryobjsize:" << 

-	//		pattern.objsize();

-

-	if( strstr(ns, ".system.") ) { 

-		cout << "\nERROR: attempt to update in system namespace " << ns << endl;

-		ss << " can't update system namespace ";

-		return;

-	}

-

-	int nscanned = 0;

-	{

-		JSMatcher matcher(pattern);

-		JSObj order;

-		auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);

-		if( c.get() == 0 )

-			c = theDataFileMgr.findAll(ns);

-		while( c->ok() ) {

-			Record *r = c->_current();

-			nscanned++;

-			JSObj js(r);

-			if( !matcher.matches(js) ) {

-				if( c->tempStopOnMiss() )

-					break;

-			}

-			else {

-				/* note: we only update one row and quit.  if you do multiple later, 

-				   be careful or multikeys in arrays could break things badly.  best 

-				   to only allow updating a single row with a multikey lookup.

-				   */

-

-				if( profile )

-					ss << " nscanned:" << nscanned;

-

-				/* look for $inc etc.  note as listed here, all fields to inc must be this type, you can't set some

-				   regular ones at the moment. */

-				if( updateobj.firstElement().fieldName()[0] == '$' ) {

-					vector<Mod> mods;

-					getMods(mods, updateobj);

-					applyMods(mods, c->currLoc().obj());

-					if( profile ) 

-						ss << " fastmod ";

-					return;

-				}

-

-				theDataFileMgr.update(ns, r, c->currLoc(), updateobj.objdata(), updateobj.objsize(), ss);

-				return;

-			}

-			c->advance();

-		}

-	}

-

-	if( profile )

-		ss << " nscanned:" << nscanned;

-

-	if( upsert ) {

-		if( updateobj.firstElement().fieldName()[0] == '$' ) {

-			/* upsert of an $inc. build a default */

-			vector<Mod> mods;

-			getMods(mods, updateobj);

-			JSObjBuilder b;

-			b.appendElements(pattern);

-			for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ )

-				b.append(i->fieldName, i->n);

-			JSObj obj = b.done();

-			theDataFileMgr.insert(ns, (void*) obj.objdata(), obj.objsize());

-			if( profile )

-				ss << " fastmodinsert ";

-			return;

-		}

-		if( profile )

-			ss << " upsert ";

-		theDataFileMgr.insert(ns, (void*) updateobj.objdata(), updateobj.objsize());

-	}

-}

-

-int queryTraceLevel = 0;

-int otherTraceLevel = 0;

-

-int initialExtentSize(int len);

-

-void clean(const char *ns, NamespaceDetails *d) {

-	for( int i = 0; i < Buckets; i++ ) 

-		d->deletedList[i].Null();

-}

-

-string validateNS(const char *ns, NamespaceDetails *d) {

-	bool valid = true;

-	stringstream ss;

-	ss << "\nvalidate\n";

-	if( d->capped )

-		ss << "  capped:" << d->capped << " max:" << d->max << '\n';

-

-	ss << "  firstExtent:" << d->firstExtent.toString() << " lastExtent:" << d->lastExtent.toString() << '\n';

-	ss << "  datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';

-

-	try { 

-

-		{

-			ss << "  first extent:\n";

-			d->firstExtent.ext()->dump(ss);

-			valid = valid && d->firstExtent.ext()->validates();

-		}

-

-		auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);

-		int n = 0;

-		long long len = 0;

-		long long nlen = 0;

-		while( c->ok() ) { 

-			n++;

-			Record *r = c->_current();

-			len += r->lengthWithHeaders;

-			nlen += r->netLength();

-			c->advance();

-		}

-		ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";

-		ss << "  " << len << " bytes record data w/headers\n";

-		ss << "  " << nlen << " bytes record data wout/headers\n";

-

-		ss << "  deletedList: ";

-		for( int i = 0; i < Buckets; i++ ) { 

-			ss << (d->deletedList[i].isNull() ? '0' : '1');

-		}

-		ss << endl;

-		int ndel = 0;

-		long long delSize = 0;

-		for( int i = 0; i < Buckets; i++ ) { 

-			DiskLoc loc = d->deletedList[i];

-			while( !loc.isNull() ) { 

-				ndel++;

-				DeletedRecord *d = loc.drec();

-				delSize += d->lengthWithHeaders;

-				loc = d->nextDeleted;

-			}

-		}

-		ss << "  deleted: n: " << ndel << " size: " << delSize << endl;

-

-		int idxn = 0;

-		try  {

-			ss << "  nIndexes:" << d->nIndexes << endl;

-			for( ; idxn < d->nIndexes; idxn++ ) {

-				ss << "    " << d->indexes[idxn].indexNamespace() << " keys:" << 

-					d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl;

-			}

-		} 

-		catch(...) { 

-			ss << "\n  exception during index validate idxn:" << idxn << endl;

-		}

-

-	}

-	catch(AssertionException) {

-		ss << "\n  exception during validate\n" << endl; 

-	}

-

-	if( !valid ) 

-		ss << "  ns corrupt, requires dbchk\n";

-

-	return ss.str();

-}

-

-bool userCreateNS(const char *ns, JSObj& j);

-

-const int edebug=1;

-

-bool dbEval(JSObj& cmd, JSObjBuilder& result) { 

-	Element e = cmd.firstElement();

-	assert( e.type() == Code );

-	const char *code = e.valuestr();

-	if ( ! JavaJS )

-		JavaJS = new JavaJSImpl();

-

-	jlong f = JavaJS->functionCreate(code);

-	if( f == 0 ) { 

-		result.append("errmsg", "compile failed");

-		return false;

-	}

-

-	Scope s;

-	s.setString("$client", client->name.c_str());

-	Element args = cmd.findElement("args");

-	if( args.type() == Array ) {

-		JSObj eo = args.embeddedObject();

-		if( edebug ) {

-			cout << "args:" << eo.toString() << endl;

-			cout << "code:\n" << code << endl;

-		}

-		s.setObject("args", eo);

-	}

-

-	int res = s.invoke(f);

-	if( res ) {

-		result.append("errno", (double) res);

-		result.append("errmsg", "invoke failed");

-		return false;

-	}

-

-	int type = s.type("return");

-	if( type == Object || type == Array )

-		result.append("retval", s.getObject("return"));

-	else if( type == Number ) 

-		result.append("retval", s.getNumber("return"));

-	else if( type == String )

-		result.append("retval", s.getString("return").c_str());

-	else if( type == Bool ) {

-		result.appendBool("retval", s.getBoolean("return"));

-	}

-

-	return true;

-}

-

-// e.g.

-//   system.cmd$.find( { queryTraceLevel: 2 } );

-// 

-// returns true if ran a cmd

-//

-inline bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { 

-

-	const char *p = strchr(ns, '.');

-	if( !p ) return false;

-	if( strcmp(p, ".$cmd") != 0 ) return false;

-

-	bool ok = false;

-	bool valid = false;

-

-	//cout << jsobj.toString() << endl;

-

-	Element e;

-	e = jsobj.firstElement();

-

-	if( e.eoo() ) goto done;

-	if( e.type() == Code ) { 

-		valid = true;

-		ok = dbEval(jsobj, anObjBuilderForYa);

-	}

-	else if( e.type() == Number ) { 

-		if( strcmp(e.fieldName(), "profile") == 0 ) { 

-			anObjBuilderForYa.append("was", (double) client->profile);

-			int p = (int) e.number();

-			valid = true;

-			if( p == -1 )

-				ok = true;

-			else if( p >= 0 && p <= 2 ) { 

-				ok = true;

-				client->profile = p;

-			}

-			else {

-				ok = false;

-			}

-		}

-		else {

-			if( strncmp(ns, "admin", p-ns) != 0 ) // admin only

-				return false;

-			if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) {

-				valid = ok = true;

-				queryTraceLevel = (int) e.number();

-			} else if( strcmp(e.fieldName(),"traceAll") == 0 ) { 

-				valid = ok = true;

-				queryTraceLevel = (int) e.number();

-				otherTraceLevel = (int) e.number();

-			}

-		}

-	}

-	else if( e.type() == String ) {

-		string us(ns, p-ns);

-

-		if( strcmp( e.fieldName(), "create") == 0 ) { 

-			valid = true;

-			string ns = us + '.' + e.valuestr();

-			ok = userCreateNS(ns.c_str(), jsobj);

-		}

-		else if( strcmp( e.fieldName(), "clean") == 0 ) { 

-			valid = true;

-			string dropNs = us + '.' + e.valuestr();

-			NamespaceDetails *d = nsdetails(dropNs.c_str());

-			cout << "CMD: clean " << dropNs << endl;

-			if( d ) { 

-				ok = true;

-				anObjBuilderForYa.append("ns", dropNs.c_str());

-				clean(dropNs.c_str(), d);

-			}

-			else {

-				anObjBuilderForYa.append("errmsg", "ns not found");

-			}

-		}

-		else if( strcmp( e.fieldName(), "drop") == 0 ) { 

-			valid = true;

-			string dropNs = us + '.' + e.valuestr();

-			NamespaceDetails *d = nsdetails(dropNs.c_str());

-			cout << "CMD: clean " << dropNs << endl;

-			if( d ) { 

-				ok = true;

-				anObjBuilderForYa.append("ns", dropNs.c_str());

-				client->namespaceIndex->kill(dropNs.c_str());

-			}

-			else {

-				anObjBuilderForYa.append("errmsg", "ns not found");

-			}

-		}

-		else if( strcmp( e.fieldName(), "validate") == 0 ) { 

-			valid = true;

-			string toValidateNs = us + '.' + e.valuestr();

-			NamespaceDetails *d = nsdetails(toValidateNs.c_str());

-			cout << "CMD: validate " << toValidateNs << endl;

-			if( d ) { 

-				ok = true;

-				anObjBuilderForYa.append("ns", toValidateNs.c_str());

-				string s = validateNS(toValidateNs.c_str(), d);

-				anObjBuilderForYa.append("result", s.c_str());

-			}

-			else {

-				anObjBuilderForYa.append("errmsg", "ns not found");

-			}

-		}

-		else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) { 

-			valid = true;

-			/* note: temp implementation.  space not reclaimed! */

-			string toDeleteNs = us + '.' + e.valuestr();

-			NamespaceDetails *d = nsdetails(toDeleteNs.c_str());

-			cout << "CMD: deleteIndexes " << toDeleteNs << endl;

-			if( d ) {

-				Element f = jsobj.findElement("index");

-				if( !f.eoo() ) { 

-					// delete a specific index

-					if( f.type() == String ) { 

-						const char *idxName = f.valuestr();

-						if( *idxName == '*' && idxName[1] == 0 ) { 

-							ok = true;

-							cout << "  d->nIndexes was " << d->nIndexes << endl;

-							anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);

-							anObjBuilderForYa.append("msg", "all indexes deleted for collection");

-							cout << "  alpha implementation, space not reclaimed" << endl;

-							d->nIndexes = 0;

-						}

-						else {

-							// delete just one index

-							int x = d->findIndexByName(idxName);

-							if( x >= 0 ) { 

-								cout << "  d->nIndexes was " << d->nIndexes << endl;

-								anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);

-								d->nIndexes--;

-								for( int i = x; i < d->nIndexes; i++ )

-									d->indexes[i] = d->indexes[i+1];

-								ok=true;

-								cout << "  alpha implementation, space not reclaimed" << endl;

-							} else { 

-								cout << "deleteIndexes: " << idxName << " not found" << endl;

-							}

-						}

-					}

-				}

-			}

-			else {

-				anObjBuilderForYa.append("errmsg", "ns not found");

-			}

-		}

-	}

-

-done:

-	if( !valid )

-		anObjBuilderForYa.append("errmsg", "no such cmd");

-	anObjBuilderForYa.append("ok", ok?1.0:0.0);

-	JSObj x = anObjBuilderForYa.done();

-	b.append((void*) x.objdata(), x.objsize());

-	return true;

-}

-

-bool runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { 

-	try {

-		return _runCommands(ns, jsobj, ss, b, anObjBuilderForYa);

-	}

-	catch( AssertionException ) {

-		;

-	}

-	ss << " assertion ";

-	anObjBuilderForYa.append("errmsg", "db assertion failure");

-	anObjBuilderForYa.append("ok", 0.0);

-	JSObj x = anObjBuilderForYa.done();

-	b.append((void*) x.objdata(), x.objsize());

-	return true;

-}

-

-int nCaught = 0;

-

-void killCursors(int n, long long *ids) {

-	int k = 0;

-	for( int i = 0; i < n; i++ ) {

-		if( ClientCursor::erase(ids[i]) )

-			k++;

-	}

-	cout << "killCursors: found " << k << " of " << n << endl;

-}

-

-auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order);

-

-QueryResult* runQuery(const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj, 

-					  auto_ptr< set<string> > filter, stringstream& ss) 

-{

-	bool wantMore = true;

-	int ntoreturn = _ntoreturn;

-	if( _ntoreturn < 0 ) { 

-		ntoreturn = -_ntoreturn;

-		wantMore = false;

-	}

-	ss << "query " << ns << " ntoreturn:" << ntoreturn;

-	if( ntoskip ) 

-		ss << " ntoskip:" << ntoskip;

-	if( client->profile )

-		ss << "<br>query: " << jsobj.toString() << ' ';

-

-	int n = 0;

-	BufBuilder b(32768);

-	JSObjBuilder cmdResBuf;

-	long long cursorid = 0;

-

-	b.skip(sizeof(QueryResult));

-

-	/* we assume you are using findOne() for running a cmd... */

-	if( ntoreturn == 1 && runCommands(ns, jsobj, ss, b, cmdResBuf) ) { 

-		n = 1;

-	}

-	else {

-

-		JSObj query = jsobj.getObjectField("query");

-		JSObj order = jsobj.getObjectField("orderby");

-		if( query.isEmpty() && order.isEmpty() )

-			query = jsobj;

-

-		auto_ptr<JSMatcher> matcher(new JSMatcher(query));

-		JSMatcher &debug1 = *matcher;

-		assert( debug1.getN() < 5000 );

-

-		int nscanned = 0;

-		auto_ptr<Cursor> c = getSpecialCursor(ns);

-

-		/*try*/{

-

-			if( c.get() == 0 ) {

-				c = getIndexCursor(ns, query, order);

-			}

-			if( c.get() == 0 ) {

-				//c = theDataFileMgr.findAll(ns);

-				c = findTableScan(ns, order);

-			}

-

-			while( c->ok() ) {

-				JSObj js = c->current();

-				if( queryTraceLevel >= 50 )

-					cout << " checking against:\n " << js.toString() << endl;

-				nscanned++;

-				bool deep;

-

-JSMatcher &debug = *matcher;

-assert( debug.getN() < 5000 );

-

-				if( !matcher->matches(js, &deep) ) {

-					if( c->tempStopOnMiss() )

-						break;

-				}

-				else if( !deep || !c->dup(c->currLoc()) ) { // i.e., check for dups on deep items only

-					// got a match.

-					if( ntoskip > 0 ) {

-						ntoskip--;

-					}

-					else {

-						bool ok = true;

-						assert( js.objsize() >= 0 ); //defensive for segfaults

-						if( filter.get() ) {

-							// we just want certain fields from the object.

-							JSObj x;

-							ok = x.addFields(js, *filter) > 0;

-							if( ok ) 

-								b.append((void*) x.objdata(), x.objsize());

-						}

-						else {

-							b.append((void*) js.objdata(), js.objsize());

-						}

-						if( ok ) {

-							n++;

-							if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||

-								(ntoreturn==0 && b.len()>1*1024*1024) ) {

-									/* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */

-									if( wantMore && ntoreturn != 1 ) {

-										c->advance();

-										if( c->ok() ) {

-											// more...so save a cursor

-											ClientCursor *cc = new ClientCursor();

-											cc->c = c;

-											cursorid = allocCursorId();

-											cc->cursorid = cursorid;

-											cc->matcher = matcher;

-											cc->ns = ns;

-											cc->pos = n;

-											ClientCursor::add(cc);

-											cc->updateLocation();

-											cc->filter = filter;

-										}

-									}

-									break;

-							}

-						}

-					}

-				}

-				c->advance();

-			}

-

-			if( client->profile )

-				ss << "  nscanned:" << nscanned << ' ';

-		}

-		/*catch( AssertionException e ) { 

-			if( n )

-				throw e;

-			if( nCaught++ >= 1000 ) { 

-				cout << "Too many query exceptions, terminating" << endl;

-				exit(-8);

-			}

-			cout << " Assertion running query, returning an empty result" << endl;

-		}*/

-	}

-

-	QueryResult *qr = (QueryResult *) b.buf();

-	qr->_data[0] = 0;

-	qr->_data[1] = 0;

-	qr->_data[2] = 0;

-	qr->_data[3] = 0;

-	qr->len = b.len();

-	ss << " reslen:" << b.len();

-	//	qr->channel = 0;

-	qr->operation = opReply;

-	qr->cursorId = cursorid;

-	qr->startingFrom = 0;

-	qr->nReturned = n;

-	b.decouple();

-

-	ss << " nreturned:" << n;

-	return qr;

-}

-

-QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid) {

-

-//	cout << "getMore ns:" << ns << " ntoreturn:" << ntoreturn << " cursorid:" << 

-//		cursorid << endl;

-

-	BufBuilder b(32768);

-

-	ClientCursor *cc = ClientCursor::find(cursorid);

-

-	b.skip(sizeof(QueryResult));

-

-	int start = 0;

-	int n = 0;

-

-	if( cc ) {

-		start = cc->pos;

-		Cursor *c = cc->c.get();

-		while( 1 ) {

-			if( !c->ok() ) {

-done:

-				// done!  kill cursor.

-				bool ok = ClientCursor::erase(cursorid);

-				assert(ok);

-				cursorid = 0;

-				cc = 0;

-				break;

-			}

-			JSObj js = c->current();

-			bool deep;

-			if( !cc->matcher->matches(js, &deep) ) {

-				if( c->tempStopOnMiss() )

-					goto done;

-			} 

-			else if( !deep || !c->dup(c->currLoc()) ) {

-				bool ok = true;

-				if( cc->filter.get() ) {

-					JSObj x;

-					ok = x.addFields(js, *cc->filter) > 0;

-					if( ok ) 

-						b.append((void*) x.objdata(), x.objsize());

-				}

-				else {

-					b.append((void*) js.objdata(), js.objsize());

-				}

-				if( ok ) {

-					n++;

-					if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||

-						(ntoreturn==0 && b.len()>1*1024*1024) ) {

-						c->advance();

-						cc->pos += n;

-						cc->updateLocation();

-						break;

-					}

-				}

-			}

-			c->advance();

-		}

-	}

-

-	QueryResult *qr = (QueryResult *) b.buf();

-	qr->cursorId = cursorid;

-	qr->startingFrom = start;

-	qr->len = b.len();

-	//	qr->reserved = 0;

-	qr->operation = opReply;

-	qr->nReturned = n;

-	b.decouple();

-

-	return qr;

-}

+// query.cpp
+
+#include "stdafx.h"
+#include "query.h"
+#include "pdfile.h"
+#include "jsobj.h"
+#include "../util/builder.h"
+#include <time.h>
+#include "introspect.h"
+#include "btree.h"
+#include "../util/lruishmap.h"
+#include "javajs.h"
+
+//ns->query->DiskLoc
+LRUishMap<JSObj,DiskLoc,5> lrutest(123);
+
+int nextCursorId = 1;
+
+#pragma pack(push)
+#pragma pack(1)
+struct EmptyObject {
+	EmptyObject() { len = 5; jstype = EOO; }
+	int len;
+	char jstype;
+} emptyObject;
+#pragma pack(pop)
+
+JSObj emptyObj((char *) &emptyObject);
+
+int getGtLtOp(Element& e);
+void appendElementHandlingGtLt(JSObjBuilder& b, Element& e);
+
+/* todo: _ cache query plans 
+         _ use index on partial match with the query
+
+   parameters
+     query - the query, e.g., { name: 'joe' }
+     order - order by spec, e.g., { name: 1 } 1=ASC, -1=DESC
+
+*/
+auto_ptr<Cursor> getIndexCursor(const char *ns, JSObj& query, JSObj& order) { 
+	NamespaceDetails *d = nsdetails(ns);
+	if( d == 0 ) return auto_ptr<Cursor>();
+
+	// queryFields, e.g. { 'name' }
+	set<string> queryFields;
+	query.getFieldNames(queryFields);
+
+	if( !order.isEmpty() ) {
+		set<string> orderFields;
+		order.getFieldNames(orderFields);
+		// order by
+		for(int i = 0; i < d->nIndexes; i++ ) { 
+			JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }
+			assert( strcmp(ns, idxInfo.getStringField("ns")) == 0 );
+			JSObj idxKey = idxInfo.getObjectField("key");
+			set<string> keyFields;
+			idxKey.getFieldNames(keyFields);
+			if( keyFields == orderFields ) {
+				bool reverse = 
+					order.firstElement().type() == Number && 
+					order.firstElement().number() < 0;
+				JSObjBuilder b;
+				/* todo: start with the right key, not just beginning of index, when query is also
+				         specified!
+				*/
+				return auto_ptr<Cursor>(new BtreeCursor(d->indexes[i].head, reverse ? maxKey : emptyObj, reverse ? -1 : 1, false));
+			}
+		}
+	}
+
+	// regular query without order by
+	for(int i = 0; i < d->nIndexes; i++ ) { 
+		JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }
+		JSObj idxKey = idxInfo.getObjectField("key");
+		set<string> keyFields;
+		idxKey.getFieldNames(keyFields);
+		if( keyFields == queryFields ) {
+			JSObjBuilder b;
+			JSObj q = query.extractFields(idxKey, b);
+			/* regexp: only supported if form is /^text/ */
+			JSObjBuilder b2;
+			JSElemIter it(q);
+			bool first = true;
+			while( it.more() ) {
+				Element e = it.next();
+				if( e.eoo() )
+					break;
+
+				// GT/LT
+				if( e.type() == Object ) { 
+					int op = getGtLtOp(e);
+					if( op ) { 
+						if( !first || !it.next().eoo() ) {
+							// compound keys with GT/LT not supported yet via index.
+							goto fail;
+						}
+						int direction = - JSMatcher::opDirection(op);
+						return auto_ptr<Cursor>( new BtreeCursor(
+							d->indexes[i].head, 
+							direction == 1 ? emptyObj : maxKey, 
+							direction, 
+							true) );
+					}
+				}
+
+				first = false;
+				if( e.type() == RegEx ) { 
+					if( *e.regexFlags() )
+						goto fail;
+					const char *re = e.regex();
+					const char *p = re;
+					if( *p++ != '^' ) goto fail;
+					while( *p ) {
+						if( *p == ' ' || (*p>='0'&&*p<='9') || (*p>='@'&&*p<='Z') || (*p>='a'&&*p<='z') )
+							;
+						else
+							goto fail;
+						p++;
+					}
+					if( it.more() && !it.next().eoo() ) // we must be the last part of the key (for now until we are smarter)
+						goto fail;
+					// ok!
+                    b2.append(e.fieldName(), re+1);
+					break;
+				}
+				else {
+					b2.append(e);
+					//appendElementHandlingGtLt(b2, e);
+				}
+			}
+			JSObj q2 = b2.done();
+			return auto_ptr<Cursor>( 
+				new BtreeCursor(d->indexes[i].head, q2, 1, true));
+		}
+	}
+
+fail:
+	return auto_ptr<Cursor>();
+}
+
+void deleteObjects(const char *ns, JSObj pattern, bool justOne) {
+//	cout << "TEMP delete ns:" << ns << " queryobjsize:" << 
+//		pattern.objsize() << endl;
+
+	if( strstr(ns, ".system.") ) {
+		if( strstr(ns, ".system.namespaces") ){ 
+			cout << "WARNING: delete on system namespace " << ns << endl;
+		}
+		else if( strstr(ns, ".system.indexes") ) {
+			cout << "WARNING: delete on system namespace " << ns << endl;
+		}
+		else { 
+			cout << "ERROR: attempt to delete in system namespace " << ns << endl;
+			return;
+		}
+	}
+
+	JSMatcher matcher(pattern);
+	JSObj order;
+	auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);
+	if( c.get() == 0 )
+		c = theDataFileMgr.findAll(ns);
+
+	Cursor &tempDebug = *c;
+	int temp = 0;
+	int tempd = 0;
+
+DiskLoc _tempDelLoc;
+
+	while( c->ok() ) {
+		temp++;
+
+		Record *r = c->_current();
+		DiskLoc rloc = c->currLoc();
+		c->advance(); // must advance before deleting as the next ptr will die
+		JSObj js(r);
+		//cout << "TEMP: " << js.toString() << endl;
+		bool deep;
+		if( !matcher.matches(js, &deep) ) {
+			if( c->tempStopOnMiss() )
+				break;
+		}
+		else { 
+			assert( !deep || !c->dup(rloc) ); // can't be a dup, we deleted it!
+//			cout << "  found match to delete" << endl;
+			if( !justOne )
+				c->noteLocation();
+_tempDelLoc = rloc;
+			theDataFileMgr.deleteRecord(ns, r, rloc);
+			tempd = temp;
+			if( justOne )
+				return;
+			c->checkLocation();
+		}
+	}
+}
+
+struct Mod { 
+	const char *fieldName;
+	double n;
+};
+
+void applyMods(vector<Mod>& mods, JSObj obj) { 
+	for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ ) { 
+		Mod& m = *i;
+		Element e = obj.findElement(m.fieldName);
+		if( e.type() == Number ) {
+			e.number() += m.n;
+		}
+	}
+}
+
+/* get special operations like $inc 
+   { $inc: { a:1, b:1 } }
+*/
+void getMods(vector<Mod>& mods, JSObj from) { 
+	JSElemIter it(from);
+	while( it.more() ) {
+		Element e = it.next();
+		if( strcmp(e.fieldName(), "$inc") == 0 && e.type() == Object ) {
+			JSObj j = e.embeddedObject();
+			JSElemIter jt(j);
+			while( jt.more() ) { 
+				Element f = jt.next();
+				if( f.eoo() )
+					break;
+				Mod m;
+				m.fieldName = f.fieldName();
+				if( f.type() == Number ) {
+					m.n = f.number();
+					mods.push_back(m);
+				} 
+			}
+		}
+	}
+}
+
+/*
+todo:
+ smart requery find record immediately
+*/
+void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss) {
+//cout << "TEMP BAD";
+//lrutest.find(updateobj);
+
+	int profile = client->profile;
+
+	//	cout << "update ns:" << ns << " objsize:" << updateobj.objsize() << " queryobjsize:" << 
+	//		pattern.objsize();
+
+	if( strstr(ns, ".system.") ) { 
+		cout << "\nERROR: attempt to update in system namespace " << ns << endl;
+		ss << " can't update system namespace ";
+		return;
+	}
+
+	int nscanned = 0;
+	{
+		JSMatcher matcher(pattern);
+		JSObj order;
+		auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);
+		if( c.get() == 0 )
+			c = theDataFileMgr.findAll(ns);
+		while( c->ok() ) {
+			Record *r = c->_current();
+			nscanned++;
+			JSObj js(r);
+			if( !matcher.matches(js) ) {
+				if( c->tempStopOnMiss() )
+					break;
+			}
+			else {
+				/* note: we only update one row and quit.  if you do multiple later, 
+				   be careful or multikeys in arrays could break things badly.  best 
+				   to only allow updating a single row with a multikey lookup.
+				   */
+
+				if( profile )
+					ss << " nscanned:" << nscanned;
+
+				/* look for $inc etc.  note as listed here, all fields to inc must be this type, you can't set some
+				   regular ones at the moment. */
+				if( updateobj.firstElement().fieldName()[0] == '$' ) {
+					vector<Mod> mods;
+					getMods(mods, updateobj);
+					applyMods(mods, c->currLoc().obj());
+					if( profile ) 
+						ss << " fastmod ";
+					return;
+				}
+
+				theDataFileMgr.update(ns, r, c->currLoc(), updateobj.objdata(), updateobj.objsize(), ss);
+				return;
+			}
+			c->advance();
+		}
+	}
+
+	if( profile )
+		ss << " nscanned:" << nscanned;
+
+	if( upsert ) {
+		if( updateobj.firstElement().fieldName()[0] == '$' ) {
+			/* upsert of an $inc. build a default */
+			vector<Mod> mods;
+			getMods(mods, updateobj);
+			JSObjBuilder b;
+			b.appendElements(pattern);
+			for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ )
+				b.append(i->fieldName, i->n);
+			JSObj obj = b.done();
+			theDataFileMgr.insert(ns, (void*) obj.objdata(), obj.objsize());
+			if( profile )
+				ss << " fastmodinsert ";
+			return;
+		}
+		if( profile )
+			ss << " upsert ";
+		theDataFileMgr.insert(ns, (void*) updateobj.objdata(), updateobj.objsize());
+	}
+}
+
+int queryTraceLevel = 0;
+int otherTraceLevel = 0;
+
+int initialExtentSize(int len);
+
+void clean(const char *ns, NamespaceDetails *d) {
+	for( int i = 0; i < Buckets; i++ ) 
+		d->deletedList[i].Null();
+}
+
+string validateNS(const char *ns, NamespaceDetails *d) {
+	bool valid = true;
+	stringstream ss;
+	ss << "\nvalidate\n";
+	if( d->capped )
+		ss << "  capped:" << d->capped << " max:" << d->max << '\n';
+
+	ss << "  firstExtent:" << d->firstExtent.toString() << " lastExtent:" << d->lastExtent.toString() << '\n';
+	ss << "  datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
+
+	try { 
+
+		{
+			ss << "  first extent:\n";
+			d->firstExtent.ext()->dump(ss);
+			valid = valid && d->firstExtent.ext()->validates();
+		}
+
+		auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+		int n = 0;
+		long long len = 0;
+		long long nlen = 0;
+		while( c->ok() ) { 
+			n++;
+			Record *r = c->_current();
+			len += r->lengthWithHeaders;
+			nlen += r->netLength();
+			c->advance();
+		}
+		ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";
+		ss << "  " << len << " bytes record data w/headers\n";
+		ss << "  " << nlen << " bytes record data wout/headers\n";
+
+		ss << "  deletedList: ";
+		for( int i = 0; i < Buckets; i++ ) { 
+			ss << (d->deletedList[i].isNull() ? '0' : '1');
+		}
+		ss << endl;
+		int ndel = 0;
+		long long delSize = 0;
+		for( int i = 0; i < Buckets; i++ ) { 
+			DiskLoc loc = d->deletedList[i];
+			while( !loc.isNull() ) { 
+				ndel++;
+				DeletedRecord *d = loc.drec();
+				delSize += d->lengthWithHeaders;
+				loc = d->nextDeleted;
+			}
+		}
+		ss << "  deleted: n: " << ndel << " size: " << delSize << endl;
+
+		int idxn = 0;
+		try  {
+			ss << "  nIndexes:" << d->nIndexes << endl;
+			for( ; idxn < d->nIndexes; idxn++ ) {
+				ss << "    " << d->indexes[idxn].indexNamespace() << " keys:" << 
+					d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl;
+			}
+		} 
+		catch(...) { 
+			ss << "\n  exception during index validate idxn:" << idxn << endl;
+		}
+
+	}
+	catch(AssertionException) {
+		ss << "\n  exception during validate\n" << endl; 
+	}
+
+	if( !valid ) 
+		ss << "  ns corrupt, requires dbchk\n";
+
+	return ss.str();
+}
+
+bool userCreateNS(const char *ns, JSObj& j);
+
+const int edebug=1;
+
+bool dbEval(JSObj& cmd, JSObjBuilder& result) { 
+	Element e = cmd.firstElement();
+	assert( e.type() == Code );
+	const char *code = e.valuestr();
+	if ( ! JavaJS )
+		JavaJS = new JavaJSImpl();
+
+	jlong f = JavaJS->functionCreate(code);
+	if( f == 0 ) { 
+		result.append("errmsg", "compile failed");
+		return false;
+	}
+
+	Scope s;
+	s.setString("$client", client->name.c_str());
+	Element args = cmd.findElement("args");
+	if( args.type() == Array ) {
+		JSObj eo = args.embeddedObject();
+		if( edebug ) {
+			cout << "args:" << eo.toString() << endl;
+			cout << "code:\n" << code << endl;
+		}
+		s.setObject("args", eo);
+	}
+
+	int res = s.invoke(f);
+	if( res ) {
+		result.append("errno", (double) res);
+		result.append("errmsg", "invoke failed");
+		return false;
+	}
+
+	int type = s.type("return");
+	if( type == Object || type == Array )
+		result.append("retval", s.getObject("return"));
+	else if( type == Number ) 
+		result.append("retval", s.getNumber("return"));
+	else if( type == String )
+		result.append("retval", s.getString("return").c_str());
+	else if( type == Bool ) {
+		result.appendBool("retval", s.getBoolean("return"));
+	}
+
+	return true;
+}
+
+// e.g.
+//   system.cmd$.find( { queryTraceLevel: 2 } );
+// 
+// returns true if ran a cmd
+//
+inline bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { 
+
+	const char *p = strchr(ns, '.');
+	if( !p ) return false;
+	if( strcmp(p, ".$cmd") != 0 ) return false;
+
+	bool ok = false;
+	bool valid = false;
+
+	//cout << jsobj.toString() << endl;
+
+	Element e;
+	e = jsobj.firstElement();
+
+	if( e.eoo() ) goto done;
+	if( e.type() == Code ) { 
+		valid = true;
+		ok = dbEval(jsobj, anObjBuilderForYa);
+	}
+	else if( e.type() == Number ) { 
+		if( strcmp(e.fieldName(), "profile") == 0 ) { 
+			anObjBuilderForYa.append("was", (double) client->profile);
+			int p = (int) e.number();
+			valid = true;
+			if( p == -1 )
+				ok = true;
+			else if( p >= 0 && p <= 2 ) { 
+				ok = true;
+				client->profile = p;
+			}
+			else {
+				ok = false;
+			}
+		}
+		else {
+			if( strncmp(ns, "admin", p-ns) != 0 ) // admin only
+				return false;
+			if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) {
+				valid = ok = true;
+				queryTraceLevel = (int) e.number();
+			} else if( strcmp(e.fieldName(),"traceAll") == 0 ) { 
+				valid = ok = true;
+				queryTraceLevel = (int) e.number();
+				otherTraceLevel = (int) e.number();
+			}
+		}
+	}
+	else if( e.type() == String ) {
+		string us(ns, p-ns);
+
+		if( strcmp( e.fieldName(), "create") == 0 ) { 
+			valid = true;
+			string ns = us + '.' + e.valuestr();
+			ok = userCreateNS(ns.c_str(), jsobj);
+		}
+		else if( strcmp( e.fieldName(), "clean") == 0 ) { 
+			valid = true;
+			string dropNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(dropNs.c_str());
+			cout << "CMD: clean " << dropNs << endl;
+			if( d ) { 
+				ok = true;
+				anObjBuilderForYa.append("ns", dropNs.c_str());
+				clean(dropNs.c_str(), d);
+			}
+			else {
+				anObjBuilderForYa.append("errmsg", "ns not found");
+			}
+		}
+		else if( strcmp( e.fieldName(), "drop") == 0 ) { 
+			valid = true;
+			string dropNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(dropNs.c_str());
+			cout << "CMD: clean " << dropNs << endl;
+			if( d ) { 
+				ok = true;
+				anObjBuilderForYa.append("ns", dropNs.c_str());
+				client->namespaceIndex->kill(dropNs.c_str());
+			}
+			else {
+				anObjBuilderForYa.append("errmsg", "ns not found");
+			}
+		}
+		else if( strcmp( e.fieldName(), "validate") == 0 ) { 
+			valid = true;
+			string toValidateNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(toValidateNs.c_str());
+			cout << "CMD: validate " << toValidateNs << endl;
+			if( d ) { 
+				ok = true;
+				anObjBuilderForYa.append("ns", toValidateNs.c_str());
+				string s = validateNS(toValidateNs.c_str(), d);
+				anObjBuilderForYa.append("result", s.c_str());
+			}
+			else {
+				anObjBuilderForYa.append("errmsg", "ns not found");
+			}
+		}
+		else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) { 
+			valid = true;
+			/* note: temp implementation.  space not reclaimed! */
+			string toDeleteNs = us + '.' + e.valuestr();
+			NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
+			cout << "CMD: deleteIndexes " << toDeleteNs << endl;
+			if( d ) {
+				Element f = jsobj.findElement("index");
+				if( !f.eoo() ) { 
+					// delete a specific index
+					if( f.type() == String ) { 
+						const char *idxName = f.valuestr();
+						if( *idxName == '*' && idxName[1] == 0 ) { 
+							ok = true;
+							cout << "  d->nIndexes was " << d->nIndexes << endl;
+							anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);
+							anObjBuilderForYa.append("msg", "all indexes deleted for collection");
+							cout << "  alpha implementation, space not reclaimed" << endl;
+							d->nIndexes = 0;
+						}
+						else {
+							// delete just one index
+							int x = d->findIndexByName(idxName);
+							if( x >= 0 ) { 
+								cout << "  d->nIndexes was " << d->nIndexes << endl;
+								anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);
+								d->nIndexes--;
+								for( int i = x; i < d->nIndexes; i++ )
+									d->indexes[i] = d->indexes[i+1];
+								ok=true;
+								cout << "  alpha implementation, space not reclaimed" << endl;
+							} else { 
+								cout << "deleteIndexes: " << idxName << " not found" << endl;
+							}
+						}
+					}
+				}
+			}
+			else {
+				anObjBuilderForYa.append("errmsg", "ns not found");
+			}
+		}
+	}
+
+done:
+	if( !valid )
+		anObjBuilderForYa.append("errmsg", "no such cmd");
+	anObjBuilderForYa.append("ok", ok?1.0:0.0);
+	JSObj x = anObjBuilderForYa.done();
+	b.append((void*) x.objdata(), x.objsize());
+	return true;
+}
+
+bool runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { 
+	try {
+		return _runCommands(ns, jsobj, ss, b, anObjBuilderForYa);
+	}
+	catch( AssertionException ) {
+		;
+	}
+	ss << " assertion ";
+	anObjBuilderForYa.append("errmsg", "db assertion failure");
+	anObjBuilderForYa.append("ok", 0.0);
+	JSObj x = anObjBuilderForYa.done();
+	b.append((void*) x.objdata(), x.objsize());
+	return true;
+}
+
+int nCaught = 0;
+
+void killCursors(int n, long long *ids) {
+	int k = 0;
+	for( int i = 0; i < n; i++ ) {
+		if( ClientCursor::erase(ids[i]) )
+			k++;
+	}
+	cout << "killCursors: found " << k << " of " << n << endl;
+}
+
+auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order);
+
+QueryResult* runQuery(const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj, 
+					  auto_ptr< set<string> > filter, stringstream& ss) 
+{
+	bool wantMore = true;
+	int ntoreturn = _ntoreturn;
+	if( _ntoreturn < 0 ) { 
+		ntoreturn = -_ntoreturn;
+		wantMore = false;
+	}
+	ss << "query " << ns << " ntoreturn:" << ntoreturn;
+	if( ntoskip ) 
+		ss << " ntoskip:" << ntoskip;
+	if( client->profile )
+		ss << "<br>query: " << jsobj.toString() << ' ';
+
+	int n = 0;
+	BufBuilder b(32768);
+	JSObjBuilder cmdResBuf;
+	long long cursorid = 0;
+
+	b.skip(sizeof(QueryResult));
+
+	/* we assume you are using findOne() for running a cmd... */
+	if( ntoreturn == 1 && runCommands(ns, jsobj, ss, b, cmdResBuf) ) { 
+		n = 1;
+	}
+	else {
+
+		JSObj query = jsobj.getObjectField("query");
+		JSObj order = jsobj.getObjectField("orderby");
+		if( query.isEmpty() && order.isEmpty() )
+			query = jsobj;
+
+		auto_ptr<JSMatcher> matcher(new JSMatcher(query));
+		JSMatcher &debug1 = *matcher;
+		assert( debug1.getN() < 5000 );
+
+		int nscanned = 0;
+		auto_ptr<Cursor> c = getSpecialCursor(ns);
+
+		/*try*/{
+
+			if( c.get() == 0 ) {
+				c = getIndexCursor(ns, query, order);
+			}
+			if( c.get() == 0 ) {
+				//c = theDataFileMgr.findAll(ns);
+				c = findTableScan(ns, order);
+			}
+
+			while( c->ok() ) {
+				JSObj js = c->current();
+				if( queryTraceLevel >= 50 )
+					cout << " checking against:\n " << js.toString() << endl;
+				nscanned++;
+				bool deep;
+
+JSMatcher &debug = *matcher;
+assert( debug.getN() < 5000 );
+
+				if( !matcher->matches(js, &deep) ) {
+					if( c->tempStopOnMiss() )
+						break;
+				}
+				else if( !deep || !c->dup(c->currLoc()) ) { // i.e., check for dups on deep items only
+					// got a match.
+					if( ntoskip > 0 ) {
+						ntoskip--;
+					}
+					else {
+						bool ok = true;
+						assert( js.objsize() >= 0 ); //defensive for segfaults
+						if( filter.get() ) {
+							// we just want certain fields from the object.
+							JSObj x;
+							ok = x.addFields(js, *filter) > 0;
+							if( ok ) 
+								b.append((void*) x.objdata(), x.objsize());
+						}
+						else {
+							b.append((void*) js.objdata(), js.objsize());
+						}
+						if( ok ) {
+							n++;
+							if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
+								(ntoreturn==0 && b.len()>1*1024*1024) ) {
+									/* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
+									if( wantMore && ntoreturn != 1 ) {
+										c->advance();
+										if( c->ok() ) {
+											// more...so save a cursor
+											ClientCursor *cc = new ClientCursor();
+											cc->c = c;
+											cursorid = allocCursorId();
+											cc->cursorid = cursorid;
+											cc->matcher = matcher;
+											cc->ns = ns;
+											cc->pos = n;
+											ClientCursor::add(cc);
+											cc->updateLocation();
+											cc->filter = filter;
+										}
+									}
+									break;
+							}
+						}
+					}
+				}
+				c->advance();
+			}
+
+			if( client->profile )
+				ss << "  nscanned:" << nscanned << ' ';
+		}
+		/*catch( AssertionException e ) { 
+			if( n )
+				throw e;
+			if( nCaught++ >= 1000 ) { 
+				cout << "Too many query exceptions, terminating" << endl;
+				exit(-8);
+			}
+			cout << " Assertion running query, returning an empty result" << endl;
+		}*/
+	}
+
+	QueryResult *qr = (QueryResult *) b.buf();
+	qr->_data[0] = 0;
+	qr->_data[1] = 0;
+	qr->_data[2] = 0;
+	qr->_data[3] = 0;
+	qr->len = b.len();
+	ss << " reslen:" << b.len();
+	//	qr->channel = 0;
+	qr->operation = opReply;
+	qr->cursorId = cursorid;
+	qr->startingFrom = 0;
+	qr->nReturned = n;
+	b.decouple();
+
+	ss << " nreturned:" << n;
+	return qr;
+}
+
+QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid) {
+
+//	cout << "getMore ns:" << ns << " ntoreturn:" << ntoreturn << " cursorid:" << 
+//		cursorid << endl;
+
+	BufBuilder b(32768);
+
+	ClientCursor *cc = ClientCursor::find(cursorid);
+
+	b.skip(sizeof(QueryResult));
+
+	int start = 0;
+	int n = 0;
+
+	if( cc ) {
+		start = cc->pos;
+		Cursor *c = cc->c.get();
+		while( 1 ) {
+			if( !c->ok() ) {
+done:
+				// done!  kill cursor.
+				bool ok = ClientCursor::erase(cursorid);
+				assert(ok);
+				cursorid = 0;
+				cc = 0;
+				break;
+			}
+			JSObj js = c->current();
+			bool deep;
+			if( !cc->matcher->matches(js, &deep) ) {
+				if( c->tempStopOnMiss() )
+					goto done;
+			} 
+			else if( !deep || !c->dup(c->currLoc()) ) {
+				bool ok = true;
+				if( cc->filter.get() ) {
+					JSObj x;
+					ok = x.addFields(js, *cc->filter) > 0;
+					if( ok ) 
+						b.append((void*) x.objdata(), x.objsize());
+				}
+				else {
+					b.append((void*) js.objdata(), js.objsize());
+				}
+				if( ok ) {
+					n++;
+					if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
+						(ntoreturn==0 && b.len()>1*1024*1024) ) {
+						c->advance();
+						cc->pos += n;
+						cc->updateLocation();
+						break;
+					}
+				}
+			}
+			c->advance();
+		}
+	}
+
+	QueryResult *qr = (QueryResult *) b.buf();
+	qr->cursorId = cursorid;
+	qr->startingFrom = start;
+	qr->len = b.len();
+	//	qr->reserved = 0;
+	qr->operation = opReply;
+	qr->nReturned = n;
+	b.decouple();
+
+	return qr;
+}
diff --git a/db/query.h b/db/query.h
index 44180f8c98c..b0f7a503fcb 100644
--- a/db/query.h
+++ b/db/query.h
@@ -1,135 +1,135 @@
-// query.h

-

-#pragma once

-

-#include "../stdafx.h"

-#include "../grid/message.h"

-#include "jsobj.h"

-#include "storage.h"

-

-/* requests:

-

-   dbDelete

-      int reserved=0;

-      string collection;

-	  int flags=0; // 1=DeleteSingle

-      JSObject query;

-   dbUpdate:

-      int reserved;

-      string collection;

-	  int flags; // 1=upsert

-      JSObject query;

-	  JSObject objectToUpdate;

-        objectToUpdate may include { $inc: <field> }.

-   dbQuery:

-      int reserved;

-      string collection;

-	  int nToSkip;

-	  int nToReturn; // how many you want back as the beginning of the cursor data

-      JSObject query;

-	  [JSObject fieldsToReturn]

-   dbGetMore:

-      int reserved;

-	  string collection; // redundant, might use for security.

-      int nToReturn;

-      int64 cursorID;

-   dbKillCursors=2007:

-      int reserved;

-      int n;

-	  int64 cursorIDs[n];

-

-   Note that on Update, there is only one object, which is different

-   from insert where you can pass a list of objects to insert in the db.

-   Note that the update field layout is very similar layout to Query.

-*/

-

-/* db response format

-

-   Query or GetMore:

-      int reserved;

-      int64 cursorID;

-      int startingFrom;

-      int nReturned; // 0=infinity

-      list of marshalled JSObjects;

-*/

-

-#pragma pack(push)

-#pragma pack(1)

-

-struct QueryResult : public MsgData {

-	long long cursorId;

-	int startingFrom;

-	int nReturned;

-	const char *data() { return (char *) (((int *)&nReturned)+1); }

-};

-

-#pragma pack(pop)

-

-QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid);

-

-// caller must free() returned QueryResult.

-QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, 

-					  JSObj j, auto_ptr< set<string> > fieldFilter,

-					  stringstream&);

-

-void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss);

-void deleteObjects(const char *ns, JSObj pattern, bool justOne);

-

-class ClientCursor;

-typedef map<long long, ClientCursor*> CCMap;

-extern CCMap clientCursors; /* cursorid -> ClientCursor */

-

-/* Cursor -- and its derived classes -- are our internal cursors.

-

-   ClientCursor is a wrapper that represents a cursorid from our client 

-   application's perspective.

-*/

-class Cursor;

-class ClientCursor {

-	friend class CursInspector;

-public:

-	ClientCursor() { 

-		cursorid=0; pos=0; nextAtThisLocation=0; 

-#if defined(_WIN32)

-	cout << "clientcursor() " << cursorid << endl;

-#endif

-	}

-	~ClientCursor();

-	long long cursorid;

-	string ns;

-	auto_ptr<JSMatcher> matcher;

-	auto_ptr<Cursor> c;

-	int pos;

-	DiskLoc lastLoc;

-	auto_ptr< set<string> > filter;

-

-	/* report to us that a new clientcursor exists so we can track it. You still

-	   do the initial updateLocation() yourself. 

-	   */

-	static void add(ClientCursor*);

-

-	static bool erase(long long cursorid);

-

-	static ClientCursor* find(long long id) {

-		CCMap::iterator it = clientCursors.find(id);

-		if( it == clientCursors.end() ) { 

-			cout << "ClientCursor::find(): cursor not found in map " << id << endl;

-			return 0;

-		}

-		return it->second;

-	}

-

-	/* call when cursor's location changes so that we can update the 

-	   cursorsbylocation map.  if you are locked and internally iterating, only 

-	   need to call when you are ready to "unlock".

-	   */

-	void updateLocation();

-

-private:

-	void addToByLocation(DiskLoc cl);

-	static void cleanupByLocation(DiskLoc loc, long long cursorid);

-public:

-	ClientCursor *nextAtThisLocation;

-};

-

-long long allocCursorId();

+// query.h
+
+#pragma once
+
+#include "../stdafx.h"
+#include "../grid/message.h"
+#include "jsobj.h"
+#include "storage.h"
+
+/* requests:
+
+   dbDelete
+      int reserved=0;
+      string collection;
+	  int flags=0; // 1=DeleteSingle
+      JSObject query;
+   dbUpdate:
+      int reserved;
+      string collection;
+	  int flags; // 1=upsert
+      JSObject query;
+	  JSObject objectToUpdate;
+        objectToUpdate may include { $inc: <field> }.
+   dbQuery:
+      int reserved;
+      string collection;
+	  int nToSkip;
+	  int nToReturn; // how many you want back as the beginning of the cursor data
+      JSObject query;
+	  [JSObject fieldsToReturn]
+   dbGetMore:
+      int reserved;
+	  string collection; // redundant, might use for security.
+      int nToReturn;
+      int64 cursorID;
+   dbKillCursors=2007:
+      int reserved;
+      int n;
+	  int64 cursorIDs[n];
+
+   Note that on Update, there is only one object, which is different
+   from insert where you can pass a list of objects to insert in the db.
+   Note that the update field layout is very similar layout to Query.
+*/
+
+/* db response format
+
+   Query or GetMore:
+      int reserved;
+      int64 cursorID;
+      int startingFrom;
+      int nReturned; // 0=infinity
+      list of marshalled JSObjects;
+*/
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct QueryResult : public MsgData {
+	long long cursorId;
+	int startingFrom;
+	int nReturned;
+	const char *data() { return (char *) (((int *)&nReturned)+1); }
+};
+
+#pragma pack(pop)
+
+QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid);
+
+// caller must free() returned QueryResult.
+QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, 
+					  JSObj j, auto_ptr< set<string> > fieldFilter,
+					  stringstream&);
+
+void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss);
+void deleteObjects(const char *ns, JSObj pattern, bool justOne);
+
+class ClientCursor;
+typedef map<long long, ClientCursor*> CCMap;
+extern CCMap clientCursors; /* cursorid -> ClientCursor */
+
+/* Cursor -- and its derived classes -- are our internal cursors.
+
+   ClientCursor is a wrapper that represents a cursorid from our client 
+   application's perspective.
+*/
+class Cursor;
+class ClientCursor {
+	friend class CursInspector;
+public:
+	ClientCursor() { 
+		cursorid=0; pos=0; nextAtThisLocation=0; 
+#if defined(_WIN32)
+	cout << "clientcursor() " << cursorid << endl;
+#endif
+	}
+	~ClientCursor();
+	long long cursorid;
+	string ns;
+	auto_ptr<JSMatcher> matcher;
+	auto_ptr<Cursor> c;
+	int pos;
+	DiskLoc lastLoc;
+	auto_ptr< set<string> > filter;
+
+	/* report to us that a new clientcursor exists so we can track it. You still
+	   do the initial updateLocation() yourself. 
+	   */
+	static void add(ClientCursor*);
+
+	static bool erase(long long cursorid);
+
+	static ClientCursor* find(long long id) {
+		CCMap::iterator it = clientCursors.find(id);
+		if( it == clientCursors.end() ) { 
+			cout << "ClientCursor::find(): cursor not found in map " << id << endl;
+			return 0;
+		}
+		return it->second;
+	}
+
+	/* call when cursor's location changes so that we can update the 
+	   cursorsbylocation map.  if you are locked and internally iterating, only 
+	   need to call when you are ready to "unlock".
+	   */
+	void updateLocation();
+
+private:
+	void addToByLocation(DiskLoc cl);
+	static void cleanupByLocation(DiskLoc loc, long long cursorid);
+public:
+	ClientCursor *nextAtThisLocation;
+};
+
+long long allocCursorId();
diff --git a/db/resource.h b/db/resource.h
index 6fafd161abb..3041137c21a 100644
--- a/db/resource.h
+++ b/db/resource.h
@@ -1,14 +1,14 @@
-//{{NO_DEPENDENCIES}}

-// Microsoft Visual C++ generated include file.

-// Used by db.rc

-

-// Next default values for new objects

-// 

-#ifdef APSTUDIO_INVOKED

-#ifndef APSTUDIO_READONLY_SYMBOLS

-#define _APS_NEXT_RESOURCE_VALUE        101

-#define _APS_NEXT_COMMAND_VALUE         40001

-#define _APS_NEXT_CONTROL_VALUE         1001

-#define _APS_NEXT_SYMED_VALUE           101

-#endif

-#endif

+//{{NO_DEPENDENCIES}}
+// Microsoft Visual C++ generated include file.
+// Used by db.rc
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE        101
+#define _APS_NEXT_COMMAND_VALUE         40001
+#define _APS_NEXT_CONTROL_VALUE         1001
+#define _APS_NEXT_SYMED_VALUE           101
+#endif
+#endif
diff --git a/db/storage.h b/db/storage.h
index 78d1dbf1a5e..fbeed51091b 100644
--- a/db/storage.h
+++ b/db/storage.h
@@ -1,90 +1,90 @@
-/* storage.h

-

-   Storage subsystem management.

-   Lays out our datafiles on disk, manages disk space.

-*/

-

-#pragma once

-

-#pragma pack(push)

-#pragma pack(1)

-

-class Record;

-class DeletedRecord;

-class Extent;

-class BtreeBucket;

-class JSObj;

-class PhysicalDataFile;

-class Bucket;

-

-class DiskLoc {

-	int fileNo; /* this will be volume, file #, etc. */

-	int ofs;

-public:

-	enum { NullOfs = -1 };

-	int a() const { return fileNo; }

-	DiskLoc(int a, int b) : fileNo(a), ofs(b) { 

-		assert(ofs!=0);

-	}

-	DiskLoc() { fileNo = -1; ofs = NullOfs; }

-

-	DiskLoc(const DiskLoc& l) { fileNo=l.fileNo; ofs=l.ofs; }

-

-	bool isNull() const { return ofs == NullOfs; }

-	void Null() { fileNo = -1; ofs = NullOfs; }

-	void assertOk() { assert(!isNull()); }

-

-	string toString() const {

-		if( isNull() ) 

-			return "null";

-		stringstream ss;

-		ss << hex << fileNo << ':' << ofs;

-		return ss.str();

-	}

-

-	int& GETOFS() { return ofs; }

-	int getOfs() const { return ofs; }

-	void set(int a, int b) { fileNo=a; ofs=b; }

-	void setOfs(int _fileNo, int _ofs) { 

-		fileNo = _fileNo;

-		ofs = _ofs;

-	}

-

-	void inc(int amt) {

-		assert( !isNull() );

-		ofs += amt;

-	}

-

-	bool sameFile(DiskLoc b) { return fileNo == b.fileNo; }

-

-	bool operator==(const DiskLoc& b) const { return fileNo==b.fileNo && ofs == b.ofs; }

-	bool operator!=(const DiskLoc& b) const { return !(*this==b); }

-	const DiskLoc& operator=(const DiskLoc& b) { 

-		fileNo=b.fileNo; ofs = b.ofs;

-		assert(ofs!=0);

-		return *this;

-	}

-	int compare(const DiskLoc& b) const { 

-		int x = fileNo - b.fileNo;

-		if( x ) 

-			return x;

-		if( ofs == b.ofs ) return 0;

-		return ofs < b.ofs ? -1 : 1;

-	}

-	bool operator<(const DiskLoc& b) const { 

-		if( fileNo == b.fileNo )

-			return ofs < b.ofs;

-		return fileNo < b.fileNo;

-	}

-

-	JSObj obj() const;

-	Record* rec() const;

-	DeletedRecord* drec() const;

-	Extent* ext() const;

-	BtreeBucket* btree() const;

-	Bucket* bucket() const;

-

-	PhysicalDataFile& pdf() const;

-};

-

-#pragma pack(pop)

+/* storage.h
+
+   Storage subsystem management.
+   Lays out our datafiles on disk, manages disk space.
+*/
+
+#pragma once
+
+#pragma pack(push)
+#pragma pack(1)
+
+class Record;
+class DeletedRecord;
+class Extent;
+class BtreeBucket;
+class JSObj;
+class PhysicalDataFile;
+class Bucket;
+
+class DiskLoc {
+	int fileNo; /* this will be volume, file #, etc. */
+	int ofs;
+public:
+	enum { NullOfs = -1 };
+	int a() const { return fileNo; }
+	DiskLoc(int a, int b) : fileNo(a), ofs(b) { 
+		assert(ofs!=0);
+	}
+	DiskLoc() { fileNo = -1; ofs = NullOfs; }
+
+	DiskLoc(const DiskLoc& l) { fileNo=l.fileNo; ofs=l.ofs; }
+
+	bool isNull() const { return ofs == NullOfs; }
+	void Null() { fileNo = -1; ofs = NullOfs; }
+	void assertOk() { assert(!isNull()); }
+
+	string toString() const {
+		if( isNull() ) 
+			return "null";
+		stringstream ss;
+		ss << hex << fileNo << ':' << ofs;
+		return ss.str();
+	}
+
+	int& GETOFS() { return ofs; }
+	int getOfs() const { return ofs; }
+	void set(int a, int b) { fileNo=a; ofs=b; }
+	void setOfs(int _fileNo, int _ofs) { 
+		fileNo = _fileNo;
+		ofs = _ofs;
+	}
+
+	void inc(int amt) {
+		assert( !isNull() );
+		ofs += amt;
+	}
+
+	bool sameFile(DiskLoc b) { return fileNo == b.fileNo; }
+
+	bool operator==(const DiskLoc& b) const { return fileNo==b.fileNo && ofs == b.ofs; }
+	bool operator!=(const DiskLoc& b) const { return !(*this==b); }
+	const DiskLoc& operator=(const DiskLoc& b) { 
+		fileNo=b.fileNo; ofs = b.ofs;
+		assert(ofs!=0);
+		return *this;
+	}
+	int compare(const DiskLoc& b) const { 
+		int x = fileNo - b.fileNo;
+		if( x ) 
+			return x;
+		if( ofs == b.ofs ) return 0;
+		return ofs < b.ofs ? -1 : 1;
+	}
+	bool operator<(const DiskLoc& b) const { 
+		if( fileNo == b.fileNo )
+			return ofs < b.ofs;
+		return fileNo < b.fileNo;
+	}
+
+	JSObj obj() const;
+	Record* rec() const;
+	DeletedRecord* drec() const;
+	Extent* ext() const;
+	BtreeBucket* btree() const;
+	Bucket* bucket() const;
+
+	PhysicalDataFile& pdf() const;
+};
+
+#pragma pack(pop)
diff --git a/grid/message.cpp b/grid/message.cpp
index 0a10667e49f..caf2cd7e8a8 100644
--- a/grid/message.cpp
+++ b/grid/message.cpp
@@ -1,196 +1,196 @@
-/* message

-

-   todo: authenticate; encrypt?

-*/

-

-#include "stdafx.h"

-#include "message.h"

-#include <time.h>

-#include "../util/goodies.h"

-

-// if you want trace output:

-#define mmm(x)

-

-/* listener ------------------------------------------------------------------- */

-

-void Listener::listen() {

-	SockAddr me(port);

-	int sock = socket(AF_INET, SOCK_STREAM, 0);

-	if( sock == INVALID_SOCKET ) {

-		cout << "ERROR: listen(): invalid socket? " << errno << endl;

-		return;

-	}

-	prebindOptions( sock );

-	if( bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) { 

-		cout << "listen(): bind() failed errno:" << errno << endl;

-		if( errno == 98 )

-			cout << "98 == addr already in use" << endl;

-		closesocket(sock);

-		return;

-	}

-

-	if( ::listen(sock, 128) != 0 ) { 

-		cout << "listen(): listen() failed " << errno << endl;

-		closesocket(sock);

-		return;

-	}

-

-	SockAddr from;

-	while( 1 ) { 

-		int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize);

-		if( s < 0 ) {

-			cout << "Listener: accept() returns " << s << " errno:" << errno << endl;

-			continue;

-		}

-		disableNagle(s);

-		cout << "Listener: connection accepted from " << from.toString() << endl;

-		accepted( new MessagingPort(s, from) );

-	}

-}

-

-/* messagingport -------------------------------------------------------------- */

-

-MSGID NextMsgId;

-struct MsgStart {

-	MsgStart() {

-		NextMsgId = (((unsigned) time(0)) << 16) ^ curTimeMillis();

-		assert(MsgDataHeaderSize == 16);

-	}

-} msgstart;

-

-MessagingPort::MessagingPort(int _sock, SockAddr& _far) : sock(_sock), farEnd(_far) { }

-

-MessagingPort::MessagingPort() {

-	sock = -1;

-}

-

-void MessagingPort::shutdown() { 

-	if( sock >= 0 ) { 

-		closesocket(sock);

-		sock = -1;

-	}

-}

-

-MessagingPort::~MessagingPort() { 

-	shutdown();

-}

-

-bool MessagingPort::connect(SockAddr& _far)

-{

-	farEnd = _far;

-

-	sock = socket(AF_INET, SOCK_STREAM, 0);

-	if( sock == INVALID_SOCKET ) {

-		cout << "ERROR: connect(): invalid socket? " << errno << endl;

-		return false;

-	}

-	if( ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize) ) { 

-		cout << "ERROR: connect(): connect() failed" << errno << endl;

-		closesocket(sock); sock = -1;

-		return false;

-	}

-	disableNagle(sock);

-	return true;

-}

-

-bool MessagingPort::recv(Message& m) {

-	mmm( cout << "*  recv() sock:" << this->sock << endl; )

-	int len = -1;

-

-	char *lenbuf = (char *) &len;

-	int lft = 4;

-	while( 1 ) {

-		int x = ::recv(sock, lenbuf, lft, 0);

-		if( x == 0 ) {

-			cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;

-			m.reset();

-			return false;

-		}

-		if( x < 0 ) { 

-			cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString()<<endl;

-			m.reset();

-			return false;

-		}

-		lft -= x;

-		if( lft == 0 )

-			break;

-		lenbuf += x;

-		cout << "MessagingPort::recv(): got " << x << " bytes wanted 4, lft=" << lft << endl;

-		assert( lft > 0 );

-	}

-

-//	assert( x == 4 );

-

-	assert( len >= 0 && len <= 16000000 );

-        

-	int z = (len+1023)&0xfffffc00; assert(z>=len);

-	MsgData *md = (MsgData *) malloc(z);

-	md->len = len;

-        

-        if ( len <= 0 ){

-          cout << "got a length of 0, something is wrong" << endl;

-          return false;

-        }

-

-	char *p = (char *) &md->id;

-	int left = len -4;

-	while( 1 ) {

-		int x = ::recv(sock, p, left, 0);

-		if( x == 0 ) {

-			cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;

-			m.reset();

-			return false;

-		}

-		if( x < 0 ) { 

-			cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString() << endl;

-			m.reset();

-			return false;

-		}

-		left -= x;

-		p += x;

-		if( left <= 0 )

-			break;

-	}

-

-	m.setData(md, true);

-	return true;

-}

-

-void MessagingPort::reply(Message& received, Message& response) {

-	say(received.from, response, received.data->id);

-}

-

-bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) {

-	mmm( cout << "*call()" << endl; )

-	MSGID old = toSend.data->id;

-	say(to, toSend);

-	while( 1 ) {

-		bool ok = recv(response);

-		if( !ok )

-			return false;

-		//cout << "got response: " << response.data->responseTo << endl;

-		if( response.data->responseTo == toSend.data->id ) 

-			break;

-		cout << "********************" << endl;

-		cout << "ERROR: MessagingPort::call() wrong id got:" << response.data->responseTo << " expect:" << toSend.data->id << endl;

-		cout << "  old:" << old << endl;

-		cout << "  response msgid:" << response.data->id << endl;

-		cout << "  response len:  " << response.data->len << endl;

-		assert(false);

-		response.reset();

-	}

-	mmm( cout << "*call() end" << endl; )

-	return true;

-}

-

-void MessagingPort::say(SockAddr& to, Message& toSend, int responseTo) {

-	mmm( cout << "*  say() sock:" << this->sock << " thr:" << GetCurrentThreadId() << endl; )

-	MSGID msgid = NextMsgId;

-	++NextMsgId;

-	toSend.data->id = msgid;

-	toSend.data->responseTo = responseTo;

-	int x = ::send(sock, (char *) toSend.data, toSend.data->len, 0);

-	if( x <= 0 ) { 

-		cout << "MessagingPort::say: send() error " << errno << ' ' << farEnd.toString() << endl;

-	}

-}

+/* message
+
+   todo: authenticate; encrypt?
+*/
+
+#include "stdafx.h"
+#include "message.h"
+#include <time.h>
+#include "../util/goodies.h"
+
+// if you want trace output:
+#define mmm(x)
+
+/* listener ------------------------------------------------------------------- */
+
+void Listener::listen() {
+	SockAddr me(port);
+	int sock = socket(AF_INET, SOCK_STREAM, 0);
+	if( sock == INVALID_SOCKET ) {
+		cout << "ERROR: listen(): invalid socket? " << errno << endl;
+		return;
+	}
+	prebindOptions( sock );
+	if( bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) { 
+		cout << "listen(): bind() failed errno:" << errno << endl;
+		if( errno == 98 )
+			cout << "98 == addr already in use" << endl;
+		closesocket(sock);
+		return;
+	}
+
+	if( ::listen(sock, 128) != 0 ) { 
+		cout << "listen(): listen() failed " << errno << endl;
+		closesocket(sock);
+		return;
+	}
+
+	SockAddr from;
+	while( 1 ) { 
+		int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize);
+		if( s < 0 ) {
+			cout << "Listener: accept() returns " << s << " errno:" << errno << endl;
+			continue;
+		}
+		disableNagle(s);
+		cout << "Listener: connection accepted from " << from.toString() << endl;
+		accepted( new MessagingPort(s, from) );
+	}
+}
+
+/* messagingport -------------------------------------------------------------- */
+
+MSGID NextMsgId;
+struct MsgStart {
+	MsgStart() {
+		NextMsgId = (((unsigned) time(0)) << 16) ^ curTimeMillis();
+		assert(MsgDataHeaderSize == 16);
+	}
+} msgstart;
+
+MessagingPort::MessagingPort(int _sock, SockAddr& _far) : sock(_sock), farEnd(_far) { }
+
+MessagingPort::MessagingPort() {
+	sock = -1;
+}
+
+void MessagingPort::shutdown() { 
+	if( sock >= 0 ) { 
+		closesocket(sock);
+		sock = -1;
+	}
+}
+
+MessagingPort::~MessagingPort() { 
+	shutdown();
+}
+
+bool MessagingPort::connect(SockAddr& _far)
+{
+	farEnd = _far;
+
+	sock = socket(AF_INET, SOCK_STREAM, 0);
+	if( sock == INVALID_SOCKET ) {
+		cout << "ERROR: connect(): invalid socket? " << errno << endl;
+		return false;
+	}
+	if( ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize) ) { 
+		cout << "ERROR: connect(): connect() failed" << errno << endl;
+		closesocket(sock); sock = -1;
+		return false;
+	}
+	disableNagle(sock);
+	return true;
+}
+
+bool MessagingPort::recv(Message& m) {
+	mmm( cout << "*  recv() sock:" << this->sock << endl; )
+	int len = -1;
+
+	char *lenbuf = (char *) &len;
+	int lft = 4;
+	while( 1 ) {
+		int x = ::recv(sock, lenbuf, lft, 0);
+		if( x == 0 ) {
+			cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;
+			m.reset();
+			return false;
+		}
+		if( x < 0 ) { 
+			cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString()<<endl;
+			m.reset();
+			return false;
+		}
+		lft -= x;
+		if( lft == 0 )
+			break;
+		lenbuf += x;
+		cout << "MessagingPort::recv(): got " << x << " bytes wanted 4, lft=" << lft << endl;
+		assert( lft > 0 );
+	}
+
+//	assert( x == 4 );
+
+	assert( len >= 0 && len <= 16000000 );
+        
+	int z = (len+1023)&0xfffffc00; assert(z>=len);
+	MsgData *md = (MsgData *) malloc(z);
+	md->len = len;
+        
+        if ( len <= 0 ){
+          cout << "got a length of 0, something is wrong" << endl;
+          return false;
+        }
+
+	char *p = (char *) &md->id;
+	int left = len -4;
+	while( 1 ) {
+		int x = ::recv(sock, p, left, 0);
+		if( x == 0 ) {
+			cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;
+			m.reset();
+			return false;
+		}
+		if( x < 0 ) { 
+			cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString() << endl;
+			m.reset();
+			return false;
+		}
+		left -= x;
+		p += x;
+		if( left <= 0 )
+			break;
+	}
+
+	m.setData(md, true);
+	return true;
+}
+
+void MessagingPort::reply(Message& received, Message& response) {
+	say(received.from, response, received.data->id);
+}
+
+bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) {
+	mmm( cout << "*call()" << endl; )
+	MSGID old = toSend.data->id;
+	say(to, toSend);
+	while( 1 ) {
+		bool ok = recv(response);
+		if( !ok )
+			return false;
+		//cout << "got response: " << response.data->responseTo << endl;
+		if( response.data->responseTo == toSend.data->id ) 
+			break;
+		cout << "********************" << endl;
+		cout << "ERROR: MessagingPort::call() wrong id got:" << response.data->responseTo << " expect:" << toSend.data->id << endl;
+		cout << "  old:" << old << endl;
+		cout << "  response msgid:" << response.data->id << endl;
+		cout << "  response len:  " << response.data->len << endl;
+		assert(false);
+		response.reset();
+	}
+	mmm( cout << "*call() end" << endl; )
+	return true;
+}
+
+void MessagingPort::say(SockAddr& to, Message& toSend, int responseTo) {
+	mmm( cout << "*  say() sock:" << this->sock << " thr:" << GetCurrentThreadId() << endl; )
+	MSGID msgid = NextMsgId;
+	++NextMsgId;
+	toSend.data->id = msgid;
+	toSend.data->responseTo = responseTo;
+	int x = ::send(sock, (char *) toSend.data, toSend.data->len, 0);
+	if( x <= 0 ) { 
+		cout << "MessagingPort::say: send() error " << errno << ' ' << farEnd.toString() << endl;
+	}
+}
diff --git a/grid/message.h b/grid/message.h
index 1395fa51e0a..604e9a67343 100644
--- a/grid/message.h
+++ b/grid/message.h
@@ -1,129 +1,129 @@
-// message.h

-

-#pragma once

-

-#include "../util/sock.h"

-

-class Message;

-class MessagingPort; 

-typedef WrappingInt MSGID;

-const int DBPort = 27017;

-

-class Listener { 

-public:

-	Listener(int p) : port(p) { } 

-	void listen(); // never returns (start a thread)

-

-	/* spawn a thread, etc., then return */

-	virtual void accepted(MessagingPort *mp) = 0;

-private:

-	int port;

-};

-

-class AbstractMessagingPort { 

-public:

-	virtual void reply(Message& received, Message& response) = 0;

-};

-

-class MessagingPort : public AbstractMessagingPort {

-public:

-	MessagingPort(int sock, SockAddr& farEnd);

-	MessagingPort();

-	~MessagingPort();

-

-	void shutdown();

-

-	bool connect(SockAddr& farEnd);

-

-	/* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's.

-	   also, the Message data will go out of scope on the subsequent recv call. 

-	*/

-	bool recv(Message& m);

-	void reply(Message& received, Message& response);

-	bool call(SockAddr& to, Message& toSend, Message& response);

-	void say(SockAddr& to, Message& toSend, int responseTo = -1);

-

-private:

-	int sock;

-	SockAddr farEnd;

-};

-

-#pragma pack(push)

-#pragma pack(1)

-

-enum Operations { 

-	opReply = 1,     /* reply. responseTo is set. */

-

-	dbMsg = 1000,    /* generic msg command followed by a string */

-

-	dbUpdate = 2001, /* update object */

-	dbInsert = 2002,

-//	dbGetByOID = 2003,

-	dbQuery = 2004,

-	dbGetMore = 2005,

-	dbDelete = 2006,

-	dbKillCursors = 2007

-};

-

-struct MsgData {

-	int len; /* len of the msg, including this field */

-	MSGID id; /* request/reply id's match... */

-	int responseTo; /* id of the message we are responding to */

-	int operation;

-	char _data[4];

-

-	int dataLen();

-};

-const int MsgDataHeaderSize = sizeof(MsgData) - 4;

-inline int MsgData::dataLen() { return len - MsgDataHeaderSize; }

-

-#pragma pack(pop)

-

-class Message {

-public:

-	Message() { data = 0; freeIt = false; }

-        Message( void * _data , bool _freeIt ){ data = (MsgData*)_data; freeIt = _freeIt; };

-	~Message() { reset(); }

-

-	SockAddr from;

-	MsgData *data;

-

-	Message& operator=(Message& r) { 

-		assert( data == 0 );

-		data = r.data;

-		assert( r.freeIt );

-		r.freeIt = false;

-		r.data = 0;

-		freeIt = true;

-		return *this;

-	}

-

-	void reset() {

-		if( freeIt && data )

-			free(data);

-		data = 0; freeIt = false;

-	}

-

-	void setData(MsgData *d, bool _freeIt) {

-		assert( data == 0 ); 

-		freeIt = _freeIt;

-		data = d;

-	}

-	void setData(int operation, const char *msgtxt) {

-		setData(operation, msgtxt, strlen(msgtxt)+1);

-	}

-	void setData(int operation, const char *msgdata, int len) {

-		assert(data == 0);

-		int dataLen = len + sizeof(MsgData) - 4;

-		MsgData *d = (MsgData *) malloc(dataLen);

-		memcpy(d->_data, msgdata, len);

-		d->len = dataLen;

-		d->operation = operation;

-		freeIt= true;

-		data = d;

-	}

-

-private:

-	bool freeIt;

-};

-

+// message.h
+
+#pragma once
+
+#include "../util/sock.h"
+
+class Message;
+class MessagingPort; 
+typedef WrappingInt MSGID;
+const int DBPort = 27017;
+
+class Listener { 
+public:
+	Listener(int p) : port(p) { } 
+	void listen(); // never returns (start a thread)
+
+	/* spawn a thread, etc., then return */
+	virtual void accepted(MessagingPort *mp) = 0;
+private:
+	int port;
+};
+
+class AbstractMessagingPort { 
+public:
+	virtual void reply(Message& received, Message& response) = 0;
+};
+
+class MessagingPort : public AbstractMessagingPort {
+public:
+	MessagingPort(int sock, SockAddr& farEnd);
+	MessagingPort();
+	~MessagingPort();
+
+	void shutdown();
+
+	bool connect(SockAddr& farEnd);
+
+	/* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's.
+	   also, the Message data will go out of scope on the subsequent recv call. 
+	*/
+	bool recv(Message& m);
+	void reply(Message& received, Message& response);
+	bool call(SockAddr& to, Message& toSend, Message& response);
+	void say(SockAddr& to, Message& toSend, int responseTo = -1);
+
+private:
+	int sock;
+	SockAddr farEnd;
+};
+
+#pragma pack(push)
+#pragma pack(1)
+
+enum Operations { 
+	opReply = 1,     /* reply. responseTo is set. */
+
+	dbMsg = 1000,    /* generic msg command followed by a string */
+
+	dbUpdate = 2001, /* update object */
+	dbInsert = 2002,
+//	dbGetByOID = 2003,
+	dbQuery = 2004,
+	dbGetMore = 2005,
+	dbDelete = 2006,
+	dbKillCursors = 2007
+};
+
+struct MsgData {
+	int len; /* len of the msg, including this field */
+	MSGID id; /* request/reply id's match... */
+	int responseTo; /* id of the message we are responding to */
+	int operation;
+	char _data[4];
+
+	int dataLen();
+};
+const int MsgDataHeaderSize = sizeof(MsgData) - 4;
+inline int MsgData::dataLen() { return len - MsgDataHeaderSize; }
+
+#pragma pack(pop)
+
+class Message {
+public:
+	Message() { data = 0; freeIt = false; }
+        Message( void * _data , bool _freeIt ){ data = (MsgData*)_data; freeIt = _freeIt; };
+	~Message() { reset(); }
+
+	SockAddr from;
+	MsgData *data;
+
+	Message& operator=(Message& r) { 
+		assert( data == 0 );
+		data = r.data;
+		assert( r.freeIt );
+		r.freeIt = false;
+		r.data = 0;
+		freeIt = true;
+		return *this;
+	}
+
+	void reset() {
+		if( freeIt && data )
+			free(data);
+		data = 0; freeIt = false;
+	}
+
+	void setData(MsgData *d, bool _freeIt) {
+		assert( data == 0 ); 
+		freeIt = _freeIt;
+		data = d;
+	}
+	void setData(int operation, const char *msgtxt) {
+		setData(operation, msgtxt, strlen(msgtxt)+1);
+	}
+	void setData(int operation, const char *msgdata, int len) {
+		assert(data == 0);
+		int dataLen = len + sizeof(MsgData) - 4;
+		MsgData *d = (MsgData *) malloc(dataLen);
+		memcpy(d->_data, msgdata, len);
+		d->len = dataLen;
+		d->operation = operation;
+		freeIt= true;
+		data = d;
+	}
+
+private:
+	bool freeIt;
+};
+
diff --git a/grid/protocol.h b/grid/protocol.h
index f98c6cde1f4..e7d0298ec9b 100644
--- a/grid/protocol.h
+++ b/grid/protocol.h
@@ -1,261 +1,261 @@
-// protocol.h

-

-#pragma once

-

-NOT USED 

-

-#include "boost/thread/mutex.hpp"

-#include "boost/thread/condition.hpp"

-#include "../util/sock.h"

-#include "../util/goodies.h"

-

-typedef WrappingInt MSGID;

-

-struct Fragment;

-

-#if 0

-#define ptrace(x) 

-#else

-#define ptrace(x) { cout << curTimeMillis() % 10000; x }

-#endif

-

-#if 1

-#define etrace(x) 

-#else

-#define etrace(x) { cout << curTimeMillis() % 10000; x }

-#endif

-

-class F; // fragment

-class MR; // message.  R=receiver side.

-class CR; // connection receiver side

-class MS; // message   S=sender side.

-class CS; // connection sender side

-class ProtocolConnection; // connection overall

-

-/* ip:port:channel 

-   We have one receiver thread per process (per ip address destination), and then 

-   multiplex messages out to multiple connections(generally one per thread) which 

-   each have a 'channel'.

-*/

-class EndPoint { 

-public:

-	EndPoint() : channel(0) { }

-	int channel;

-	SockAddr sa;

-	bool operator<(const EndPoint& r) const {

-		if( channel != r.channel )

-			return channel < r.channel;

-		return sa < r.sa;

-	}

-	string toString() { 

-		stringstream out;

-		out << sa.toString() << ':';

-		if( channel == -2 ) out << "ANYCHANNEL";

-		else if( channel == -1 ) out << "AUTOASSIGNCHANNEL";

-		else out << channel;

-		return out.str();

-	}

-};

-

-/* the double underscore stuff here is the actual implementation glue.

-   wanted to keep that stuff clean and separate.  so put your implementation

-   of these in protoimpl.h.

-*/

-

-void __sendRESET(ProtocolConnection *pc);

-

-// sender -> 

-void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *, bool retran=false); // transmit a fragment

-void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, MSGID msgid, int fragNo); // transmit the REQUEST ACK msg

-

-// receiver -> 

-void __sendACK(ProtocolConnection *pc, MSGID msgid); // transmit ACK

-void __sendMISSING(ProtocolConnection *pc, EndPoint& to, MSGID msgid, vector<short>& ids); // transmit MISSING

-

-// -> receiver

-F* __recv(UDPConnection& c, SockAddr& from); /* recv from socket a fragment and pass back */

-

-class F { 

-public:

-	F(Fragment *f);

-	~F();

-	int __num();   //frag #

-	int __len();

-	MSGID __msgid();

-	int __channel();

-	bool __isREQUESTACK(); // if true, this is just a request for acknowledgement not real data

-	int __firstFragMsgLen(); // only works on first fragment

-

-	// sender side:

-	bool __isACK(); // if true, this is an ack of a message

-	bool __isMISSING(); // if true, list of frags to retransmit

-	short* __getMissing(int& n); // get the missing fragno list

-

-	Fragment *internals;

-	enum { NORMAL, ACK, MISSING, REQUESTACK, RESET } op;

-};

-

-class MR { 

-public:

-	MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from);

-	~MR() { freeFrags(); }

-	void freeFrags() { 

-		for( unsigned i = 0; i < f.size(); i++ )

-			delete f[i];

-	}

-	bool got(F *f, EndPoint& from); // received a fragment

-	bool gotFirst() { return f[0] != 0; }

-	ProtocolConnection& pc;

-	void removeFromReceivingList();

-	bool complete();

-	const MSGID msgid;

-	int n() { return f.size(); }

-public:

-	int messageLenExpected;

-	int nExpected, nReceived;

-	void reportMissings(bool reportAll);

-	vector<F*> f;

-	vector<unsigned> reportTimes;

-	EndPoint from;

-};

-

-/* this is for knowing what is already received.  we might get dup packets later and need 

-   to ignore them. */

-class MsgTracker {

-public:

-	std::list<MSGID> recentlyReceivedList;

-	std::set<MSGID> recentlyReceived;

-	MSGID lastFullyReceived;

-

-	void reset() { 

-		recentlyReceivedList.clear();

-		recentlyReceived.clear();

-		lastFullyReceived = 0;

-	}

-

-	void got(MSGID m) {

-		unsigned sz = recentlyReceived.size();

-		if( sz > 256 ) {

-			recentlyReceived.erase(recentlyReceivedList.front());

-			recentlyReceivedList.pop_front();

-		}

-		recentlyReceivedList.push_back(m);

-		recentlyReceived.insert(m);

-		if( m > lastFullyReceived || sz == 0 )

-			lastFullyReceived = m;

-	}

-};

-

-class CR { 

-	friend class MR;

-public:

-	~CR() { ptrace( cout << ".warning: ~CR() not implemented" << endl; ) }

-	CR(ProtocolConnection& _pc) : pc(_pc) { }

-	MR* recv();

-public:

-	MR* getPendingMsg(F *fr);

-	bool oldMessageId(int channel, MSGID m);

-	void queueReceived(MR*);

-

-	ProtocolConnection& pc;

-	boost::condition receivedSome;

- 	vector<MR*> received; /* ready to dequeue and use */

-	map<int,MR*> pendingMessages; /* partly received msgs */

-	MsgTracker oldMsgTracker;

-};

-

-/* -- sender side ------------------------------------------------*/

-

-class CS { 

-public:

-	CS(ProtocolConnection& _pc);

-

-	ProtocolConnection& pc;

-	vector<MS*> pendingSend;

-	boost::condition msgSent;

-	void resetIt();

-

-	double delayMax;

-	double delay;

-	void delayGotMissing() { 

-		double delayOld = delay;

-		if( delay == 0 )

-			delay = 2.0;

-		else

-			delay = delay * 1.25;

-		if( delay > delayMax ) delay = delayMax;

-		if( delay != delayOld )

-			cout << ".DELAY INCREASED TO " << delay << endl;

-	}

-	void delaySentMsg() { 

-		if( delay != 0.0 ) {

-			delay = delay * 0.5;

-			if( delay<0.5 ) delay = 0;

-			cout << ".DELAY DECREASED TO " << delay << endl;

-		}

-	}

-};

-

-typedef map<EndPoint,ProtocolConnection*> EndPointToPC;

-extern EndPointToPC pcMap; /* the *far* endpoint -> pc */

-

-/* -- overall Connection object ----------------------------------*/

-

-#pragma warning( disable: 4355 )

-

-class ProtocolConnection { 

-public:

-	string toString();

-

-	ProtocolConnection(ProtocolConnection& par, EndPoint& to);

-	ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd);

-	~ProtocolConnection();

-

-	void shutdown();

-	bool acceptAnyChannel() const;

-	UDPConnection& udpConnection;

-	/* note the channel for myEnd might be "any" for the any pc -

-  	   so you can't use that channel for sending.  Use MS/MR 

-	   for that.

-	   */

-	EndPoint myEnd;

-	EndPoint farEnd;

-

-	/* if this was instantiated automatically for an acceptAnyChannel(), 

-	   keep a ptr back to it and queue received msgs there. 

-	*/

-	ProtocolConnection *parent;

-

-	CR cr;

-	CS cs;

-	bool first; // true if yet to send first message on this conn

-

-private:

-	void init();

-};

-

-/* -- sender side ------------------------------------------------*/

-

-class MS { 

-public:

-	MS(ProtocolConnection *_pc, EndPoint &_to, MSGID _msgid) : 

-	  pc(_pc), to(_to), msgid(_msgid), complainInterval(50) { }

-	~MS() { 

-		for( unsigned i = 0; i < fragments.size(); i++ )

-			delete fragments[i];

-	}

-

-	/* init fragments, then call this */

-	void send();

-

-	vector<F*> fragments;

-

-	/* request retrainsmissions. */

-	bool complain(unsigned now);

-

-	ProtocolConnection* pc;

-	EndPoint to;

-	const MSGID msgid;

-	unsigned lastComplainTime;

-	unsigned complainInterval;

-};

+// protocol.h
+
+#pragma once
+
+NOT USED 
+
+#include "boost/thread/mutex.hpp"
+#include "boost/thread/condition.hpp"
+#include "../util/sock.h"
+#include "../util/goodies.h"
+
+typedef WrappingInt MSGID;
+
+struct Fragment;
+
+#if 0
+#define ptrace(x) 
+#else
+#define ptrace(x) { cout << curTimeMillis() % 10000; x }
+#endif
+
+#if 1
+#define etrace(x) 
+#else
+#define etrace(x) { cout << curTimeMillis() % 10000; x }
+#endif
+
+class F; // fragment
+class MR; // message.  R=receiver side.
+class CR; // connection receiver side
+class MS; // message   S=sender side.
+class CS; // connection sender side
+class ProtocolConnection; // connection overall
+
+/* ip:port:channel 
+   We have one receiver thread per process (per ip address destination), and then 
+   multiplex messages out to multiple connections(generally one per thread) which 
+   each have a 'channel'.
+*/
+class EndPoint { 
+public:
+	EndPoint() : channel(0) { }
+	int channel;
+	SockAddr sa;
+	bool operator<(const EndPoint& r) const {
+		if( channel != r.channel )
+			return channel < r.channel;
+		return sa < r.sa;
+	}
+	string toString() { 
+		stringstream out;
+		out << sa.toString() << ':';
+		if( channel == -2 ) out << "ANYCHANNEL";
+		else if( channel == -1 ) out << "AUTOASSIGNCHANNEL";
+		else out << channel;
+		return out.str();
+	}
+};
+
+/* the double underscore stuff here is the actual implementation glue.
+   wanted to keep that stuff clean and separate.  so put your implementation
+   of these in protoimpl.h.
+*/
+
+void __sendRESET(ProtocolConnection *pc);
+
+// sender -> 
+void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *, bool retran=false); // transmit a fragment
+void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, MSGID msgid, int fragNo); // transmit the REQUEST ACK msg
+
+// receiver -> 
+void __sendACK(ProtocolConnection *pc, MSGID msgid); // transmit ACK
+void __sendMISSING(ProtocolConnection *pc, EndPoint& to, MSGID msgid, vector<short>& ids); // transmit MISSING
+
+// -> receiver
+F* __recv(UDPConnection& c, SockAddr& from); /* recv from socket a fragment and pass back */
+
+class F { 
+public:
+	F(Fragment *f);
+	~F();
+	int __num();   //frag #
+	int __len();
+	MSGID __msgid();
+	int __channel();
+	bool __isREQUESTACK(); // if true, this is just a request for acknowledgement not real data
+	int __firstFragMsgLen(); // only works on first fragment
+
+	// sender side:
+	bool __isACK(); // if true, this is an ack of a message
+	bool __isMISSING(); // if true, list of frags to retransmit
+	short* __getMissing(int& n); // get the missing fragno list
+
+	Fragment *internals;
+	enum { NORMAL, ACK, MISSING, REQUESTACK, RESET } op;
+};
+
+class MR { 
+public:
+	MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from);
+	~MR() { freeFrags(); }
+	void freeFrags() { 
+		for( unsigned i = 0; i < f.size(); i++ )
+			delete f[i];
+	}
+	bool got(F *f, EndPoint& from); // received a fragment
+	bool gotFirst() { return f[0] != 0; }
+	ProtocolConnection& pc;
+	void removeFromReceivingList();
+	bool complete();
+	const MSGID msgid;
+	int n() { return f.size(); }
+public:
+	int messageLenExpected;
+	int nExpected, nReceived;
+	void reportMissings(bool reportAll);
+	vector<F*> f;
+	vector<unsigned> reportTimes;
+	EndPoint from;
+};
+
+/* this is for knowing what is already received.  we might get dup packets later and need 
+   to ignore them. */
+class MsgTracker {
+public:
+	std::list<MSGID> recentlyReceivedList;
+	std::set<MSGID> recentlyReceived;
+	MSGID lastFullyReceived;
+
+	void reset() { 
+		recentlyReceivedList.clear();
+		recentlyReceived.clear();
+		lastFullyReceived = 0;
+	}
+
+	void got(MSGID m) {
+		unsigned sz = recentlyReceived.size();
+		if( sz > 256 ) {
+			recentlyReceived.erase(recentlyReceivedList.front());
+			recentlyReceivedList.pop_front();
+		}
+		recentlyReceivedList.push_back(m);
+		recentlyReceived.insert(m);
+		if( m > lastFullyReceived || sz == 0 )
+			lastFullyReceived = m;
+	}
+};
+
+class CR { 
+	friend class MR;
+public:
+	~CR() { ptrace( cout << ".warning: ~CR() not implemented" << endl; ) }
+	CR(ProtocolConnection& _pc) : pc(_pc) { }
+	MR* recv();
+public:
+	MR* getPendingMsg(F *fr);
+	bool oldMessageId(int channel, MSGID m);
+	void queueReceived(MR*);
+
+	ProtocolConnection& pc;
+	boost::condition receivedSome;
+ 	vector<MR*> received; /* ready to dequeue and use */
+	map<int,MR*> pendingMessages; /* partly received msgs */
+	MsgTracker oldMsgTracker;
+};
+
+/* -- sender side ------------------------------------------------*/
+
+class CS { 
+public:
+	CS(ProtocolConnection& _pc);
+
+	ProtocolConnection& pc;
+	vector<MS*> pendingSend;
+	boost::condition msgSent;
+	void resetIt();
+
+	double delayMax;
+	double delay;
+	void delayGotMissing() { 
+		double delayOld = delay;
+		if( delay == 0 )
+			delay = 2.0;
+		else
+			delay = delay * 1.25;
+		if( delay > delayMax ) delay = delayMax;
+		if( delay != delayOld )
+			cout << ".DELAY INCREASED TO " << delay << endl;
+	}
+	void delaySentMsg() { 
+		if( delay != 0.0 ) {
+			delay = delay * 0.5;
+			if( delay<0.5 ) delay = 0;
+			cout << ".DELAY DECREASED TO " << delay << endl;
+		}
+	}
+};
+
+typedef map<EndPoint,ProtocolConnection*> EndPointToPC;
+extern EndPointToPC pcMap; /* the *far* endpoint -> pc */
+
+/* -- overall Connection object ----------------------------------*/
+
+#pragma warning( disable: 4355 )
+
+class ProtocolConnection { 
+public:
+	string toString();
+
+	ProtocolConnection(ProtocolConnection& par, EndPoint& to);
+	ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd);
+	~ProtocolConnection();
+
+	void shutdown();
+	bool acceptAnyChannel() const;
+	UDPConnection& udpConnection;
+	/* note the channel for myEnd might be "any" for the any pc -
+  	   so you can't use that channel for sending.  Use MS/MR 
+	   for that.
+	   */
+	EndPoint myEnd;
+	EndPoint farEnd;
+
+	/* if this was instantiated automatically for an acceptAnyChannel(), 
+	   keep a ptr back to it and queue received msgs there. 
+	*/
+	ProtocolConnection *parent;
+
+	CR cr;
+	CS cs;
+	bool first; // true if yet to send first message on this conn
+
+private:
+	void init();
+};
+
+/* -- sender side ------------------------------------------------*/
+
+class MS { 
+public:
+	MS(ProtocolConnection *_pc, EndPoint &_to, MSGID _msgid) : 
+	  pc(_pc), to(_to), msgid(_msgid), complainInterval(50) { }
+	~MS() { 
+		for( unsigned i = 0; i < fragments.size(); i++ )
+			delete fragments[i];
+	}
+
+	/* init fragments, then call this */
+	void send();
+
+	vector<F*> fragments;
+
+	/* request retrainsmissions. */
+	bool complain(unsigned now);
+
+	ProtocolConnection* pc;
+	EndPoint to;
+	const MSGID msgid;
+	unsigned lastComplainTime;
+	unsigned complainInterval;
+};
diff --git a/grid/protoimpl.h b/grid/protoimpl.h
index 5ad19a47250..9ca5277e527 100644
--- a/grid/protoimpl.h
+++ b/grid/protoimpl.h
@@ -1,242 +1,242 @@
-// protoimpl.h

-

-#pragma once

-

-/* packet dumping level of detail. */

-const bool dumpPackets = false; // this must be true to get anything at all

-const bool dumpIP = false; // output the ip address

-const bool dumpBytesDetailed = false; // more data output

-

-#include "message.h"

-

-extern boost::mutex coutmutex;

-

-const int FragMax = 1480;

-const int FragHeader = 10;

-const int MSS = FragMax - FragHeader;

-

-#pragma pack(push)

-#pragma pack(1)

-

-struct Fragment {

-	enum { MinFragmentLen = FragHeader + 1 };

-	MSGID msgId;

-	short channel;

-	short fragmentLen;

-	short fragmentNo;

-	char data[16];

-	int fragmentDataLen() { return fragmentLen - FragHeader; }

-	char* fragmentData() { return data; }

-

-	bool ok(int nRead) { 

-		if( nRead < MinFragmentLen || fragmentLen > nRead || fragmentLen < MinFragmentLen ) {

-			ptrace( cout << ".recv: fragment bad. fragmentLen:" << fragmentLen << " nRead:" << nRead << endl; )

-			return false;

-		}

-		if( fragmentNo == 0 && fragmentLen < MinFragmentLen + MsgDataHeaderSize ) { 

-			ptrace( cout << ".recv: bad first fragment. fragmentLen:" << fragmentLen << endl; )

-			return false;

-		}

-		return true;

-	} 

-

-	MsgData* startOfMsgData() { assert(fragmentNo == 0); return (MsgData *) data; }

-};

-#pragma pack(pop)

-

-inline void DUMP(Fragment& f, SockAddr& t, const char *tabs) { 

-	if( !dumpPackets )

-		return;

-	cout << tabs << curTimeMillis() % 10000 << ' ';

-	short s = f.fragmentNo;

-	if( s == -32768 ) 

-		cout << "ACK M:" << f.msgId % 1000;

-	else if( s == -32767 )

-		cout << "MISSING";

-	else if( s == -32766 )

-		cout << "RESET ch:" << f.channel;

-	else if( s < 0 )

-		cout << "REQUESTACK";

-	else

-		cout << '#' << s << ' ' << f.fragmentLen << " M:" << f.msgId % 1000;

-	cout << ' ';

-	if( dumpIP )

-		cout << t.toString();

-}

-

-inline void DUMPDATA(Fragment& f, const char *tabs) { 

-	if( !dumpPackets )

-		return;

-	if( f.fragmentNo >= 0 ) { 

-		cout << '\n' << tabs;

-		int x = f.fragmentDataLen();

-		if( dumpBytesDetailed ) { 

-			char *p = (char *) &f;

-			cout << hex << *((unsigned*)p) << ' '; p+=4;

-			cout << *((short*)p) << ' '; p+=2;

-			cout << *((short*)p) << ' '; p+=2;

-			cout << *((short*)p) << '|'; p+=2;

-			if( x < 16 ) cout << "???";

-			else {

-				for( int i = 0; i < 4; i++ ) { // MSGDATA

-					cout << *((unsigned*)p);

-					cout << (i < 3 ? ' ' : '|');

-					p += 4;

-				}

-				cout << '\n' << tabs;

-				x -= 16;

-				if( x > 32 ) x = 32;

-				while( x-- > 0 ) { cout << (unsigned) (unsigned char) *p++ << ' '; } 

-			}

-		}

-		else { 

-			char *p = f.data;

-			if( f.fragmentNo == 0 ) { 

-				p += 16; x -= 16;

-			}

-			if( x > 28 ) x = 28;

-			for( int i = 0; i < x; i++ ) {

-				if( *p == 0 ) cout << (char) 0xb0;

-				else cout << (*p >= 32 ? *p : '.');

-				p++;

-			}

-		}

-	}

-	cout << dec << endl;

-}

-

-inline void SEND(UDPConnection& c, Fragment &f, SockAddr& to, const char *extra="") { 

-	lock lk(coutmutex);

-	DUMP(f, to, "\t\t\t\t\t>");

-	c.sendto((char *) &f, f.fragmentLen, to);

-	if( dumpPackets )

-		cout << extra;

-	DUMPDATA(f, "\t\t\t\t\t      ");

-}

-

-// sender ->

-inline void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *f, bool retran) {

-	assert( f->internals->channel == to.channel );

-    ptrace( cout << ".sendfrag " << f->__num() << ' ' << retran << endl; )

-	SEND(pc->udpConnection, *f->internals, to.sa, retran ? " retran" : "");

-}

-

-inline void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, 

-							 MSGID msgid, int fragNo) { 

-	Fragment f;

-	f.msgId = msgid;

-	f.channel = to.channel; assert( f.channel >= 0 );

-	f.fragmentNo = ((short) -fragNo) -1;

-	f.fragmentLen = FragHeader;

-	ptrace( cout << ".requesting ack, fragno=" << f.fragmentNo << " msg:" << f.msgId << ' ' << to.toString() << endl; )

-	SEND(pc->udpConnection, f, to.sa);

-}

-

-// receiver -> 

-inline void __sendACK(ProtocolConnection *pc, MSGID msgid) {

-	ptrace( cout << "...__sendACK() to:" << pc->farEnd.toString() << " msg:" << msgid << endl; )

-	Fragment f;

-	f.msgId = msgid;

-	f.channel = pc->farEnd.channel; assert( f.channel >= 0 );

-	f.fragmentNo = -32768;

-	f.fragmentLen = FragHeader;

-	SEND(pc->udpConnection, f, pc->farEnd.sa);

-}

-

-/* this is to clear old state for the channel in terms of what msgids are 

-   already sent. 

-*/

-inline void __sendRESET(ProtocolConnection *pc) {

-	Fragment f;

-	f.msgId = -1;

-	f.channel = pc->farEnd.channel; assert( f.channel >= 0 );

-	f.fragmentNo = -32766;

-	f.fragmentLen = FragHeader;

-	ptrace( cout << "...__sendRESET() to:" << pc->farEnd.toString() << endl; )

-	SEND(pc->udpConnection, f, pc->farEnd.sa);

-}

-

-inline void __sendMISSING(ProtocolConnection *pc, EndPoint& to, 

-						  MSGID msgid, vector<short>& ids) {

-	int n = ids.size(); 

-	ptrace( cout << "..sendMISSING n:" << n << " firstmissing:" << ids[0] << " last:" << ids[ids.size()-1] << to.toString() << endl; )

-	if( n > 256 ) {

-		ptrace( cout << "\t..sendMISSING limiting to 256 ids" << endl; )

-		n = 256;

-	}

-	Fragment *f = (Fragment*) malloc(FragHeader + n*2);

-	f->msgId = msgid;

-	f->channel = to.channel; assert( f->channel >= 0 );

-	f->fragmentNo = -32767;

-	f->fragmentLen = FragHeader + n*2;

-	short *s = (short *) f->data;

-	for( int i = 0; i < n; i++ )

-		*s++ = ids[i];

-//	ptrace( cout << "...sendMISSING fraglen:" << f->fragmentLen << endl; )

-	SEND(pc->udpConnection, *f, to.sa);

-	free(f);

-}

-

-// -> receiver

-inline F* __recv(UDPConnection& c, SockAddr& from) {

-	Fragment *f = (Fragment *) malloc(MaxMTU);

-	int n;

-	while( 1 ) {

-//		n = c.recvfrom((char*) f, c.mtu(), from);

-		n = c.recvfrom((char*) f, MaxMTU, from);

-//		cout << "recvfrom returned " << n << endl;

-		if( n >= 0 )

-			break;

-		if( !goingAway ) {

-			cout << ".recvfrom returned error " << getLastError() << " socket:" << c.sock << endl;

-			cout << "sleeping 2 seconds " << endl;

-			sleepsecs(2);

-		}

-	}

-	assert( f->fragmentLen == n );

-	if( f->fragmentNo > 0 ) {

-		// don't waste tons of space if the maxmtu is 16k but we get 1480

-		unsigned newsz = (f->fragmentLen + 255) & 0xffffff00;

-		if( newsz < MaxMTU )

-			f = (Fragment *) realloc(f, newsz);

-	}

-	{

-		lock lk(coutmutex);

-		DUMP(*f, from, "\t\t\t\t\t\t\t\t\t\t<");

-		DUMPDATA(*f,   "\t\t\t\t\t\t\t\t\t\t      ");

-	}

-	return new F(f);

-}

-

-inline F::F(Fragment *f) : internals(f), op(NORMAL) { 

-	if( internals->fragmentNo < 0 ) { 

-		if( internals->fragmentNo == -32768 ) {

-			op = ACK;

-			ptrace( cout << ".got ACK msg:" << internals->msgId << endl; )

-		} else if( internals->fragmentNo == -32767 ) {

-			op = MISSING;

-			ptrace( cout << ".got MISSING" << endl; )

-		} else if( internals->fragmentNo == -32766 ) {

-			op = RESET;

-		} else {

-			op = REQUESTACK;

-			internals->fragmentNo = -(internals->fragmentNo+1);

-			ptrace( cout << ".got REQUESTACK frag:" << internals->fragmentNo << " msg:" << internals->msgId << endl; )

-		}

-	}

-}

-inline F::~F() { free(internals); internals=0; }

-inline int F::__num() { return internals->fragmentNo; }

-inline int F::__len() { return internals->fragmentLen; }

-inline MSGID F::__msgid() { return internals->msgId; }

-inline int F::__channel() { return internals->channel; }

-inline bool F::__isREQUESTACK() { return op == REQUESTACK; }

-inline bool F::__isACK() { return op == ACK; }

-inline bool F::__isMISSING() { return op == MISSING; }

-inline short* F::__getMissing(int& n) { 

-	n = internals->fragmentDataLen() / 2;

-	return (short *) internals->fragmentData();

-}

-inline int F::__firstFragMsgLen() { 

-	return internals->startOfMsgData()->len;

-}

+// protoimpl.h
+
+#pragma once
+
+/* packet dumping level of detail. */
+const bool dumpPackets = false; // this must be true to get anything at all
+const bool dumpIP = false; // output the ip address
+const bool dumpBytesDetailed = false; // more data output
+
+#include "message.h"
+
+extern boost::mutex coutmutex;
+
+const int FragMax = 1480;
+const int FragHeader = 10;
+const int MSS = FragMax - FragHeader;
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct Fragment {
+	enum { MinFragmentLen = FragHeader + 1 };
+	MSGID msgId;
+	short channel;
+	short fragmentLen;
+	short fragmentNo;
+	char data[16];
+	int fragmentDataLen() { return fragmentLen - FragHeader; }
+	char* fragmentData() { return data; }
+
+	bool ok(int nRead) { 
+		if( nRead < MinFragmentLen || fragmentLen > nRead || fragmentLen < MinFragmentLen ) {
+			ptrace( cout << ".recv: fragment bad. fragmentLen:" << fragmentLen << " nRead:" << nRead << endl; )
+			return false;
+		}
+		if( fragmentNo == 0 && fragmentLen < MinFragmentLen + MsgDataHeaderSize ) { 
+			ptrace( cout << ".recv: bad first fragment. fragmentLen:" << fragmentLen << endl; )
+			return false;
+		}
+		return true;
+	} 
+
+	MsgData* startOfMsgData() { assert(fragmentNo == 0); return (MsgData *) data; }
+};
+#pragma pack(pop)
+
+inline void DUMP(Fragment& f, SockAddr& t, const char *tabs) { 
+	if( !dumpPackets )
+		return;
+	cout << tabs << curTimeMillis() % 10000 << ' ';
+	short s = f.fragmentNo;
+	if( s == -32768 ) 
+		cout << "ACK M:" << f.msgId % 1000;
+	else if( s == -32767 )
+		cout << "MISSING";
+	else if( s == -32766 )
+		cout << "RESET ch:" << f.channel;
+	else if( s < 0 )
+		cout << "REQUESTACK";
+	else
+		cout << '#' << s << ' ' << f.fragmentLen << " M:" << f.msgId % 1000;
+	cout << ' ';
+	if( dumpIP )
+		cout << t.toString();
+}
+
+inline void DUMPDATA(Fragment& f, const char *tabs) { 
+	if( !dumpPackets )
+		return;
+	if( f.fragmentNo >= 0 ) { 
+		cout << '\n' << tabs;
+		int x = f.fragmentDataLen();
+		if( dumpBytesDetailed ) { 
+			char *p = (char *) &f;
+			cout << hex << *((unsigned*)p) << ' '; p+=4;
+			cout << *((short*)p) << ' '; p+=2;
+			cout << *((short*)p) << ' '; p+=2;
+			cout << *((short*)p) << '|'; p+=2;
+			if( x < 16 ) cout << "???";
+			else {
+				for( int i = 0; i < 4; i++ ) { // MSGDATA
+					cout << *((unsigned*)p);
+					cout << (i < 3 ? ' ' : '|');
+					p += 4;
+				}
+				cout << '\n' << tabs;
+				x -= 16;
+				if( x > 32 ) x = 32;
+				while( x-- > 0 ) { cout << (unsigned) (unsigned char) *p++ << ' '; } 
+			}
+		}
+		else { 
+			char *p = f.data;
+			if( f.fragmentNo == 0 ) { 
+				p += 16; x -= 16;
+			}
+			if( x > 28 ) x = 28;
+			for( int i = 0; i < x; i++ ) {
+				if( *p == 0 ) cout << (char) 0xb0;
+				else cout << (*p >= 32 ? *p : '.');
+				p++;
+			}
+		}
+	}
+	cout << dec << endl;
+}
+
+inline void SEND(UDPConnection& c, Fragment &f, SockAddr& to, const char *extra="") { 
+	lock lk(coutmutex);
+	DUMP(f, to, "\t\t\t\t\t>");
+	c.sendto((char *) &f, f.fragmentLen, to);
+	if( dumpPackets )
+		cout << extra;
+	DUMPDATA(f, "\t\t\t\t\t      ");
+}
+
+// sender ->
+inline void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *f, bool retran) {
+	assert( f->internals->channel == to.channel );
+    ptrace( cout << ".sendfrag " << f->__num() << ' ' << retran << endl; )
+	SEND(pc->udpConnection, *f->internals, to.sa, retran ? " retran" : "");
+}
+
+inline void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, 
+							 MSGID msgid, int fragNo) { 
+	Fragment f;
+	f.msgId = msgid;
+	f.channel = to.channel; assert( f.channel >= 0 );
+	f.fragmentNo = ((short) -fragNo) -1;
+	f.fragmentLen = FragHeader;
+	ptrace( cout << ".requesting ack, fragno=" << f.fragmentNo << " msg:" << f.msgId << ' ' << to.toString() << endl; )
+	SEND(pc->udpConnection, f, to.sa);
+}
+
+// receiver -> 
+inline void __sendACK(ProtocolConnection *pc, MSGID msgid) {
+	ptrace( cout << "...__sendACK() to:" << pc->farEnd.toString() << " msg:" << msgid << endl; )
+	Fragment f;
+	f.msgId = msgid;
+	f.channel = pc->farEnd.channel; assert( f.channel >= 0 );
+	f.fragmentNo = -32768;
+	f.fragmentLen = FragHeader;
+	SEND(pc->udpConnection, f, pc->farEnd.sa);
+}
+
+/* this is to clear old state for the channel in terms of what msgids are 
+   already sent. 
+*/
+inline void __sendRESET(ProtocolConnection *pc) {
+	Fragment f;
+	f.msgId = -1;
+	f.channel = pc->farEnd.channel; assert( f.channel >= 0 );
+	f.fragmentNo = -32766;
+	f.fragmentLen = FragHeader;
+	ptrace( cout << "...__sendRESET() to:" << pc->farEnd.toString() << endl; )
+	SEND(pc->udpConnection, f, pc->farEnd.sa);
+}
+
+inline void __sendMISSING(ProtocolConnection *pc, EndPoint& to, 
+						  MSGID msgid, vector<short>& ids) {
+	int n = ids.size(); 
+	ptrace( cout << "..sendMISSING n:" << n << " firstmissing:" << ids[0] << " last:" << ids[ids.size()-1] << to.toString() << endl; )
+	if( n > 256 ) {
+		ptrace( cout << "\t..sendMISSING limiting to 256 ids" << endl; )
+		n = 256;
+	}
+	Fragment *f = (Fragment*) malloc(FragHeader + n*2);
+	f->msgId = msgid;
+	f->channel = to.channel; assert( f->channel >= 0 );
+	f->fragmentNo = -32767;
+	f->fragmentLen = FragHeader + n*2;
+	short *s = (short *) f->data;
+	for( int i = 0; i < n; i++ )
+		*s++ = ids[i];
+//	ptrace( cout << "...sendMISSING fraglen:" << f->fragmentLen << endl; )
+	SEND(pc->udpConnection, *f, to.sa);
+	free(f);
+}
+
+// -> receiver
+inline F* __recv(UDPConnection& c, SockAddr& from) {
+	Fragment *f = (Fragment *) malloc(MaxMTU);
+	int n;
+	while( 1 ) {
+//		n = c.recvfrom((char*) f, c.mtu(), from);
+		n = c.recvfrom((char*) f, MaxMTU, from);
+//		cout << "recvfrom returned " << n << endl;
+		if( n >= 0 )
+			break;
+		if( !goingAway ) {
+			cout << ".recvfrom returned error " << getLastError() << " socket:" << c.sock << endl;
+			cout << "sleeping 2 seconds " << endl;
+			sleepsecs(2);
+		}
+	}
+	assert( f->fragmentLen == n );
+	if( f->fragmentNo > 0 ) {
+		// don't waste tons of space if the maxmtu is 16k but we get 1480
+		unsigned newsz = (f->fragmentLen + 255) & 0xffffff00;
+		if( newsz < MaxMTU )
+			f = (Fragment *) realloc(f, newsz);
+	}
+	{
+		lock lk(coutmutex);
+		DUMP(*f, from, "\t\t\t\t\t\t\t\t\t\t<");
+		DUMPDATA(*f,   "\t\t\t\t\t\t\t\t\t\t      ");
+	}
+	return new F(f);
+}
+
+inline F::F(Fragment *f) : internals(f), op(NORMAL) { 
+	if( internals->fragmentNo < 0 ) { 
+		if( internals->fragmentNo == -32768 ) {
+			op = ACK;
+			ptrace( cout << ".got ACK msg:" << internals->msgId << endl; )
+		} else if( internals->fragmentNo == -32767 ) {
+			op = MISSING;
+			ptrace( cout << ".got MISSING" << endl; )
+		} else if( internals->fragmentNo == -32766 ) {
+			op = RESET;
+		} else {
+			op = REQUESTACK;
+			internals->fragmentNo = -(internals->fragmentNo+1);
+			ptrace( cout << ".got REQUESTACK frag:" << internals->fragmentNo << " msg:" << internals->msgId << endl; )
+		}
+	}
+}
+inline F::~F() { free(internals); internals=0; }
+inline int F::__num() { return internals->fragmentNo; }
+inline int F::__len() { return internals->fragmentLen; }
+inline MSGID F::__msgid() { return internals->msgId; }
+inline int F::__channel() { return internals->channel; }
+inline bool F::__isREQUESTACK() { return op == REQUESTACK; }
+inline bool F::__isACK() { return op == ACK; }
+inline bool F::__isMISSING() { return op == MISSING; }
+inline short* F::__getMissing(int& n) { 
+	n = internals->fragmentDataLen() / 2;
+	return (short *) internals->fragmentData();
+}
+inline int F::__firstFragMsgLen() { 
+	return internals->startOfMsgData()->len;
+}
diff --git a/grid/protorecv.cpp b/grid/protorecv.cpp
index 0639939ff08..49200d8ab56 100644
--- a/grid/protorecv.cpp
+++ b/grid/protorecv.cpp
@@ -1,386 +1,386 @@
-// protorecv.cpp

-

-#include "stdafx.h"

-#include "protocol.h"

-#include "boost/thread.hpp"

-#include "../util/goodies.h"

-#include "../util/sock.h"

-#include "protoimpl.h"

-#include "../db/introspect.h"

-

-boost::mutex biglock;

-boost::mutex coutmutex;

-boost::mutex threadStarterMutex;

-boost::condition threadActivate; // creating a new thread, grabbing threadUseThisOne

-ProtocolConnection *threadUseThisOne = 0;

-void receiverThread();

-

-map<SockAddr,ProtocolConnection*> firstPCForThisAddress;

-

-/* todo: eventually support multiple listeners on diff ports.  not 

-         bothering yet.

-*/

-ProtocolConnection *any = 0;

-

-EndPointToPC pcMap;

-

-class GeneralInspector : public SingleResultObjCursor { 

-	Cursor* clone() { return new GeneralInspector(*this); }

-	void fill() {

-		b.append("version", "1.0.0.1");

-		b.append("versionDesc", "none");

-		b.append("nConnections", pcMap.size());

-	}

-public:

-	GeneralInspector() { reg("intr.general"); }

-} _geninspectorproto;

-

-#include <signal.h>

-

-/* our version of netstat */

-void sighandler(int x) { 

-	cout << "ProtocolConnections:" << endl;

-	lock lk(biglock);

-	EndPointToPC::iterator it = pcMap.begin();

-	while( it != pcMap.end() ) { 

-		cout << "  conn " << it->second->toString() << endl;

-		it++;

-	}

-	cout << "any: ";

-	if( any ) cout << any->toString();

-	cout << "\ndone" << endl;

-}

-

-struct SetSignal { 

-	SetSignal() { 

-#if !defined(_WIN32)

-		signal(SIGUSR2, sighandler);

-#endif

-	}

-} setSignal;

-

-string ProtocolConnection::toString() { 

-	stringstream out;

-	out << myEnd.toString() << " <> " <<

-		farEnd.toString() << " rcvd.size:" << cr.received.size() <<

-		" pndngMsgs.size:" << cr.pendingMessages.size() << 

-		" pndngSnd.size:" << cs.pendingSend.size();

-	return out.str();

-} 

-

-ProtocolConnection::~ProtocolConnection() { 

-	cout << ".warning: ~ProtocolConnection() not implemented (leaks mem etc)" << endl;

-	if( any == this )

-		any = 0;

-}

-

-void ProtocolConnection::shutdown() { 

-	ptrace( cout << ".shutdown()" << endl; )

-	if( acceptAnyChannel() || first ) 

-		return;

-	ptrace( cout << ".   to:" << to.toString() << endl; )

-	__sendRESET(this);

-}

-

-inline ProtocolConnection::ProtocolConnection(ProtocolConnection& par, EndPoint& _to) :

-  udpConnection(par.udpConnection), myEnd(par.myEnd), cs(*this), cr(*this) 

-{ 

-	parent = &par;

-	farEnd = _to;

-	first = true;

-	// todo: LOCK

-

-	assert(pcMap.count(farEnd) == 0);

-//	pcMap[myEnd] = this;

-}

-

-inline bool ProtocolConnection::acceptAnyChannel() const { 

-	return myEnd.channel == MessagingPort::ANYCHANNEL; 

-}

-

-inline void ProtocolConnection::init() {

-	first = true;

-	lock lk(biglock);

-	lock tslk(threadStarterMutex);

-

-	if( acceptAnyChannel() ) {

-		assert( any == 0 );

-		any = this;

-	}

-	else {

-		pcMap[farEnd] = this;

-	}

-

-	if( firstPCForThisAddress.count(myEnd.sa) == 0 ) { 

-		firstPCForThisAddress[myEnd.sa] = this;

-		// need a receiver thread.  one per port # we listen on. shared by channels.

-		boost::thread receiver(receiverThread);

-		threadUseThisOne = this;

-		threadActivate.notify_one();

-		return;

-	}

-}

-

-ProtocolConnection::ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd) : 

-  udpConnection(c), myEnd(e), cs(*this), cr(*this) 

-{ 

-	parent = this;

-	if( _farEnd ) { 

-		farEnd.channel = myEnd.channel;

-		farEnd.sa = *_farEnd;

-	}

-	init();

-}

-

-/* find message for fragment */

-MR* CR::getPendingMsg(F *fr) {

-	MR *m;

-	map<int,MR*>::iterator i = pendingMessages.find(fr->__msgid());

-	if( i == pendingMessages.end() ) {

-		if( pendingMessages.size() > 20 ) {		

-			cout << ".warning: pendingMessages.size()>20, ignoring msg until we dequeue" << endl;

-			return 0;

-		}

-		m = new MR(&pc, fr->__msgid(), pc.farEnd);

-		pendingMessages[fr->__msgid()] = m;

-	}

-	else

-		m = i->second;

-	return m;

-/*

-	MR*& m = pendingMessages[fr->__msgid()];

-	if( m == 0 )		

-		m = new MR(&pc, fr->__msgid(), fromAddr);

-	return m;

-*/

-}

-

-void MR::removeFromReceivingList() {

-	pc.cr.pendingMessages.erase(msgid);

-}

-

-MR::MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from) : 

-        pc(*_pc), msgid(_msgid), from(_from), nReceived(0)

-{

-	messageLenExpected = nExpected = -1;

-	f.push_back(0);

-}

-

-void MR::reportMissings(bool reportAll) { 

-	vector<short> missing;

-	unsigned t = curTimeMillis();

-	for( unsigned i = 0; i < f.size(); i++ ) { 

-		if( f[i] == 0 ) {

-			int diff = tdiff(reportTimes[i],t);

-			assert( diff >= 0 || reportTimes[i] == 0 );

-			if( diff > 100 || reportTimes[i]==0 || 

-				(reportAll && diff > 1) ) {

-				reportTimes[i] = t;

-				assert( i < 25000 );

-				missing.push_back(i);

-			}

-		}

-	}

-	if( !missing.empty() )

-		__sendMISSING(&pc, from, msgid, missing);

-}

-

-inline bool MR::complete() { 

-	if( nReceived == nExpected ) {

-		assert( nExpected == (int) f.size() );

-		assert( f[0] != 0 );

-		return true;

-	}

-	return false;

-/*

-	if( nExpected > (int) f.size() || nExpected == -1 )

-		return false;

-	for( unsigned i = 0; i < f.size(); i++ )

-		if( f[i] == 0 )

-			return false;

-	return true;

-*/

-}

-

-/* true=msg complete */

-bool MR::got(F *frag, EndPoint& fromAddr) {

-	MSGID msgid = frag->__msgid();

-	int i = frag->__num();

-	if( i == 544 && !frag->__isREQUESTACK() ) { 

-		cout << "************ GOT LAST FRAGMENT #544" << endl;

-	}

-	if( nExpected < 0 && i == 0 ) {

-		messageLenExpected = frag->__firstFragMsgLen();

-		if( messageLenExpected == frag->__len()-FragHeader )

-			nExpected = 1;

-		else {

-			int mss = frag->__len()-FragHeader;

-			assert( messageLenExpected > mss );

-			nExpected = (messageLenExpected + mss - 1) / mss;

-			ptrace( cout << ".got first frag, expect:" << nExpected << "packets, expectedLen:" << messageLenExpected << endl; )

-		}

-	}

-	if( i >= (int) f.size() )

-		f.resize(i+1, 0);

-	if( frag->__isREQUESTACK() ) { 

-		ptrace( cout << "...got(): processing a REQUESTACK" << endl; )

-		/* we're simply seeing if we got the data, and then acking. */

-		delete frag;

-		frag = 0;

-		reportTimes.resize(f.size(), 0);

-		if( complete() ) {

-			__sendACK(&pc, msgid);

-			return true;

-		} 

-		reportMissings(true);

-		return false;

-	}

-	else if( f[i] != 0 ) { 

-		ptrace( cout << "\t.dup packet i:" << i << ' ' << from.toString() << endl; )

-		delete frag;

-		return false;

-	}

-	if( frag ) { 

-		f[i] = frag;

-		nReceived++;

-	}

-	reportTimes.resize(f.size(), 0);

-

-	if( !complete() )

-		return false;

-	__sendACK(&pc, msgid);

-	return true;

-

-/*

-	if( f[0] == 0 || f[i] == 0 ) {

-//		reportMissings(frag == 0);

-		return false;

-	}

-	if( i+1 < nExpected ) {

-//cout << "TEMP COMMENT" << endl;

-//		if( i > 0 && f[i-1] == 0 )

-//			reportMissings(frag == 0);

-		return false;

-	}

-	// last fragment received

-	if( !complete() ) {

-//		reportMissings(frag == 0);

-		return false;

-	}

-	__sendACK(&pc, fromAddr, msgid);

-	return frag != 0;

-*/

-}

-

-MR* CR::recv() { 

-	MR *x;

-	{

-		lock lk(biglock);

-		while( received.empty() )

-			receivedSome.wait(lk);

-		x = received.back();

-		received.pop_back();

-	}

-	return x;

-}

-

-// this is how we tell protosend.cpp we got these

-void gotACK(F*, ProtocolConnection *);

-void gotMISSING(F*, ProtocolConnection *);

-

-void receiverThread() {

-	ProtocolConnection *startingConn; // this thread manages many; this is just initiator or the parent for acceptany

-	UDPConnection *uc;

-	{

-		lock lk(threadStarterMutex);

-		while( 1 ) {

-			if( threadUseThisOne != 0 ) {

-				uc = &threadUseThisOne->udpConnection;

-				startingConn = threadUseThisOne;

-				threadUseThisOne = 0;

-				break;

-			}

-			threadActivate.wait(lk);

-		}

-	}

-

-	cout << "\n.Activating a new receiverThread\n   " << startingConn->toString() << '\n' << endl;

-

-	EndPoint fromAddr;

-	while( 1 ) {

-		F *f = __recv(*uc, fromAddr.sa);

-		lock l(biglock);

-		fromAddr.channel = f->__channel();

-		ptrace( cout << "..__recv() from:" << fromAddr.toString() << " frag:" << f->__num() << endl; )

-		assert( fromAddr.channel >= 0 );

-		EndPointToPC::iterator it = pcMap.find(fromAddr);

-		ProtocolConnection *mypc;

-		if( it == pcMap.end() ) {

-			if( !startingConn->acceptAnyChannel() ) {

-				cout << ".WARNING: got packet from an unknown endpoint:" << fromAddr.toString() << endl;

-			    cout << ".  this may be ok if you just restarted" << endl;

-				delete f;

-				continue;

-			}

-			cout << ".New connection accepted from " << fromAddr.toString() << endl;

-			mypc = new ProtocolConnection(*startingConn, fromAddr);

-			pcMap[fromAddr] = mypc;

-		}

-		else

-			mypc = it->second;

-

-		assert( fromAddr.channel == mypc->farEnd.channel );

-		MsgTracker& track = mypc->cr.oldMsgTracker;

-

-		if( f->op != F::NORMAL ) {

-			if( f->__isACK() ) { 

-				gotACK(f, mypc);

-				delete f;

-				continue;

-			}

-			if( f->__isMISSING() ) { 

-				gotMISSING(f, mypc);

-				delete f;

-				continue;

-			}

-			if( f->op == F::RESET ) { 

-				ptrace( cout << ".got RESET" << endl; )

-				track.reset();

-				mypc->cs.resetIt();

-				delete f;

-				continue;

-			}

-		}

-

-		if( track.recentlyReceived.count(f->__msgid()) ) { 

-			// already done with it.  ignore, other than acking.

-			if( f->__isREQUESTACK() )

-				__sendACK(mypc, f->__msgid());

-			else { 

-				ptrace( cout << ".ignoring packet about msg already received msg:" << f->__msgid() << " op:" << f->op << endl; )

-			}

-			delete f;

-			continue;

-		}

-

-		if( f->__msgid() <= track.lastFullyReceived && !track.recentlyReceivedList.empty() ) {

-			// reconnect on an old channel?

-			ptrace( cout << ".warning: strange msgid:" << f->__msgid() << " received, last:" << track->lastFullyReceived << " conn:" << fromAddr.toString() << endl; )

-		}

-

-		MR *m = mypc->cr.getPendingMsg(f); /* todo: optimize for single fragment case? */

-		if( m == 0 ) { 

-			ptrace( cout << "..getPendingMsg() returns 0" << endl; )

-			delete f;

-			continue;

-		}

-		if( m->got(f, fromAddr) ) {

-			track.got(m->msgid);

-			m->removeFromReceivingList();

-			{

-				mypc->parent->cr.received.push_back(m);

-				mypc->parent->cr.receivedSome.notify_one();

-			}

-		}

-	}

-}

+// protorecv.cpp
+
+#include "stdafx.h"
+#include "protocol.h"
+#include "boost/thread.hpp"
+#include "../util/goodies.h"
+#include "../util/sock.h"
+#include "protoimpl.h"
+#include "../db/introspect.h"
+
+boost::mutex biglock;
+boost::mutex coutmutex;
+boost::mutex threadStarterMutex;
+boost::condition threadActivate; // creating a new thread, grabbing threadUseThisOne
+ProtocolConnection *threadUseThisOne = 0;
+void receiverThread();
+
+map<SockAddr,ProtocolConnection*> firstPCForThisAddress;
+
+/* todo: eventually support multiple listeners on diff ports.  not 
+         bothering yet.
+*/
+ProtocolConnection *any = 0;
+
+EndPointToPC pcMap;
+
+class GeneralInspector : public SingleResultObjCursor { 
+	Cursor* clone() { return new GeneralInspector(*this); }
+	void fill() {
+		b.append("version", "1.0.0.1");
+		b.append("versionDesc", "none");
+		b.append("nConnections", pcMap.size());
+	}
+public:
+	GeneralInspector() { reg("intr.general"); }
+} _geninspectorproto;
+
+#include <signal.h>
+
+/* our version of netstat */
+void sighandler(int x) { 
+	cout << "ProtocolConnections:" << endl;
+	lock lk(biglock);
+	EndPointToPC::iterator it = pcMap.begin();
+	while( it != pcMap.end() ) { 
+		cout << "  conn " << it->second->toString() << endl;
+		it++;
+	}
+	cout << "any: ";
+	if( any ) cout << any->toString();
+	cout << "\ndone" << endl;
+}
+
+struct SetSignal { 
+	SetSignal() { 
+#if !defined(_WIN32)
+		signal(SIGUSR2, sighandler);
+#endif
+	}
+} setSignal;
+
+string ProtocolConnection::toString() { 
+	stringstream out;
+	out << myEnd.toString() << " <> " <<
+		farEnd.toString() << " rcvd.size:" << cr.received.size() <<
+		" pndngMsgs.size:" << cr.pendingMessages.size() << 
+		" pndngSnd.size:" << cs.pendingSend.size();
+	return out.str();
+} 
+
+ProtocolConnection::~ProtocolConnection() { 
+	cout << ".warning: ~ProtocolConnection() not implemented (leaks mem etc)" << endl;
+	if( any == this )
+		any = 0;
+}
+
+void ProtocolConnection::shutdown() { 
+	ptrace( cout << ".shutdown()" << endl; )
+	if( acceptAnyChannel() || first ) 
+		return;
+	ptrace( cout << ".   to:" << to.toString() << endl; )
+	__sendRESET(this);
+}
+
+inline ProtocolConnection::ProtocolConnection(ProtocolConnection& par, EndPoint& _to) :
+  udpConnection(par.udpConnection), myEnd(par.myEnd), cs(*this), cr(*this) 
+{ 
+	parent = &par;
+	farEnd = _to;
+	first = true;
+	// todo: LOCK
+
+	assert(pcMap.count(farEnd) == 0);
+//	pcMap[myEnd] = this;
+}
+
+inline bool ProtocolConnection::acceptAnyChannel() const { 
+	return myEnd.channel == MessagingPort::ANYCHANNEL; 
+}
+
+inline void ProtocolConnection::init() {
+	first = true;
+	lock lk(biglock);
+	lock tslk(threadStarterMutex);
+
+	if( acceptAnyChannel() ) {
+		assert( any == 0 );
+		any = this;
+	}
+	else {
+		pcMap[farEnd] = this;
+	}
+
+	if( firstPCForThisAddress.count(myEnd.sa) == 0 ) { 
+		firstPCForThisAddress[myEnd.sa] = this;
+		// need a receiver thread.  one per port # we listen on. shared by channels.
+		boost::thread receiver(receiverThread);
+		threadUseThisOne = this;
+		threadActivate.notify_one();
+		return;
+	}
+}
+
+ProtocolConnection::ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd) : 
+  udpConnection(c), myEnd(e), cs(*this), cr(*this) 
+{ 
+	parent = this;
+	if( _farEnd ) { 
+		farEnd.channel = myEnd.channel;
+		farEnd.sa = *_farEnd;
+	}
+	init();
+}
+
+/* find message for fragment */
+MR* CR::getPendingMsg(F *fr) {
+	MR *m;
+	map<int,MR*>::iterator i = pendingMessages.find(fr->__msgid());
+	if( i == pendingMessages.end() ) {
+		if( pendingMessages.size() > 20 ) {		
+			cout << ".warning: pendingMessages.size()>20, ignoring msg until we dequeue" << endl;
+			return 0;
+		}
+		m = new MR(&pc, fr->__msgid(), pc.farEnd);
+		pendingMessages[fr->__msgid()] = m;
+	}
+	else
+		m = i->second;
+	return m;
+/*
+	MR*& m = pendingMessages[fr->__msgid()];
+	if( m == 0 )		
+		m = new MR(&pc, fr->__msgid(), fromAddr);
+	return m;
+*/
+}
+
+void MR::removeFromReceivingList() {
+	pc.cr.pendingMessages.erase(msgid);
+}
+
+MR::MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from) : 
+        pc(*_pc), msgid(_msgid), from(_from), nReceived(0)
+{
+	messageLenExpected = nExpected = -1;
+	f.push_back(0);
+}
+
+void MR::reportMissings(bool reportAll) { 
+	vector<short> missing;
+	unsigned t = curTimeMillis();
+	for( unsigned i = 0; i < f.size(); i++ ) { 
+		if( f[i] == 0 ) {
+			int diff = tdiff(reportTimes[i],t);
+			assert( diff >= 0 || reportTimes[i] == 0 );
+			if( diff > 100 || reportTimes[i]==0 || 
+				(reportAll && diff > 1) ) {
+				reportTimes[i] = t;
+				assert( i < 25000 );
+				missing.push_back(i);
+			}
+		}
+	}
+	if( !missing.empty() )
+		__sendMISSING(&pc, from, msgid, missing);
+}
+
+inline bool MR::complete() { 
+	if( nReceived == nExpected ) {
+		assert( nExpected == (int) f.size() );
+		assert( f[0] != 0 );
+		return true;
+	}
+	return false;
+/*
+	if( nExpected > (int) f.size() || nExpected == -1 )
+		return false;
+	for( unsigned i = 0; i < f.size(); i++ )
+		if( f[i] == 0 )
+			return false;
+	return true;
+*/
+}
+
+/* true=msg complete */
+bool MR::got(F *frag, EndPoint& fromAddr) {
+	MSGID msgid = frag->__msgid();
+	int i = frag->__num();
+	if( i == 544 && !frag->__isREQUESTACK() ) { 
+		cout << "************ GOT LAST FRAGMENT #544" << endl;
+	}
+	if( nExpected < 0 && i == 0 ) {
+		messageLenExpected = frag->__firstFragMsgLen();
+		if( messageLenExpected == frag->__len()-FragHeader )
+			nExpected = 1;
+		else {
+			int mss = frag->__len()-FragHeader;
+			assert( messageLenExpected > mss );
+			nExpected = (messageLenExpected + mss - 1) / mss;
+			ptrace( cout << ".got first frag, expect:" << nExpected << "packets, expectedLen:" << messageLenExpected << endl; )
+		}
+	}
+	if( i >= (int) f.size() )
+		f.resize(i+1, 0);
+	if( frag->__isREQUESTACK() ) { 
+		ptrace( cout << "...got(): processing a REQUESTACK" << endl; )
+		/* we're simply seeing if we got the data, and then acking. */
+		delete frag;
+		frag = 0;
+		reportTimes.resize(f.size(), 0);
+		if( complete() ) {
+			__sendACK(&pc, msgid);
+			return true;
+		} 
+		reportMissings(true);
+		return false;
+	}
+	else if( f[i] != 0 ) { 
+		ptrace( cout << "\t.dup packet i:" << i << ' ' << from.toString() << endl; )
+		delete frag;
+		return false;
+	}
+	if( frag ) { 
+		f[i] = frag;
+		nReceived++;
+	}
+	reportTimes.resize(f.size(), 0);
+
+	if( !complete() )
+		return false;
+	__sendACK(&pc, msgid);
+	return true;
+
+/*
+	if( f[0] == 0 || f[i] == 0 ) {
+//		reportMissings(frag == 0);
+		return false;
+	}
+	if( i+1 < nExpected ) {
+//cout << "TEMP COMMENT" << endl;
+//		if( i > 0 && f[i-1] == 0 )
+//			reportMissings(frag == 0);
+		return false;
+	}
+	// last fragment received
+	if( !complete() ) {
+//		reportMissings(frag == 0);
+		return false;
+	}
+	__sendACK(&pc, fromAddr, msgid);
+	return frag != 0;
+*/
+}
+
+MR* CR::recv() { 
+	MR *x;
+	{
+		lock lk(biglock);
+		while( received.empty() )
+			receivedSome.wait(lk);
+		x = received.back();
+		received.pop_back();
+	}
+	return x;
+}
+
+// this is how we tell protosend.cpp we got these
+void gotACK(F*, ProtocolConnection *);
+void gotMISSING(F*, ProtocolConnection *);
+
+void receiverThread() {
+	ProtocolConnection *startingConn; // this thread manages many; this is just initiator or the parent for acceptany
+	UDPConnection *uc;
+	{
+		lock lk(threadStarterMutex);
+		while( 1 ) {
+			if( threadUseThisOne != 0 ) {
+				uc = &threadUseThisOne->udpConnection;
+				startingConn = threadUseThisOne;
+				threadUseThisOne = 0;
+				break;
+			}
+			threadActivate.wait(lk);
+		}
+	}
+
+	cout << "\n.Activating a new receiverThread\n   " << startingConn->toString() << '\n' << endl;
+
+	EndPoint fromAddr;
+	while( 1 ) {
+		F *f = __recv(*uc, fromAddr.sa);
+		lock l(biglock);
+		fromAddr.channel = f->__channel();
+		ptrace( cout << "..__recv() from:" << fromAddr.toString() << " frag:" << f->__num() << endl; )
+		assert( fromAddr.channel >= 0 );
+		EndPointToPC::iterator it = pcMap.find(fromAddr);
+		ProtocolConnection *mypc;
+		if( it == pcMap.end() ) {
+			if( !startingConn->acceptAnyChannel() ) {
+				cout << ".WARNING: got packet from an unknown endpoint:" << fromAddr.toString() << endl;
+			    cout << ".  this may be ok if you just restarted" << endl;
+				delete f;
+				continue;
+			}
+			cout << ".New connection accepted from " << fromAddr.toString() << endl;
+			mypc = new ProtocolConnection(*startingConn, fromAddr);
+			pcMap[fromAddr] = mypc;
+		}
+		else
+			mypc = it->second;
+
+		assert( fromAddr.channel == mypc->farEnd.channel );
+		MsgTracker& track = mypc->cr.oldMsgTracker;
+
+		if( f->op != F::NORMAL ) {
+			if( f->__isACK() ) { 
+				gotACK(f, mypc);
+				delete f;
+				continue;
+			}
+			if( f->__isMISSING() ) { 
+				gotMISSING(f, mypc);
+				delete f;
+				continue;
+			}
+			if( f->op == F::RESET ) { 
+				ptrace( cout << ".got RESET" << endl; )
+				track.reset();
+				mypc->cs.resetIt();
+				delete f;
+				continue;
+			}
+		}
+
+		if( track.recentlyReceived.count(f->__msgid()) ) { 
+			// already done with it.  ignore, other than acking.
+			if( f->__isREQUESTACK() )
+				__sendACK(mypc, f->__msgid());
+			else { 
+				ptrace( cout << ".ignoring packet about msg already received msg:" << f->__msgid() << " op:" << f->op << endl; )
+			}
+			delete f;
+			continue;
+		}
+
+		if( f->__msgid() <= track.lastFullyReceived && !track.recentlyReceivedList.empty() ) {
+			// reconnect on an old channel?
+			ptrace( cout << ".warning: strange msgid:" << f->__msgid() << " received, last:" << track->lastFullyReceived << " conn:" << fromAddr.toString() << endl; )
+		}
+
+		MR *m = mypc->cr.getPendingMsg(f); /* todo: optimize for single fragment case? */
+		if( m == 0 ) { 
+			ptrace( cout << "..getPendingMsg() returns 0" << endl; )
+			delete f;
+			continue;
+		}
+		if( m->got(f, fromAddr) ) {
+			track.got(m->msgid);
+			m->removeFromReceivingList();
+			{
+				mypc->parent->cr.received.push_back(m);
+				mypc->parent->cr.receivedSome.notify_one();
+			}
+		}
+	}
+}
diff --git a/grid/protosend.cpp b/grid/protosend.cpp
index 2d22450ced1..dfd31036127 100644
--- a/grid/protosend.cpp
+++ b/grid/protosend.cpp
@@ -1,188 +1,188 @@
-// protosend.cpp

-

-/* todo: redo locking! */

-

-#include "stdafx.h"

-#include "protocol.h"

-#include "boost/thread.hpp"

-#include "../util/goodies.h"

-#include "../util/sock.h"

-#include "protoimpl.h"

-using namespace boost;

-typedef boost::mutex::scoped_lock lock;

-

-/* todo: granularity? */

-extern boost::mutex biglock;

-

-void senderComplainThread();

-

-boost::thread sender(senderComplainThread);

-

-bool erasePending(ProtocolConnection *pc, int id);

-

-inline bool MS::complain(unsigned now) {

-	if( tdiff(lastComplainTime, now) < (int) complainInterval )

-		return false;

-	if( complainInterval > 4000 ) { 

-		etrace( cout << ".no ack of send after " << complainInterval/1000 << " seconds " << to.toString() << endl; )

-		if( complainInterval > 35000 ) { 

-			etrace( cout << ".GIVING UP on sending message" << endl; )

-			erasePending(pc, msgid);

-			return true;

-		}

-	}

-	complainInterval *= 2;

-	lastComplainTime = now;

-	__sendREQUESTACK(pc, to, msgid, fragments.size()-1);

-	return false; 

-}

-

-// where's my ack?

-void senderComplainThread() { 

-	sleepmillis(200);

-

-	while( 1 ) { 

-		sleepmillis(50);

-		{

-			lock lk(biglock);

-			unsigned now = curTimeMillis();

-			/* todo: more efficient data structure here. */

-			for( EndPointToPC::iterator i = pcMap.begin(); i != pcMap.end(); i++ ) {

-				ProtocolConnection *pc = i->second;

-				for( vector<MS*>::iterator j = pc->cs.pendingSend.begin(); j < pc->cs.pendingSend.end(); j++ )

-					if( (*j)->complain(now) ) 

-						break;

-			}

-		}

-	}

-}

-

-inline MS* _slocked_getMSForFrag(F* fr, ProtocolConnection *pc) { 

-	int id = fr->__msgid();

-	vector<MS*>::iterator it;

-	for( it = pc->cs.pendingSend.begin(); it<pc->cs.pendingSend.end(); it++ ) {

-		MS *m = *it;

-		if( m->msgid == id )

-			return m;

-	}

-	return 0;

-}

-

-// received a MISSING message from the other end.  retransmit.

-void gotMISSING(F* fr, ProtocolConnection *pc) {

-	pc->cs.delayGotMissing();

-

-	ptrace( cout << ".gotMISSING() msgid:" << fr->__msgid() << endl; )

-	MS *m = _slocked_getMSForFrag(fr, pc);

-	if( m ) {

-		{ 

-			int df = tdiff(m->lastComplainTime, curTimeMillis()) * 2;

-			if( df < 10 )

-				df = 10;

-			if( df > 2000 )

-				df = 2000;

-			m->complainInterval = (unsigned) df;

-			cout << "TEMP: set complainInterval to " << m->complainInterval << endl;

-		}

-

-		int n;

-		short* s = fr->__getMissing(n);

-		assert( n > 0 && n < 5000 );

-		for( int i = 0; i < n; i++ ) {

-			//				ptrace( cout << "..  resending frag #" << s[i] << ' ' << m->to.toString() << endl; )

-			__sendFrag(pc, m->to, m->fragments[s[i]], true);

-			if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {

-				ptrace( cout << "SLEEP" << endl; )

-				sleepmillis((int) pc->cs.delay);

-			}

-		}

-		return;

-	}

-	ptrace( cout << "\t.warning: gotMISSING for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; )

-}

-

-/* done sending a msg, clean up and notify sender to unblock */

-bool erasePending(ProtocolConnection *pc, int id) {

-	vector<MS*>::iterator it;

-	CS& cs = pc->cs;

-	for( it = cs.pendingSend.begin(); it < cs.pendingSend.end(); it++ ) {

-		MS *m = *it;

-		if( m->msgid == id ) { 

-			cs.pendingSend.erase(it);

-			ptrace( cout << "..gotACK/erase: found pendingSend msg:" << id << endl; )

-			delete m;

-			cs.msgSent.notify_one();

-			return true;

-		}

-	}

-	return false;

-}

-

-// received an ACK message. so we can discard our saved copy we were trying to send, we

-// are done with it.

-void gotACK(F* fr, ProtocolConnection *pc) {

-	if( erasePending(pc, fr->__msgid()) )

-		return;

-	ptrace( cout << ".warning: got ack for an unknown msg id:" << fr->__msgid()	<< ' ' << pc->farEnd.toString() << endl; )

-}

-

-void MS::send() {

-	/* flow control */

-	lock lk(biglock);

-	ptrace( cout << "..MS::send() pending=" << pc->cs.pendingSend.size() << endl; )

-

-	if( pc->acceptAnyChannel() ) { 

-		EndPointToPC::iterator it = pcMap.find(to);

-		if( it == pcMap.end() ) {

-			cout << ".ERROR: can't find ProtocolConnection object for " << to.toString() << endl;

-			assert(false);

-			return;

-		}

-		/* switch to the child protocolconnection */

-		pc = it->second;

-	}

-	else {

-		assert(pc->myEnd.channel == to.channel);

-	}

-

-	if( pc->first ) { 

-		pc->first = false;

-		if( pc->myEnd.channel >= 0 )

-			__sendRESET(pc);

-		assert( pc->farEnd.channel == to.channel);

-		if( pc->farEnd.sa.isLocalHost() )

-			pc->cs.delayMax = 1.0;

-	}

-

-	// not locked here, probably ok to call size()

-	if( pc->cs.pendingSend.size() >= 1 ) {

-		cout << ".waiting for queued sends to complete " << pc->cs.pendingSend.size() << endl;

-		while( pc->cs.pendingSend.size() >= 1 )

-			pc->cs.msgSent.wait(lk);

-		cout << ".waitend" << endl;

-	}

-

-	lastComplainTime = curTimeMillis();

-	pc->cs.pendingSend.push_back(this);

-	/* todo: pace */

-	for( unsigned i = 0; i < fragments.size(); i++ ) {

-		__sendFrag(pc, to, fragments[i]);

-		if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {

-			ptrace( cout << ".sleep " << pc->cs.delay << endl; )

-			sleepmillis((int) pc->cs.delay);

-		}

-	}

-

-	pc->cs.delaySentMsg();

-}

-

-CS::CS(ProtocolConnection& _pc) : 

-    pc(_pc), delay(0), delayMax(10)

-{ 

-}

-

-void CS::resetIt() { 

-	for( vector<MS*>::iterator i = pendingSend.begin(); i < pendingSend.end(); i++ )

-		delete (*i);

-	pendingSend.clear();

-}

+// protosend.cpp
+
+/* todo: redo locking! */
+
+#include "stdafx.h"
+#include "protocol.h"
+#include "boost/thread.hpp"
+#include "../util/goodies.h"
+#include "../util/sock.h"
+#include "protoimpl.h"
+using namespace boost;
+typedef boost::mutex::scoped_lock lock;
+
+/* todo: granularity? */
+extern boost::mutex biglock;
+
+void senderComplainThread();
+
+boost::thread sender(senderComplainThread);
+
+bool erasePending(ProtocolConnection *pc, int id);
+
+inline bool MS::complain(unsigned now) {
+	if( tdiff(lastComplainTime, now) < (int) complainInterval )
+		return false;
+	if( complainInterval > 4000 ) { 
+		etrace( cout << ".no ack of send after " << complainInterval/1000 << " seconds " << to.toString() << endl; )
+		if( complainInterval > 35000 ) { 
+			etrace( cout << ".GIVING UP on sending message" << endl; )
+			erasePending(pc, msgid);
+			return true;
+		}
+	}
+	complainInterval *= 2;
+	lastComplainTime = now;
+	__sendREQUESTACK(pc, to, msgid, fragments.size()-1);
+	return false; 
+}
+
+// where's my ack?
+void senderComplainThread() { 
+	sleepmillis(200);
+
+	while( 1 ) { 
+		sleepmillis(50);
+		{
+			lock lk(biglock);
+			unsigned now = curTimeMillis();
+			/* todo: more efficient data structure here. */
+			for( EndPointToPC::iterator i = pcMap.begin(); i != pcMap.end(); i++ ) {
+				ProtocolConnection *pc = i->second;
+				for( vector<MS*>::iterator j = pc->cs.pendingSend.begin(); j < pc->cs.pendingSend.end(); j++ )
+					if( (*j)->complain(now) ) 
+						break;
+			}
+		}
+	}
+}
+
+inline MS* _slocked_getMSForFrag(F* fr, ProtocolConnection *pc) { 
+	int id = fr->__msgid();
+	vector<MS*>::iterator it;
+	for( it = pc->cs.pendingSend.begin(); it<pc->cs.pendingSend.end(); it++ ) {
+		MS *m = *it;
+		if( m->msgid == id )
+			return m;
+	}
+	return 0;
+}
+
+// received a MISSING message from the other end.  retransmit.
+void gotMISSING(F* fr, ProtocolConnection *pc) {
+	pc->cs.delayGotMissing();
+
+	ptrace( cout << ".gotMISSING() msgid:" << fr->__msgid() << endl; )
+	MS *m = _slocked_getMSForFrag(fr, pc);
+	if( m ) {
+		{ 
+			int df = tdiff(m->lastComplainTime, curTimeMillis()) * 2;
+			if( df < 10 )
+				df = 10;
+			if( df > 2000 )
+				df = 2000;
+			m->complainInterval = (unsigned) df;
+			cout << "TEMP: set complainInterval to " << m->complainInterval << endl;
+		}
+
+		int n;
+		short* s = fr->__getMissing(n);
+		assert( n > 0 && n < 5000 );
+		for( int i = 0; i < n; i++ ) {
+			//				ptrace( cout << "..  resending frag #" << s[i] << ' ' << m->to.toString() << endl; )
+			__sendFrag(pc, m->to, m->fragments[s[i]], true);
+			if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {
+				ptrace( cout << "SLEEP" << endl; )
+				sleepmillis((int) pc->cs.delay);
+			}
+		}
+		return;
+	}
+	ptrace( cout << "\t.warning: gotMISSING for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; )
+}
+
+/* done sending a msg, clean up and notify sender to unblock */
+bool erasePending(ProtocolConnection *pc, int id) {
+	vector<MS*>::iterator it;
+	CS& cs = pc->cs;
+	for( it = cs.pendingSend.begin(); it < cs.pendingSend.end(); it++ ) {
+		MS *m = *it;
+		if( m->msgid == id ) { 
+			cs.pendingSend.erase(it);
+			ptrace( cout << "..gotACK/erase: found pendingSend msg:" << id << endl; )
+			delete m;
+			cs.msgSent.notify_one();
+			return true;
+		}
+	}
+	return false;
+}
+
+// received an ACK message. so we can discard our saved copy we were trying to send, we
+// are done with it.
+void gotACK(F* fr, ProtocolConnection *pc) {
+	if( erasePending(pc, fr->__msgid()) )
+		return;
+	ptrace( cout << ".warning: got ack for an unknown msg id:" << fr->__msgid()	<< ' ' << pc->farEnd.toString() << endl; )
+}
+
+void MS::send() {
+	/* flow control */
+	lock lk(biglock);
+	ptrace( cout << "..MS::send() pending=" << pc->cs.pendingSend.size() << endl; )
+
+	if( pc->acceptAnyChannel() ) { 
+		EndPointToPC::iterator it = pcMap.find(to);
+		if( it == pcMap.end() ) {
+			cout << ".ERROR: can't find ProtocolConnection object for " << to.toString() << endl;
+			assert(false);
+			return;
+		}
+		/* switch to the child protocolconnection */
+		pc = it->second;
+	}
+	else {
+		assert(pc->myEnd.channel == to.channel);
+	}
+
+	if( pc->first ) { 
+		pc->first = false;
+		if( pc->myEnd.channel >= 0 )
+			__sendRESET(pc);
+		assert( pc->farEnd.channel == to.channel);
+		if( pc->farEnd.sa.isLocalHost() )
+			pc->cs.delayMax = 1.0;
+	}
+
+	// not locked here, probably ok to call size()
+	if( pc->cs.pendingSend.size() >= 1 ) {
+		cout << ".waiting for queued sends to complete " << pc->cs.pendingSend.size() << endl;
+		while( pc->cs.pendingSend.size() >= 1 )
+			pc->cs.msgSent.wait(lk);
+		cout << ".waitend" << endl;
+	}
+
+	lastComplainTime = curTimeMillis();
+	pc->cs.pendingSend.push_back(this);
+	/* todo: pace */
+	for( unsigned i = 0; i < fragments.size(); i++ ) {
+		__sendFrag(pc, to, fragments[i]);
+		if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {
+			ptrace( cout << ".sleep " << pc->cs.delay << endl; )
+			sleepmillis((int) pc->cs.delay);
+		}
+	}
+
+	pc->cs.delaySentMsg();
+}
+
+CS::CS(ProtocolConnection& _pc) : 
+    pc(_pc), delay(0), delayMax(10)
+{ 
+}
+
+void CS::resetIt() { 
+	for( vector<MS*>::iterator i = pendingSend.begin(); i < pendingSend.end(); i++ )
+		delete (*i);
+	pendingSend.clear();
+}
diff --git a/stdafx.cpp b/stdafx.cpp
index 0d4d4346506..c61f9d1ab85 100644
--- a/stdafx.cpp
+++ b/stdafx.cpp
@@ -1,33 +1,33 @@
-// stdafx.cpp : source file that includes just the standard includes

-// db.pch will be the pre-compiled header

-// stdafx.obj will contain the pre-compiled type information

-

-#include "stdafx.h"

-

-// TODO: reference any additional headers you need in STDAFX.H

-// and not in this file

-

-struct MyAsserts {

-	MyAsserts() {

-

-	}

-} myassertsstdafx;

-

-#undef assert

-

-#undef yassert

-#include "assert.h"

-

-void sayDbContext(const char *p = 0);

-

-void wasserted(const char *msg, const char *file, unsigned line) { 

-	problem() << "Assertion failure " << msg << ' ' << file << ' ' << line << endl;

-	cout << "Assertion failure " << msg << endl;

-	cout << ' ' << file << ' ' << line << endl;

-	sayDbContext();

-}

-

-void asserted(const char *msg, const char *file, unsigned line) { 

-	wasserted(msg, file, line);

-	throw AssertionException();

-}

+// stdafx.cpp : source file that includes just the standard includes
+// db.pch will be the pre-compiled header
+// stdafx.obj will contain the pre-compiled type information
+
+#include "stdafx.h"
+
+// TODO: reference any additional headers you need in STDAFX.H
+// and not in this file
+
+struct MyAsserts {
+	MyAsserts() {
+
+	}
+} myassertsstdafx;
+
+#undef assert
+
+#undef yassert
+#include "assert.h"
+
+void sayDbContext(const char *p = 0);
+
+void wasserted(const char *msg, const char *file, unsigned line) { 
+	problem() << "Assertion failure " << msg << ' ' << file << ' ' << line << endl;
+	cout << "Assertion failure " << msg << endl;
+	cout << ' ' << file << ' ' << line << endl;
+	sayDbContext();
+}
+
+void asserted(const char *msg, const char *file, unsigned line) { 
+	wasserted(msg, file, line);
+	throw AssertionException();
+}
diff --git a/stdafx.h b/stdafx.h
index 2c27004912a..0ccb6b17d99 100644
--- a/stdafx.h
+++ b/stdafx.h
@@ -1,111 +1,111 @@
-// stdafx.h : include file for standard system include files,

-// or project specific include files that are used frequently, but

-// are changed infrequently

-//

-

-#pragma once

-

-#if defined(_WIN32)

-const bool debug=true;

-#else

-const bool debug=false;

-#endif

-

-#include "targetver.h"

-

-//#include "assert.h"

-

-// you can catch these

-class AssertionException { 

-public:

-	AssertionException() { }

-};

-

-void asserted(const char *msg, const char *file, unsigned line);

-void wasserted(const char *msg, const char *file, unsigned line);

-

-#ifdef assert

-#undef assert

-#endif

-

-#define assert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )

-

-#define xassert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )

-

-#define yassert 1

-

-/* warning only - keeps going */

-#define wassert(_Expression) (void)( (!!(_Expression)) || (wasserted(#_Expression, __FILE__, __LINE__), 0) )

-

-#include <stdio.h>

-#include <sstream>

-#include <signal.h>

-

-typedef char _TCHAR;

-

-#include <iostream>

-#include <fstream>

-using namespace std;

-

-#include "time.h"

-#include <map>

-#include <string>

-#include <vector>

-#include <set>

-

-#if !defined(_WIN32)

-typedef int HANDLE;

-inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); }

-#else

-typedef void *HANDLE;

-#endif

-

-//#if defined(CHAR)

-//#error CHAR already defined?

-//#endif

-

-//#if defined(_WIN32_WINNT)

-//typedef wchar_t CHAR;

-//#else

-// more to be done...linux unicode is 32 bit.

-//typedef unsigned short CHAR; // 16 bit unicode

-//#endif

-

-#define null (0)

-

-#include <vector>

-

-// for debugging

-typedef struct _Ints { int i[100]; } *Ints;

-typedef struct _Chars { char c[200]; } *Chars;

-

-typedef char CHARS[400];

-

-typedef struct _OWS {

-	int size;

-	char type;

-	char string[400];

-} *OWS;

-

-extern ofstream problems;

-

-class Client;

-extern Client *client;

-extern const char *curNs;

-

-// not threadsafe

-inline ofstream& problem() { 

-	time_t t;

-	time(&t);

-	string now(ctime(&t),0,20);

-	problems << now;

-	if( client ) 

-		problems << curNs << ' ';

-	return problems;

-}

-

-#define DEBUGGING if( 0 ) 

-

-extern unsigned occasion; 

-

-#define OCCASIONALLY if( ++occasion % 16 == 0 ) 

+// stdafx.h : include file for standard system include files,
+// or project specific include files that are used frequently, but
+// are changed infrequently
+//
+
+#pragma once
+
+#if defined(_WIN32)
+const bool debug=true;
+#else
+const bool debug=false;
+#endif
+
+#include "targetver.h"
+
+//#include "assert.h"
+
+// you can catch these
+class AssertionException { 
+public:
+	AssertionException() { }
+};
+
+void asserted(const char *msg, const char *file, unsigned line);
+void wasserted(const char *msg, const char *file, unsigned line);
+
+#ifdef assert
+#undef assert
+#endif
+
+#define assert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )
+
+#define xassert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )
+
+#define yassert 1
+
+/* warning only - keeps going */
+#define wassert(_Expression) (void)( (!!(_Expression)) || (wasserted(#_Expression, __FILE__, __LINE__), 0) )
+
+#include <stdio.h>
+#include <sstream>
+#include <signal.h>
+
+typedef char _TCHAR;
+
+#include <iostream>
+#include <fstream>
+using namespace std;
+
+#include "time.h"
+#include <map>
+#include <string>
+#include <vector>
+#include <set>
+
+#if !defined(_WIN32)
+typedef int HANDLE;
+inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); }
+#else
+typedef void *HANDLE;
+#endif
+
+//#if defined(CHAR)
+//#error CHAR already defined?
+//#endif
+
+//#if defined(_WIN32_WINNT)
+//typedef wchar_t CHAR;
+//#else
+// more to be done...linux unicode is 32 bit.
+//typedef unsigned short CHAR; // 16 bit unicode
+//#endif
+
+#define null (0)
+
+#include <vector>
+
+// for debugging
+typedef struct _Ints { int i[100]; } *Ints;
+typedef struct _Chars { char c[200]; } *Chars;
+
+typedef char CHARS[400];
+
+typedef struct _OWS {
+	int size;
+	char type;
+	char string[400];
+} *OWS;
+
+extern ofstream problems;
+
+class Client;
+extern Client *client;
+extern const char *curNs;
+
+// not threadsafe
+inline ofstream& problem() { 
+	time_t t;
+	time(&t);
+	string now(ctime(&t),0,20);
+	problems << now;
+	if( client ) 
+		problems << curNs << ' ';
+	return problems;
+}
+
+#define DEBUGGING if( 0 ) 
+
+extern unsigned occasion; 
+
+#define OCCASIONALLY if( ++occasion % 16 == 0 ) 
diff --git a/targetver.h b/targetver.h
index 3965af59bcd..11ff16bf5aa 100644
--- a/targetver.h
+++ b/targetver.h
@@ -1,5 +1,5 @@
-#pragma once

-#ifndef _WIN32_WINNT            // Allow use of features specific to Windows Vista or later.

-#define _WIN32_WINNT 0x0600     // Change this to the appropriate value to target other versions of Windows.

-#endif

-

+#pragma once
+#ifndef _WIN32_WINNT            // Allow use of features specific to Windows Vista or later.
+#define _WIN32_WINNT 0x0600     // Change this to the appropriate value to target other versions of Windows.
+#endif
+
diff --git a/util/builder.h b/util/builder.h
index ddc49ec2c91..67b8c38d87f 100644
--- a/util/builder.h
+++ b/util/builder.h
@@ -1,67 +1,67 @@
-/* builder.h

-

-*/

-

-#pragma once

-

-#include "../stdafx.h"

-

-class BufBuilder {

-public:

-	BufBuilder(int initsize = 512) : size(initsize) { 

-		data = (char *) malloc(size); assert(data);

-		l = 0;

-	}

-	~BufBuilder() { kill(); } 

-

-	void kill() {

-		if( data ) {

-			free(data);

-			data = 0;

-		}

-	}

-

-	/* leave room for some stuff later */

-	void skip(int n) { grow(n); }

-

-	/* note this may be deallocated (realloced) if you keep writing. */

-	char* buf() { return data; }

-

-	/* assume ownership of the buffer - you must then free it */

-	void decouple() { data = 0;	}

-

-	template<class T> void append(T j) { *((T*)grow(sizeof(T))) = j; }

-	void append(short j) { append<short>(j); }

-	void append(int j) { append<int>(j); }

-	void append(unsigned j) { append<unsigned>(j); }

-	void append(bool j) { append<bool>(j); }

-	void append(double j) { append<double>(j); }

-

-	void append(void *src, int len) { memcpy(grow(len), src, len); }

-

-	void append(const char *str) { 

-		append((void*) str, strlen(str)+1);

-	}

-

-	int len() { return l; }

-

-private:

-	/* returns the pre-grow write position */

-	char* grow(int by) {

-		int oldlen = l;

-		l += by;

-		if( l > size ) {

-			int a = size * 2;

-			if( l > a )

-				a = l + 16 * 1024;

-			assert( a < 64 * 1024 * 1024 );

-			data = (char *) realloc(data, a);

-			size= a;

-		}

-		return data + oldlen;

-	}

-

-	char *data;

-	int l;

-	int size;

-};

+/* builder.h
+
+*/
+
+#pragma once
+
+#include "../stdafx.h"
+
+class BufBuilder {
+public:
+	BufBuilder(int initsize = 512) : size(initsize) { 
+		data = (char *) malloc(size); assert(data);
+		l = 0;
+	}
+	~BufBuilder() { kill(); } 
+
+	void kill() {
+		if( data ) {
+			free(data);
+			data = 0;
+		}
+	}
+
+	/* leave room for some stuff later */
+	void skip(int n) { grow(n); }
+
+	/* note this may be deallocated (realloced) if you keep writing. */
+	char* buf() { return data; }
+
+	/* assume ownership of the buffer - you must then free it */
+	void decouple() { data = 0;	}
+
+	template<class T> void append(T j) { *((T*)grow(sizeof(T))) = j; }
+	void append(short j) { append<short>(j); }
+	void append(int j) { append<int>(j); }
+	void append(unsigned j) { append<unsigned>(j); }
+	void append(bool j) { append<bool>(j); }
+	void append(double j) { append<double>(j); }
+
+	void append(void *src, int len) { memcpy(grow(len), src, len); }
+
+	void append(const char *str) { 
+		append((void*) str, strlen(str)+1);
+	}
+
+	int len() { return l; }
+
+private:
+	/* returns the pre-grow write position */
+	char* grow(int by) {
+		int oldlen = l;
+		l += by;
+		if( l > size ) {
+			int a = size * 2;
+			if( l > a )
+				a = l + 16 * 1024;
+			assert( a < 64 * 1024 * 1024 );
+			data = (char *) realloc(data, a);
+			size= a;
+		}
+		return data + oldlen;
+	}
+
+	char *data;
+	int l;
+	int size;
+};
diff --git a/util/goodies.h b/util/goodies.h
index 79933a9a2b8..94ca721aa8b 100644
--- a/util/goodies.h
+++ b/util/goodies.h
@@ -1,128 +1,128 @@
-// goodies.h

-// miscellaneous junk

-

-#pragma once

-

-#include "../stdafx.h"

-

-#if !defined(_WIN32)

-#include <pthread.h>

-inline pthread_t GetCurrentThreadId() { return pthread_self(); }

-#endif

-

-/* set to TRUE if we are exiting */

-extern bool goingAway;

-

-bool isPrime(int n); 

-int nextPrime(int n);

-

-inline void dumpmemory(const char *data, int len) { 

-	if( len > 1024 )

-		len = 1024;

-	try {

-	const char *q = data;

-	const char *p = q;

-	while( len > 0 ) {

-		for( int i = 0; i < 16; i++ ) { 

-			if( *p >= 32 && *p <= 126 )

-				cout << *p;

-			else 

-				cout << '.';

-			p++;

-		}

-		cout << "  ";

-		p -= 16;

-		for( int i = 0; i < 16; i++ )

-			cout << (unsigned) ((unsigned char)*p++) << ' ';

-		cout << endl;

-		len -= 16;

-	}

-	} catch(...) {

-	}

-}

-

-#undef yassert

-#include <boost/thread/thread.hpp>

-#include <boost/thread/xtime.hpp>

-#undef assert

-#define assert xassert

-#define yassert 1

-

-struct WrappingInt {

-	WrappingInt() { x = 0; } 

-	WrappingInt(unsigned z) : x(z) { }

-	unsigned x;

-	operator unsigned() const { return x; }

-	WrappingInt& operator++() { x++; return *this; }

-	static int diff(unsigned a, unsigned b) { return a-b; }

-	bool operator<=(WrappingInt r) { 

-		// platform dependent

-		int df = (r.x - x);

-		return df >= 0;

-	}

-	bool operator>(WrappingInt r) { return !(r<=*this); }

-};

-

-#include <ctime>

-

-inline void time_t_to_String(time_t t, char *buf) {

-#if defined(_WIN32)

-	ctime_s(buf, 64, &t);

-#else

-	ctime_r(&t, buf);

-#endif

-	buf[24] = 0; // don't want the \n

-}

-

-inline void sleepsecs(int s) { 

-	boost::xtime xt;

-	boost::xtime_get(&xt, boost::TIME_UTC);

-	xt.sec += s;

-	boost::thread::sleep(xt);

-}

-inline void sleepmillis(int s) { 

-	boost::xtime xt;

-	boost::xtime_get(&xt, boost::TIME_UTC);

-	xt.nsec += s * 1000000;

-	boost::thread::sleep(xt);

-}

-// note this wraps

-inline int tdiff(unsigned told, unsigned tnew) { 

-	return WrappingInt::diff(tnew, told);

-}

-inline unsigned curTimeMillis() {

-	boost::xtime xt;

-	boost::xtime_get(&xt, boost::TIME_UTC);

-	unsigned t = xt.nsec / 1000000;

-	return (xt.sec & 0xfffff) * 1000 + t;

-}

-inline unsigned long long jsTime() {

-	boost::xtime xt;

-	boost::xtime_get(&xt, boost::TIME_UTC);

-	unsigned long long t = xt.nsec / 1000000;

-	return ((unsigned long long) xt.sec * 1000) + t;

-}

-// measures up to 1024 seconds.  or, 512 seconds with tdiff that is...

-inline unsigned curTimeMicros() {

-	boost::xtime xt;

-	boost::xtime_get(&xt, boost::TIME_UTC);

-	unsigned t = xt.nsec / 1000;

-	unsigned secs = xt.sec % 1024;

-	t = secs*1000000 + t;

-	return t;

-}

-using namespace boost;

-typedef boost::mutex::scoped_lock lock;

-

-// simple scoped timer

-class Timer { 

-public:

-	Timer() { old = curTimeMicros(); }

-	int millis() { return micros() / 1000; }

-	int micros() { 

-		unsigned n = curTimeMicros();

-		return tdiff(old, n); 

-	}

-private:

-	unsigned old;

-};

+// goodies.h
+// miscellaneous junk
+
+#pragma once
+
+#include "../stdafx.h"
+
+#if !defined(_WIN32)
+#include <pthread.h>
+inline pthread_t GetCurrentThreadId() { return pthread_self(); }
+#endif
+
+/* set to TRUE if we are exiting */
+extern bool goingAway;
+
+bool isPrime(int n); 
+int nextPrime(int n);
+
+inline void dumpmemory(const char *data, int len) { 
+	if( len > 1024 )
+		len = 1024;
+	try {
+	const char *q = data;
+	const char *p = q;
+	while( len > 0 ) {
+		for( int i = 0; i < 16; i++ ) { 
+			if( *p >= 32 && *p <= 126 )
+				cout << *p;
+			else 
+				cout << '.';
+			p++;
+		}
+		cout << "  ";
+		p -= 16;
+		for( int i = 0; i < 16; i++ )
+			cout << (unsigned) ((unsigned char)*p++) << ' ';
+		cout << endl;
+		len -= 16;
+	}
+	} catch(...) {
+	}
+}
+
+#undef yassert
+#include <boost/thread/thread.hpp>
+#include <boost/thread/xtime.hpp>
+#undef assert
+#define assert xassert
+#define yassert 1
+
+struct WrappingInt {
+	WrappingInt() { x = 0; } 
+	WrappingInt(unsigned z) : x(z) { }
+	unsigned x;
+	operator unsigned() const { return x; }
+	WrappingInt& operator++() { x++; return *this; }
+	static int diff(unsigned a, unsigned b) { return a-b; }
+	bool operator<=(WrappingInt r) { 
+		// platform dependent
+		int df = (r.x - x);
+		return df >= 0;
+	}
+	bool operator>(WrappingInt r) { return !(r<=*this); }
+};
+
+#include <ctime>
+
+inline void time_t_to_String(time_t t, char *buf) {
+#if defined(_WIN32)
+	ctime_s(buf, 64, &t);
+#else
+	ctime_r(&t, buf);
+#endif
+	buf[24] = 0; // don't want the \n
+}
+
+inline void sleepsecs(int s) { 
+	boost::xtime xt;
+	boost::xtime_get(&xt, boost::TIME_UTC);
+	xt.sec += s;
+	boost::thread::sleep(xt);
+}
+inline void sleepmillis(int s) { 
+	boost::xtime xt;
+	boost::xtime_get(&xt, boost::TIME_UTC);
+	xt.nsec += s * 1000000;
+	boost::thread::sleep(xt);
+}
+// note this wraps
+inline int tdiff(unsigned told, unsigned tnew) { 
+	return WrappingInt::diff(tnew, told);
+}
+inline unsigned curTimeMillis() {
+	boost::xtime xt;
+	boost::xtime_get(&xt, boost::TIME_UTC);
+	unsigned t = xt.nsec / 1000000;
+	return (xt.sec & 0xfffff) * 1000 + t;
+}
+inline unsigned long long jsTime() {
+	boost::xtime xt;
+	boost::xtime_get(&xt, boost::TIME_UTC);
+	unsigned long long t = xt.nsec / 1000000;
+	return ((unsigned long long) xt.sec * 1000) + t;
+}
+// measures up to 1024 seconds.  or, 512 seconds with tdiff that is...
+inline unsigned curTimeMicros() {
+	boost::xtime xt;
+	boost::xtime_get(&xt, boost::TIME_UTC);
+	unsigned t = xt.nsec / 1000;
+	unsigned secs = xt.sec % 1024;
+	t = secs*1000000 + t;
+	return t;
+}
+using namespace boost;
+typedef boost::mutex::scoped_lock lock;
+
+// simple scoped timer
+class Timer { 
+public:
+	Timer() { old = curTimeMicros(); }
+	int millis() { return micros() / 1000; }
+	int micros() { 
+		unsigned n = curTimeMicros();
+		return tdiff(old, n); 
+	}
+private:
+	unsigned old;
+};
diff --git a/util/hashtab.h b/util/hashtab.h
index 8b8b53b182e..9ea3c2342d5 100644
--- a/util/hashtab.h
+++ b/util/hashtab.h
@@ -1,110 +1,110 @@
-/* hashtab.h

-

-   Simple, fixed size hash table.  Darn simple.

-

-   Uses a contiguous block of memory, so you can put it in a memory mapped file very easily.

-*/

-

-#pragma once 

-

-#include "../stdafx.h"

-#include <map>

-

-#pragma pack(push)

-#pragma pack(1)

-

-/* you should define:

-

-   int Key::hash() return > 0 always.

-*/

-

-template <

-	class Key,

-	class Type

->

-class HashTable {

-public:

-	const char *name;

-	struct Node {

-		int hash;

-		Key k;

-		Type value;

-		bool inUse() { return hash != 0; }

-	} *nodes;

-	int n;

-

-	int _find(const Key& k, bool& found) {

-		found = false;

-		int h = k.hash();

-		int i = h % n;

-		int start = i;

-		int chain = 0;

-		while( 1 ) {

-			if( !nodes[i].inUse() ) {

-				return i;

-			}

-			if( nodes[i].hash == h && nodes[i].k == k ) {

-				found = true;

-				return i;

-			}

-			chain++;

-			i = (i+1) % n;

-			if( i == start ) { 

-				cout << "warning: hashtable is full " << name << endl;

-				return -1;

-			}

-			if( chain == 200 )

-				cout << "warning: hashtable long chain " << name << endl;

-		}

-	}

-

-public:

-	/* buf must be all zeroes on initialization. */

-	HashTable(void *buf, int buflen, const char *_name) : name(_name) { 

-		int m = sizeof(Node);

-		// cout << "hashtab init, buflen:" << buflen << " m:" << m << endl;

-		n = buflen / m;

-		if( (n & 1) == 0 )

-			n--;

-		nodes = (Node *) buf;

-		assert(nodes[n-1].hash == 0);

-		assert(nodes[0].hash == 0);

-

-		assert( sizeof(Node) == 628 );

-		//cout << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << endl;

-	}

-

-	Type* get(const Key& k) { 

-		bool found;

-		int i = _find(k, found);

-		if( found )

-			return &nodes[i].value;

-		return 0;

-	}

-

-	void kill(const Key& k) { 

-		bool found;

-		int i = _find(k, found);

-		if( i >= 0 && found )

-			nodes[i].k.kill();

-	}

-

-	void put(const Key& k, const Type& value) {

-		bool found;

-		int i = _find(k, found);

-		if( i < 0 )

-			return;

-		if( !found ) {

-			nodes[i].k = k;

-			nodes[i].hash = k.hash();

-		}

-		else {

-			assert( nodes[i].hash == k.hash() );

-		}

-		nodes[i].value = value;

-	}

-

-};

-

-#pragma pack(pop)

-

+/* hashtab.h
+
+   Simple, fixed size hash table.  Darn simple.
+
+   Uses a contiguous block of memory, so you can put it in a memory mapped file very easily.
+*/
+
+#pragma once 
+
+#include "../stdafx.h"
+#include <map>
+
+#pragma pack(push)
+#pragma pack(1)
+
+/* you should define:
+
+   int Key::hash() return > 0 always.
+*/
+
+template <
+	class Key,
+	class Type
+>
+class HashTable {
+public:
+	const char *name;
+	struct Node {
+		int hash;
+		Key k;
+		Type value;
+		bool inUse() { return hash != 0; }
+	} *nodes;
+	int n;
+
+	int _find(const Key& k, bool& found) {
+		found = false;
+		int h = k.hash();
+		int i = h % n;
+		int start = i;
+		int chain = 0;
+		while( 1 ) {
+			if( !nodes[i].inUse() ) {
+				return i;
+			}
+			if( nodes[i].hash == h && nodes[i].k == k ) {
+				found = true;
+				return i;
+			}
+			chain++;
+			i = (i+1) % n;
+			if( i == start ) { 
+				cout << "warning: hashtable is full " << name << endl;
+				return -1;
+			}
+			if( chain == 200 )
+				cout << "warning: hashtable long chain " << name << endl;
+		}
+	}
+
+public:
+	/* buf must be all zeroes on initialization. */
+	HashTable(void *buf, int buflen, const char *_name) : name(_name) { 
+		int m = sizeof(Node);
+		// cout << "hashtab init, buflen:" << buflen << " m:" << m << endl;
+		n = buflen / m;
+		if( (n & 1) == 0 )
+			n--;
+		nodes = (Node *) buf;
+		assert(nodes[n-1].hash == 0);
+		assert(nodes[0].hash == 0);
+
+		assert( sizeof(Node) == 628 );
+		//cout << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << endl;
+	}
+
+	Type* get(const Key& k) { 
+		bool found;
+		int i = _find(k, found);
+		if( found )
+			return &nodes[i].value;
+		return 0;
+	}
+
+	void kill(const Key& k) { 
+		bool found;
+		int i = _find(k, found);
+		if( i >= 0 && found )
+			nodes[i].k.kill();
+	}
+
+	void put(const Key& k, const Type& value) {
+		bool found;
+		int i = _find(k, found);
+		if( i < 0 )
+			return;
+		if( !found ) {
+			nodes[i].k = k;
+			nodes[i].hash = k.hash();
+		}
+		else {
+			assert( nodes[i].hash == k.hash() );
+		}
+		nodes[i].value = value;
+	}
+
+};
+
+#pragma pack(pop)
+
diff --git a/util/lruishmap.h b/util/lruishmap.h
index 1fd63d577fb..eca27609897 100644
--- a/util/lruishmap.h
+++ b/util/lruishmap.h
@@ -1,59 +1,59 @@
-// lru-ish map.h

-

-#pragma once

-

-#include "../stdafx.h"

-#include "../util/goodies.h"

-

-/* Your K object must define:

-	 int hash() - must always return > 0.

-	 operator==

-*/

-

-template <class K, class V, int MaxChain>

-class LRUishMap { 

-public:

-	LRUishMap(int _n) { 

-		n = nextPrime(_n);

-		keys = new K[n];

-		hashes = new int[n];

-		for( int i = 0; i < n; i++ ) hashes[i] = 0;

-	}

-	~LRUishMap() {

-		delete[] keys;

-		delete[] hashes;

-	}

-

-	int _find(const K& k, bool& found) {

-		int h = k.hash(); 

-		assert( h > 0 );

-		int j = h % n;

-		int first = j;

-		for( int i = 0; i < MaxChain; i++ ) { 

-			if( hashes[j] == h ) { 

-				if( keys[j] == k ) { 

-					found = true;

-					return j;

-				}

-			}

-			else if( hashes[j] == 0 ) { 

-				found = false;

-				return j;

-			}

-		}

-		found = false;

-		return first;

-	}

-

-	V* find(const K& k) { 

-		bool found;

-		int j = _find(k, found);

-		return found ? &values[j] : 0;

-	}

-

-private:

-	int n;

-	K *keys;

-	int *hashes;

-	V *values;

-};

+// lru-ish map.h
+
+#pragma once
+
+#include "../stdafx.h"
+#include "../util/goodies.h"
+
+/* Your K object must define:
+	 int hash() - must always return > 0.
+	 operator==
+*/
+
+template <class K, class V, int MaxChain>
+class LRUishMap { 
+public:
+	LRUishMap(int _n) { 
+		n = nextPrime(_n);
+		keys = new K[n];
+		hashes = new int[n];
+		for( int i = 0; i < n; i++ ) hashes[i] = 0;
+	}
+	~LRUishMap() {
+		delete[] keys;
+		delete[] hashes;
+	}
+
+	int _find(const K& k, bool& found) {
+		int h = k.hash(); 
+		assert( h > 0 );
+		int j = h % n;
+		int first = j;
+		for( int i = 0; i < MaxChain; i++ ) { 
+			if( hashes[j] == h ) { 
+				if( keys[j] == k ) { 
+					found = true;
+					return j;
+				}
+			}
+			else if( hashes[j] == 0 ) { 
+				found = false;
+				return j;
+			}
+		}
+		found = false;
+		return first;
+	}
+
+	V* find(const K& k) { 
+		bool found;
+		int j = _find(k, found);
+		return found ? &values[j] : 0;
+	}
+
+private:
+	int n;
+	K *keys;
+	int *hashes;
+	V *values;
+};
diff --git a/util/mmap.cpp b/util/mmap.cpp
index 6955d1ca96c..f034b2a3cea 100644
--- a/util/mmap.cpp
+++ b/util/mmap.cpp
@@ -1,144 +1,144 @@
-// mmap.cpp

-

-#include "stdafx.h"

-#include "mmap.h"

-

-#if defined(_WIN32) 

-

-#include "windows.h"

-

-MemoryMappedFile::MemoryMappedFile() {

-	fd = 0; maphandle = 0; view = 0;

-}

-

-MemoryMappedFile::~MemoryMappedFile() {

-	if( view )

-		UnmapViewOfFile(view);

-	view = 0;

-	if( maphandle )

-		CloseHandle(maphandle);

-	maphandle = 0;

-	if( fd )

-		CloseHandle(fd);

-	fd = 0;

-}

-

-std::wstring toWideString(const char *s) {

- //const std::basic_string<TCHAR> s) {

-  std::basic_ostringstream<TCHAR> buf;

-  buf << s;

-  return buf.str();

-}

-

-unsigned mapped = 0;

-

-void* MemoryMappedFile::map(const char *filename, int length) {

-	std::wstring filenamew = toWideString(filename);

-

-	fd = CreateFile(

-		filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ,

-		NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);

-	if( fd == INVALID_HANDLE_VALUE ) {

-		cout << "CreateFile failed " << filename << endl;

-		return 0;

-	}

-

-#if defined(_WIN32)

-	if( mapped > 500000000 ) { 

-		cout << "too much mem mapped for win32" << endl;

-		if( length > 50000000 )

-			length = 50000000;

-	}

-	mapped += length;

-#endif

-

-	maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL);

-	if( maphandle == NULL ) {

-		cout << "CreateFileMapping failed " << filename << endl;

-		return 0;

-	}

-

-	view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0);

-	if( view == 0 )

-		cout << "MapViewOfFile failed " << filename << ' ' << GetLastError() << endl;

-

-	return view;

-}

-

-void MemoryMappedFile::flush(bool) {

-}

-

-#else

-

-#include <errno.h>

-#include <sys/mman.h>

-#include <sys/types.h>

-#include <sys/stat.h>

-#include <fcntl.h>

-

-MemoryMappedFile::MemoryMappedFile() {

-	fd = 0; maphandle = 0; view = 0; len = 0;

-}

-

-MemoryMappedFile::~MemoryMappedFile() {

-	if( view )

-		munmap(view, len);

-	view = 0;

-

-	if( fd )

-		close(fd);

-	fd = 0;

-}

-

-#ifndef O_NOATIME

-#warning NO O_NOATIME

-#define O_NOATIME 0

-#endif

-

-void* MemoryMappedFile::map(const char *filename, int length) {

-	len = length;

-

-        fd = open(filename, O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);

-	if( !fd ) {

-		cout << "couldn't open " << filename << ' ' << errno << endl;

-		return 0;

-	}

-

-	/* make sure the file is the full desired length */

-	off_t filelen = lseek(fd, 0, SEEK_END);

-	if( filelen < length ) { 

-		cout << "map: file length=" << filelen << " want:" << length << endl;

-		if( filelen != 0 ) { 

-			cout << "  failing mapping" << endl;

-			return 0;

-		}

-		cout << "  writing file to full length with zeroes..." << endl;

-		int z = 8192;

-		char buf[z];

-		memset(buf, 0, z);

-		int left = length;

-		while( 1 ) { 

-			if( left <= z ) {

-				write(fd, buf, left);

-				break;

-			}

-			write(fd, buf, z);

-			left -= z;

-		}

-		cout << "  done" << endl;

-	}

-

-	lseek(fd, length, SEEK_SET);

-	write(fd, "", 1);

-

-	view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);

-	return view;

-}

-

-void MemoryMappedFile::flush(bool sync) {

-	if( msync(view, len, sync ? MS_SYNC : MS_ASYNC) )

-		cout << "msync error " << errno << endl;

-}

-

-#endif

-

+// mmap.cpp
+
+#include "stdafx.h"
+#include "mmap.h"
+
+#if defined(_WIN32) 
+
+#include "windows.h"
+
+MemoryMappedFile::MemoryMappedFile() {
+	fd = 0; maphandle = 0; view = 0;
+}
+
+MemoryMappedFile::~MemoryMappedFile() {
+	if( view )
+		UnmapViewOfFile(view);
+	view = 0;
+	if( maphandle )
+		CloseHandle(maphandle);
+	maphandle = 0;
+	if( fd )
+		CloseHandle(fd);
+	fd = 0;
+}
+
+std::wstring toWideString(const char *s) {
+ //const std::basic_string<TCHAR> s) {
+  std::basic_ostringstream<TCHAR> buf;
+  buf << s;
+  return buf.str();
+}
+
+unsigned mapped = 0;
+
+void* MemoryMappedFile::map(const char *filename, int length) {
+	std::wstring filenamew = toWideString(filename);
+
+	fd = CreateFile(
+		filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ,
+		NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+	if( fd == INVALID_HANDLE_VALUE ) {
+		cout << "CreateFile failed " << filename << endl;
+		return 0;
+	}
+
+#if defined(_WIN32)
+	if( mapped > 500000000 ) { 
+		cout << "too much mem mapped for win32" << endl;
+		if( length > 50000000 )
+			length = 50000000;
+	}
+	mapped += length;
+#endif
+
+	maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL);
+	if( maphandle == NULL ) {
+		cout << "CreateFileMapping failed " << filename << endl;
+		return 0;
+	}
+
+	view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0);
+	if( view == 0 )
+		cout << "MapViewOfFile failed " << filename << ' ' << GetLastError() << endl;
+
+	return view;
+}
+
+void MemoryMappedFile::flush(bool) {
+}
+
+#else
+
+#include <errno.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+MemoryMappedFile::MemoryMappedFile() {
+	fd = 0; maphandle = 0; view = 0; len = 0;
+}
+
+MemoryMappedFile::~MemoryMappedFile() {
+	if( view )
+		munmap(view, len);
+	view = 0;
+
+	if( fd )
+		close(fd);
+	fd = 0;
+}
+
+#ifndef O_NOATIME
+#warning NO O_NOATIME
+#define O_NOATIME 0
+#endif
+
+void* MemoryMappedFile::map(const char *filename, int length) {
+	len = length;
+
+        fd = open(filename, O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);
+	if( !fd ) {
+		cout << "couldn't open " << filename << ' ' << errno << endl;
+		return 0;
+	}
+
+	/* make sure the file is the full desired length */
+	off_t filelen = lseek(fd, 0, SEEK_END);
+	if( filelen < length ) { 
+		cout << "map: file length=" << filelen << " want:" << length << endl;
+		if( filelen != 0 ) { 
+			cout << "  failing mapping" << endl;
+			return 0;
+		}
+		cout << "  writing file to full length with zeroes..." << endl;
+		int z = 8192;
+		char buf[z];
+		memset(buf, 0, z);
+		int left = length;
+		while( 1 ) { 
+			if( left <= z ) {
+				write(fd, buf, left);
+				break;
+			}
+			write(fd, buf, z);
+			left -= z;
+		}
+		cout << "  done" << endl;
+	}
+
+	lseek(fd, length, SEEK_SET);
+	write(fd, "", 1);
+
+	view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	return view;
+}
+
+void MemoryMappedFile::flush(bool sync) {
+	if( msync(view, len, sync ? MS_SYNC : MS_ASYNC) )
+		cout << "msync error " << errno << endl;
+}
+
+#endif
+
diff --git a/util/mmap.h b/util/mmap.h
index aae4eed0f32..15f638787d8 100644
--- a/util/mmap.h
+++ b/util/mmap.h
@@ -1,22 +1,22 @@
-// mmap.h

-

-#pragma once

-

-class MemoryMappedFile {

-public:

-	MemoryMappedFile();

-	~MemoryMappedFile();

-

-	/* only smart enough right now to deal with files of a fixed length. 

-	   creates if DNE

-	*/

-	void* map(const char *filename, int length);

-

-	void flush(bool sync);

-

-private:

-	HANDLE fd;

-	HANDLE maphandle;

-	void *view;

-	int len;

-};

+// mmap.h
+
+#pragma once
+
+class MemoryMappedFile {
+public:
+	MemoryMappedFile();
+	~MemoryMappedFile();
+
+	/* only smart enough right now to deal with files of a fixed length. 
+	   creates if DNE
+	*/
+	void* map(const char *filename, int length);
+
+	void flush(bool sync);
+
+private:
+	HANDLE fd;
+	HANDLE maphandle;
+	void *view;
+	int len;
+};
diff --git a/util/sock.cpp b/util/sock.cpp
index 7b0c20e7ff5..1b86edd4f5e 100644
--- a/util/sock.cpp
+++ b/util/sock.cpp
@@ -1,161 +1,161 @@
-// sock.cpp

-

-#include "stdafx.h"

-#include "sock.h"

-

-void sendtest() { 

-	cout << "sendtest\n"; 

-	SockAddr me(27016);

-	SockAddr dest("127.0.0.1", 27015);

-	UDPConnection c;

-	if( c.init(me) ) {

-		char buf[256];

-		cout << "sendto: ";

-		cout << c.sendto(buf, sizeof(buf), dest) << " errno:" << h_errno << endl;

-	}

-	cout << "end\n";

-}

-

-void listentest() { 

-	cout << "listentest\n"; 

-	SockAddr me(27015);

-	SockAddr sender;

-	UDPConnection c;

-	if( c.init(me) ) {

-		char buf[256];

-		cout << "recvfrom: ";

-		cout << c.recvfrom(buf, sizeof(buf), sender) << " errno:" << h_errno << endl;

-	}

-	cout << "end listentest\n";

-}

-

-void xmain();

-struct SockStartupTests {

-	SockStartupTests() {

-#if defined(_WIN32)

-        WSADATA d;

-		if( WSAStartup(MAKEWORD(2,2), &d) != 0 ) {

-			cout << "ERROR: wsastartup failed " << errno << endl;

-			problem() << "ERROR: wsastartup failed " << errno << endl;

-			exit(1);

-		}

-#endif

-		//cout << "ntohl:" << ntohl(256) << endl;

-		//sendtest();

-		//listentest();

-	}

-} sstests;

-

-#if 0

-void smain() {

-

-  WSADATA wsaData;

-  SOCKET RecvSocket;

-  sockaddr_in RecvAddr;

-  int Port = 27015;

-  char RecvBuf[1024];

-  int  BufLen = 1024;

-  sockaddr_in SenderAddr;

-  int SenderAddrSize = sizeof(SenderAddr);

-

-  //-----------------------------------------------

-  // Initialize Winsock

-  WSAStartup(MAKEWORD(2,2), &wsaData);

-

-  //-----------------------------------------------

-  // Create a receiver socket to receive datagrams

-  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

-  prebindOptions( RecvSocket );

-

-  //-----------------------------------------------

-  // Bind the socket to any address and the specified port.

-  RecvAddr.sin_family = AF_INET;

-  RecvAddr.sin_port = htons(Port);

-  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);

-

-  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));

-

-  //-----------------------------------------------

-  // Call the recvfrom function to receive datagrams

-  // on the bound socket.

-  printf("Receiving datagrams...\n");

-  recvfrom(RecvSocket, 

-    RecvBuf, 

-    BufLen, 

-    0, 

-    (SOCKADDR *)&SenderAddr, 

-    &SenderAddrSize);

-

-  //-----------------------------------------------

-  // Close the socket when finished receiving datagrams

-  printf("Finished receiving. Closing socket.\n");

-  closesocket(RecvSocket);

-

-  //-----------------------------------------------

-  // Clean up and exit.

-  printf("Exiting.\n");

-  WSACleanup();

-  return;

-}

-

-

-

-

-void xmain() {

-

-  WSADATA wsaData;

-  SOCKET RecvSocket;

-  sockaddr_in RecvAddr;

-  int Port = 27015;

-  char RecvBuf[1024];

-  int  BufLen = 1024;

-  sockaddr_in SenderAddr;

-  int SenderAddrSize = sizeof(SenderAddr);

-

-  //-----------------------------------------------

-  // Initialize Winsock

-  WSAStartup(MAKEWORD(2,2), &wsaData);

-

-  //-----------------------------------------------

-  // Create a receiver socket to receive datagrams

-

-  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

-  prebindOptions( RecvSocket );

-

-  //-----------------------------------------------

-  // Bind the socket to any address and the specified port.

-  RecvAddr.sin_family = AF_INET;

-  RecvAddr.sin_port = htons(Port);

-  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);

-

-  SockAddr a(Port);

-  bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize);

-//  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));

-

-  SockAddr b;

-

-  //-----------------------------------------------

-  // Call the recvfrom function to receive datagrams

-  // on the bound socket.

-  printf("Receiving datagrams...\n");

-  recvfrom(RecvSocket, 

-    RecvBuf, 

-    BufLen, 

-    0,

-	(SOCKADDR *) &b.sa, &b.addressSize);

-//    (SOCKADDR *)&SenderAddr, 

-//    &SenderAddrSize);

-

-  //-----------------------------------------------

-  // Close the socket when finished receiving datagrams

-  printf("Finished receiving. Closing socket.\n");

-  closesocket(RecvSocket);

-

-  //-----------------------------------------------

-  // Clean up and exit.

-  printf("Exiting.\n");

-  WSACleanup();

-  return;

-}

-

-#endif

+// sock.cpp
+
+#include "stdafx.h"
+#include "sock.h"
+
+void sendtest() { 
+	cout << "sendtest\n"; 
+	SockAddr me(27016);
+	SockAddr dest("127.0.0.1", 27015);
+	UDPConnection c;
+	if( c.init(me) ) {
+		char buf[256];
+		cout << "sendto: ";
+		cout << c.sendto(buf, sizeof(buf), dest) << " errno:" << h_errno << endl;
+	}
+	cout << "end\n";
+}
+
+void listentest() { 
+	cout << "listentest\n"; 
+	SockAddr me(27015);
+	SockAddr sender;
+	UDPConnection c;
+	if( c.init(me) ) {
+		char buf[256];
+		cout << "recvfrom: ";
+		cout << c.recvfrom(buf, sizeof(buf), sender) << " errno:" << h_errno << endl;
+	}
+	cout << "end listentest\n";
+}
+
+void xmain();
+struct SockStartupTests {
+	SockStartupTests() {
+#if defined(_WIN32)
+        WSADATA d;
+		if( WSAStartup(MAKEWORD(2,2), &d) != 0 ) {
+			cout << "ERROR: wsastartup failed " << errno << endl;
+			problem() << "ERROR: wsastartup failed " << errno << endl;
+			exit(1);
+		}
+#endif
+		//cout << "ntohl:" << ntohl(256) << endl;
+		//sendtest();
+		//listentest();
+	}
+} sstests;
+
+#if 0
+void smain() {
+
+  WSADATA wsaData;
+  SOCKET RecvSocket;
+  sockaddr_in RecvAddr;
+  int Port = 27015;
+  char RecvBuf[1024];
+  int  BufLen = 1024;
+  sockaddr_in SenderAddr;
+  int SenderAddrSize = sizeof(SenderAddr);
+
+  //-----------------------------------------------
+  // Initialize Winsock
+  WSAStartup(MAKEWORD(2,2), &wsaData);
+
+  //-----------------------------------------------
+  // Create a receiver socket to receive datagrams
+  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+  prebindOptions( RecvSocket );
+
+  //-----------------------------------------------
+  // Bind the socket to any address and the specified port.
+  RecvAddr.sin_family = AF_INET;
+  RecvAddr.sin_port = htons(Port);
+  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+
+  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));
+
+  //-----------------------------------------------
+  // Call the recvfrom function to receive datagrams
+  // on the bound socket.
+  printf("Receiving datagrams...\n");
+  recvfrom(RecvSocket, 
+    RecvBuf, 
+    BufLen, 
+    0, 
+    (SOCKADDR *)&SenderAddr, 
+    &SenderAddrSize);
+
+  //-----------------------------------------------
+  // Close the socket when finished receiving datagrams
+  printf("Finished receiving. Closing socket.\n");
+  closesocket(RecvSocket);
+
+  //-----------------------------------------------
+  // Clean up and exit.
+  printf("Exiting.\n");
+  WSACleanup();
+  return;
+}
+
+
+
+
+void xmain() {
+
+  WSADATA wsaData;
+  SOCKET RecvSocket;
+  sockaddr_in RecvAddr;
+  int Port = 27015;
+  char RecvBuf[1024];
+  int  BufLen = 1024;
+  sockaddr_in SenderAddr;
+  int SenderAddrSize = sizeof(SenderAddr);
+
+  //-----------------------------------------------
+  // Initialize Winsock
+  WSAStartup(MAKEWORD(2,2), &wsaData);
+
+  //-----------------------------------------------
+  // Create a receiver socket to receive datagrams
+
+  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+  prebindOptions( RecvSocket );
+
+  //-----------------------------------------------
+  // Bind the socket to any address and the specified port.
+  RecvAddr.sin_family = AF_INET;
+  RecvAddr.sin_port = htons(Port);
+  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+
+  SockAddr a(Port);
+  bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize);
+//  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));
+
+  SockAddr b;
+
+  //-----------------------------------------------
+  // Call the recvfrom function to receive datagrams
+  // on the bound socket.
+  printf("Receiving datagrams...\n");
+  recvfrom(RecvSocket, 
+    RecvBuf, 
+    BufLen, 
+    0,
+	(SOCKADDR *) &b.sa, &b.addressSize);
+//    (SOCKADDR *)&SenderAddr, 
+//    &SenderAddrSize);
+
+  //-----------------------------------------------
+  // Close the socket when finished receiving datagrams
+  printf("Finished receiving. Closing socket.\n");
+  closesocket(RecvSocket);
+
+  //-----------------------------------------------
+  // Clean up and exit.
+  printf("Exiting.\n");
+  WSACleanup();
+  return;
+}
+
+#endif
diff --git a/util/sock.h b/util/sock.h
index def64035c80..3c00fe6a83f 100644
--- a/util/sock.h
+++ b/util/sock.h
@@ -1,161 +1,161 @@
-// sock.h

-

-#pragma once

- 

-#include <stdio.h>

-#include <sstream>

-#include "goodies.h"

-

-#if defined(_WIN32)

-#include <winsock2.h>

-#include <ws2tcpip.h>

-typedef int socklen_t;

-inline int getLastError() { return WSAGetLastError(); }

-inline void disableNagle(int sock) { 

-	int x = 1;

-	if( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) ) 

-		cout << "ERROR: disableNagle failed" << endl;

-}

-inline void prebindOptions( int sock ){

-}

-#else

-#include <sys/socket.h>

-#include <sys/types.h>

-#include <sys/socket.h>

-#include <netinet/in.h>

-#include <netinet/tcp.h>

-#include <arpa/inet.h>

-#include <errno.h>

-inline void closesocket(int s) { close(s); }

-const int INVALID_SOCKET = -1;

-typedef int SOCKET;

-#define h_errno errno

-inline int getLastError() { return errno; }

-inline void disableNagle(int sock) { 

-  int x = 1;

-
-  #ifdef SOL_TCP

-  int level = SOL_TCP;

-  #else

-  int level = SOL_SOCKET;

-  #endif

+// sock.h
+
+#pragma once
+ 
+#include <stdio.h>
+#include <sstream>
+#include "goodies.h"
+
+#if defined(_WIN32)
+#include <winsock2.h>
+#include <ws2tcpip.h>
+typedef int socklen_t;
+inline int getLastError() { return WSAGetLastError(); }
+inline void disableNagle(int sock) { 
+	int x = 1;
+	if( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) ) 
+		cout << "ERROR: disableNagle failed" << endl;
+}
+inline void prebindOptions( int sock ){
+}
+#else
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <errno.h>
+inline void closesocket(int s) { close(s); }
+const int INVALID_SOCKET = -1;
+typedef int SOCKET;
+#define h_errno errno
+inline int getLastError() { return errno; }
+inline void disableNagle(int sock) { 
+  int x = 1;
+
+  #ifdef SOL_TCP
+  int level = SOL_TCP;
+  #else
+  int level = SOL_SOCKET;
+  #endif
 
   if( setsockopt(sock, level, TCP_NODELAY, (char *) &x, sizeof(x)) )
-    cout << "ERROR: disableNagle failed" << endl; 

+    cout << "ERROR: disableNagle failed" << endl; 
+
+}
+inline void prebindOptions( int sock ){
+  cout << "doing prebind option" << endl;
+  int x = 1;
+  if ( setsockopt( sock , SOL_SOCKET, SO_REUSEADDR, &x, sizeof(x)) < 0 )
+    cout << "Failed to set socket opt, SO_REUSEADDR" << endl;
+}
+
+
+#endif
+
+struct SockAddr {
+	SockAddr() { addressSize = sizeof(sockaddr_in); memset(&sa, 0, sizeof(sa)); }
+	SockAddr(int sourcePort); /* listener side */ 
+	SockAddr(const char *ip, int port); /* EndPoint (remote) side, or if you want to specify which interface locally */
+
+	struct sockaddr_in sa;
+	socklen_t addressSize;
+
+	bool isLocalHost() const { 
+#if defined(_WIN32)
+		return sa.sin_addr.S_un.S_addr == 0x100007f;
+#else
+		return sa.sin_addr.s_addr == 0x100007f; 
+#endif
+	}
+
+	string toString() { 
+		stringstream out;
+		out << inet_ntoa(sa.sin_addr) << ':' 
+			<< sa.sin_port;
+		return out.str();
+	}
+
+	unsigned getPort() { return sa.sin_port; }
+
+	bool operator==(const SockAddr& r) const { 
+		return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr &&
+			sa.sin_port == r.sa.sin_port;
+	}
+	bool operator!=(const SockAddr& r) const { return !(*this == r); }
+	bool operator<(const SockAddr& r) const { 
+		if( sa.sin_port >= r.sa.sin_port )
+			return false;
+		return sa.sin_addr.s_addr < r.sa.sin_addr.s_addr;
+	}
+};
+
+const int MaxMTU = 16384;
+
+class UDPConnection {
+public:
+	UDPConnection() { sock = 0; }
+	~UDPConnection() { if( sock ) { closesocket(sock); sock = 0; } }
+	bool init(const SockAddr& myAddr);
+	int recvfrom(char *buf, int len, SockAddr& sender);
+	int sendto(char *buf, int len, const SockAddr& EndPoint);
+	int mtu(const SockAddr& sa) { 
+		return sa.isLocalHost() ? 16384 : 1480;
+	}
+
+	SOCKET sock;
+};
+
+inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) {
+	return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize);
+}
+
+inline int UDPConnection::sendto(char *buf, int len, const SockAddr& EndPoint) {
+	if( 0 && rand() < (RAND_MAX>>4) ) { 
+		cout << " NOTSENT ";
+		//		cout << curTimeMillis() << " .TEST: NOT SENDING PACKET" << endl;
+		return 0;
+	}
+	return ::sendto(sock, buf, len, 0, (sockaddr *) &EndPoint.sa, EndPoint.addressSize);
+}
+
+inline bool UDPConnection::init(const SockAddr& myAddr) {
+	sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+	if( sock == INVALID_SOCKET ) {
+		cout << "invalid socket? " << errno << endl;
+		return false;
+	}
+	//cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;
+	if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) { 
+		cout << "udp init failed" << endl;
+		closesocket(sock);
+		sock = 0;
+		return false;
+	}
+	socklen_t optLen;
+	int rcvbuf;
+	if (getsockopt(sock,
+		SOL_SOCKET, 
+		SO_RCVBUF, 
+		(char*)&rcvbuf, 
+		&optLen) != -1)
+		cout << "SO_RCVBUF:" << rcvbuf << endl;
+	return true;
+}
+
+inline SockAddr::SockAddr(int sourcePort) {
+	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));
+	sa.sin_family = AF_INET;
+	sa.sin_port = htons(sourcePort);
+	sa.sin_addr.s_addr = htonl(INADDR_ANY);
+	addressSize = sizeof(sa);
+}
 
+inline SockAddr::SockAddr(const char *ip, int port) {
+	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));
+	sa.sin_family = AF_INET;
+	sa.sin_port = htons(port);
+	sa.sin_addr.s_addr = inet_addr(ip);
+	addressSize = sizeof(sa);
 }
-inline void prebindOptions( int sock ){

-  cout << "doing prebind option" << endl;

-  int x = 1;

-  if ( setsockopt( sock , SOL_SOCKET, SO_REUSEADDR, &x, sizeof(x)) < 0 )

-    cout << "Failed to set socket opt, SO_REUSEADDR" << endl;

-}

-

-

-#endif

-

-struct SockAddr {

-	SockAddr() { addressSize = sizeof(sockaddr_in); memset(&sa, 0, sizeof(sa)); }

-	SockAddr(int sourcePort); /* listener side */ 

-	SockAddr(const char *ip, int port); /* EndPoint (remote) side, or if you want to specify which interface locally */

-

-	struct sockaddr_in sa;

-	socklen_t addressSize;

-

-	bool isLocalHost() const { 

-#if defined(_WIN32)

-		return sa.sin_addr.S_un.S_addr == 0x100007f;

-#else

-		return sa.sin_addr.s_addr == 0x100007f; 

-#endif

-	}

-

-	string toString() { 

-		stringstream out;

-		out << inet_ntoa(sa.sin_addr) << ':' 

-			<< sa.sin_port;

-		return out.str();

-	}

-

-	unsigned getPort() { return sa.sin_port; }

-

-	bool operator==(const SockAddr& r) const { 

-		return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr &&

-			sa.sin_port == r.sa.sin_port;

-	}

-	bool operator!=(const SockAddr& r) const { return !(*this == r); }

-	bool operator<(const SockAddr& r) const { 

-		if( sa.sin_port >= r.sa.sin_port )

-			return false;

-		return sa.sin_addr.s_addr < r.sa.sin_addr.s_addr;

-	}

-};

-

-const int MaxMTU = 16384;

-

-class UDPConnection {

-public:

-	UDPConnection() { sock = 0; }

-	~UDPConnection() { if( sock ) { closesocket(sock); sock = 0; } }

-	bool init(const SockAddr& myAddr);

-	int recvfrom(char *buf, int len, SockAddr& sender);

-	int sendto(char *buf, int len, const SockAddr& EndPoint);

-	int mtu(const SockAddr& sa) { 

-		return sa.isLocalHost() ? 16384 : 1480;

-	}

-

-	SOCKET sock;

-};

-

-inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) {

-	return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize);

-}

-

-inline int UDPConnection::sendto(char *buf, int len, const SockAddr& EndPoint) {

-	if( 0 && rand() < (RAND_MAX>>4) ) { 

-		cout << " NOTSENT ";

-		//		cout << curTimeMillis() << " .TEST: NOT SENDING PACKET" << endl;

-		return 0;

-	}

-	return ::sendto(sock, buf, len, 0, (sockaddr *) &EndPoint.sa, EndPoint.addressSize);

-}

-

-inline bool UDPConnection::init(const SockAddr& myAddr) {

-	sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

-	if( sock == INVALID_SOCKET ) {

-		cout << "invalid socket? " << errno << endl;

-		return false;

-	}

-	//cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;

-	if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) { 

-		cout << "udp init failed" << endl;

-		closesocket(sock);

-		sock = 0;

-		return false;

-	}

-	socklen_t optLen;

-	int rcvbuf;

-	if (getsockopt(sock,

-		SOL_SOCKET, 

-		SO_RCVBUF, 

-		(char*)&rcvbuf, 

-		&optLen) != -1)

-		cout << "SO_RCVBUF:" << rcvbuf << endl;

-	return true;

-}

-

-inline SockAddr::SockAddr(int sourcePort) {

-	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));

-	sa.sin_family = AF_INET;

-	sa.sin_port = htons(sourcePort);

-	sa.sin_addr.s_addr = htonl(INADDR_ANY);

-	addressSize = sizeof(sa);

-}

-

-inline SockAddr::SockAddr(const char *ip, int port) {

-	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));

-	sa.sin_family = AF_INET;

-	sa.sin_port = htons(port);

-	sa.sin_addr.s_addr = inet_addr(ip);

-	addressSize = sizeof(sa);

-}

diff --git a/util/util.cpp b/util/util.cpp
index 612a27f589c..92e0edb7423 100644
--- a/util/util.cpp
+++ b/util/util.cpp
@@ -1,48 +1,48 @@
-#include "stdafx.h"

-#include "goodies.h"

-

-unsigned occasion = 0;

-bool goingAway = false;

-

-bool isPrime(int n) { 

-	int z = 2;

-	while( 1 ) { 

-		if( z*z > n )

-			break;

-		if( n % z == 0 )

-			return false;

-		z++;

-	}

-	return true;

-}

-

-int nextPrime(int n) { 

-	n |= 1; // 2 goes to 3...don't care...

-	while( !isPrime(n) )

-		n += 2;

-	return n;

-}

-

-struct UtilTest { 

-	UtilTest() { 

-		assert( WrappingInt(0) <= WrappingInt(0) );

-		assert( WrappingInt(0) <= WrappingInt(1) );

-		assert( !(WrappingInt(1) <= WrappingInt(0)) );

-		assert( (WrappingInt(0xf0000000) <= WrappingInt(0)) );

-		assert( (WrappingInt(0xf0000000) <= WrappingInt(9000)) );

-		assert( !(WrappingInt(300) <= WrappingInt(0xe0000000)) );

-

-		assert( tdiff(3, 4) == 1 ); 

-		assert( tdiff(4, 3) == -1 ); 

-		assert( tdiff(0xffffffff, 0) == 1 ); 

-

-		assert( isPrime(3) );

-		assert( isPrime(2) );

-		assert( isPrime(13) );

-		assert( isPrime(17) );

-		assert( !isPrime(9) );

-		assert( !isPrime(6) );

-		assert( nextPrime(4) == 5 );

-		assert( nextPrime(8) == 11 );

-	}

-} utilTest;

+#include "stdafx.h"
+#include "goodies.h"
+
+unsigned occasion = 0;
+bool goingAway = false;
+
+bool isPrime(int n) { 
+	int z = 2;
+	while( 1 ) { 
+		if( z*z > n )
+			break;
+		if( n % z == 0 )
+			return false;
+		z++;
+	}
+	return true;
+}
+
+int nextPrime(int n) { 
+	n |= 1; // 2 goes to 3...don't care...
+	while( !isPrime(n) )
+		n += 2;
+	return n;
+}
+
+struct UtilTest { 
+	UtilTest() { 
+		assert( WrappingInt(0) <= WrappingInt(0) );
+		assert( WrappingInt(0) <= WrappingInt(1) );
+		assert( !(WrappingInt(1) <= WrappingInt(0)) );
+		assert( (WrappingInt(0xf0000000) <= WrappingInt(0)) );
+		assert( (WrappingInt(0xf0000000) <= WrappingInt(9000)) );
+		assert( !(WrappingInt(300) <= WrappingInt(0xe0000000)) );
+
+		assert( tdiff(3, 4) == 1 ); 
+		assert( tdiff(4, 3) == -1 ); 
+		assert( tdiff(0xffffffff, 0) == 1 ); 
+
+		assert( isPrime(3) );
+		assert( isPrime(2) );
+		assert( isPrime(13) );
+		assert( isPrime(17) );
+		assert( !isPrime(9) );
+		assert( !isPrime(6) );
+		assert( nextPrime(4) == 5 );
+		assert( nextPrime(8) == 11 );
+	}
+} utilTest;