more defensive code and smarter validate functionr0.0.7_rc1

7 files changed, 97 insertions, 44 deletions

diff --git a/db/btree.cpp b/db/btree.cpp
index d7f0c1e7c02..bbe0d1a33f6 100644
--- a/db/btree.cpp
+++ b/db/btree.cpp
@@ -244,7 +244,6 @@ void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, in
 	DiskLoc loc = thisLoc;
 	while( 1 ) { 
 		BtreeBucket *b = loc.btree();
-//		b->dump();
 		if( !b->nextChild.isNull() ) { 
 			loc = b->nextChild;
 			continue;
@@ -347,6 +346,7 @@ found:
 
 /* note: may delete the entire bucket!  this invalid upon return sometimes. */
 void BtreeBucket::delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p) { 
+	dassert( thisLoc.btree() == this );
 	assert(n>0);
 	DiskLoc left = childForPos(p);
 
@@ -403,6 +403,7 @@ inline void fix(const DiskLoc& thisLoc, const DiskLoc& child) {
 
 /* this sucks.  maybe get rid of parent ptrs. */
 void BtreeBucket::fixParentPtrs(const DiskLoc& thisLoc) { 
+	dassert( thisLoc.btree() == this );
 	fix(thisLoc, nextChild);
 	for( int i = 0; i < n; i++ )
 		fix(thisLoc, k(i).prevChildBucket);
@@ -414,6 +415,7 @@ void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos,
 							 DiskLoc recordLoc, JSObj& key,
 							 DiskLoc lchild, DiskLoc rchild, IndexDetails& idx) 
 {
+	dassert( thisLoc.btree() == this );
 	if( insert_debug )
 		cout << "   " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' ' 
 		<< lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;
diff --git a/db/db.cpp b/db/db.cpp
index 660b7b697ee..32ce707857b 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -347,7 +347,7 @@ public:
    115 replay, opLogging
 */
 void listen(int port) { 
-	const char *Version = "db version: 118 27jun2008";
+	const char *Version = "db version: 119 30jun2008";
 	problem() << Version << endl;
 	cout << Version << endl;
 	pdfileInit();
@@ -510,16 +510,22 @@ void connThread()
 			client = 0;
 			curOp = 0;
 
+			int ms;
 			bool log = false;
 			curOp = m.data->operation;
 
-			/* use this if you only want to process operations for a particular namespace.  maybe add to cmd line parms or a #define 
-			   DbMessage ddd(m);
-			   if( strncmp(ddd.getns(), "grid", 4) != 0 ) { 
-			   cout << "TEMP skip " << ddd.getns() << '\n';
-			   }
-			   else
-			*/
+#if 0
+				/* use this if you only want to process operations for a particular namespace.  
+				maybe add to cmd line parms or something fancier.
+				*/
+				DbMessage ddd(m);
+				if( strncmp(ddd.getns(), "clusterstock", 12) != 0 ) { 
+					static int q;
+					if( ++q < 20 ) 
+						cout << "TEMP skip " << ddd.getns() << endl;
+					goto skip;
+				}
+#endif
 
 			if( m.data->operation == dbMsg ) { 
 				ss << "msg ";
@@ -611,19 +617,21 @@ void connThread()
 				assert(false);
 			}
 
-			int ms = t.millis();
+			ms = t.millis();
 			log = log || ctr++ % 512 == 0;
 			DEV log = true;
 			if( log || ms > 100 ) {
 				ss << ' ' << t.millis() << "ms";
 				cout << ss.str().c_str() << endl;
 			}
+skip:
 			if( client && client->profile >= 1 ) { 
 				if( client->profile >= 2 || ms >= 100 ) { 
 					// profile it
 					profile(ss.str().c_str()+20/*skip ts*/, ms);
 				}
 			}
+
 		} /* end lock */
 		if( dbresponse.response ) 
 			dbMsgPort.reply(m, *dbresponse.response, dbresponse.responseTo);
diff --git a/db/db.vcproj b/db/db.vcproj
index 60d4da0b402..5020802bb37 100644
--- a/db/db.vcproj
+++ b/db/db.vcproj
@@ -329,6 +329,10 @@
 				>

 			</File>

 			<File

+				RelativePath=".\repl.h"

+				>

+			</File>

+			<File

 				RelativePath=".\resource.h"

 				>

 			</File>

diff --git a/db/namespace.h b/db/namespace.h
index 2c02662e26d..f35b84acdd2 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -24,6 +24,7 @@ public:
 		buf[0] = 0x7f;
 	}
 
+	bool operator==(const char *r) { return strcmp(buf, r) == 0; }
 	bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; }
 	int hash() const {
 		unsigned x = 0;
@@ -170,8 +171,6 @@ public:
 
 	/* just for diagnostics */
 	size_t detailsOffset(NamespaceDetails *d) { 
-	  cout << d << ' ' << ht->nodes << endl;
-	  cout << f.viewOfs() << endl;
 	  return ((char *) d) -  (char *) ht->nodes;
 	}
 
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
index 4c9ea6e3985..816f91a4015 100644
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -72,6 +72,7 @@ int bucketSizes[] = {
 //NamespaceIndexMgr namespaceIndexMgr;
 
 void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) { 
+	dassert( dloc.drec() == d );
 	DEBUGGING cout << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl;
 	int b = bucket(d->lengthWithHeaders);
 	DiskLoc& list = deletedList[b];
@@ -262,6 +263,7 @@ void NamespaceDetails::compact() {
 }
 
 /* alloc with capped table handling. */
+int n_complaints_cap = 0;
 DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
 	if( !capped )
 		return __stdAlloc(len);
@@ -273,6 +275,9 @@ DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
 	DiskLoc loc;
 
 	// delete records until we have room and the max # objects limit achieved.
+	Extent *theExtent = firstExtent.ext(); // only one extent if capped.
+	dassert( theExtent->ns == ns ); 
+	theExtent->assertOk();
 	while( 1 ) {
 		if( nrecords < max ) { 
 			loc = __stdAlloc(len);
@@ -280,11 +285,16 @@ DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
 				break;
 		}
 
-		DiskLoc fr = firstExtent.ext()->firstRecord;
+		DiskLoc fr = theExtent->firstRecord;
 		if( fr.isNull() ) { 
-			cout << "couldn't make room for new record in capped ns " << ns 
-				<< " len: " << len << " extentsize:" << lastExtentSize << '\n';
-			assert( len * 5 > lastExtentSize ); // assume it is unusually large record; if not, something is broken
+			if( ++n_complaints_cap < 8 ) {
+				cout << "couldn't make room for new record in capped ns " << ns << '\n'
+					<< "  len: " << len << " extentsize:" << lastExtentSize << '\n';
+				cout << "  magic: " << hex << theExtent->magic << " extent->ns: " << theExtent->ns.buf << '\n';
+				cout << "  fr: " << theExtent->firstRecord.toString() << 
+					" lr: " << theExtent->lastRecord.toString() << " extent->len: " << theExtent->length << '\n';
+				assert( len * 5 > lastExtentSize ); // assume it is unusually large record; if not, something is broken
+			}
 			return DiskLoc();
 		}
 
@@ -584,7 +594,7 @@ auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order) {
 	if( el.type() != Number || el.number() >= 0 )
 		return DataFileMgr::findAll(ns);
 
-	// "reverse natural order"		// "reverse natural order"
+	// "reverse natural order"
 	NamespaceDetails *d = nsdetails(ns);
 	if( !d )
 		return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
@@ -676,6 +686,8 @@ void  unindexRecord(const char *ns, NamespaceDetails *d, Record *todelete, const
 
 void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK) 
 {
+	dassert( todelete == dl.rec() );
+
 	int tempextofs = todelete->extentOfs;
 
 	NamespaceDetails* d = nsdetails(ns);
@@ -755,6 +767,8 @@ void DataFileMgr::update(
 		Record *toupdate, const DiskLoc& dl,
 		const char *buf, int len, stringstream& ss) 
 {
+	dassert( toupdate == dl.rec() );
+
 	NamespaceDetails *d = nsdetails(ns);
 
 	if(  toupdate->netLength() < len ) {
diff --git a/db/pdfile.h b/db/pdfile.h
index 103aba93825..17933eb246f 100644
--- a/db/pdfile.h
+++ b/db/pdfile.h
@@ -33,7 +33,6 @@ public:
 	PhysicalDataFile(int fn) : fileNo(fn) { }
 	void open(int fileNo, const char *filename);
 
-
 	Extent* newExtent(const char *ns, int approxSize, int loops = 0);
 private:
 	Extent* getExtent(DiskLoc loc);
@@ -42,8 +41,8 @@ private:
 
 	MemoryMappedFile mmf;
 	PDFHeader *header;
-int __unUsEd;
-//	int length;
+	int __unUsEd;
+	//	int length;
 	int fileNo;
 };
 
@@ -100,7 +99,7 @@ public:
 	DiskLoc getPrev(const DiskLoc& myLoc);
 };
 
-/* extents are regions where all the records within the region 
+/* extents are datafile regions where all the records within the region 
    belong to the same namespace.
 */
 class Extent {
@@ -110,7 +109,6 @@ public:
 	DiskLoc xnext, xprev; /* next/prev extent for this namespace */
 	Namespace ns; /* which namespace this extent is for.  this is just for troubleshooting really */
 	int length;   /* size of the extent, including these fields */
-	//	DiskLoc firstEmptyRegion;
 	DiskLoc firstRecord, lastRecord;
 	char extentData[4];
 
@@ -275,6 +273,7 @@ public:
 	DiskLoc curr;
 };
 
+/* used for order { $natural: -1 } */
 class ReverseCursor : public BasicCursor {
 public:
 	bool advance() { 
@@ -434,7 +433,7 @@ inline void _deleteDataFiles(const char *client) {
 	q = p / (c+"ns");
 	bool ok = boost::filesystem::remove(q);
 	if( ok ) 
-		cout << "removed file " << q.string() << endl;
+		cout << "  removed file " << q.string() << endl;
 	int i = 0;
 	int extra = 10; // should not be necessary, this is defensive in case there are missing files
 	while( 1 ) { 
@@ -443,9 +442,9 @@ inline void _deleteDataFiles(const char *client) {
 		ss << c << i;
 		q = p / ss.str();
 		if( boost::filesystem::remove(q) ) {
-			cout << "removed file " << q.string() << endl;
+			cout << "  removed file " << q.string() << endl;
 			if( extra != 10 ) 
-				cout << "WARNING: extra == " << extra << endl;
+				cout << "  WARNING: extra == " << extra << endl;
 		}
 		else if( --extra <= 0 ) 
 			break;
@@ -496,4 +495,3 @@ inline Record* DataFileMgr::getRecord(const DiskLoc& dl) {
 	assert( dl.a() != -1 );
 	return client->getFile(dl.a())->recordAt(dl);
 }
-
diff --git a/db/query.cpp b/db/query.cpp
index da78bd4d789..81b9a35549a 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -12,6 +12,11 @@
 #include "javajs.h"
 #include "json.h"
 
+/* We cut off further objects once we cross this threshold; thus, you might get 
+   a little bit more than this, it is a threshold rather than a limit.
+*/
+const int MaxBytesToReturnToClientAtOnce = 4 * 1024 * 1024;
+
 //ns->query->DiskLoc
 LRUishMap<JSObj,DiskLoc,5> lrutest(123);
 
@@ -343,7 +348,13 @@ string validateNS(const char *ns, NamespaceDetails *d) {
 	if( d->capped )
 		ss << "  capped:" << d->capped << " max:" << d->max << '\n';
 
-	ss << "  firstExtent:" << d->firstExtent.toString() << " lastExtent:" << d->lastExtent.toString() << '\n';
+	ss << "  firstExtent:" << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->ns.buf << '\n';
+	ss << "  lastExtent:" << d->lastExtent.toString()    << " ns:" << d->lastExtent.ext()->ns.buf << '\n';
+	try { 
+		d->firstExtent.ext()->assertOk();
+		d->lastExtent.ext()->assertOk(); 
+	} catch(...) { valid=false; ss << " extent asserted "; }
+
 	ss << "  datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
 	ss << "  padding:" << d->paddingFactor << '\n';
 	try { 
@@ -362,25 +373,37 @@ string validateNS(const char *ns, NamespaceDetails *d) {
 		long long len = 0;
 		long long nlen = 0;
 		set<DiskLoc> recs;
+		int outOfOrder = 0;
+		DiskLoc cl_last;
 		while( c->ok() ) { 
 			n++;
 
+			DiskLoc cl = c->currLoc();
 			if( n < 1000000 )
-				recs.insert(c->currLoc());
-
-//			if( c->currLoc() == DiskLoc(0, 0x476878) ) {
-//				cout << "************** target exists in base collection \n";
-//				cout << c->current().toString() << endl;
-//			}
+				recs.insert(cl);
+			if( d->capped ) {
+				if( cl < cl_last )
+					outOfOrder++;
+				cl_last = cl;
+			}
 
 			Record *r = c->_current();
 			len += r->lengthWithHeaders;
 			nlen += r->netLength();
 			c->advance();
 		}
+		if( d->capped ) { 
+			ss << "  capped outOfOrder:" << outOfOrder;
+			if( outOfOrder > 1 ) { 
+				valid = false;
+				ss << " ???";
+			}
+			else ss << " (OK)";
+			ss << '\n';
+		}
 		ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";
-		ss << "  " << len << " bytes record data w/headers\n";
-		ss << "  " << nlen << " bytes record data wout/headers\n";
+		ss << "  " << len << " bytes data w/headers\n";
+		ss << "  " << nlen << " bytes data wout/headers\n";
 
 		ss << "  deletedList: ";
 		for( int i = 0; i < Buckets; i++ ) { 
@@ -521,13 +544,18 @@ inline bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuil
 	}
 	else if( e.type() == Number ) { 
 		if( strcmp(e.fieldName(), "dropDatabase") == 0 ) { 
-			valid = true;
-			int p = (int) e.number();
-			if( p != 1 ) {
-				ok = false;
-			} else { 
-				dropDatabase(ns);
-				ok = true;
+			if( 1 ) {
+				valid = true;
+				int p = (int) e.number();
+				if( p != 1 ) {
+					ok = false;
+				} else { 
+					dropDatabase(ns);
+					ok = true;
+				}
+			}
+			else { 
+				cout << "TEMP CODE: dropdatabase commented out " << endl;
 			}
 		}
 		else if( strcmp(e.fieldName(), "profile") == 0 ) { 
@@ -836,7 +864,7 @@ assert( debug.getN() < 5000 );
 						}
 						if( ok ) {
 							n++;
-							if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
+							if( (ntoreturn>0 && (n >= ntoreturn || b.len() > MaxBytesToReturnToClientAtOnce)) ||
 								(ntoreturn==0 && b.len()>1*1024*1024) ) {
 									/* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
 									if( wantMore && ntoreturn != 1 ) {
@@ -964,7 +992,7 @@ done:
 					}
 					if( ok ) {
 						n++;
-						if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
+						if( (ntoreturn>0 && (n >= ntoreturn || b.len() > MaxBytesToReturnToClientAtOnce)) ||
 							(ntoreturn==0 && b.len()>1*1024*1024) ) {
 								c->advance();
 								cc->pos += n;