diff options
author | Dwight <dmerriman@gmail.com> | 2008-06-06 09:43:15 -0400 |
---|---|---|
committer | Dwight <dmerriman@gmail.com> | 2008-06-06 09:43:15 -0400 |
commit | 3051b961cac30f9bf81ac72b816ddb5e8e3c2ee9 (patch) | |
tree | 85a5b1cb376b067eee5cf668d42deff78a870627 | |
parent | 877b72efcdd55f9fc9b271c707d4b489e551793d (diff) | |
download | mongo-3051b961cac30f9bf81ac72b816ddb5e8e3c2ee9.tar.gz |
dos2unix
-rw-r--r-- | db/btree.cpp | 1802 | ||||
-rw-r--r-- | db/btree.h | 376 | ||||
-rw-r--r-- | db/clientcursor.cpp | 428 | ||||
-rw-r--r-- | db/db.cpp | 1504 | ||||
-rw-r--r-- | db/db.h | 8 | ||||
-rw-r--r-- | db/introspect.cpp | 78 | ||||
-rw-r--r-- | db/introspect.h | 92 | ||||
-rw-r--r-- | db/javajs.cpp | 42 | ||||
-rw-r--r-- | db/jsobj.cpp | 1620 | ||||
-rw-r--r-- | db/jsobj.h | 1028 | ||||
-rw-r--r-- | db/minilex.h | 194 | ||||
-rw-r--r-- | db/namespace.h | 450 | ||||
-rw-r--r-- | db/objwrappers.h | 18 | ||||
-rw-r--r-- | db/pdfile.cpp | 1938 | ||||
-rw-r--r-- | db/pdfile.h | 912 | ||||
-rw-r--r-- | db/query.cpp | 1708 | ||||
-rw-r--r-- | db/query.h | 270 | ||||
-rw-r--r-- | db/resource.h | 28 | ||||
-rw-r--r-- | db/storage.h | 180 | ||||
-rw-r--r-- | grid/message.cpp | 392 | ||||
-rw-r--r-- | grid/message.h | 258 | ||||
-rw-r--r-- | grid/protocol.h | 522 | ||||
-rw-r--r-- | grid/protoimpl.h | 484 | ||||
-rw-r--r-- | grid/protorecv.cpp | 772 | ||||
-rw-r--r-- | grid/protosend.cpp | 376 | ||||
-rw-r--r-- | stdafx.cpp | 66 | ||||
-rw-r--r-- | stdafx.h | 222 | ||||
-rw-r--r-- | targetver.h | 10 | ||||
-rw-r--r-- | util/builder.h | 134 | ||||
-rw-r--r-- | util/goodies.h | 256 | ||||
-rw-r--r-- | util/hashtab.h | 220 | ||||
-rw-r--r-- | util/lruishmap.h | 118 | ||||
-rw-r--r-- | util/mmap.cpp | 288 | ||||
-rw-r--r-- | util/mmap.h | 44 | ||||
-rw-r--r-- | util/sock.cpp | 322 | ||||
-rw-r--r-- | util/sock.h | 314 | ||||
-rw-r--r-- | util/util.cpp | 96 |
37 files changed, 8785 insertions, 8785 deletions
diff --git a/db/btree.cpp b/db/btree.cpp index 8e2b982d079..2cbfe9811b4 100644 --- a/db/btree.cpp +++ b/db/btree.cpp @@ -1,901 +1,901 @@ -// btree.cpp
-
-#include "stdafx.h"
-#include "btree.h"
-#include "pdfile.h"
-
-/* it is easy to do custom sizes for a namespace - all the same for now */
-const int BucketSize = 8192;
-const int KeyMax = BucketSize / 10;
-
-int ninserts = 0;
-extern int otherTraceLevel;
-int split_debug = 0;
-int insert_debug = 0;
-DiskLoc maxDiskLoc(0x7fffffff, 0x7fffffff);
-DiskLoc minDiskLoc(0, 1);
-
-inline KeyNode::KeyNode(BucketBasics& bb, _KeyNode &k) :
- prevChildBucket(k.prevChildBucket),
- recordLoc(k.recordLoc), key(bb.data+k.keyDataOfs())
-{ }
-
-/* BucketBasics --------------------------------------------------- */
-
-inline void BucketBasics::setNotPacked() { flags &= ~Packed; }
-inline void BucketBasics::setPacked() { flags |= Packed; }
-
-void BucketBasics::_shape(int level, stringstream& ss) {
- for( int i = 0; i < level; i++ ) ss << ' ';
- ss << "*\n";
- for( int i = 0; i < n; i++ )
- if( !k(i).prevChildBucket.isNull() )
- k(i).prevChildBucket.btree()->_shape(level+1,ss);
- if( !nextChild.isNull() )
- nextChild.btree()->_shape(level+1,ss);
-}
-
-int bt_fv=0;
-int bt_dmp=0;
-
-void BucketBasics::dumpTree(DiskLoc thisLoc) {
- bt_dmp=1;
- fullValidate(thisLoc);
- bt_dmp=0;
-}
-
-int BucketBasics::fullValidate(const DiskLoc& thisLoc) {
- assertValid(true);
-// if( bt_fv==0 )
-// return;
-
- if( bt_dmp ) {
- cout << thisLoc.toString() << ' ';
- ((BtreeBucket *) this)->dump();
- }
-
- // keycount
- int kc = 0;
-
- for( int i = 0; i < n; i++ ) {
- _KeyNode& kn = k(i);
- if( kn.isUsed() ) kc++;
- if( !kn.prevChildBucket.isNull() ) {
- DiskLoc left = kn.prevChildBucket;
- BtreeBucket *b = left.btree();
- wassert( b->parent == thisLoc );
- kc += b->fullValidate(kn.prevChildBucket);
- }
- }
- if( !nextChild.isNull() ) {
- BtreeBucket *b = nextChild.btree();
- wassert( b->parent == thisLoc );
- kc += b->fullValidate(nextChild);
- }
-
- return kc;
-}
-
-int nDumped = 0;
-
-void BucketBasics::assertValid(bool force) {
- if( !debug && !force )
- return;
- wassert( n >= 0 && n < BucketSize );
- wassert( emptySize >= 0 && emptySize < BucketSize );
- wassert( topSize >= n && topSize <= BucketSize );
- wassert( Size == BucketSize );
- if( 1 ) {
- // slow:
- for( int i = 0; i < n-1; i++ ) {
- JSObj k1 = keyNode(i).key;
- JSObj k2 = keyNode(i+1).key;
- int z = k1.woCompare(k2); //OK
- if( z > 0 ) {
- cout << "ERROR: btree key order corrupt. Keys:" << endl;
- if( ++nDumped < 5 ) {
- for( int j = 0; j < n; j++ ) {
- cout << " " << keyNode(j).key.toString() << endl;
- }
- ((BtreeBucket *) this)->dump();
- }
- wassert(false);
- break;
- }
- else if( z == 0 ) {
- wassert( k(i).recordLoc < k(i+1).recordLoc );
- }
- }
- }
- else {
- //faster:
- if( n > 1 ) {
- JSObj k1 = keyNode(0).key;
- JSObj k2 = keyNode(n-1).key;
- int z = k1.woCompare(k2);
- wassert( z <= 0 );
- }
- }
-}
-
-inline void BucketBasics::markUnused(int keypos) {
- assert( keypos >= 0 && keypos < n );
- k(keypos).setUnused();
-}
-
-inline int BucketBasics::totalDataSize() const {
- return Size - (data-(char*)this);
-}
-
-void BucketBasics::init(){
- parent.Null(); nextChild.Null();
- Size = BucketSize;
- flags = Packed;
- n = 0;
- emptySize = totalDataSize(); topSize = 0;
- reserved = 0;
-}
-
-/* we allocate space from the end of the buffer for data.
- the keynodes grow from the front.
-*/
-inline int BucketBasics::_alloc(int bytes) {
- topSize += bytes;
- emptySize -= bytes;
- int ofs = totalDataSize() - topSize;
- assert( ofs > 0 );
- return ofs;
-}
-
-void BucketBasics::_delKeyAtPos(int keypos) {
- assert( keypos >= 0 && keypos <= n );
- assert( childForPos(keypos).isNull() );
- n--;
- assert( n > 0 || nextChild.isNull() );
- for( int j = keypos; j < n; j++ )
- k(j) = k(j+1);
- emptySize += sizeof(_KeyNode);
- setNotPacked();
-}
-
-/* add a key. must be > all existing. be careful to set next ptr right. */
-void BucketBasics::pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild) {
- int bytesNeeded = key.objsize() + sizeof(_KeyNode);
- assert( bytesNeeded <= emptySize );
- assert( n == 0 || keyNode(n-1).key.woCompare(key) <= 0 );
- emptySize -= sizeof(_KeyNode);
- _KeyNode& kn = k(n++);
- kn.prevChildBucket = prevChild;
- kn.recordLoc = recordLoc;
- kn.setKeyDataOfs( (short) _alloc(key.objsize()) );
- char *p = dataAt(kn.keyDataOfs());
- memcpy(p, key.objdata(), key.objsize());
-}
-
-bool BucketBasics::basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key) {
- assert( keypos >= 0 && keypos <= n );
- int bytesNeeded = key.objsize() + sizeof(_KeyNode);
- if( bytesNeeded > emptySize ) {
- pack();
- if( bytesNeeded > emptySize )
- return false;
- }
- for( int j = n; j > keypos; j-- ) // make room
- k(j) = k(j-1);
- n++;
- emptySize -= sizeof(_KeyNode);
- _KeyNode& kn = k(keypos);
- kn.prevChildBucket.Null();
- kn.recordLoc = recordLoc;
- kn.setKeyDataOfs((short) _alloc(key.objsize()) );
- char *p = dataAt(kn.keyDataOfs());
- memcpy(p, key.objdata(), key.objsize());
- return true;
-}
-
-/* when we delete things we just leave empty space until the node is
- full and then we repack it.
-*/
-void BucketBasics::pack() {
- if( flags & Packed )
- return;
-
- int tdz = totalDataSize();
- char temp[BucketSize];
- int ofs = tdz;
- topSize = 0;
- for( int j = 0; j < n; j++ ) {
- short ofsold = k(j).keyDataOfs();
- int sz = keyNode(j).key.objsize();
- ofs -= sz;
- topSize += sz;
- memcpy(temp+ofs, dataAt(ofsold), sz);
- k(j).setKeyDataOfsSavingUse( ofs );
- }
- int dataUsed = tdz - ofs;
- memcpy(data + ofs, temp + ofs, dataUsed);
- emptySize = tdz - dataUsed - n * sizeof(_KeyNode);
- assert( emptySize >= 0 );
-
- setPacked();
- assertValid();
-}
-
-inline void BucketBasics::truncateTo(int N) {
- n = N;
- setNotPacked();
- pack();
-}
-
-/* - BtreeBucket --------------------------------------------------- */
-
-/* return largest key in the subtree. */
-void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey) {
- DiskLoc loc = thisLoc;
- while( 1 ) {
- BtreeBucket *b = loc.btree();
-// b->dump();
- if( !b->nextChild.isNull() ) {
- loc = b->nextChild;
- continue;
- }
-
- assert(b->n>0);
- largestLoc = loc;
- largestKey = b->n-1;
-
- break;
- }
-}
-
-/* pos: for existing keys k0...kn-1.
- returns # it goes BEFORE. so key[pos-1] < key < key[pos]
- returns n if it goes after the last existing key.
- note result might be Unused!
-*/
-bool BtreeBucket::find(JSObj& key, DiskLoc recordLoc, int& pos) {
- /* binary search for this key */
- int l=0; int h=n-1;
- while( l <= h ) {
- int m = (l+h)/2;
- KeyNode M = keyNode(m);
- int x = key.woCompare(M.key);
- if( x == 0 )
- x = recordLoc.compare(M.recordLoc);
- if( x < 0 ) // key < M.key
- h = m-1;
- else if( x > 0 )
- l = m+1;
- else {
- // found it. however, if dup keys are here, be careful we might have
- // found one in the middle. we want find() to return the leftmost instance.
-/*
- while( m >= 1 && keyNode(m-1).key.woEqual(key) )
- m--;
-*/
-
- pos = m;
-
-/*
- DiskLoc ch = k(m).prevChildBucket;
- if( !ch.isNull() ) {
- // if dup keys, might be dups to the left.
- DiskLoc largestLoc;
- int largestKey;
- ch.btree()->findLargestKey(ch, largestLoc, largestKey);
- if( !largestLoc.isNull() ) {
- if( largestLoc.btree()->keyAt(largestKey).woEqual(key) )
- return false;
- }
- }
-*/
-
- return true;
- }
-//? x = key.woCompare(M.key);
- }
- // not found
- pos = l;
- if( pos != n ) {
- JSObj keyatpos = keyNode(pos).key;
- wassert( key.woCompare(keyatpos) <= 0 );
- if( pos > 0 ) {
- wassert( keyNode(pos-1).key.woCompare(key) <= 0 );
- }
- }
-
- return false;
-}
-
-void BtreeBucket::delBucket(const DiskLoc& thisLoc, IndexDetails& id) {
- assert( !isHead() );
-
- BtreeBucket *p = parent.btree();
- if( p->nextChild == thisLoc ) {
- p->nextChild.Null();
- }
- else {
- for( int i = 0; i < p->n; i++ ) {
- if( p->k(i).prevChildBucket == thisLoc ) {
- p->k(i).prevChildBucket.Null();
- goto found;
- }
- }
- cout << "ERROR: can't find ref to deleted bucket.\n";
- cout << "To delete:\n";
- dump();
- cout << "Parent:\n";
- p->dump();
- assert(false);
- }
-found:
- //defensive:
- n = -1;
- parent.Null();
- theDataFileMgr.deleteRecord(id.indexNamespace().c_str(), thisLoc.rec(), thisLoc);
-}
-
-/* note: may delete the entire bucket! this invalid upon return sometimes. */
-void BtreeBucket::delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p) {
- assert(n>0);
- DiskLoc left = childForPos(p);
-
- if( n == 1 ) {
- if( left.isNull() && nextChild.isNull() ) {
- if( isHead() )
- _delKeyAtPos(p); // we don't delete the top bucket ever
- else
- delBucket(thisLoc, id);
- return;
- }
- markUnused(p);
- return;
- }
-
- if( left.isNull() )
- _delKeyAtPos(p);
- else
- markUnused(p);
-}
-
-int verbose = 0;
-int qqq = 0;
-
-bool BtreeBucket::unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc ) {
- if( key.objsize() > KeyMax ) {
- problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << ' ' << key.toString() << endl;
- return false;
- }
-
- int pos;
- bool found;
- DiskLoc loc = locate(thisLoc, key, pos, found, recordLoc, 1);
- if( found ) {
- loc.btree()->delKeyAtPos(loc, id, pos);
- return true;
- }
- return false;
-}
-
-BtreeBucket* BtreeBucket::allocTemp() {
- BtreeBucket *b = (BtreeBucket*) malloc(BucketSize);
- b->init();
- return b;
-}
-
-inline void fix(const DiskLoc& thisLoc, const DiskLoc& child) {
- if( !child.isNull() ) {
- if( insert_debug )
- cout << " " << child.toString() << ".parent=" << thisLoc.toString() << endl;
- child.btree()->parent = thisLoc;
- }
-}
-
-/* this sucks. maybe get rid of parent ptrs. */
-void BtreeBucket::fixParentPtrs(const DiskLoc& thisLoc) {
- fix(thisLoc, nextChild);
- for( int i = 0; i < n; i++ )
- fix(thisLoc, k(i).prevChildBucket);
-}
-
-/* keypos - where to insert the key i3n range 0..n. 0=make leftmost, n=make rightmost.
-*/
-void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos,
- DiskLoc recordLoc, JSObj& key,
- DiskLoc lchild, DiskLoc rchild, IndexDetails& idx)
-{
- if( insert_debug )
- cout << " " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' '
- << lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;
-
- DiskLoc oldLoc = thisLoc;
-
- if( basicInsert(keypos, recordLoc, key) ) {
- _KeyNode& kn = k(keypos);
- if( keypos+1 == n ) { // last key
- if( nextChild != lchild ) {
- cout << "ERROR nextChild != lchild" << endl;
- cout << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;
- cout << " keyPos: " << keypos << " n:" << n << endl;
- cout << " nextChild: " << nextChild.toString() << " lchild: " << lchild.toString() << endl;
- cout << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;
- cout << " key: " << key.toString() << endl;
- dump();
-#if defined(_WIN32)
- cout << "\n\nDUMPING FULL INDEX" << endl;
- bt_dmp=1;
- bt_fv=1;
- idx.head.btree()->fullValidate(idx.head);
-#endif
- assert(false);
- }
- kn.prevChildBucket = nextChild;
- assert( kn.prevChildBucket == lchild );
- nextChild = rchild;
- if( !rchild.isNull() )
- rchild.btree()->parent = thisLoc;
- }
- else {
- k(keypos).prevChildBucket = lchild;
- if( k(keypos+1).prevChildBucket != lchild ) {
- cout << "ERROR k(keypos+1).prevChildBucket != lchild" << endl;
- cout << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl;
- cout << " keyPos: " << keypos << " n:" << n << endl;
- cout << " k(keypos+1).pcb: " << k(keypos+1).prevChildBucket.toString() << " lchild: " << lchild.toString() << endl;
- cout << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl;
- cout << " key: " << key.toString() << endl;
- dump();
-#if defined(_WIN32)
- cout << "\n\nDUMPING FULL INDEX" << endl;
- bt_dmp=1;
- bt_fv=1;
- idx.head.btree()->fullValidate(idx.head);
-#endif
- assert(false);
- }
- k(keypos+1).prevChildBucket = rchild;
- if( !rchild.isNull() )
- rchild.btree()->parent = thisLoc;
- }
- return;
- }
-
- // split
- if( split_debug )
- cout << " " << thisLoc.toString() << ".split" << endl;
-
- int mid = n / 2;
-
- /* on duplicate key, we need to ensure that they all end up on the RHS */
- if( 0 ) {
- assert(mid>0);
- while( 1 ) {
- KeyNode mn = keyNode(mid);
- KeyNode left = keyNode(mid-1);
- if( left.key < mn.key )
- break;
- mid--;
- if( mid < 3 ) {
- problem() << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;
- cout << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl;
- cout << " ns:" << idx.indexNamespace() << endl;
- cout << " key:" << mn.key.toString() << endl;
- break;
- }
- }
- }
-
- BtreeBucket *r = allocTemp();
- DiskLoc rLoc;
-
- if( split_debug )
- cout << " mid:" << mid << ' ' << keyNode(mid).key.toString() << " n:" << n << endl;
- for( int i = mid+1; i < n; i++ ) {
- KeyNode kn = keyNode(i);
- if( i == keypos ) {
- // slip in the new one
- r->pushBack(recordLoc, key, kn.prevChildBucket);
- r->pushBack(kn.recordLoc, kn.key, rchild);
- }
- else
- r->pushBack(kn.recordLoc, kn.key, kn.prevChildBucket);
- }
- r->nextChild = nextChild;
- r->assertValid();
-//r->dump();
- rLoc = theDataFileMgr.insert(idx.indexNamespace().c_str(), r, r->Size, true);
- if( split_debug )
- cout << " new rLoc:" << rLoc.toString() << endl;
- free(r); r = 0;
- rLoc.btree()->fixParentPtrs(rLoc);
-
- {
- KeyNode middle = keyNode(mid);
- nextChild = middle.prevChildBucket; // middle key gets promoted, its children will be thisLoc (l) and rLoc (r)
- if( split_debug ) {
- //rLoc.btree()->dump();
- cout << " middle key:" << middle.key.toString() << endl;
- }
-
- // promote middle to a parent node
- if( parent.isNull() ) {
- // make a new parent if we were the root
- BtreeBucket *p = allocTemp();
- p->pushBack(middle.recordLoc, middle.key, thisLoc);
- p->nextChild = rLoc;
- p->assertValid();
- parent = idx.head = theDataFileMgr.insert(idx.indexNamespace().c_str(), p, p->Size, true);
- if( split_debug )
- cout << " we were root, making new root:" << hex << parent.getOfs() << dec << endl;
- free(p);
- rLoc.btree()->parent = parent;
- }
- else {
- /* set this before calling _insert - if it splits it will do fixParent() logic and fix the value,
- so we don't want to overwrite that if it happens.
- */
- rLoc.btree()->parent = parent;
- if( split_debug )
- cout << " promoting middle key " << middle.key.toString() << endl;
- parent.btree()->_insert(parent, middle.recordLoc, middle.key, false, thisLoc, rLoc, idx);
- }
- BtreeBucket *br = rLoc.btree();
-//br->dump();
-
-//parent.btree()->dump();
-//idx.head.btree()->dump();
-
- }
-
- // mark on left that we no longer have anything from midpoint on.
- bool highest = keypos == n;
- truncateTo(mid); // note this may trash middle.key! thus we had to promote it before finishing up here.
-
- // add our new key, there is room now
- {
-
-//dump();
-
- if( keypos <= mid ) {
-// if( keypos < mid ) {
- if( split_debug )
- cout << " keypos<mid, insertHere() the new key" << endl;
- insertHere(thisLoc, keypos, recordLoc, key, lchild, rchild, idx);
-//dump();
- } else if( highest ) {
- // else handled above already.
- int kp = keypos-mid-1; assert(kp>=0);
- rLoc.btree()->insertHere(rLoc, kp, recordLoc, key, lchild, rchild, idx);
-// set a bp here.
-// if( !lchild.isNull() ) cout << lchild.btree()->parent.toString() << endl;
-// if( !rchild.isNull() ) cout << rchild.btree()->parent.toString() << endl;
- }
- }
-
- if( split_debug )
- cout << " split end " << hex << thisLoc.getOfs() << dec << endl;
-}
-
-DiskLoc BtreeBucket::addHead(IndexDetails& id) {
- BtreeBucket *p = allocTemp();
- DiskLoc loc = theDataFileMgr.insert(id.indexNamespace().c_str(), p, p->Size, true);
- return loc;
-}
-
-DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) {
- DiskLoc p = thisLoc;
- while( !p.btree()->isHead() )
- p = p.btree()->parent;
- return p;
-}
-
-DiskLoc BtreeBucket::advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) {
- if( keyOfs < 0 || keyOfs >= n ) {
- cout << "ASSERT failure BtreeBucket::advance, caller: " << caller << endl;
- cout << " thisLoc: " << thisLoc.toString() << endl;
- cout << " keyOfs: " << keyOfs << " n:" << n << " direction: " << direction << endl;
- cout << bucketSummary() << endl;
- assert( keyOfs >= 0 && keyOfs < n );
- }
- int adj = direction < 0 ? 1 : 0;
- int ko = keyOfs + direction;
- DiskLoc nextDown = childForPos(ko+adj);
- if( !nextDown.isNull() ) {
-// nextDown.btree()->dump();//TEMP:
- while( 1 ) {
- keyOfs = direction>0 ? 0 : nextDown.btree()->n - 1;
- DiskLoc loc= nextDown.btree()->childForPos(keyOfs + adj);
- if( loc.isNull() )
- break;
- nextDown = loc;
- }
- return nextDown;
- }
-
- if( ko < n && ko >= 0 ) {
- keyOfs = ko;
- return thisLoc;
- }
-
- // end of bucket. traverse back up.
- DiskLoc childLoc = thisLoc;
- DiskLoc ancestor = parent;
- while( 1 ) {
- if( ancestor.isNull() )
- break;
- BtreeBucket *an = ancestor.btree();
- for( int i = 0; i < an->n; i++ ) {
- if( an->childForPos(i+adj) == childLoc ) {
- keyOfs = i;
- return ancestor;
- }
- }
- assert( direction<0 || an->nextChild == childLoc );
- // parent exhausted also, keep going up
- childLoc = ancestor;
- ancestor = an->parent;
- }
-
- return DiskLoc();
-}
-
-DiskLoc BtreeBucket::locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction) {
- int p;
- found = find(key, recordLoc, p);
- if( found ) {
- pos = p;
- return thisLoc;
- }
-
- DiskLoc child = childForPos(p);
-
- if( !child.isNull() ) {
- DiskLoc l = child.btree()->locate(child, key, pos, found, recordLoc, direction);
- if( !l.isNull() )
- return l;
- }
-
- if( direction == -1 && p == n && n ) {
- p--;
- }
-
- pos = p;
- return pos == n ? DiskLoc() /*theend*/ : thisLoc;
-}
-
-/* thisloc is the location of this bucket object. you must pass that in. */
-int BtreeBucket::_insert(DiskLoc thisLoc, DiskLoc recordLoc,
- JSObj& key, bool dupsAllowed,
- DiskLoc lChild, DiskLoc rChild, IndexDetails& idx) {
- if( key.objsize() > KeyMax ) {
- problem() << "ERROR: key too large len:" << key.objsize() << " max:" << KeyMax << ' ' << idx.indexNamespace() << endl;
- return 2;
- }
- assert( key.objsize() > 0 );
-
- int pos;
- bool found = find(key, recordLoc, pos);
- if( insert_debug ) {
- cout << " " << thisLoc.toString() << '.' << "_insert " <<
- key.toString() << '/' << recordLoc.toString() <<
- " l:" << lChild.toString() << " r:" << rChild.toString() << endl;
- cout << " found:" << found << " pos:" << pos << " n:" << n << endl;
- }
-
- if( found ) {
- if( k(pos).isUnused() ) {
- cout << "an unused already occupying keyslot, write more code.\n";
- cout << " index may be corrupt (missing data) now.\n";
- }
-
- cout << "_insert(): key already exists in index\n";
- cout << " " << idx.indexNamespace().c_str() << " thisLoc:" << thisLoc.toString() << '\n';
- cout << " " << key.toString() << '\n';
- cout << " " << "recordLoc:" << recordLoc.toString() << " pos:" << pos << endl;
- cout << " old l r: " << childForPos(pos).toString() << ' ' << childForPos(pos+1).toString() << endl;
- cout << " new l r: " << lChild.toString() << ' ' << rChild.toString() << endl;
- assert(false);
-
- // on a dup key always insert on the right or else you will be broken.
-// pos++;
- // on a promotion, find the right point to update if dup keys.
- /* not needed: we always insert right after the first key so we are ok with just pos++...
- if( !rChild.isNull() ) {
- while( pos < n && k(pos).prevChildBucket != lchild ) {
- pos++;
- cout << "looking for the right dup key" << endl;
- }
- }
- */
- }
-
- DEBUGGING cout << "TEMP: key: " << key.toString() << endl;
- DiskLoc& child = getChild(pos);
- if( insert_debug )
- cout << " getChild(" << pos << "): " << child.toString() << endl;
- if( child.isNull() || !rChild.isNull() /* means an 'internal' insert */ ) {
- insertHere(thisLoc, pos, recordLoc, key, lChild, rChild, idx);
- return 0;
- }
-
- return child.btree()->insert(child, recordLoc, key, dupsAllowed, idx, false);
-}
-
-void BtreeBucket::dump() {
- cout << "DUMP btreebucket: ";
- cout << " parent:" << hex << parent.getOfs() << dec;
- for( int i = 0; i < n; i++ ) {
- cout << '\n';
- KeyNode k = keyNode(i);
- cout << '\t' << i << '\t' << k.key.toString() << "\tleft:" << hex <<
- k.prevChildBucket.getOfs() << "\trec:" << k.recordLoc.getOfs() << dec;
- if( this->k(i).isUnused() )
- cout << " UNUSED";
- }
- cout << " right:" << hex << nextChild.getOfs() << dec << endl;
-}
-
-JSObj *music = 0;
-
-void tempMusic(DiskLoc thisLoc)
-{
- BtreeCursor c(thisLoc, *music, 1, true);
- while( c.ok() ) {
- KeyNode kn = c.currKeyNode();
- if( !kn.key.woEqual(*music) )
- break;
- if( kn.recordLoc.getOfs() == 0x4c8d7c0 ) {
- cout << "*** found it" << endl;
- // c.bucket.btree()->dump();
- return;
- }
- c.advance();
- }
-
- cout << "*** NOT FOUND" << endl;
-}
-
-/* todo: meaning of return code unclear clean up */
-int BtreeBucket::insert(DiskLoc thisLoc, DiskLoc recordLoc,
- JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel)
-{
- if( toplevel ) {
- if( key.objsize() > KeyMax ) {
- problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.toString() << '\n';
- return 3;
- }
- ++ninserts;
- /*
- if( ninserts % 1000 == 0 ) {
- cout << "ninserts: " << ninserts << endl;
- if( 0 && ninserts >= 127287 ) {
- cout << "debug?" << endl;
- split_debug = 1;
- }
- }
- */
- }
-
- bool chk = false;
-
- int x = _insert(thisLoc, recordLoc, key, dupsAllowed, DiskLoc(), DiskLoc(), idx);
- assertValid();
-
-/* if( toplevel ) {
- if( recordLoc.getOfs() == 0x4c8d7c0 ) {
- if( key.toString() == "{ _searchIndex: \"music\" }" ) {
- tempMusic(thisLoc);
- }
- }
- }
-*/
- return x;
-}
-
-void BtreeBucket::shape(stringstream& ss) {
- _shape(0, ss);
-}
-
-/* - BtreeCursor --------------------------------------------------- */
-
-BtreeCursor::BtreeCursor(DiskLoc head, JSObj& k, int _direction, bool sm) :
- direction(_direction), stopmiss(sm)
-{
-//otherTraceLevel = 999;
-
- bool found;
- if( otherTraceLevel >= 12 ) {
- if( otherTraceLevel >= 200 ) {
- cout << "::BtreeCursor() qtl>200. validating entire index." << endl;
- head.btree()->fullValidate(head);
- }
- else {
- cout << "BTreeCursor(). dumping head bucket" << endl;
- head.btree()->dump();
- }
- }
- bucket = head.btree()->locate(head, k, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
- checkUnused();
-}
-
-int zzz = 0;
-
-/* skip unused keys. */
-void BtreeCursor::checkUnused() {
- int u = 0;
- while( 1 ) {
- if( !ok() )
- break;
- BtreeBucket *b = bucket.btree();
- _KeyNode& kn = b->k(keyOfs);
- if( kn.isUsed() )
- break;
- bucket = b->advance(bucket, keyOfs, direction, "checkUnused");
- u++;
- }
- if( u > 10 && ++zzz % 16 == 0 )
- cout << "btree unused skipped:" << u << endl;
-}
-
-/*DiskLoc BtreeCursor::currLoc() {
- assert( !bucket.isNull() );
- _KeyNode& kn = bucket.btree()->k(keyOfs);
- assert( kn.isUsed() );
- return kn.recordLoc;
-}*/
-
-bool BtreeCursor::advance() {
- if( bucket.isNull() )
- return false;
- bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance");
- checkUnused();
- return !bucket.isNull();
-}
-
-void BtreeCursor::noteLocation() {
- if( !eof() ) {
- JSObj o = bucket.btree()->keyAt(keyOfs).copy();
- keyAtKeyOfs = o;
- locAtKeyOfs = bucket.btree()->k(keyOfs).recordLoc;
- }
-}
-
-int clctr = 0;
-
-/* see if things moved around (deletes, splits, inserts) */
-void BtreeCursor::checkLocation() {
- try {
- if( eof() )
- return;
- BtreeBucket *b = bucket.btree();
- if( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) &&
- b->k(keyOfs).recordLoc == locAtKeyOfs ) {
- if( !b->k(keyOfs).isUsed() )
- checkUnused();
- return;
- }
- }
- catch( AssertionException ) {
- cout << "Caught exception in checkLocation(), that's maybe ok" << endl;
- }
-
- bool found;
- DiskLoc bold = bucket;
-
-/* TODO: Switch to keep indexdetails and do idx.head! */
- /* didn't find, check from the top */
- DiskLoc head = bold.btree()->getHead(bold);
- bucket = head.btree()->locate(head, keyAtKeyOfs, keyOfs, found, locAtKeyOfs, direction);
- if( clctr++ % 128 == 0 )
- cout << " key seems to have moved in the index, refinding. found:" << found << endl;
- if( found )
- checkUnused();
-}
-
-/* ----------------------------------------------------------------------------- */
-
-struct BtreeUnitTest {
- BtreeUnitTest() {
- assert( minDiskLoc.compare(maxDiskLoc) < 0 );
- }
-} btut;
+// btree.cpp + +#include "stdafx.h" +#include "btree.h" +#include "pdfile.h" + +/* it is easy to do custom sizes for a namespace - all the same for now */ +const int BucketSize = 8192; +const int KeyMax = BucketSize / 10; + +int ninserts = 0; +extern int otherTraceLevel; +int split_debug = 0; +int insert_debug = 0; +DiskLoc maxDiskLoc(0x7fffffff, 0x7fffffff); +DiskLoc minDiskLoc(0, 1); + +inline KeyNode::KeyNode(BucketBasics& bb, _KeyNode &k) : + prevChildBucket(k.prevChildBucket), + recordLoc(k.recordLoc), key(bb.data+k.keyDataOfs()) +{ } + +/* BucketBasics --------------------------------------------------- */ + +inline void BucketBasics::setNotPacked() { flags &= ~Packed; } +inline void BucketBasics::setPacked() { flags |= Packed; } + +void BucketBasics::_shape(int level, stringstream& ss) { + for( int i = 0; i < level; i++ ) ss << ' '; + ss << "*\n"; + for( int i = 0; i < n; i++ ) + if( !k(i).prevChildBucket.isNull() ) + k(i).prevChildBucket.btree()->_shape(level+1,ss); + if( !nextChild.isNull() ) + nextChild.btree()->_shape(level+1,ss); +} + +int bt_fv=0; +int bt_dmp=0; + +void BucketBasics::dumpTree(DiskLoc thisLoc) { + bt_dmp=1; + fullValidate(thisLoc); + bt_dmp=0; +} + +int BucketBasics::fullValidate(const DiskLoc& thisLoc) { + assertValid(true); +// if( bt_fv==0 ) +// return; + + if( bt_dmp ) { + cout << thisLoc.toString() << ' '; + ((BtreeBucket *) this)->dump(); + } + + // keycount + int kc = 0; + + for( int i = 0; i < n; i++ ) { + _KeyNode& kn = k(i); + if( kn.isUsed() ) kc++; + if( !kn.prevChildBucket.isNull() ) { + DiskLoc left = kn.prevChildBucket; + BtreeBucket *b = left.btree(); + wassert( b->parent == thisLoc ); + kc += b->fullValidate(kn.prevChildBucket); + } + } + if( !nextChild.isNull() ) { + BtreeBucket *b = nextChild.btree(); + wassert( b->parent == thisLoc ); + kc += b->fullValidate(nextChild); + } + + return kc; +} + +int nDumped = 0; + +void BucketBasics::assertValid(bool force) { + if( !debug && !force ) + return; + wassert( n >= 0 && n < BucketSize ); + wassert( emptySize >= 0 && emptySize < BucketSize ); + wassert( topSize >= n && topSize <= BucketSize ); + wassert( Size == BucketSize ); + if( 1 ) { + // slow: + for( int i = 0; i < n-1; i++ ) { + JSObj k1 = keyNode(i).key; + JSObj k2 = keyNode(i+1).key; + int z = k1.woCompare(k2); //OK + if( z > 0 ) { + cout << "ERROR: btree key order corrupt. Keys:" << endl; + if( ++nDumped < 5 ) { + for( int j = 0; j < n; j++ ) { + cout << " " << keyNode(j).key.toString() << endl; + } + ((BtreeBucket *) this)->dump(); + } + wassert(false); + break; + } + else if( z == 0 ) { + wassert( k(i).recordLoc < k(i+1).recordLoc ); + } + } + } + else { + //faster: + if( n > 1 ) { + JSObj k1 = keyNode(0).key; + JSObj k2 = keyNode(n-1).key; + int z = k1.woCompare(k2); + wassert( z <= 0 ); + } + } +} + +inline void BucketBasics::markUnused(int keypos) { + assert( keypos >= 0 && keypos < n ); + k(keypos).setUnused(); +} + +inline int BucketBasics::totalDataSize() const { + return Size - (data-(char*)this); +} + +void BucketBasics::init(){ + parent.Null(); nextChild.Null(); + Size = BucketSize; + flags = Packed; + n = 0; + emptySize = totalDataSize(); topSize = 0; + reserved = 0; +} + +/* we allocate space from the end of the buffer for data. + the keynodes grow from the front. +*/ +inline int BucketBasics::_alloc(int bytes) { + topSize += bytes; + emptySize -= bytes; + int ofs = totalDataSize() - topSize; + assert( ofs > 0 ); + return ofs; +} + +void BucketBasics::_delKeyAtPos(int keypos) { + assert( keypos >= 0 && keypos <= n ); + assert( childForPos(keypos).isNull() ); + n--; + assert( n > 0 || nextChild.isNull() ); + for( int j = keypos; j < n; j++ ) + k(j) = k(j+1); + emptySize += sizeof(_KeyNode); + setNotPacked(); +} + +/* add a key. must be > all existing. be careful to set next ptr right. */ +void BucketBasics::pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild) { + int bytesNeeded = key.objsize() + sizeof(_KeyNode); + assert( bytesNeeded <= emptySize ); + assert( n == 0 || keyNode(n-1).key.woCompare(key) <= 0 ); + emptySize -= sizeof(_KeyNode); + _KeyNode& kn = k(n++); + kn.prevChildBucket = prevChild; + kn.recordLoc = recordLoc; + kn.setKeyDataOfs( (short) _alloc(key.objsize()) ); + char *p = dataAt(kn.keyDataOfs()); + memcpy(p, key.objdata(), key.objsize()); +} + +bool BucketBasics::basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key) { + assert( keypos >= 0 && keypos <= n ); + int bytesNeeded = key.objsize() + sizeof(_KeyNode); + if( bytesNeeded > emptySize ) { + pack(); + if( bytesNeeded > emptySize ) + return false; + } + for( int j = n; j > keypos; j-- ) // make room + k(j) = k(j-1); + n++; + emptySize -= sizeof(_KeyNode); + _KeyNode& kn = k(keypos); + kn.prevChildBucket.Null(); + kn.recordLoc = recordLoc; + kn.setKeyDataOfs((short) _alloc(key.objsize()) ); + char *p = dataAt(kn.keyDataOfs()); + memcpy(p, key.objdata(), key.objsize()); + return true; +} + +/* when we delete things we just leave empty space until the node is + full and then we repack it. +*/ +void BucketBasics::pack() { + if( flags & Packed ) + return; + + int tdz = totalDataSize(); + char temp[BucketSize]; + int ofs = tdz; + topSize = 0; + for( int j = 0; j < n; j++ ) { + short ofsold = k(j).keyDataOfs(); + int sz = keyNode(j).key.objsize(); + ofs -= sz; + topSize += sz; + memcpy(temp+ofs, dataAt(ofsold), sz); + k(j).setKeyDataOfsSavingUse( ofs ); + } + int dataUsed = tdz - ofs; + memcpy(data + ofs, temp + ofs, dataUsed); + emptySize = tdz - dataUsed - n * sizeof(_KeyNode); + assert( emptySize >= 0 ); + + setPacked(); + assertValid(); +} + +inline void BucketBasics::truncateTo(int N) { + n = N; + setNotPacked(); + pack(); +} + +/* - BtreeBucket --------------------------------------------------- */ + +/* return largest key in the subtree. */ +void BtreeBucket::findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey) { + DiskLoc loc = thisLoc; + while( 1 ) { + BtreeBucket *b = loc.btree(); +// b->dump(); + if( !b->nextChild.isNull() ) { + loc = b->nextChild; + continue; + } + + assert(b->n>0); + largestLoc = loc; + largestKey = b->n-1; + + break; + } +} + +/* pos: for existing keys k0...kn-1. + returns # it goes BEFORE. so key[pos-1] < key < key[pos] + returns n if it goes after the last existing key. + note result might be Unused! +*/ +bool BtreeBucket::find(JSObj& key, DiskLoc recordLoc, int& pos) { + /* binary search for this key */ + int l=0; int h=n-1; + while( l <= h ) { + int m = (l+h)/2; + KeyNode M = keyNode(m); + int x = key.woCompare(M.key); + if( x == 0 ) + x = recordLoc.compare(M.recordLoc); + if( x < 0 ) // key < M.key + h = m-1; + else if( x > 0 ) + l = m+1; + else { + // found it. however, if dup keys are here, be careful we might have + // found one in the middle. we want find() to return the leftmost instance. +/* + while( m >= 1 && keyNode(m-1).key.woEqual(key) ) + m--; +*/ + + pos = m; + +/* + DiskLoc ch = k(m).prevChildBucket; + if( !ch.isNull() ) { + // if dup keys, might be dups to the left. + DiskLoc largestLoc; + int largestKey; + ch.btree()->findLargestKey(ch, largestLoc, largestKey); + if( !largestLoc.isNull() ) { + if( largestLoc.btree()->keyAt(largestKey).woEqual(key) ) + return false; + } + } +*/ + + return true; + } +//? x = key.woCompare(M.key); + } + // not found + pos = l; + if( pos != n ) { + JSObj keyatpos = keyNode(pos).key; + wassert( key.woCompare(keyatpos) <= 0 ); + if( pos > 0 ) { + wassert( keyNode(pos-1).key.woCompare(key) <= 0 ); + } + } + + return false; +} + +void BtreeBucket::delBucket(const DiskLoc& thisLoc, IndexDetails& id) { + assert( !isHead() ); + + BtreeBucket *p = parent.btree(); + if( p->nextChild == thisLoc ) { + p->nextChild.Null(); + } + else { + for( int i = 0; i < p->n; i++ ) { + if( p->k(i).prevChildBucket == thisLoc ) { + p->k(i).prevChildBucket.Null(); + goto found; + } + } + cout << "ERROR: can't find ref to deleted bucket.\n"; + cout << "To delete:\n"; + dump(); + cout << "Parent:\n"; + p->dump(); + assert(false); + } +found: + //defensive: + n = -1; + parent.Null(); + theDataFileMgr.deleteRecord(id.indexNamespace().c_str(), thisLoc.rec(), thisLoc); +} + +/* note: may delete the entire bucket! this invalid upon return sometimes. */ +void BtreeBucket::delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p) { + assert(n>0); + DiskLoc left = childForPos(p); + + if( n == 1 ) { + if( left.isNull() && nextChild.isNull() ) { + if( isHead() ) + _delKeyAtPos(p); // we don't delete the top bucket ever + else + delBucket(thisLoc, id); + return; + } + markUnused(p); + return; + } + + if( left.isNull() ) + _delKeyAtPos(p); + else + markUnused(p); +} + +int verbose = 0; +int qqq = 0; + +bool BtreeBucket::unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc ) { + if( key.objsize() > KeyMax ) { + problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << ' ' << key.toString() << endl; + return false; + } + + int pos; + bool found; + DiskLoc loc = locate(thisLoc, key, pos, found, recordLoc, 1); + if( found ) { + loc.btree()->delKeyAtPos(loc, id, pos); + return true; + } + return false; +} + +BtreeBucket* BtreeBucket::allocTemp() { + BtreeBucket *b = (BtreeBucket*) malloc(BucketSize); + b->init(); + return b; +} + +inline void fix(const DiskLoc& thisLoc, const DiskLoc& child) { + if( !child.isNull() ) { + if( insert_debug ) + cout << " " << child.toString() << ".parent=" << thisLoc.toString() << endl; + child.btree()->parent = thisLoc; + } +} + +/* this sucks. maybe get rid of parent ptrs. */ +void BtreeBucket::fixParentPtrs(const DiskLoc& thisLoc) { + fix(thisLoc, nextChild); + for( int i = 0; i < n; i++ ) + fix(thisLoc, k(i).prevChildBucket); +} + +/* keypos - where to insert the key i3n range 0..n. 0=make leftmost, n=make rightmost. +*/ +void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos, + DiskLoc recordLoc, JSObj& key, + DiskLoc lchild, DiskLoc rchild, IndexDetails& idx) +{ + if( insert_debug ) + cout << " " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' ' + << lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl; + + DiskLoc oldLoc = thisLoc; + + if( basicInsert(keypos, recordLoc, key) ) { + _KeyNode& kn = k(keypos); + if( keypos+1 == n ) { // last key + if( nextChild != lchild ) { + cout << "ERROR nextChild != lchild" << endl; + cout << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl; + cout << " keyPos: " << keypos << " n:" << n << endl; + cout << " nextChild: " << nextChild.toString() << " lchild: " << lchild.toString() << endl; + cout << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl; + cout << " key: " << key.toString() << endl; + dump(); +#if defined(_WIN32) + cout << "\n\nDUMPING FULL INDEX" << endl; + bt_dmp=1; + bt_fv=1; + idx.head.btree()->fullValidate(idx.head); +#endif + assert(false); + } + kn.prevChildBucket = nextChild; + assert( kn.prevChildBucket == lchild ); + nextChild = rchild; + if( !rchild.isNull() ) + rchild.btree()->parent = thisLoc; + } + else { + k(keypos).prevChildBucket = lchild; + if( k(keypos+1).prevChildBucket != lchild ) { + cout << "ERROR k(keypos+1).prevChildBucket != lchild" << endl; + cout << " thisLoc: " << thisLoc.toString() << ' ' << idx.indexNamespace() << endl; + cout << " keyPos: " << keypos << " n:" << n << endl; + cout << " k(keypos+1).pcb: " << k(keypos+1).prevChildBucket.toString() << " lchild: " << lchild.toString() << endl; + cout << " recordLoc: " << recordLoc.toString() << " rchild: " << rchild.toString() << endl; + cout << " key: " << key.toString() << endl; + dump(); +#if defined(_WIN32) + cout << "\n\nDUMPING FULL INDEX" << endl; + bt_dmp=1; + bt_fv=1; + idx.head.btree()->fullValidate(idx.head); +#endif + assert(false); + } + k(keypos+1).prevChildBucket = rchild; + if( !rchild.isNull() ) + rchild.btree()->parent = thisLoc; + } + return; + } + + // split + if( split_debug ) + cout << " " << thisLoc.toString() << ".split" << endl; + + int mid = n / 2; + + /* on duplicate key, we need to ensure that they all end up on the RHS */ + if( 0 ) { + assert(mid>0); + while( 1 ) { + KeyNode mn = keyNode(mid); + KeyNode left = keyNode(mid-1); + if( left.key < mn.key ) + break; + mid--; + if( mid < 3 ) { + problem() << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl; + cout << "Assertion failure - mid<3: duplicate key bug not fixed yet" << endl; + cout << " ns:" << idx.indexNamespace() << endl; + cout << " key:" << mn.key.toString() << endl; + break; + } + } + } + + BtreeBucket *r = allocTemp(); + DiskLoc rLoc; + + if( split_debug ) + cout << " mid:" << mid << ' ' << keyNode(mid).key.toString() << " n:" << n << endl; + for( int i = mid+1; i < n; i++ ) { + KeyNode kn = keyNode(i); + if( i == keypos ) { + // slip in the new one + r->pushBack(recordLoc, key, kn.prevChildBucket); + r->pushBack(kn.recordLoc, kn.key, rchild); + } + else + r->pushBack(kn.recordLoc, kn.key, kn.prevChildBucket); + } + r->nextChild = nextChild; + r->assertValid(); +//r->dump(); + rLoc = theDataFileMgr.insert(idx.indexNamespace().c_str(), r, r->Size, true); + if( split_debug ) + cout << " new rLoc:" << rLoc.toString() << endl; + free(r); r = 0; + rLoc.btree()->fixParentPtrs(rLoc); + + { + KeyNode middle = keyNode(mid); + nextChild = middle.prevChildBucket; // middle key gets promoted, its children will be thisLoc (l) and rLoc (r) + if( split_debug ) { + //rLoc.btree()->dump(); + cout << " middle key:" << middle.key.toString() << endl; + } + + // promote middle to a parent node + if( parent.isNull() ) { + // make a new parent if we were the root + BtreeBucket *p = allocTemp(); + p->pushBack(middle.recordLoc, middle.key, thisLoc); + p->nextChild = rLoc; + p->assertValid(); + parent = idx.head = theDataFileMgr.insert(idx.indexNamespace().c_str(), p, p->Size, true); + if( split_debug ) + cout << " we were root, making new root:" << hex << parent.getOfs() << dec << endl; + free(p); + rLoc.btree()->parent = parent; + } + else { + /* set this before calling _insert - if it splits it will do fixParent() logic and fix the value, + so we don't want to overwrite that if it happens. + */ + rLoc.btree()->parent = parent; + if( split_debug ) + cout << " promoting middle key " << middle.key.toString() << endl; + parent.btree()->_insert(parent, middle.recordLoc, middle.key, false, thisLoc, rLoc, idx); + } + BtreeBucket *br = rLoc.btree(); +//br->dump(); + +//parent.btree()->dump(); +//idx.head.btree()->dump(); + + } + + // mark on left that we no longer have anything from midpoint on. + bool highest = keypos == n; + truncateTo(mid); // note this may trash middle.key! thus we had to promote it before finishing up here. + + // add our new key, there is room now + { + +//dump(); + + if( keypos <= mid ) { +// if( keypos < mid ) { + if( split_debug ) + cout << " keypos<mid, insertHere() the new key" << endl; + insertHere(thisLoc, keypos, recordLoc, key, lchild, rchild, idx); +//dump(); + } else if( highest ) { + // else handled above already. + int kp = keypos-mid-1; assert(kp>=0); + rLoc.btree()->insertHere(rLoc, kp, recordLoc, key, lchild, rchild, idx); +// set a bp here. +// if( !lchild.isNull() ) cout << lchild.btree()->parent.toString() << endl; +// if( !rchild.isNull() ) cout << rchild.btree()->parent.toString() << endl; + } + } + + if( split_debug ) + cout << " split end " << hex << thisLoc.getOfs() << dec << endl; +} + +DiskLoc BtreeBucket::addHead(IndexDetails& id) { + BtreeBucket *p = allocTemp(); + DiskLoc loc = theDataFileMgr.insert(id.indexNamespace().c_str(), p, p->Size, true); + return loc; +} + +DiskLoc BtreeBucket::getHead(const DiskLoc& thisLoc) { + DiskLoc p = thisLoc; + while( !p.btree()->isHead() ) + p = p.btree()->parent; + return p; +} + +DiskLoc BtreeBucket::advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller) { + if( keyOfs < 0 || keyOfs >= n ) { + cout << "ASSERT failure BtreeBucket::advance, caller: " << caller << endl; + cout << " thisLoc: " << thisLoc.toString() << endl; + cout << " keyOfs: " << keyOfs << " n:" << n << " direction: " << direction << endl; + cout << bucketSummary() << endl; + assert( keyOfs >= 0 && keyOfs < n ); + } + int adj = direction < 0 ? 1 : 0; + int ko = keyOfs + direction; + DiskLoc nextDown = childForPos(ko+adj); + if( !nextDown.isNull() ) { +// nextDown.btree()->dump();//TEMP: + while( 1 ) { + keyOfs = direction>0 ? 0 : nextDown.btree()->n - 1; + DiskLoc loc= nextDown.btree()->childForPos(keyOfs + adj); + if( loc.isNull() ) + break; + nextDown = loc; + } + return nextDown; + } + + if( ko < n && ko >= 0 ) { + keyOfs = ko; + return thisLoc; + } + + // end of bucket. traverse back up. + DiskLoc childLoc = thisLoc; + DiskLoc ancestor = parent; + while( 1 ) { + if( ancestor.isNull() ) + break; + BtreeBucket *an = ancestor.btree(); + for( int i = 0; i < an->n; i++ ) { + if( an->childForPos(i+adj) == childLoc ) { + keyOfs = i; + return ancestor; + } + } + assert( direction<0 || an->nextChild == childLoc ); + // parent exhausted also, keep going up + childLoc = ancestor; + ancestor = an->parent; + } + + return DiskLoc(); +} + +DiskLoc BtreeBucket::locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction) { + int p; + found = find(key, recordLoc, p); + if( found ) { + pos = p; + return thisLoc; + } + + DiskLoc child = childForPos(p); + + if( !child.isNull() ) { + DiskLoc l = child.btree()->locate(child, key, pos, found, recordLoc, direction); + if( !l.isNull() ) + return l; + } + + if( direction == -1 && p == n && n ) { + p--; + } + + pos = p; + return pos == n ? DiskLoc() /*theend*/ : thisLoc; +} + +/* thisloc is the location of this bucket object. you must pass that in. */ +int BtreeBucket::_insert(DiskLoc thisLoc, DiskLoc recordLoc, + JSObj& key, bool dupsAllowed, + DiskLoc lChild, DiskLoc rChild, IndexDetails& idx) { + if( key.objsize() > KeyMax ) { + problem() << "ERROR: key too large len:" << key.objsize() << " max:" << KeyMax << ' ' << idx.indexNamespace() << endl; + return 2; + } + assert( key.objsize() > 0 ); + + int pos; + bool found = find(key, recordLoc, pos); + if( insert_debug ) { + cout << " " << thisLoc.toString() << '.' << "_insert " << + key.toString() << '/' << recordLoc.toString() << + " l:" << lChild.toString() << " r:" << rChild.toString() << endl; + cout << " found:" << found << " pos:" << pos << " n:" << n << endl; + } + + if( found ) { + if( k(pos).isUnused() ) { + cout << "an unused already occupying keyslot, write more code.\n"; + cout << " index may be corrupt (missing data) now.\n"; + } + + cout << "_insert(): key already exists in index\n"; + cout << " " << idx.indexNamespace().c_str() << " thisLoc:" << thisLoc.toString() << '\n'; + cout << " " << key.toString() << '\n'; + cout << " " << "recordLoc:" << recordLoc.toString() << " pos:" << pos << endl; + cout << " old l r: " << childForPos(pos).toString() << ' ' << childForPos(pos+1).toString() << endl; + cout << " new l r: " << lChild.toString() << ' ' << rChild.toString() << endl; + assert(false); + + // on a dup key always insert on the right or else you will be broken. +// pos++; + // on a promotion, find the right point to update if dup keys. + /* not needed: we always insert right after the first key so we are ok with just pos++... + if( !rChild.isNull() ) { + while( pos < n && k(pos).prevChildBucket != lchild ) { + pos++; + cout << "looking for the right dup key" << endl; + } + } + */ + } + + DEBUGGING cout << "TEMP: key: " << key.toString() << endl; + DiskLoc& child = getChild(pos); + if( insert_debug ) + cout << " getChild(" << pos << "): " << child.toString() << endl; + if( child.isNull() || !rChild.isNull() /* means an 'internal' insert */ ) { + insertHere(thisLoc, pos, recordLoc, key, lChild, rChild, idx); + return 0; + } + + return child.btree()->insert(child, recordLoc, key, dupsAllowed, idx, false); +} + +void BtreeBucket::dump() { + cout << "DUMP btreebucket: "; + cout << " parent:" << hex << parent.getOfs() << dec; + for( int i = 0; i < n; i++ ) { + cout << '\n'; + KeyNode k = keyNode(i); + cout << '\t' << i << '\t' << k.key.toString() << "\tleft:" << hex << + k.prevChildBucket.getOfs() << "\trec:" << k.recordLoc.getOfs() << dec; + if( this->k(i).isUnused() ) + cout << " UNUSED"; + } + cout << " right:" << hex << nextChild.getOfs() << dec << endl; +} + +JSObj *music = 0; + +void tempMusic(DiskLoc thisLoc) +{ + BtreeCursor c(thisLoc, *music, 1, true); + while( c.ok() ) { + KeyNode kn = c.currKeyNode(); + if( !kn.key.woEqual(*music) ) + break; + if( kn.recordLoc.getOfs() == 0x4c8d7c0 ) { + cout << "*** found it" << endl; + // c.bucket.btree()->dump(); + return; + } + c.advance(); + } + + cout << "*** NOT FOUND" << endl; +} + +/* todo: meaning of return code unclear clean up */ +int BtreeBucket::insert(DiskLoc thisLoc, DiskLoc recordLoc, + JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel) +{ + if( toplevel ) { + if( key.objsize() > KeyMax ) { + problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.toString() << '\n'; + return 3; + } + ++ninserts; + /* + if( ninserts % 1000 == 0 ) { + cout << "ninserts: " << ninserts << endl; + if( 0 && ninserts >= 127287 ) { + cout << "debug?" << endl; + split_debug = 1; + } + } + */ + } + + bool chk = false; + + int x = _insert(thisLoc, recordLoc, key, dupsAllowed, DiskLoc(), DiskLoc(), idx); + assertValid(); + +/* if( toplevel ) { + if( recordLoc.getOfs() == 0x4c8d7c0 ) { + if( key.toString() == "{ _searchIndex: \"music\" }" ) { + tempMusic(thisLoc); + } + } + } +*/ + return x; +} + +void BtreeBucket::shape(stringstream& ss) { + _shape(0, ss); +} + +/* - BtreeCursor --------------------------------------------------- */ + +BtreeCursor::BtreeCursor(DiskLoc head, JSObj& k, int _direction, bool sm) : + direction(_direction), stopmiss(sm) +{ +//otherTraceLevel = 999; + + bool found; + if( otherTraceLevel >= 12 ) { + if( otherTraceLevel >= 200 ) { + cout << "::BtreeCursor() qtl>200. validating entire index." << endl; + head.btree()->fullValidate(head); + } + else { + cout << "BTreeCursor(). dumping head bucket" << endl; + head.btree()->dump(); + } + } + bucket = head.btree()->locate(head, k, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction); + checkUnused(); +} + +int zzz = 0; + +/* skip unused keys. */ +void BtreeCursor::checkUnused() { + int u = 0; + while( 1 ) { + if( !ok() ) + break; + BtreeBucket *b = bucket.btree(); + _KeyNode& kn = b->k(keyOfs); + if( kn.isUsed() ) + break; + bucket = b->advance(bucket, keyOfs, direction, "checkUnused"); + u++; + } + if( u > 10 && ++zzz % 16 == 0 ) + cout << "btree unused skipped:" << u << endl; +} + +/*DiskLoc BtreeCursor::currLoc() { + assert( !bucket.isNull() ); + _KeyNode& kn = bucket.btree()->k(keyOfs); + assert( kn.isUsed() ); + return kn.recordLoc; +}*/ + +bool BtreeCursor::advance() { + if( bucket.isNull() ) + return false; + bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance"); + checkUnused(); + return !bucket.isNull(); +} + +void BtreeCursor::noteLocation() { + if( !eof() ) { + JSObj o = bucket.btree()->keyAt(keyOfs).copy(); + keyAtKeyOfs = o; + locAtKeyOfs = bucket.btree()->k(keyOfs).recordLoc; + } +} + +int clctr = 0; + +/* see if things moved around (deletes, splits, inserts) */ +void BtreeCursor::checkLocation() { + try { + if( eof() ) + return; + BtreeBucket *b = bucket.btree(); + if( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) && + b->k(keyOfs).recordLoc == locAtKeyOfs ) { + if( !b->k(keyOfs).isUsed() ) + checkUnused(); + return; + } + } + catch( AssertionException ) { + cout << "Caught exception in checkLocation(), that's maybe ok" << endl; + } + + bool found; + DiskLoc bold = bucket; + +/* TODO: Switch to keep indexdetails and do idx.head! */ + /* didn't find, check from the top */ + DiskLoc head = bold.btree()->getHead(bold); + bucket = head.btree()->locate(head, keyAtKeyOfs, keyOfs, found, locAtKeyOfs, direction); + if( clctr++ % 128 == 0 ) + cout << " key seems to have moved in the index, refinding. found:" << found << endl; + if( found ) + checkUnused(); +} + +/* ----------------------------------------------------------------------------- */ + +struct BtreeUnitTest { + BtreeUnitTest() { + assert( minDiskLoc.compare(maxDiskLoc) < 0 ); + } +} btut; diff --git a/db/btree.h b/db/btree.h index 407a954ca27..063196f67b4 100644 --- a/db/btree.h +++ b/db/btree.h @@ -1,188 +1,188 @@ -// btree.h
-
-#pragma once
-
-#include "../stdafx.h"
-#include "jsobj.h"
-#include "storage.h"
-#include "pdfile.h"
-
-#pragma pack(push)
-#pragma pack(1)
-
-struct _KeyNode {
- DiskLoc prevChildBucket;
- DiskLoc recordLoc;
- short keyDataOfs() { return (short) _kdo; }
- unsigned short _kdo;
- void setKeyDataOfs(short s) { _kdo = s; assert(s>=0); }
- void setKeyDataOfsSavingUse(short s) { _kdo = s; assert(s>=0); }
- void setUnused() { recordLoc.GETOFS() |= 1; }
-// void setUsed() { _kdo &= 0x7fff; }
- int isUnused() { return (recordLoc.getOfs() & 1); }
- int isUsed() { return !isUnused(); }
-};
-
-#pragma pack(pop)
-
-class BucketBasics;
-
-/* wrapper - this is our in memory representation of the key. _KeyNode is the disk representation. */
-class KeyNode {
-public:
- KeyNode(BucketBasics& bb, _KeyNode &k);
- DiskLoc& prevChildBucket;
- DiskLoc& recordLoc;
- JSObj key;
-};
-
-#pragma pack(push)
-#pragma pack(1)
-
-/* this class is all about the storage management */
-class BucketBasics {
- friend class KeyNode;
-public:
- void dumpTree(DiskLoc thisLoc);
- bool isHead() { return parent.isNull(); }
- void assertValid(bool force = false);
- int fullValidate(const DiskLoc& thisLoc); /* traverses everything */
-protected:
- DiskLoc& getChild(int pos) {
- assert( pos >= 0 && pos <= n );
- return pos == n ? nextChild : k(pos).prevChildBucket;
- }
- KeyNode keyNode(int i) {
- assert( i < n );
- return KeyNode(*this, k(i));
- }
-
- char * dataAt(short ofs) { return data + ofs; }
-
- void init(); // initialize a new node
-
- /* returns false if node is full and must be split
- keypos is where to insert -- inserted after that key #. so keypos=0 is the leftmost one.
- */
- bool basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key);
- void pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild);
- void _delKeyAtPos(int keypos); // low level version that doesn't deal with child ptrs.
-
- /* !Packed means there is deleted fragment space within the bucket.
- We "repack" when we run out of space before considering the node
- to be full.
- */
- enum Flags { Packed=1 };
-
- DiskLoc childForPos(int p) {
- return p == n ? nextChild : k(p).prevChildBucket;
- }
-
- int totalDataSize() const;
- void pack(); void setNotPacked(); void setPacked();
- int _alloc(int bytes);
- void truncateTo(int N);
- void markUnused(int keypos);
-public:
- DiskLoc parent;
-
- string bucketSummary() const {
- stringstream ss;
- ss << " Bucket info:" << endl;
- ss << " n: " << n << endl;
- ss << " parent: " << parent.toString() << endl;
- ss << " nextChild: " << parent.toString() << endl;
- ss << " Size: " << Size << " flags:" << flags << endl;
- ss << " emptySize: " << emptySize << " topSize: " << topSize << endl;
- return ss.str();
- }
-
-protected:
- void _shape(int level, stringstream&);
- DiskLoc nextChild; // child bucket off and to the right of the highest key.
- int Size; // total size of this btree node in bytes. constant.
- int flags;
- int emptySize; // size of the empty region
- int topSize; // size of the data at the top of the bucket (keys are at the beginning or 'bottom')
- int n; // # of keys so far.
- int reserved;
- _KeyNode& k(int i) { return ((_KeyNode*)data)[i]; }
- char data[4];
-};
-
-class BtreeBucket : public BucketBasics {
- friend class BtreeCursor;
-public:
- void dump();
-
- static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */
- int insert(DiskLoc thisLoc, DiskLoc recordLoc,
- JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel);
-
- bool unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc);
-
- /* locate may return an "unused" key that is just a marker. so be careful.
- looks for a key:recordloc pair.
- */
- DiskLoc locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction=1);
-
- /* advance one key position in the index: */
- DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller);
- DiskLoc getHead(const DiskLoc& thisLoc);
-
- /* get tree shape */
- void shape(stringstream&);
-private:
- void fixParentPtrs(const DiskLoc& thisLoc);
- void delBucket(const DiskLoc& thisLoc, IndexDetails&);
- void delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p);
- JSObj keyAt(int keyOfs) { return keyOfs >= n ? JSObj() : keyNode(keyOfs).key; }
- static BtreeBucket* allocTemp(); /* caller must release with free() */
- void insertHere(DiskLoc thisLoc, int keypos,
- DiskLoc recordLoc, JSObj& key,
- DiskLoc lchild, DiskLoc rchild, IndexDetails&);
- int _insert(DiskLoc thisLoc, DiskLoc recordLoc,
- JSObj& key, bool dupsAllowed,
- DiskLoc lChild, DiskLoc rChild, IndexDetails&);
- bool find(JSObj& key, DiskLoc recordLoc, int& pos);
- static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey);
-};
-
-class BtreeCursor : public Cursor {
- friend class BtreeBucket;
-public:
- BtreeCursor(DiskLoc head, JSObj& startKey, int direction, bool stopmiss);
- virtual bool ok() { return !bucket.isNull(); }
- bool eof() { return !ok(); }
- virtual bool advance();
- virtual bool tempStopOnMiss() { return stopmiss; }
- virtual void noteLocation(); // updates keyAtKeyOfs...
- virtual void checkLocation();
-
- _KeyNode& _currKeyNode() {
- assert( !bucket.isNull() );
- _KeyNode& kn = bucket.btree()->k(keyOfs);
- assert( kn.isUsed() );
- return kn;
- }
- KeyNode currKeyNode() {
- assert( !bucket.isNull() );
- return bucket.btree()->keyNode(keyOfs);
- }
-
- virtual DiskLoc currLoc() { return !bucket.isNull() ? _currKeyNode().recordLoc : DiskLoc(); }
- virtual Record* _current() { return currLoc().rec(); }
- virtual JSObj current() { return JSObj(_current()); }
- virtual const char * toString() { return "BtreeCursor"; }
-
-private:
- void checkUnused();
- DiskLoc bucket;
- int keyOfs;
- int direction; // 1=fwd,-1=reverse
- bool stopmiss;
- JSObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call
- DiskLoc locAtKeyOfs;
-};
-
-#pragma pack(pop)
+// btree.h + +#pragma once + +#include "../stdafx.h" +#include "jsobj.h" +#include "storage.h" +#include "pdfile.h" + +#pragma pack(push) +#pragma pack(1) + +struct _KeyNode { + DiskLoc prevChildBucket; + DiskLoc recordLoc; + short keyDataOfs() { return (short) _kdo; } + unsigned short _kdo; + void setKeyDataOfs(short s) { _kdo = s; assert(s>=0); } + void setKeyDataOfsSavingUse(short s) { _kdo = s; assert(s>=0); } + void setUnused() { recordLoc.GETOFS() |= 1; } +// void setUsed() { _kdo &= 0x7fff; } + int isUnused() { return (recordLoc.getOfs() & 1); } + int isUsed() { return !isUnused(); } +}; + +#pragma pack(pop) + +class BucketBasics; + +/* wrapper - this is our in memory representation of the key. _KeyNode is the disk representation. */ +class KeyNode { +public: + KeyNode(BucketBasics& bb, _KeyNode &k); + DiskLoc& prevChildBucket; + DiskLoc& recordLoc; + JSObj key; +}; + +#pragma pack(push) +#pragma pack(1) + +/* this class is all about the storage management */ +class BucketBasics { + friend class KeyNode; +public: + void dumpTree(DiskLoc thisLoc); + bool isHead() { return parent.isNull(); } + void assertValid(bool force = false); + int fullValidate(const DiskLoc& thisLoc); /* traverses everything */ +protected: + DiskLoc& getChild(int pos) { + assert( pos >= 0 && pos <= n ); + return pos == n ? nextChild : k(pos).prevChildBucket; + } + KeyNode keyNode(int i) { + assert( i < n ); + return KeyNode(*this, k(i)); + } + + char * dataAt(short ofs) { return data + ofs; } + + void init(); // initialize a new node + + /* returns false if node is full and must be split + keypos is where to insert -- inserted after that key #. so keypos=0 is the leftmost one. + */ + bool basicInsert(int keypos, const DiskLoc& recordLoc, JSObj& key); + void pushBack(const DiskLoc& recordLoc, JSObj& key, DiskLoc prevChild); + void _delKeyAtPos(int keypos); // low level version that doesn't deal with child ptrs. + + /* !Packed means there is deleted fragment space within the bucket. + We "repack" when we run out of space before considering the node + to be full. + */ + enum Flags { Packed=1 }; + + DiskLoc childForPos(int p) { + return p == n ? nextChild : k(p).prevChildBucket; + } + + int totalDataSize() const; + void pack(); void setNotPacked(); void setPacked(); + int _alloc(int bytes); + void truncateTo(int N); + void markUnused(int keypos); +public: + DiskLoc parent; + + string bucketSummary() const { + stringstream ss; + ss << " Bucket info:" << endl; + ss << " n: " << n << endl; + ss << " parent: " << parent.toString() << endl; + ss << " nextChild: " << parent.toString() << endl; + ss << " Size: " << Size << " flags:" << flags << endl; + ss << " emptySize: " << emptySize << " topSize: " << topSize << endl; + return ss.str(); + } + +protected: + void _shape(int level, stringstream&); + DiskLoc nextChild; // child bucket off and to the right of the highest key. + int Size; // total size of this btree node in bytes. constant. + int flags; + int emptySize; // size of the empty region + int topSize; // size of the data at the top of the bucket (keys are at the beginning or 'bottom') + int n; // # of keys so far. + int reserved; + _KeyNode& k(int i) { return ((_KeyNode*)data)[i]; } + char data[4]; +}; + +class BtreeBucket : public BucketBasics { + friend class BtreeCursor; +public: + void dump(); + + static DiskLoc addHead(IndexDetails&); /* start a new index off, empty */ + int insert(DiskLoc thisLoc, DiskLoc recordLoc, + JSObj& key, bool dupsAllowed, IndexDetails& idx, bool toplevel); + + bool unindex(const DiskLoc& thisLoc, IndexDetails& id, JSObj& key, const DiskLoc& recordLoc); + + /* locate may return an "unused" key that is just a marker. so be careful. + looks for a key:recordloc pair. + */ + DiskLoc locate(const DiskLoc& thisLoc, JSObj& key, int& pos, bool& found, DiskLoc recordLoc, int direction=1); + + /* advance one key position in the index: */ + DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller); + DiskLoc getHead(const DiskLoc& thisLoc); + + /* get tree shape */ + void shape(stringstream&); +private: + void fixParentPtrs(const DiskLoc& thisLoc); + void delBucket(const DiskLoc& thisLoc, IndexDetails&); + void delKeyAtPos(const DiskLoc& thisLoc, IndexDetails& id, int p); + JSObj keyAt(int keyOfs) { return keyOfs >= n ? JSObj() : keyNode(keyOfs).key; } + static BtreeBucket* allocTemp(); /* caller must release with free() */ + void insertHere(DiskLoc thisLoc, int keypos, + DiskLoc recordLoc, JSObj& key, + DiskLoc lchild, DiskLoc rchild, IndexDetails&); + int _insert(DiskLoc thisLoc, DiskLoc recordLoc, + JSObj& key, bool dupsAllowed, + DiskLoc lChild, DiskLoc rChild, IndexDetails&); + bool find(JSObj& key, DiskLoc recordLoc, int& pos); + static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey); +}; + +class BtreeCursor : public Cursor { + friend class BtreeBucket; +public: + BtreeCursor(DiskLoc head, JSObj& startKey, int direction, bool stopmiss); + virtual bool ok() { return !bucket.isNull(); } + bool eof() { return !ok(); } + virtual bool advance(); + virtual bool tempStopOnMiss() { return stopmiss; } + virtual void noteLocation(); // updates keyAtKeyOfs... + virtual void checkLocation(); + + _KeyNode& _currKeyNode() { + assert( !bucket.isNull() ); + _KeyNode& kn = bucket.btree()->k(keyOfs); + assert( kn.isUsed() ); + return kn; + } + KeyNode currKeyNode() { + assert( !bucket.isNull() ); + return bucket.btree()->keyNode(keyOfs); + } + + virtual DiskLoc currLoc() { return !bucket.isNull() ? _currKeyNode().recordLoc : DiskLoc(); } + virtual Record* _current() { return currLoc().rec(); } + virtual JSObj current() { return JSObj(_current()); } + virtual const char * toString() { return "BtreeCursor"; } + +private: + void checkUnused(); + DiskLoc bucket; + int keyOfs; + int direction; // 1=fwd,-1=reverse + bool stopmiss; + JSObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call + DiskLoc locAtKeyOfs; +}; + +#pragma pack(pop) diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp index c9d21d48aa8..d90762b90c5 100644 --- a/db/clientcursor.cpp +++ b/db/clientcursor.cpp @@ -1,214 +1,214 @@ -// clientcursor.cpp
-
-/* Cursor -- and its derived classes -- are our internal cursors.
-
- ClientCursor is a wrapper that represents a cursorid from our client
- application's perspective.
-*/
-
-#include "stdafx.h"
-#include "query.h"
-#include "introspect.h"
-#include <time.h>
-
-/* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */
-
-typedef map<DiskLoc, ClientCursor*> DiskToCC;
-map<DiskLoc, ClientCursor*> byLocation;
-//HashTable<DiskLoc,ClientCursor*> byLocation(malloc(10000000), 10000000, "bylocation");
-
-CCMap clientCursors;
-
-class CursInspector : public SingleResultObjCursor {
- Cursor* clone() {
- return new CursInspector();
- }
-// Cursor* clone() { return new CursInspector(*this); }
- void fill() {
- b.append("byLocation_size", byLocation.size());
- b.append("clientCursors_size", clientCursors.size());
-
-
-
- stringstream ss;
- ss << '\n';
- int x = 40;
- DiskToCC::iterator it = byLocation.begin();
- while( it != byLocation.end() ) {
- DiskLoc dl = it->first;
- ClientCursor *cc = it->second;
- ss << dl.toString() << " -> \n";
-
- while( cc ) {
- ss << " cid:" << cc->cursorid << ' ' << cc->ns << " pos:" << cc->pos << " LL:" << cc->lastLoc.toString();
- try {
- setClient(cc->ns.c_str());
- Record *r = dl.rec();
- ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec << ' ' << cc->c->toString();
- if( r->nextOfs >= 0 && r->nextOfs < 16 )
- ss << " DELETED??? (!)";
- }
- catch(...) {
- ss << " EXCEPTION";
- }
- ss << "\n";
- cc = cc->nextAtThisLocation;
- }
- if( --x <= 0 ) {
- ss << "only first 40 shown\n" << endl;
- break;
- }
- it++;
- }
- b.append("dump", ss.str().c_str());
- }
-public:
- CursInspector() { reg("intr.cursors"); }
-} _ciproto;
-
-/* must call this when a btree node is updated */
-void removedKey(const DiskLoc& btreeLoc, int keyPos) {
-// TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-}
-
-/* must call this on a delete so we clean up the cursors. */
-void aboutToDelete(const DiskLoc& dl) {
- DiskToCC::iterator it = byLocation.find(dl);
-// cout << "atd:" << dl.toString() << endl;
- if( it != byLocation.end() ) {
- ClientCursor *cc = it->second;
- byLocation.erase(it);
-
- assert( cc != 0 );
- int z = 0;
- while( cc ) {
- z++;
-// cout << "cc: " << cc->ns << endl;
- ClientCursor *nxt = cc->nextAtThisLocation;
- cc->nextAtThisLocation = 0; // updateLocation will manipulate linked list ptrs, so clean that up first.
- cc->c->checkLocation();
- cc->c->advance();
- cc->lastLoc.Null(); // so updateLocation doesn't try to remove, just to be faster -- we handled that.
- cc->updateLocation();
- cc = nxt;
- }
-// cout << "z:" << z << endl;
- }
-}
-
-void ClientCursor::cleanupByLocation(DiskLoc loc, long long cursorid) {
- if( loc.isNull() )
- return;
-
- DiskToCC::iterator it = byLocation.find(loc);
- if( it != byLocation.end() ) {
- ClientCursor *first = it->second;
- ClientCursor *cc = first;
- ClientCursor *prev = 0;
-
- while( 1 ) {
- if( cc == 0 )
- break;
- if( cc->cursorid == cursorid ) {
- // found one to remove.
- if( prev == 0 ) {
- if( cc->nextAtThisLocation )
- it->second = cc->nextAtThisLocation;
- else
- byLocation.erase(it);
- }
- else {
- prev->nextAtThisLocation = cc->nextAtThisLocation;
- }
- cc->nextAtThisLocation = 0;
- return;
- }
- cc = cc->nextAtThisLocation;
- }
- }
-
- // not found!
- //cout << "Assertion failure - cleanupByLocation: not found " << cursorid << endl;
-}
-
-ClientCursor::~ClientCursor() {
-#if defined(_WIN32)
- cout << "~clientcursor " << cursorid << endl;
-#endif
- assert( pos != -2 );
-
- cleanupByLocation(lastLoc, cursorid);
-
- assert( pos != -2 );
-
- // defensive
- lastLoc.Null();
- cursorid = -1;
- pos = -2;
- nextAtThisLocation = 0;
-}
-
-// note this doesn't set lastLoc -- caller should.
-void ClientCursor::addToByLocation(DiskLoc cl) {
- assert( cursorid );
-
- if( nextAtThisLocation ) {
- wassert( nextAtThisLocation == 0 );
- return;
- }
-
- DiskToCC::iterator j = byLocation.find(cl);
- nextAtThisLocation = j == byLocation.end() ? 0 : j->second;
- byLocation[cl] = this;
-}
-
-void ClientCursor::updateLocation() {
- assert( cursorid );
-
- DiskLoc cl = c->currLoc();
- // cout<< " TEMP: updateLocation last:" << lastLoc.toString() << " cl:" << cl.toString() << '\n';
-
- if( !lastLoc.isNull() )
- cleanupByLocation(lastLoc, cursorid);
-
- if( !cl.isNull() )
- addToByLocation(cl);
-
- lastLoc = cl;
- c->noteLocation();
-}
-
-/* report to us that a new clientcursor exists so we can track it.
- note you still must call updateLocation (which likely should be changed)
-*/
-void ClientCursor::add(ClientCursor* cc) {
- clientCursors[cc->cursorid] = cc;
-}
-
-// todo: delete the ClientCursor.
-// todo: other map
-bool ClientCursor::erase(long long id) {
- CCMap::iterator it = clientCursors.find(id);
- if( it != clientCursors.end() ) {
- ClientCursor *cc = it->second;
- it->second = 0; // defensive
- clientCursors.erase(it);
- delete cc; // destructor will fix byLocation map
- return true;
- }
- return false;
-}
-
-long long allocCursorId() {
- long long x;
- while( 1 ) {
- x = (((long long)rand()) << 32);
- x = x | (int) curTimeMillis() | 0x80000000; // OR to make sure not zero
- if( clientCursors.count(x) == 0 )
- break;
- }
-#if defined(_WIN32)
- cout << "alloccursorid " << x << endl;
-#endif
- return x;
-}
+// clientcursor.cpp + +/* Cursor -- and its derived classes -- are our internal cursors. + + ClientCursor is a wrapper that represents a cursorid from our client + application's perspective. +*/ + +#include "stdafx.h" +#include "query.h" +#include "introspect.h" +#include <time.h> + +/* TODO: FIX cleanup of clientCursors when hit the end. (ntoreturn insufficient) */ + +typedef map<DiskLoc, ClientCursor*> DiskToCC; +map<DiskLoc, ClientCursor*> byLocation; +//HashTable<DiskLoc,ClientCursor*> byLocation(malloc(10000000), 10000000, "bylocation"); + +CCMap clientCursors; + +class CursInspector : public SingleResultObjCursor { + Cursor* clone() { + return new CursInspector(); + } +// Cursor* clone() { return new CursInspector(*this); } + void fill() { + b.append("byLocation_size", byLocation.size()); + b.append("clientCursors_size", clientCursors.size()); + + + + stringstream ss; + ss << '\n'; + int x = 40; + DiskToCC::iterator it = byLocation.begin(); + while( it != byLocation.end() ) { + DiskLoc dl = it->first; + ClientCursor *cc = it->second; + ss << dl.toString() << " -> \n"; + + while( cc ) { + ss << " cid:" << cc->cursorid << ' ' << cc->ns << " pos:" << cc->pos << " LL:" << cc->lastLoc.toString(); + try { + setClient(cc->ns.c_str()); + Record *r = dl.rec(); + ss << " lwh:" << hex << r->lengthWithHeaders << " nxt:" << r->nextOfs << " prv:" << r->prevOfs << dec << ' ' << cc->c->toString(); + if( r->nextOfs >= 0 && r->nextOfs < 16 ) + ss << " DELETED??? (!)"; + } + catch(...) { + ss << " EXCEPTION"; + } + ss << "\n"; + cc = cc->nextAtThisLocation; + } + if( --x <= 0 ) { + ss << "only first 40 shown\n" << endl; + break; + } + it++; + } + b.append("dump", ss.str().c_str()); + } +public: + CursInspector() { reg("intr.cursors"); } +} _ciproto; + +/* must call this when a btree node is updated */ +void removedKey(const DiskLoc& btreeLoc, int keyPos) { +// TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +} + +/* must call this on a delete so we clean up the cursors. */ +void aboutToDelete(const DiskLoc& dl) { + DiskToCC::iterator it = byLocation.find(dl); +// cout << "atd:" << dl.toString() << endl; + if( it != byLocation.end() ) { + ClientCursor *cc = it->second; + byLocation.erase(it); + + assert( cc != 0 ); + int z = 0; + while( cc ) { + z++; +// cout << "cc: " << cc->ns << endl; + ClientCursor *nxt = cc->nextAtThisLocation; + cc->nextAtThisLocation = 0; // updateLocation will manipulate linked list ptrs, so clean that up first. + cc->c->checkLocation(); + cc->c->advance(); + cc->lastLoc.Null(); // so updateLocation doesn't try to remove, just to be faster -- we handled that. + cc->updateLocation(); + cc = nxt; + } +// cout << "z:" << z << endl; + } +} + +void ClientCursor::cleanupByLocation(DiskLoc loc, long long cursorid) { + if( loc.isNull() ) + return; + + DiskToCC::iterator it = byLocation.find(loc); + if( it != byLocation.end() ) { + ClientCursor *first = it->second; + ClientCursor *cc = first; + ClientCursor *prev = 0; + + while( 1 ) { + if( cc == 0 ) + break; + if( cc->cursorid == cursorid ) { + // found one to remove. + if( prev == 0 ) { + if( cc->nextAtThisLocation ) + it->second = cc->nextAtThisLocation; + else + byLocation.erase(it); + } + else { + prev->nextAtThisLocation = cc->nextAtThisLocation; + } + cc->nextAtThisLocation = 0; + return; + } + cc = cc->nextAtThisLocation; + } + } + + // not found! + //cout << "Assertion failure - cleanupByLocation: not found " << cursorid << endl; +} + +ClientCursor::~ClientCursor() { +#if defined(_WIN32) + cout << "~clientcursor " << cursorid << endl; +#endif + assert( pos != -2 ); + + cleanupByLocation(lastLoc, cursorid); + + assert( pos != -2 ); + + // defensive + lastLoc.Null(); + cursorid = -1; + pos = -2; + nextAtThisLocation = 0; +} + +// note this doesn't set lastLoc -- caller should. +void ClientCursor::addToByLocation(DiskLoc cl) { + assert( cursorid ); + + if( nextAtThisLocation ) { + wassert( nextAtThisLocation == 0 ); + return; + } + + DiskToCC::iterator j = byLocation.find(cl); + nextAtThisLocation = j == byLocation.end() ? 0 : j->second; + byLocation[cl] = this; +} + +void ClientCursor::updateLocation() { + assert( cursorid ); + + DiskLoc cl = c->currLoc(); + // cout<< " TEMP: updateLocation last:" << lastLoc.toString() << " cl:" << cl.toString() << '\n'; + + if( !lastLoc.isNull() ) + cleanupByLocation(lastLoc, cursorid); + + if( !cl.isNull() ) + addToByLocation(cl); + + lastLoc = cl; + c->noteLocation(); +} + +/* report to us that a new clientcursor exists so we can track it. + note you still must call updateLocation (which likely should be changed) +*/ +void ClientCursor::add(ClientCursor* cc) { + clientCursors[cc->cursorid] = cc; +} + +// todo: delete the ClientCursor. +// todo: other map +bool ClientCursor::erase(long long id) { + CCMap::iterator it = clientCursors.find(id); + if( it != clientCursors.end() ) { + ClientCursor *cc = it->second; + it->second = 0; // defensive + clientCursors.erase(it); + delete cc; // destructor will fix byLocation map + return true; + } + return false; +} + +long long allocCursorId() { + long long x; + while( 1 ) { + x = (((long long)rand()) << 32); + x = x | (int) curTimeMillis() | 0x80000000; // OR to make sure not zero + if( clientCursors.count(x) == 0 ) + break; + } +#if defined(_WIN32) + cout << "alloccursorid " << x << endl; +#endif + return x; +} diff --git a/db/db.cpp b/db/db.cpp index ebdedc8ce50..c50c5769420 100644 --- a/db/db.cpp +++ b/db/db.cpp @@ -1,757 +1,757 @@ -// db.cpp : Defines the entry point for the console application.
-//
-
-#include "stdafx.h"
-#include "db.h"
-#include "../grid/message.h"
-#include "../util/mmap.h"
-#include "../util/hashtab.h"
-#include "../util/goodies.h"
-#include "pdfile.h"
-#include "jsobj.h"
-#include "javajs.h"
-#include "query.h"
-#include "introspect.h"
-
-extern const char *dbpath;
-extern int curOp;
-
-boost::mutex dbMutex;
-
-struct MyStartupTests {
- MyStartupTests() {
- assert( sizeof(OID) == 12 );
- }
-} mystartupdbcpp;
-
-/* example for
- var zz = { x: 3, y: "foo", v: { z:5, arr: [1, 2] } }
- zz.v.arr.prop = "zoo";
-*/
-
-void quicktest() {
- cout << "quicktest()\n";
-
- MemoryMappedFile mmf;
- char *m = (char *) mmf.map("/tmp/quicktest", 16384);
- // cout << "mmf reads: " << m << endl;
- strcpy_s(m, 1000, "hello worldz");
-}
-
-void pdfileInit();
-
-class DbMessage {
-public:
- DbMessage(Message& _m) : m(_m) {
- theEnd = _m.data->_data + _m.data->dataLen();
- int *r = (int *) _m.data->_data;
- reserved = *r;
- r++;
- data = (const char *) r;
- nextjsobj = data;
- }
-
- const char * getns() { return data; }
- void getns(Namespace& ns) {
- ns = data;
- }
-
- int pullInt() {
- if( nextjsobj == data )
- nextjsobj += strlen(data) + 1; // skip namespace
- int i = *((int *)nextjsobj);
- nextjsobj += 4;
- return i;
- }
- long long pullInt64() {
- if( nextjsobj == data )
- nextjsobj += strlen(data) + 1; // skip namespace
- long long i = *((long long *)nextjsobj);
- nextjsobj += 8;
- return i;
- }
-
- OID* getOID() {
- return (OID *) (data + strlen(data) + 1); // skip namespace
- }
-
- void getQueryStuff(const char *&query, int& ntoreturn) {
- int *i = (int *) (data + strlen(data) + 1);
- ntoreturn = *i;
- i++;
- query = (const char *) i;
- }
-
- /* for insert and update msgs */
- bool moreJSObjs() { return nextjsobj != 0; }
- JSObj nextJsObj() {
- if( nextjsobj == data )
- nextjsobj += strlen(data) + 1; // skip namespace
- JSObj js(nextjsobj);
- assert( js.objsize() < ( theEnd - data ) );
- if( js.objsize() <= 0 )
- nextjsobj = null;
- else {
- nextjsobj += js.objsize();
- if( nextjsobj >= theEnd )
- nextjsobj = 0;
- }
- return js;
- }
-
-private:
- Message& m;
- int reserved;
- const char *data;
- const char *nextjsobj;
- const char *theEnd;
-};
-
-void killCursors(int n, long long *ids);
-void receivedKillCursors(Message& m) {
- int *x = (int *) m.data->_data;
- x++; // reserved
- int n = *x++;
- assert( n >= 1 );
- if( n > 2000 ) {
- problem() << "Assertion failure, receivedKillCursors, n=" << n << endl;
- cout << "Assertion failure, receivedKillCursors, n=" << n << endl;
- assert( n < 30000 );
- }
- killCursors(n, (long long *) x);
-}
-
-void receivedUpdate(Message& m, stringstream& ss) {
- DbMessage d(m);
- const char *ns = d.getns();
- assert(*ns);
- setClient(ns);
- if( client->profile )
- ss << ns << ' ';
- int flags = d.pullInt();
- JSObj query = d.nextJsObj();
-
- assert( d.moreJSObjs() );
- assert( query.objsize() < m.data->dataLen() );
- JSObj toupdate = d.nextJsObj();
-
- assert( toupdate.objsize() < m.data->dataLen() );
- assert( query.objsize() + toupdate.objsize() < m.data->dataLen() );
- updateObjects(ns, toupdate, query, flags & 1, ss);
-}
-
-void receivedDelete(Message& m) {
- DbMessage d(m);
- const char *ns = d.getns();
- assert(*ns);
- setClient(ns);
- int flags = d.pullInt();
- assert( d.moreJSObjs() );
- JSObj pattern = d.nextJsObj();
- deleteObjects(ns, pattern, flags & 1);
-}
-
-void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
- DbMessage d(m);
- const char *ns = d.getns();
- setClient(ns);
- int ntoskip = d.pullInt();
- int ntoreturn = d.pullInt();
- JSObj query = d.nextJsObj();
- auto_ptr< set<string> > fields;
- if( d.moreJSObjs() ) {
- fields = auto_ptr< set<string> >(new set<string>());
- d.nextJsObj().getFieldNames(*fields);
- }
- QueryResult* msgdata;
-
- try {
- msgdata = runQuery(ns, ntoskip, ntoreturn, query, fields, ss);
- }
- catch( AssertionException ) {
- ss << " exception ";
- problem() << " Caught Assertion in runQuery " << ns << endl;
- cout << " Caught Assertion in runQuery, continuing" << endl;
- cout << " ntoskip:" << ntoskip << " ntoreturn:" << ntoreturn << endl;
- cout << " ns:" << ns << endl;
- cout << " query:" << query.toString() << endl;
- msgdata = (QueryResult*) malloc(sizeof(QueryResult));
- QueryResult *qr = msgdata;
- qr->_data[0] = 0;
- qr->_data[1] = 0;
- qr->_data[2] = 0;
- qr->_data[3] = 0;
- qr->len = sizeof(QueryResult);
- qr->operation = opReply;
- qr->cursorId = 0;
- qr->startingFrom = 0;
- qr->nReturned = 0;
- }
- Message resp;
- resp.setData(msgdata, true); // transport will free
- if( client ) {
- if( client->profile )
- ss << " bytes:" << resp.data->dataLen();
- }
- else {
- cout << "ERROR: client is null; ns=" << ns << endl;
- }
- dbMsgPort.reply(m, resp);
-}
-
-void receivedGetMore(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) {
- DbMessage d(m);
- const char *ns = d.getns();
- ss << ns;
- setClient(ns);
- int ntoreturn = d.pullInt();
- long long cursorid = d.pullInt64();
- ss << " cid:" << cursorid;
- ss << " ntoreturn:" << ntoreturn;
- QueryResult* msgdata = getMore(ns, ntoreturn, cursorid);
- Message resp;
- resp.setData(msgdata, true);
- ss << " bytes:" << resp.data->dataLen();
- ss << " nreturned:" << msgdata->nReturned;
- dbMsgPort.reply(m, resp);
-}
-
-void receivedInsert(Message& m, stringstream& ss) {
-// cout << "GOT MSG id:" << m.data->id << endl;
- DbMessage d(m);
- while( d.moreJSObjs() ) {
- JSObj js = d.nextJsObj();
-// cout << " temp dbinsert: got js object, size=" << js.objsize() << " ns:" << d.getns() << endl;
- const char *ns = d.getns();
- assert(*ns);
- setClient(ns);
- ss << ns;
- theDataFileMgr.insert(ns, (void*) js.objdata(), js.objsize());
- }
-}
-
-void testTheDb() {
- stringstream ss;
-
- setClient("sys.unittest.pdfile");
-
- /* this is not validly formatted, if you query this namespace bad things will happen */
- theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
- theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
-
- JSObj j1((const char *) &js1);
- deleteObjects("sys.unittest.delete", j1, false);
- theDataFileMgr.insert("sys.unittest.delete", &js1, sizeof(js1));
- deleteObjects("sys.unittest.delete", j1, false);
- updateObjects("sys.unittest.delete", j1, j1, true,ss);
- updateObjects("sys.unittest.delete", j1, j1, false,ss);
-
- auto_ptr<Cursor> c = theDataFileMgr.findAll("sys.unittest.pdfile");
- while( c->ok() ) {
- Record* r = c->_current();
- c->advance();
- }
- cout << endl;
-
- client = 0;
-}
-
-int port = DBPort;
-
-MessagingPort *grab = 0;
-void connThread();
-
-class OurListener : public Listener {
-public:
- OurListener(int p) : Listener(p) { }
- virtual void accepted(MessagingPort *mp) {
- assert( grab == 0 );
- grab = mp;
- boost::thread thr(connThread);
- while( grab )
- sleepmillis(1);
- }
-};
-
-void listen(int port) {
- const char *Version = "db version: 110 2jun2008 embedded obj queries";
- problem() << Version << endl;
- cout << Version << endl;
- pdfileInit();
- testTheDb();
- cout << curTimeMillis() % 10000 << " waiting for connections on port " << port << " ...\n" << endl;
- OurListener l(port);
- l.listen();
-}
-
-int ctr = 0;
-extern int callDepth;
-
-class JniMessagingPort : public AbstractMessagingPort {
-public:
- JniMessagingPort(Message& _container) : container(_container) { }
- void reply(Message& received, Message& response) {
- container = response;
- }
- Message & container;
-};
-
-/* a call from java/js to the database locally.
-
- m - inbound message
- out - outbound message, if there is any, will be set here.
- if there is one, out.data will be non-null on return.
- The out.data buffer will automatically clean up when out
- goes out of scope (out.freeIt==true)
-
- note we should already be in the mutex lock from connThread() at this point.
-*/
-void jniCallback(Message& m, Message& out)
-{
- Client *clientOld = client;
-
- JniMessagingPort jmp(out);
- callDepth++;
- int curOpOld = curOp;
-
- try {
-
- stringstream ss;
- char buf[64];
- time_t_to_String(time(0), buf);
- buf[20] = 0; // don't want the year
- ss << buf << " dbjs ";
-
- {
- Timer t;
-
- bool log = false;
- curOp = m.data->operation;
- if( m.data->operation == dbQuery ) {
- receivedQuery(jmp, m, ss);
- }
- else if( m.data->operation == dbInsert ) {
- ss << "insert ";
- receivedInsert(m, ss);
- }
- else if( m.data->operation == dbUpdate ) {
- ss << "update ";
- receivedUpdate(m, ss);
- }
- else if( m.data->operation == dbDelete ) {
- ss << "remove ";
- receivedDelete(m);
- }
- else if( m.data->operation == dbGetMore ) {
- log = true;
- ss << "getmore ";
- receivedGetMore(jmp, m, ss);
- }
- else if( m.data->operation == dbKillCursors ) {
- try {
- log = true;
- ss << "killcursors ";
- receivedKillCursors(m);
- }
- catch( AssertionException ) {
- problem() << "Caught Assertion in kill cursors, continuing" << endl;
- cout << "Caught Assertion in kill cursors, continuing" << endl;
- ss << " exception ";
- }
- }
- else {
- cout << " jnicall: operation isn't supported: " << m.data->operation << endl;
- assert(false);
- }
-
- int ms = t.millis();
- log = log || ctr++ % 128 == 0;
- if( log || ms > 100 ) {
- ss << ' ' << t.millis() << "ms";
- cout << ss.str().c_str() << endl;
- }
- if( client && client->profile >= 1 ) {
- if( client->profile >= 2 || ms >= 100 ) {
- // profile it
- profile(ss.str().c_str()+20/*skip ts*/, ms);
- }
- }
- }
-
- }
- catch( AssertionException ) {
- problem() << "Caught AssertionException in jniCall()" << endl;
- cout << "Caught AssertionException in jniCall()" << endl;
- }
-
- curOp = curOpOld;
- callDepth--;
-
- if( client != clientOld ) {
- client = clientOld;
- wassert(false);
- }
-}
-
-void connThread()
-{
- try {
-
- MessagingPort& dbMsgPort = *grab;
- grab = 0;
-
- Message m;
- while( 1 ) {
- m.reset();
- stringstream ss;
-
- if( !dbMsgPort.recv(m) ) {
- cout << "MessagingPort::recv(): returned false" << endl;
- dbMsgPort.shutdown();
- break;
- }
-
- char buf[64];
- time_t_to_String(time(0), buf);
- buf[20] = 0; // don't want the year
- ss << buf;
- // ss << curTimeMillis() % 10000 << ' ';
-
- {
- lock lk(dbMutex);
- Timer t;
- client = 0;
- curOp = 0;
-
- bool log = false;
- curOp = m.data->operation;
- if( m.data->operation == dbMsg ) {
- ss << "msg ";
- char *p = m.data->_data;
- int len = strlen(p);
- if( len > 400 )
- cout << curTimeMillis() % 10000 <<
- " long msg received, len:" << len <<
- " ends with: " << p + len - 10 << endl;
- bool end = strcmp("end", p) == 0;
- Message resp;
- resp.setData(opReply, "i am fine");
- dbMsgPort.reply(m, resp);
- if( end ) {
- cout << curTimeMillis() % 10000 << " end msg" << endl;
- dbMsgPort.shutdown();
- sleepmillis(500);
- problem() << "exiting end msg" << endl;
- exit(EXIT_SUCCESS);
- }
- }
- else if( m.data->operation == dbQuery ) {
-#if defined(_WIN32)
- log = true;
-#endif
- receivedQuery(dbMsgPort, m, ss);
- }
- else if( m.data->operation == dbInsert ) {
- try {
- ss << "insert ";
- receivedInsert(m, ss);
- }
- catch( AssertionException ) {
- problem() << " Caught Assertion insert, continuing" << endl;
- cout << "Caught Assertion, continuing" << endl;
- ss << " exception ";
- }
- }
- else if( m.data->operation == dbUpdate ) {
- try {
- ss << "update ";
- receivedUpdate(m, ss);
- }
- catch( AssertionException ) {
- problem() << " Caught Assertion update, continuing" << endl;
- cout << "Caught Assertion update, continuing" << endl;
- ss << " exception ";
- }
- }
- else if( m.data->operation == dbDelete ) {
- try {
- ss << "remove ";
- receivedDelete(m);
- }
- catch( AssertionException ) {
- problem() << " Caught Assertion receviedDelete, continuing" << endl;
- cout << "Caught Assertion receviedDelete, continuing" << endl;
- ss << " exception ";
- }
- }
- else if( m.data->operation == dbGetMore ) {
- log = true;
- ss << "getmore ";
- receivedGetMore(dbMsgPort, m, ss);
- }
- else if( m.data->operation == dbKillCursors ) {
- try {
- log = true;
- ss << "killcursors ";
- receivedKillCursors(m);
- }
- catch( AssertionException ) {
- cout << "Caught Assertion in kill cursors, continuing" << endl;
- problem() << " Caught Assertion in kill cursors, continuing" << endl;
- ss << " exception ";
- }
- }
- else {
- cout << " operation isn't supported: " << m.data->operation << endl;
- assert(false);
- }
-
- int ms = t.millis();
- log = log || ctr++ % 128 == 0;
- if( log || ms > 100 ) {
- ss << ' ' << t.millis() << "ms";
- cout << ss.str().c_str() << endl;
- }
- if( client && client->profile >= 1 ) {
- if( client->profile >= 2 || ms >= 100 ) {
- // profile it
- profile(ss.str().c_str()+20/*skip ts*/, ms);
- }
- }
- }
- }
-
- }
- catch( AssertionException ) {
- cout << "Caught AssertionException, terminating" << endl;
- problem() << "Caught AssertionException, terminating" << endl;
- exit(-7);
- }
-}
-
-
-void msg(const char *m, const char *address, int port, int extras = 0) {
-
- SockAddr db(address, port);
-
-// SockAddr db("127.0.0.1", DBPort);
-// SockAddr db("192.168.37.1", MessagingPort::DBPort);
-// SockAddr db("10.0.21.60", MessagingPort::DBPort);
-// SockAddr db("172.16.0.179", MessagingPort::DBPort);
-
- MessagingPort p;
- if( !p.connect(db) )
- return;
-
- for( int q = 0; q < 3; q++ ) {
- Message send;
- Message response;
-
- send.setData( dbMsg , m);
- int len = send.data->dataLen();
-
- for( int i = 0; i < extras; i++ )
- p.say(db, send);
-
- Timer t;
- bool ok = p.call(db, send, response);
- double tm = t.micros() + 1;
- cout << " ****ok. response.data:" << ok << " time:" << tm / 1000.0 << "ms " <<
- ((double) len) * 8 / 1000000 / (tm/1000000) << "Mbps" << endl;
- if( q+1 < 3 ) {
- cout << "\t\tSLEEP 8 then sending again" << endl;
- sleepsecs(8);
- }
- }
-
- p.shutdown();
-}
-
-void msg(const char *m, int extras = 0) {
- msg(m, "127.0.0.1", DBPort, extras);
-}
-
-#if !defined(_WIN32)
-
-#include <signal.h>
-
-void pipeSigHandler( int signal ) {
- psignal( signal, "Signal Received : ");
-}
-
-void segvhandler(int x) {
- cout << "got SIGSEGV " << x << ", terminating :-(" << endl;
- problem() << "got SIGSEGV " << x << ", terminating :-(" << endl;
- exit(-9);
+// db.cpp : Defines the entry point for the console application. +// + +#include "stdafx.h" +#include "db.h" +#include "../grid/message.h" +#include "../util/mmap.h" +#include "../util/hashtab.h" +#include "../util/goodies.h" +#include "pdfile.h" +#include "jsobj.h" +#include "javajs.h" +#include "query.h" +#include "introspect.h" + +extern const char *dbpath; +extern int curOp; + +boost::mutex dbMutex; + +struct MyStartupTests { + MyStartupTests() { + assert( sizeof(OID) == 12 ); + } +} mystartupdbcpp; + +/* example for + var zz = { x: 3, y: "foo", v: { z:5, arr: [1, 2] } } + zz.v.arr.prop = "zoo"; +*/ + +void quicktest() { + cout << "quicktest()\n"; + + MemoryMappedFile mmf; + char *m = (char *) mmf.map("/tmp/quicktest", 16384); + // cout << "mmf reads: " << m << endl; + strcpy_s(m, 1000, "hello worldz"); } -
-void mysighandler(int x) {
- // [dm] not working. why?
- // [gmj] because the VM catches sig 3 to do a thread dump
- cout << "got kill or ctrl c signal " << x << ", terminating" << endl;
- problem() << "got kill or ctrl c signal " << x << ", terminating" << endl;
- exit(0);
-}
-
-void setupSignals() {
- cout << "SETUPSIGNALS " << signal(SIGINT, mysighandler) << endl;
+ +void pdfileInit(); + +class DbMessage { +public: + DbMessage(Message& _m) : m(_m) { + theEnd = _m.data->_data + _m.data->dataLen(); + int *r = (int *) _m.data->_data; + reserved = *r; + r++; + data = (const char *) r; + nextjsobj = data; + } + + const char * getns() { return data; } + void getns(Namespace& ns) { + ns = data; + } + + int pullInt() { + if( nextjsobj == data ) + nextjsobj += strlen(data) + 1; // skip namespace + int i = *((int *)nextjsobj); + nextjsobj += 4; + return i; + } + long long pullInt64() { + if( nextjsobj == data ) + nextjsobj += strlen(data) + 1; // skip namespace + long long i = *((long long *)nextjsobj); + nextjsobj += 8; + return i; + } + + OID* getOID() { + return (OID *) (data + strlen(data) + 1); // skip namespace + } + + void getQueryStuff(const char *&query, int& ntoreturn) { + int *i = (int *) (data + strlen(data) + 1); + ntoreturn = *i; + i++; + query = (const char *) i; + } + + /* for insert and update msgs */ + bool moreJSObjs() { return nextjsobj != 0; } + JSObj nextJsObj() { + if( nextjsobj == data ) + nextjsobj += strlen(data) + 1; // skip namespace + JSObj js(nextjsobj); + assert( js.objsize() < ( theEnd - data ) ); + if( js.objsize() <= 0 ) + nextjsobj = null; + else { + nextjsobj += js.objsize(); + if( nextjsobj >= theEnd ) + nextjsobj = 0; + } + return js; + } + +private: + Message& m; + int reserved; + const char *data; + const char *nextjsobj; + const char *theEnd; +}; + +void killCursors(int n, long long *ids); +void receivedKillCursors(Message& m) { + int *x = (int *) m.data->_data; + x++; // reserved + int n = *x++; + assert( n >= 1 ); + if( n > 2000 ) { + problem() << "Assertion failure, receivedKillCursors, n=" << n << endl; + cout << "Assertion failure, receivedKillCursors, n=" << n << endl; + assert( n < 30000 ); + } + killCursors(n, (long long *) x); +} + +void receivedUpdate(Message& m, stringstream& ss) { + DbMessage d(m); + const char *ns = d.getns(); + assert(*ns); + setClient(ns); + if( client->profile ) + ss << ns << ' '; + int flags = d.pullInt(); + JSObj query = d.nextJsObj(); + + assert( d.moreJSObjs() ); + assert( query.objsize() < m.data->dataLen() ); + JSObj toupdate = d.nextJsObj(); + + assert( toupdate.objsize() < m.data->dataLen() ); + assert( query.objsize() + toupdate.objsize() < m.data->dataLen() ); + updateObjects(ns, toupdate, query, flags & 1, ss); +} + +void receivedDelete(Message& m) { + DbMessage d(m); + const char *ns = d.getns(); + assert(*ns); + setClient(ns); + int flags = d.pullInt(); + assert( d.moreJSObjs() ); + JSObj pattern = d.nextJsObj(); + deleteObjects(ns, pattern, flags & 1); +} + +void receivedQuery(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) { + DbMessage d(m); + const char *ns = d.getns(); + setClient(ns); + int ntoskip = d.pullInt(); + int ntoreturn = d.pullInt(); + JSObj query = d.nextJsObj(); + auto_ptr< set<string> > fields; + if( d.moreJSObjs() ) { + fields = auto_ptr< set<string> >(new set<string>()); + d.nextJsObj().getFieldNames(*fields); + } + QueryResult* msgdata; + + try { + msgdata = runQuery(ns, ntoskip, ntoreturn, query, fields, ss); + } + catch( AssertionException ) { + ss << " exception "; + problem() << " Caught Assertion in runQuery " << ns << endl; + cout << " Caught Assertion in runQuery, continuing" << endl; + cout << " ntoskip:" << ntoskip << " ntoreturn:" << ntoreturn << endl; + cout << " ns:" << ns << endl; + cout << " query:" << query.toString() << endl; + msgdata = (QueryResult*) malloc(sizeof(QueryResult)); + QueryResult *qr = msgdata; + qr->_data[0] = 0; + qr->_data[1] = 0; + qr->_data[2] = 0; + qr->_data[3] = 0; + qr->len = sizeof(QueryResult); + qr->operation = opReply; + qr->cursorId = 0; + qr->startingFrom = 0; + qr->nReturned = 0; + } + Message resp; + resp.setData(msgdata, true); // transport will free + if( client ) { + if( client->profile ) + ss << " bytes:" << resp.data->dataLen(); + } + else { + cout << "ERROR: client is null; ns=" << ns << endl; + } + dbMsgPort.reply(m, resp); +} + +void receivedGetMore(AbstractMessagingPort& dbMsgPort, Message& m, stringstream& ss) { + DbMessage d(m); + const char *ns = d.getns(); + ss << ns; + setClient(ns); + int ntoreturn = d.pullInt(); + long long cursorid = d.pullInt64(); + ss << " cid:" << cursorid; + ss << " ntoreturn:" << ntoreturn; + QueryResult* msgdata = getMore(ns, ntoreturn, cursorid); + Message resp; + resp.setData(msgdata, true); + ss << " bytes:" << resp.data->dataLen(); + ss << " nreturned:" << msgdata->nReturned; + dbMsgPort.reply(m, resp); +} + +void receivedInsert(Message& m, stringstream& ss) { +// cout << "GOT MSG id:" << m.data->id << endl; + DbMessage d(m); + while( d.moreJSObjs() ) { + JSObj js = d.nextJsObj(); +// cout << " temp dbinsert: got js object, size=" << js.objsize() << " ns:" << d.getns() << endl; + const char *ns = d.getns(); + assert(*ns); + setClient(ns); + ss << ns; + theDataFileMgr.insert(ns, (void*) js.objdata(), js.objsize()); + } +} + +void testTheDb() { + stringstream ss; + + setClient("sys.unittest.pdfile"); + + /* this is not validly formatted, if you query this namespace bad things will happen */ + theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13); + theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13); + + JSObj j1((const char *) &js1); + deleteObjects("sys.unittest.delete", j1, false); + theDataFileMgr.insert("sys.unittest.delete", &js1, sizeof(js1)); + deleteObjects("sys.unittest.delete", j1, false); + updateObjects("sys.unittest.delete", j1, j1, true,ss); + updateObjects("sys.unittest.delete", j1, j1, false,ss); + + auto_ptr<Cursor> c = theDataFileMgr.findAll("sys.unittest.pdfile"); + while( c->ok() ) { + Record* r = c->_current(); + c->advance(); + } + cout << endl; + + client = 0; +} + +int port = DBPort; + +MessagingPort *grab = 0; +void connThread(); + +class OurListener : public Listener { +public: + OurListener(int p) : Listener(p) { } + virtual void accepted(MessagingPort *mp) { + assert( grab == 0 ); + grab = mp; + boost::thread thr(connThread); + while( grab ) + sleepmillis(1); + } +}; + +void listen(int port) { + const char *Version = "db version: 110 2jun2008 embedded obj queries"; + problem() << Version << endl; + cout << Version << endl; + pdfileInit(); + testTheDb(); + cout << curTimeMillis() % 10000 << " waiting for connections on port " << port << " ...\n" << endl; + OurListener l(port); + l.listen(); +} + +int ctr = 0; +extern int callDepth; + +class JniMessagingPort : public AbstractMessagingPort { +public: + JniMessagingPort(Message& _container) : container(_container) { } + void reply(Message& received, Message& response) { + container = response; + } + Message & container; +}; + +/* a call from java/js to the database locally. + + m - inbound message + out - outbound message, if there is any, will be set here. + if there is one, out.data will be non-null on return. + The out.data buffer will automatically clean up when out + goes out of scope (out.freeIt==true) + + note we should already be in the mutex lock from connThread() at this point. +*/ +void jniCallback(Message& m, Message& out) +{ + Client *clientOld = client; + + JniMessagingPort jmp(out); + callDepth++; + int curOpOld = curOp; + + try { + + stringstream ss; + char buf[64]; + time_t_to_String(time(0), buf); + buf[20] = 0; // don't want the year + ss << buf << " dbjs "; + + { + Timer t; + + bool log = false; + curOp = m.data->operation; + if( m.data->operation == dbQuery ) { + receivedQuery(jmp, m, ss); + } + else if( m.data->operation == dbInsert ) { + ss << "insert "; + receivedInsert(m, ss); + } + else if( m.data->operation == dbUpdate ) { + ss << "update "; + receivedUpdate(m, ss); + } + else if( m.data->operation == dbDelete ) { + ss << "remove "; + receivedDelete(m); + } + else if( m.data->operation == dbGetMore ) { + log = true; + ss << "getmore "; + receivedGetMore(jmp, m, ss); + } + else if( m.data->operation == dbKillCursors ) { + try { + log = true; + ss << "killcursors "; + receivedKillCursors(m); + } + catch( AssertionException ) { + problem() << "Caught Assertion in kill cursors, continuing" << endl; + cout << "Caught Assertion in kill cursors, continuing" << endl; + ss << " exception "; + } + } + else { + cout << " jnicall: operation isn't supported: " << m.data->operation << endl; + assert(false); + } + + int ms = t.millis(); + log = log || ctr++ % 128 == 0; + if( log || ms > 100 ) { + ss << ' ' << t.millis() << "ms"; + cout << ss.str().c_str() << endl; + } + if( client && client->profile >= 1 ) { + if( client->profile >= 2 || ms >= 100 ) { + // profile it + profile(ss.str().c_str()+20/*skip ts*/, ms); + } + } + } + + } + catch( AssertionException ) { + problem() << "Caught AssertionException in jniCall()" << endl; + cout << "Caught AssertionException in jniCall()" << endl; + } + + curOp = curOpOld; + callDepth--; + + if( client != clientOld ) { + client = clientOld; + wassert(false); + } +} + +void connThread() +{ + try { + + MessagingPort& dbMsgPort = *grab; + grab = 0; + + Message m; + while( 1 ) { + m.reset(); + stringstream ss; + + if( !dbMsgPort.recv(m) ) { + cout << "MessagingPort::recv(): returned false" << endl; + dbMsgPort.shutdown(); + break; + } + + char buf[64]; + time_t_to_String(time(0), buf); + buf[20] = 0; // don't want the year + ss << buf; + // ss << curTimeMillis() % 10000 << ' '; + + { + lock lk(dbMutex); + Timer t; + client = 0; + curOp = 0; + + bool log = false; + curOp = m.data->operation; + if( m.data->operation == dbMsg ) { + ss << "msg "; + char *p = m.data->_data; + int len = strlen(p); + if( len > 400 ) + cout << curTimeMillis() % 10000 << + " long msg received, len:" << len << + " ends with: " << p + len - 10 << endl; + bool end = strcmp("end", p) == 0; + Message resp; + resp.setData(opReply, "i am fine"); + dbMsgPort.reply(m, resp); + if( end ) { + cout << curTimeMillis() % 10000 << " end msg" << endl; + dbMsgPort.shutdown(); + sleepmillis(500); + problem() << "exiting end msg" << endl; + exit(EXIT_SUCCESS); + } + } + else if( m.data->operation == dbQuery ) { +#if defined(_WIN32) + log = true; +#endif + receivedQuery(dbMsgPort, m, ss); + } + else if( m.data->operation == dbInsert ) { + try { + ss << "insert "; + receivedInsert(m, ss); + } + catch( AssertionException ) { + problem() << " Caught Assertion insert, continuing" << endl; + cout << "Caught Assertion, continuing" << endl; + ss << " exception "; + } + } + else if( m.data->operation == dbUpdate ) { + try { + ss << "update "; + receivedUpdate(m, ss); + } + catch( AssertionException ) { + problem() << " Caught Assertion update, continuing" << endl; + cout << "Caught Assertion update, continuing" << endl; + ss << " exception "; + } + } + else if( m.data->operation == dbDelete ) { + try { + ss << "remove "; + receivedDelete(m); + } + catch( AssertionException ) { + problem() << " Caught Assertion receviedDelete, continuing" << endl; + cout << "Caught Assertion receviedDelete, continuing" << endl; + ss << " exception "; + } + } + else if( m.data->operation == dbGetMore ) { + log = true; + ss << "getmore "; + receivedGetMore(dbMsgPort, m, ss); + } + else if( m.data->operation == dbKillCursors ) { + try { + log = true; + ss << "killcursors "; + receivedKillCursors(m); + } + catch( AssertionException ) { + cout << "Caught Assertion in kill cursors, continuing" << endl; + problem() << " Caught Assertion in kill cursors, continuing" << endl; + ss << " exception "; + } + } + else { + cout << " operation isn't supported: " << m.data->operation << endl; + assert(false); + } + + int ms = t.millis(); + log = log || ctr++ % 128 == 0; + if( log || ms > 100 ) { + ss << ' ' << t.millis() << "ms"; + cout << ss.str().c_str() << endl; + } + if( client && client->profile >= 1 ) { + if( client->profile >= 2 || ms >= 100 ) { + // profile it + profile(ss.str().c_str()+20/*skip ts*/, ms); + } + } + } + } + + } + catch( AssertionException ) { + cout << "Caught AssertionException, terminating" << endl; + problem() << "Caught AssertionException, terminating" << endl; + exit(-7); + } +} + + +void msg(const char *m, const char *address, int port, int extras = 0) { + + SockAddr db(address, port); + +// SockAddr db("127.0.0.1", DBPort); +// SockAddr db("192.168.37.1", MessagingPort::DBPort); +// SockAddr db("10.0.21.60", MessagingPort::DBPort); +// SockAddr db("172.16.0.179", MessagingPort::DBPort); + + MessagingPort p; + if( !p.connect(db) ) + return; + + for( int q = 0; q < 3; q++ ) { + Message send; + Message response; + + send.setData( dbMsg , m); + int len = send.data->dataLen(); + + for( int i = 0; i < extras; i++ ) + p.say(db, send); + + Timer t; + bool ok = p.call(db, send, response); + double tm = t.micros() + 1; + cout << " ****ok. response.data:" << ok << " time:" << tm / 1000.0 << "ms " << + ((double) len) * 8 / 1000000 / (tm/1000000) << "Mbps" << endl; + if( q+1 < 3 ) { + cout << "\t\tSLEEP 8 then sending again" << endl; + sleepsecs(8); + } + } + + p.shutdown(); +} + +void msg(const char *m, int extras = 0) { + msg(m, "127.0.0.1", DBPort, extras); +} + +#if !defined(_WIN32) + +#include <signal.h> + +void pipeSigHandler( int signal ) { + psignal( signal, "Signal Received : "); +} + +void segvhandler(int x) { + cout << "got SIGSEGV " << x << ", terminating :-(" << endl; + problem() << "got SIGSEGV " << x << ", terminating :-(" << endl; + exit(-9); +} + +void mysighandler(int x) { + // [dm] not working. why? + // [gmj] because the VM catches sig 3 to do a thread dump + cout << "got kill or ctrl c signal " << x << ", terminating" << endl; + problem() << "got kill or ctrl c signal " << x << ", terminating" << endl; + exit(0); +} + +void setupSignals() { + cout << "SETUPSIGNALS " << signal(SIGINT, mysighandler) << endl; // assert( signal(SIGINT, mysighandler) != SIG_ERR ); assert( signal(SIGTERM, mysighandler) != SIG_ERR ); assert( signal(SIGQUIT, mysighandler) != SIG_ERR ); assert( signal(SIGSEGV, segvhandler) != SIG_ERR ); -}
-
-#else
-void setupSignals() {}
-#endif
-
-void initAndListen(int listenPort, const char *dbPath, const char *appserverLoc = null) {
-
- setupSignals();
-
- /*
- * ensure that the dbpath ends w/ '/' as that's key in preventing things like
- * /data/dbadmin.ns
- */
-
- if (dbPath && dbPath[strlen(dbPath)-1] != '/') {
- char *t = (char *) malloc(strlen(dbPath) + 2);
-
- strcpy(t, dbPath);
- strcat(t, "/");
- dbPath = t;
- }
-
- dbpath = dbPath;
-
-#if !defined(_WIN32)
- pid_t pid = 0;
- pid = getpid();
-#else
- int pid=0;
-#endif
-
- cout << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;
- problem() << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl;
-
- JavaJS = new JavaJSImpl(appserverLoc);
- javajstest();
-
- listen(listenPort);
-}
-
-ofstream problems("dbproblems.log", ios_base::app | ios_base::out);
-
-int main(int argc, char* argv[], char *envp[] )
-{
-#if !defined(_WIN32)
- signal(SIGPIPE, pipeSigHandler);
-#endif
- srand(curTimeMillis());
-
- if( argc >= 2 ) {
- if( strcmp(argv[1], "quicktest") == 0 ) {
- quicktest();
- return 0;
- }
- if( strcmp(argv[1], "msg") == 0 ) {
-
- // msg(argc >= 3 ? argv[2] : "ping");
-
- const char *m = "ping";
- int thePort = DBPort;
-
- if (argc >= 3) {
- m = argv[2];
-
- if (argc > 3) {
- thePort = atoi(argv[3]);
- }
- }
-
- msg(m, "127.0.0.1", thePort);
-
- goingAway = true;
- return 0;
- }
- if( strcmp(argv[1], "msglots") == 0 ) {
- msg(argc >= 3 ? argv[2] : "ping", 1000);
- goingAway = true;
- return 0;
- }
- if( strcmp(argv[1], "dev") == 0 ) {
- dbpath = "/home/dwight/db/";
- cout << "dev mode: expect db files in " << dbpath << endl;
- quicktest();
- port++;
- cout << "listening on port " << port << endl;
- listen(port);
- goingAway = true;
- return 0;
- }
- if( strcmp(argv[1], "run") == 0 ) {
-
- initAndListen(port, dbpath);
-
- goingAway = true;
- return 0;
- }
- if( strcmp(argv[1], "longmsg") == 0 ) {
- char buf[800000];
- memset(buf, 'a', 799999);
- buf[799999] = 0;
- buf[799998] = 'b';
- buf[0] = 'c';
- msg(buf);
- goingAway = true;
- return 0;
- }
-
- /*
- * *** POST STANDARD SWITCH METHOD - if we don't satisfy, we switch to a
- * slightly different mode where "run" is assumed and we can set values
- */
-
- char *appsrvPath = null;
-
- for (int i = 1; i < argc; i++) {
-
- char *s = argv[i];
-
- if (s && strcmp(s, "--port") == 0) {
- port = atoi(argv[++i]);
- }
- else if (s && strcmp(s, "--dbpath") == 0) {
- dbpath = argv[++i];
- }
- else if (s && strcmp(s, "--appsrvpath") == 0) {
- appsrvPath = argv[++i];
- }
- }
-
- initAndListen(port, dbpath, appsrvPath);
-
- goingAway = true;
- return 0;
- }
-
- cout << "usage:\n";
- cout << " quicktest just check basic assertions and exit" << endl;
- cout << " msg [msg] [port] send a request to the db server listening on port (or default)" << endl;
- cout << " msg end [port] shut down db server listening on port (or default)" << endl;
- cout << " run run db" << endl;
- cout << " dev run in dev mode (diff db loc, diff port #)" << endl;
- cout << " longmsg send a long test message to the db server" << endl;
- cout << " msglots send a bunch of test messages, and then wait for answer on the last one" << endl;
- cout << endl << "Alternate Usage :" << endl;
- cout << " --port <portno> --dbpath <root> --appsrvpath <root of appsrv>" << endl << endl;
-
- goingAway = true;
- return 0;
-}
-
-//#if !defined(_WIN32)
-//int main( int argc, char *argv[], char *envp[] ) {
-// return _tmain(argc, 0);
-//}
-//#endif
+} + +#else +void setupSignals() {} +#endif + +void initAndListen(int listenPort, const char *dbPath, const char *appserverLoc = null) { + + setupSignals(); + + /* + * ensure that the dbpath ends w/ '/' as that's key in preventing things like + * /data/dbadmin.ns + */ + + if (dbPath && dbPath[strlen(dbPath)-1] != '/') { + char *t = (char *) malloc(strlen(dbPath) + 2); + + strcpy(t, dbPath); + strcat(t, "/"); + dbPath = t; + } + + dbpath = dbPath; + +#if !defined(_WIN32) + pid_t pid = 0; + pid = getpid(); +#else + int pid=0; +#endif + + cout << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl; + problem() << "10Gen DB : starting : pid = " << pid << " port = " << port << " dbpath = " << dbpath << endl; + + JavaJS = new JavaJSImpl(appserverLoc); + javajstest(); + + listen(listenPort); +} + +ofstream problems("dbproblems.log", ios_base::app | ios_base::out); + +int main(int argc, char* argv[], char *envp[] ) +{ +#if !defined(_WIN32) + signal(SIGPIPE, pipeSigHandler); +#endif + srand(curTimeMillis()); + + if( argc >= 2 ) { + if( strcmp(argv[1], "quicktest") == 0 ) { + quicktest(); + return 0; + } + if( strcmp(argv[1], "msg") == 0 ) { + + // msg(argc >= 3 ? argv[2] : "ping"); + + const char *m = "ping"; + int thePort = DBPort; + + if (argc >= 3) { + m = argv[2]; + + if (argc > 3) { + thePort = atoi(argv[3]); + } + } + + msg(m, "127.0.0.1", thePort); + + goingAway = true; + return 0; + } + if( strcmp(argv[1], "msglots") == 0 ) { + msg(argc >= 3 ? argv[2] : "ping", 1000); + goingAway = true; + return 0; + } + if( strcmp(argv[1], "dev") == 0 ) { + dbpath = "/home/dwight/db/"; + cout << "dev mode: expect db files in " << dbpath << endl; + quicktest(); + port++; + cout << "listening on port " << port << endl; + listen(port); + goingAway = true; + return 0; + } + if( strcmp(argv[1], "run") == 0 ) { + + initAndListen(port, dbpath); + + goingAway = true; + return 0; + } + if( strcmp(argv[1], "longmsg") == 0 ) { + char buf[800000]; + memset(buf, 'a', 799999); + buf[799999] = 0; + buf[799998] = 'b'; + buf[0] = 'c'; + msg(buf); + goingAway = true; + return 0; + } + + /* + * *** POST STANDARD SWITCH METHOD - if we don't satisfy, we switch to a + * slightly different mode where "run" is assumed and we can set values + */ + + char *appsrvPath = null; + + for (int i = 1; i < argc; i++) { + + char *s = argv[i]; + + if (s && strcmp(s, "--port") == 0) { + port = atoi(argv[++i]); + } + else if (s && strcmp(s, "--dbpath") == 0) { + dbpath = argv[++i]; + } + else if (s && strcmp(s, "--appsrvpath") == 0) { + appsrvPath = argv[++i]; + } + } + + initAndListen(port, dbpath, appsrvPath); + + goingAway = true; + return 0; + } + + cout << "usage:\n"; + cout << " quicktest just check basic assertions and exit" << endl; + cout << " msg [msg] [port] send a request to the db server listening on port (or default)" << endl; + cout << " msg end [port] shut down db server listening on port (or default)" << endl; + cout << " run run db" << endl; + cout << " dev run in dev mode (diff db loc, diff port #)" << endl; + cout << " longmsg send a long test message to the db server" << endl; + cout << " msglots send a bunch of test messages, and then wait for answer on the last one" << endl; + cout << endl << "Alternate Usage :" << endl; + cout << " --port <portno> --dbpath <root> --appsrvpath <root of appsrv>" << endl << endl; + + goingAway = true; + return 0; +} + +//#if !defined(_WIN32) +//int main( int argc, char *argv[], char *envp[] ) { +// return _tmain(argc, 0); +//} +//#endif @@ -1,4 +1,4 @@ -#include "../stdafx.h"
-#include "../grid/message.h"
-
-void jniCallback(Message& m, Message& out);
+#include "../stdafx.h" +#include "../grid/message.h" + +void jniCallback(Message& m, Message& out); diff --git a/db/introspect.cpp b/db/introspect.cpp index 95bbc73879a..34db42a143d 100644 --- a/db/introspect.cpp +++ b/db/introspect.cpp @@ -1,39 +1,39 @@ -// introspect.cpp
-
-#include "stdafx.h"
-#include "introspect.h"
-#include "../util/builder.h"
-#include "../util/goodies.h"
-#include "pdfile.h"
-#include "jsobj.h"
-#include "pdfile.h"
-
-typedef map<string,Cursor*> StringToCursor;
-StringToCursor *specialNamespaces;
-
-auto_ptr<Cursor> getSpecialCursor(const char *ns) {
- StringToCursor::iterator it = specialNamespaces->find(ns);
- return auto_ptr<Cursor>
- (it == specialNamespaces->end() ?
- 0 : it->second->clone());
-}
-
-void SingleResultObjCursor::reg(const char *as) {
- if( specialNamespaces == 0 )
- specialNamespaces = new StringToCursor();
- if( specialNamespaces->count(as) == 0 ) {
- (*specialNamespaces)[as] = this;
- }
-}
-
-void profile(const char *str,
- int millis)
-{
- JSObjBuilder b;
- b.appendDate("ts", jsTime());
- b.append("info", str);
- b.append("millis", (double) millis);
- JSObj p = b.done();
- theDataFileMgr.insert(client->profileName.c_str(),
- p.objdata(), p.objsize(), true);
-}
+// introspect.cpp + +#include "stdafx.h" +#include "introspect.h" +#include "../util/builder.h" +#include "../util/goodies.h" +#include "pdfile.h" +#include "jsobj.h" +#include "pdfile.h" + +typedef map<string,Cursor*> StringToCursor; +StringToCursor *specialNamespaces; + +auto_ptr<Cursor> getSpecialCursor(const char *ns) { + StringToCursor::iterator it = specialNamespaces->find(ns); + return auto_ptr<Cursor> + (it == specialNamespaces->end() ? + 0 : it->second->clone()); +} + +void SingleResultObjCursor::reg(const char *as) { + if( specialNamespaces == 0 ) + specialNamespaces = new StringToCursor(); + if( specialNamespaces->count(as) == 0 ) { + (*specialNamespaces)[as] = this; + } +} + +void profile(const char *str, + int millis) +{ + JSObjBuilder b; + b.appendDate("ts", jsTime()); + b.append("info", str); + b.append("millis", (double) millis); + JSObj p = b.done(); + theDataFileMgr.insert(client->profileName.c_str(), + p.objdata(), p.objsize(), true); +} diff --git a/db/introspect.h b/db/introspect.h index 10bcfd96925..1dd7f3c8ee4 100644 --- a/db/introspect.h +++ b/db/introspect.h @@ -1,46 +1,46 @@ -// introspect.h
-// system management stuff.
-
-#pragma once
-
-#include "../stdafx.h"
-#include "jsobj.h"
-#include "pdfile.h"
-
-auto_ptr<Cursor> getSpecialCursor(const char *ns);
-
-class SingleResultObjCursor : public Cursor {
- int i;
-protected:
- JSObjBuilder b;
- void reg(const char *as); /* register as a certain namespace */
-public:
- SingleResultObjCursor() { i = 0; }
- virtual bool ok() { return i == 0; }
- virtual Record* _current() { assert(false); return 0; }
- virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }
-
- virtual void fill() = 0;
-
- virtual JSObj current() {
- assert(i == 0);
- fill();
- return b.done();
- }
-
- virtual bool advance() {
- i++;
- return false;
- }
-
- virtual const char * toString() { return "SingleResultObjCursor"; }
-
-};
-
-/* --- profiling --------------------------------------------
- do when client->profile is set
-*/
-
-void profile(const char *str,
- int millis);
-
+// introspect.h +// system management stuff. + +#pragma once + +#include "../stdafx.h" +#include "jsobj.h" +#include "pdfile.h" + +auto_ptr<Cursor> getSpecialCursor(const char *ns); + +class SingleResultObjCursor : public Cursor { + int i; +protected: + JSObjBuilder b; + void reg(const char *as); /* register as a certain namespace */ +public: + SingleResultObjCursor() { i = 0; } + virtual bool ok() { return i == 0; } + virtual Record* _current() { assert(false); return 0; } + virtual DiskLoc currLoc() { assert(false); return DiskLoc(); } + + virtual void fill() = 0; + + virtual JSObj current() { + assert(i == 0); + fill(); + return b.done(); + } + + virtual bool advance() { + i++; + return false; + } + + virtual const char * toString() { return "SingleResultObjCursor"; } + +}; + +/* --- profiling -------------------------------------------- + do when client->profile is set +*/ + +void profile(const char *str, + int millis); + diff --git a/db/javajs.cpp b/db/javajs.cpp index 5ad2aa1f7ad..86a0a9a2ac1 100644 --- a/db/javajs.cpp +++ b/db/javajs.cpp @@ -24,27 +24,27 @@ using namespace boost::filesystem; using namespace std; -#if defined(_WIN32)
-/* [dm] this being undefined without us adding it here means there is
- no tss cleanup on windows for boost lib?
- we don't care for now esp on windows only
-
- the boost source says:
-
- This function's sole purpose is to cause a link error in cases where
- automatic tss cleanup is not implemented by Boost.Threads as a
- reminder that user code is responsible for calling the necessary
- functions at the appropriate times (and for implementing an a
- tss_cleanup_implemented() function to eliminate the linker's
- missing symbol error).
-
- If Boost.Threads later implements automatic tss cleanup in cases
- where it currently doesn't (which is the plan), the duplicate
- symbol error will warn the user that their custom solution is no
- longer needed and can be removed.
-*/
-extern "C" void tss_cleanup_implemented(void) {
- //cout << "tss_cleanup_implemented called" << endl;
+#if defined(_WIN32) +/* [dm] this being undefined without us adding it here means there is + no tss cleanup on windows for boost lib? + we don't care for now esp on windows only + + the boost source says: + + This function's sole purpose is to cause a link error in cases where + automatic tss cleanup is not implemented by Boost.Threads as a + reminder that user code is responsible for calling the necessary + functions at the appropriate times (and for implementing an a + tss_cleanup_implemented() function to eliminate the linker's + missing symbol error). + + If Boost.Threads later implements automatic tss cleanup in cases + where it currently doesn't (which is the plan), the duplicate + symbol error will warn the user that their custom solution is no + longer needed and can be removed. +*/ +extern "C" void tss_cleanup_implemented(void) { + //cout << "tss_cleanup_implemented called" << endl; } #endif diff --git a/db/jsobj.cpp b/db/jsobj.cpp index 26a9881fccf..e55de77d335 100644 --- a/db/jsobj.cpp +++ b/db/jsobj.cpp @@ -1,810 +1,810 @@ -// jsobj.cpp
-
-#include "stdafx.h"
-#include "jsobj.h"
-#include "../util/goodies.h"
-#include "javajs.h"
-
-#if defined(_WIN32)
-
-#include <hash_map>
-using namespace stdext;
-
-typedef const char * MyStr;
-struct less_str {
- bool operator()(const MyStr & x, const MyStr & y) const {
- if ( strcmp(x, y) > 0)
- return true;
-
- return false;
- }
-};
-
-typedef hash_map<const char*, int, hash_compare<const char *, less_str> > strhashmap;
-
-#else
-
-#include <ext/hash_map>
-using namespace __gnu_cxx;
-
-typedef const char * MyStr;
-struct eq_str {
- bool operator()(const MyStr & x, const MyStr & y) const {
- if ( strcmp(x, y) == 0)
- return true;
-
- return false;
- }
-};
-
-typedef hash_map<const char*, int, hash<const char *>, eq_str > strhashmap;
-
-#endif
-
-#include "minilex.h"
-
-MiniLex minilex;
-
-class Where {
-public:
- Where() { codeCopy = 0; }
- ~Where() {
- JavaJS->scopeFree(scope);
- delete codeCopy;
- scope = 0; func = 0; codeCopy = 0;
- }
- jlong scope, func;
- strhashmap fields;
-// map<string,int> fields;
- bool fullObject;
- int nFields;
- char *codeCopy;
-
- void setFunc(const char *code) {
- codeCopy = new char[strlen(code)+1];
- strcpy(codeCopy,code);
- func = JavaJS->functionCreate( code );
- minilex.grabVariables(codeCopy, fields);
- // if user references db, eg db.foo.save(obj),
- // we make sure we have the whole thing.
- fullObject = fields.count("fullObject") +
- fields.count("db") > 0;
- nFields = fields.size();
- }
-
- void buildSubset(JSObj& src, JSObjBuilder& dst) {
- JSElemIter it(src);
- int n = 0;
- if( !it.more() ) return;
- while( 1 ) {
- Element e = it.next();
- if( e.eoo() )
- break;
- if( //n == 0 &&
- fields.find(e.fieldName()) != fields.end()
- //fields.count(e.fieldName())
- ) {
- dst.append(e);
- if( ++n >= nFields )
- break;
- }
- }
- }
-};
-
-JSMatcher::~JSMatcher() {
- for( int i = 0; i < nBuilders; i++ )
- delete builders[i];
- delete where;
-}
-
-Element nullElement;
-
-string Element::toString() {
- stringstream s;
- switch( type() ) {
- case EOO:
- return "EOO";
- case Date:
- s << fieldName() << ": Date(" << hex << date() << ')'; break;
- case Number:
- s << fieldName() << ": " << number(); break;
- case Bool:
- s << fieldName() << ": " << boolean() ? "true" : "false"; break;
- case Object:
- case Array:
- s << fieldName() << ": " << embeddedObject().toString(); break;
- case Undefined:
- s << fieldName() << ": undefined"; break;
- case jstNULL:
- s << fieldName() << ": null"; break;
- case MaxKey:
- s << fieldName() << ": MaxKey"; break;
- case Code:
- s << fieldName() << ": ";
- if( valuestrsize() > 80 )
- s << string(valuestr()).substr(0, 70) << "...";
- else {
- s << valuestr();
- }
- break;
- case String:
- s << fieldName() << ": ";
- if( valuestrsize() > 80 )
- s << '"' << string(valuestr()).substr(0, 70) << "...\"";
- else {
- s << '"' << valuestr() << '"';
- }
- break;
- case jstOID:
- s << fieldName() << " : ObjId(";
- s << hex << oid().a << hex << oid().b << ')';
- break;
- default:
- s << fieldName() << ": ?type=" << type();
- break;
- }
- return s.str();
-}
-
-int Element::size() {
- if( totalSize >= 0 )
- return totalSize;
-
- int x = 1;
- switch( type() ) {
- case EOO:
- case Undefined:
- case jstNULL:
- case MaxKey:
- break;
- case Bool:
- x = 2;
- break;
- case Date:
- case Number:
- x = 9;
- break;
- case jstOID:
- x = 13;
- break;
- case Code:
- case String:
- x = valuestrsize() + 4 + 1;
- break;
- case DBRef:
- x = valuestrsize() + 4 + 12 + 1;
- break;
- case Object:
- case Array:
- x = objsize() + 1;
- break;
- case BinData:
- x = valuestrsize() + 4 + 1 + 1/*subtype*/;
- break;
- case RegEx:
- {
- const char *p = value();
- int len1 = strlen(p);
- p = p + len1 + 1;
- x = 1 + len1 + strlen(p) + 2;
- }
- break;
- default:
- cout << "Element: bad type " << (int) type() << endl;
- assert(false);
- }
- totalSize = x + fieldNameSize;
-
- if( !eoo() ) {
- const char *next = data + totalSize;
- if( *next < 0 || *next > JSTypeMax ) {
- // bad type.
- cout << "*********************************************\n";
- cout << "Bad data or size in Element::size()" << endl;
- cout << "bad type:" << (int) *next << endl;
- cout << "totalsize:" << totalSize << " fieldnamesize:" << fieldNameSize << endl;
- cout << "lastrec:" << endl;
- dumpmemory(data, totalSize + 15);
- assert(false);
- }
- }
-
- return totalSize;
-}
-
-/* must be same type! */
-inline int compareElementValues(Element& l, Element& r) {
- int f;
- double x;
- switch( l.type() ) {
- case EOO:
- case Undefined:
- case jstNULL:
- case MaxKey:
- f = l.type() - r.type();
- if( f<0 ) return -1;
- return f==0 ? 0 : 1;
- case Bool:
- return *l.value() - *r.value();
- case Date:
- if( l.date() < r.date() )
- return -1;
- return l.date() == r.date() ? 0 : 1;
- case Number:
- x = l.number() - r.number();
- if( x < 0 ) return -1;
- return x == 0 ? 0 : 1;
- case jstOID:
- return memcmp(l.value(), r.value(), 12);
- case Code:
- case String:
- /* todo: utf version */
- return strcmp(l.valuestr(), r.valuestr());
- case Object:
- case Array:
- case DBRef:
- {
- int lsz = l.valuesize();
- int rsz = r.valuesize();
- if( lsz - rsz != 0 ) return lsz - rsz;
- return memcmp(l.value(), r.value(), lsz);
- }
- case BinData:
- case RegEx:
- cout << "compareElementValues: can't compare this type:" << (int) l.type() << endl;
- assert(false);
- break;
- default:
- cout << "compareElementValues: bad type " << (int) l.type() << endl;
- assert(false);
- }
- return -1;
-}
-
-/* JSMatcher --------------------------------------*/
-
-// If the element is something like:
-// a : { $gt : 3 }
-// we append
-// a : 3
-// else we just append the element.
-//
-void appendElementHandlingGtLt(JSObjBuilder& b, Element& e) {
- if( e.type() == Object ) {
- Element fe = e.embeddedObject().firstElement();
- const char *fn = fe.fieldName();
- if( fn[0] == '$' && fn[1] && fn[2] == 't' ) {
- b.appendAs(fe, e.fieldName());
- return;
- }
- }
- b.append(e);
-}
-
-int getGtLtOp(Element& e) {
- int op = JSMatcher::Equality;
- if( e.type() != Object )
- return op;
-
- Element fe = e.embeddedObject().firstElement();
- const char *fn = fe.fieldName();
- if( fn[0] == '$' && fn[1] && fn[2] == 't' ) {
- if( fn[1] == 'g' ) {
- if( fn[3] == 0 ) op = JSMatcher::GT;
- else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::GTE;
- }
- else if( fn[1] == 'l' ) {
- if( fn[3] == 0 ) op = JSMatcher::LT;
- else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::LTE;
- }
- }
- return op;
-}
-
-#include "pdfile.h"
-
-JSMatcher::JSMatcher(JSObj &_jsobj) :
- where(0), jsobj(_jsobj), nRegex(0)
-{
- nBuilders = 0;
-
- JSElemIter i(jsobj);
- n = 0;
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() )
- break;
-
- if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) {
- // $where: function()...
- assert( where == 0 );
- where = new Where();
- const char *code = e.valuestr();
- assert( JavaJS );
- where->scope = JavaJS->scopeCreate();
- JavaJS->scopeSetString(where->scope, "$client", client->name.c_str());
- where->setFunc(code);
- continue;
- }
-
- if( e.type() == RegEx ) {
- if( nRegex >= 4 ) {
- cout << "ERROR: too many regexes in query" << endl;
- }
- else {
- pcrecpp::RE_Options options;
- options.set_utf8(true);
- const char *flags = e.regexFlags();
- while( flags && *flags ) {
- if( *flags == 'i' )
- options.set_caseless(true);
- else if( *flags == 'm' )
- options.set_multiline(true);
- else if( *flags == 'x' )
- options.set_extended(true);
- flags++;
- }
- regexs[nRegex].re = new pcrecpp::RE(e.regex(), options);
- regexs[nRegex].fieldName = e.fieldName();
- nRegex++;
- }
- continue;
- }
-
- // greater than / less than...
- // { a : { $gt: 3 } }
- if( e.type() == Object ) {
- Element fe = e.embeddedObject().firstElement();
- const char *fn = fe.fieldName();
- if( fn[0] == '$' && fn[1] && fn[2] == 't' ) {
- int op = Equality;
- if( fn[1] == 'g' ) {
- if( fn[3] == 0 ) op = GT;
- else if( fn[3] == 'e' && fn[4] == 0 ) op = GTE;
- }
- else if( fn[1] == 'l' ) {
- if( fn[3] == 0 ) op = LT;
- else if( fn[3] == 'e' && fn[4] == 0 ) op = LTE;
- }
- if( op && nBuilders < 8) {
- JSObjBuilder *b = new JSObjBuilder();
- builders[nBuilders++] = b;
- b->appendAs(fe, e.fieldName());
- toMatch.push_back( b->done().firstElement() );
- compareOp.push_back(op);
- n++;
- continue;
- }
- }
- }
-
- {
- toMatch.push_back(e);
- compareOp.push_back(Equality);
- n++;
- }
- }
-}
-
-inline int JSMatcher::valuesMatch(Element& l, Element& r, int op) {
- if( op == 0 )
- return l.valuesEqual(r);
-
- if( l.type() != r.type() )
- return false;
-
- int c = compareElementValues(l, r);
- int z = 1 << (c+1);
- return (op & z);
-}
-
-/* return value
- -1 mismatch
- 0 missing element
- 1 match
-*/
-int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj, int compareOp, bool *deep, bool isArr) {
- {
- const char *p = strchr(fieldName, '.');
- if( p ) {
- string left(fieldName, p-fieldName);
-
- Element e = obj.getField(left.c_str());
- if( e.eoo() )
- return 0;
- if( e.type() != Object && e.type() != Array )
- return -1;
-
- JSObj eo = e.embeddedObject();
- return matchesDotted(p+1, toMatch, eo, compareOp, deep, e.type() == Array);
- }
- }
-
- Element e = obj.getField(fieldName);
-
- if( valuesMatch(e, toMatch, compareOp) ) {
- return 1;
- }
- else if( e.type() == Array ) {
- JSElemIter ai(e.embeddedObject());
- while( ai.more() ) {
- Element z = ai.next();
- if( valuesMatch( z, toMatch, compareOp) ) {
- if( deep )
- *deep = true;
- return 1;
- }
- }
- }
- else if( isArr ) {
- JSElemIter ai(obj);
- while( ai.more() ) {
- Element z = ai.next();
- if( z.type() == Object ) {
- JSObj eo = z.embeddedObject();
- int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep);
- if( cmp > 0 ) {
- if( deep ) *deep = true;
- return 1;
- }
- }
- }
- }
- else if( e.eoo() ) {
- return 0;
- }
- return -1;
-}
-
-/* deep means we looked into arrays for a match */
-bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
- if( deep )
- *deep = false;
-
- /* assuming there is usually only one thing to match. if more this
- could be slow sometimes. */
-
- for( int r = 0; r < nRegex; r++ ) {
- RegexMatcher& rm = regexs[r];
- Element e = jsobj.getFieldDotted(rm.fieldName);
- if( e.eoo() )
- return false;
- {
- char buf[64];
- const char *p = buf;
- if( e.type() == String )
- p = e.valuestr();
- else if( e.type() == Number ) {
- sprintf(buf, "%f", e.number());
- }
- else if( e.type() == Date ) {
- unsigned long long d = e.date();
- time_t t = (d/1000);
- time_t_to_String(t, buf);
- }
- else
- return false;
- if( !rm.re->PartialMatch(p) )
- return false;
- }
- }
-
- // check normal non-regex cases:
- for( int i = 0; i < n; i++ ) {
- Element& m = toMatch[i];
-
- int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep);
-
- /* missing is ok iff we were looking for null */
- if( cmp < 0 )
- return false;
- if( cmp == 0 && (m.type() != jstNULL && m.type() != Undefined ) )
- return false;
- }
-
-/*
- Element e = jsobj.getFieldDotted(m.fieldName(), arrayElName);
- if( !e.eoo() ) {
- if( valuesMatch(e, m, compareOp[i]) ) {
- goto ok;
- }
- else if( e.type() == Array ) {
- JSElemIter ai(e.embeddedObject());
- while( ai.more() ) {
- Element z = ai.next();
- if( valuesMatch( z, m, compareOp[i]) ) {
- if( deep )
- *deep = true;
- goto ok;
- }
- }
- }
- return false;
- }
-*/
-
- /* missing. that is ok iff we were looking for null */
-// if( m.type() == jstNULL || m.type() == Undefined )
-// ;
-////// else
-// return false;
-//ok:
-// ;
-// }
-
- if( where ) {
- if( where->func == 0 )
- return false; // didn't compile
- if( jsobj.objsize() < 200 || where->fullObject ) {
- JavaJS->scopeSetObject(where->scope, "obj", &jsobj);
- } else {
- JSObjBuilder b;
- where->buildSubset(jsobj, b);
- JSObj temp = b.done();
- JavaJS->scopeSetObject(where->scope, "obj", &temp);
- }
- if( JavaJS->invoke(where->scope, where->func) )
- return false;
- return JavaJS->scopeGetBoolean(where->scope, "return") != 0;
- }
-
- return true;
-}
-
-/* JSObj ------------------------------------------------------------*/
-
-string JSObj::toString() const {
- stringstream s;
- s << "{ ";
- JSElemIter i(*this);
- Element e = i.next();
- if( !e.eoo() )
- while( 1 ) {
- s << e.toString();
- e = i.next();
- if( e.eoo() )
- break;
- s << ", ";
- }
- s << " }";
- return s.str();
-}
-
-/* well ordered compare */
-int JSObj::woCompare(const JSObj& r) const {
- assert( _objdata );
- if( isEmpty() )
- return r.isEmpty() ? 0 : -1;
- if( r.isEmpty() )
- return 1;
-
- JSElemIter i(*this);
- JSElemIter j(r);
- while( 1 ) {
- // so far, equal...
-
- Element l = i.next();
- Element r = j.next();
-
- if( l == r ) {
- if( l.eoo() )
- return 0;
- continue;
- }
-
- int x = (int) l.type() - (int) r.type();
- if( x != 0 )
- return x;
- x = strcmp(l.fieldName(), r.fieldName());
- if( x != 0 )
- return x;
- x = compareElementValues(l, r);
- assert(x != 0);
- return x;
- }
- return -1;
-}
-
-/* return has eoo() true if no match
- supports "." notation to reach into embedded objects
-*/
-Element JSObj::getFieldDotted(const char *name) {
- {
- const char *p = strchr(name, '.');
- if( p ) {
- string left(name, p-name);
- JSObj sub = getObjectField(left.c_str());
- return sub.isEmpty() ? nullElement : sub.getFieldDotted(p+1);
- }
- }
-
- JSElemIter i(*this);
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() )
- break;
- if( strcmp(e.fieldName(), name) == 0 )
- return e;
- }
- return nullElement;
-}
-
-Element JSObj::getField(const char *name) {
- JSElemIter i(*this);
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() )
- break;
- if( strcmp(e.fieldName(), name) == 0 )
- return e;
- }
- return nullElement;
-}
-
-/* makes a new JSObj with the fields specified in pattern.
- fields returned in the order they appear in pattern.
- if any field missing, you get back an empty object overall.
-
- n^2 implementation bad if pattern and object have lots
- of fields - normally pattern doesn't so should be fine.
-*/
-JSObj JSObj::extractFields(JSObj pattern, JSObjBuilder& b) {
- JSElemIter i(pattern);
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() )
- break;
- Element x = getField(e.fieldName());
- if( x.eoo() )
- return JSObj();
- b.append(x);
- }
- return b.done();
-}
-
-const char * JSObj::getStringField(const char *name) {
- Element e = getField(name);
- return e.type() == String ? e.valuestr() : 0;
-}
-
-JSObj JSObj::getObjectField(const char *name) {
- Element e = getField(name);
- JSType t = e.type();
- return t == Object || t == Array ? e.embeddedObject() : JSObj();
-}
-
-int JSObj::getFieldNames(set<string>& fields) {
- int n = 0;
- JSElemIter i(*this);
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() )
- break;
- fields.insert(e.fieldName());
- n++;
- }
- return n;
-}
-
-/* note: addFields always adds _id even if not specified
- returns n added not counting _id unless requested.
-*/
-int JSObj::addFields(JSObj& from, set<string>& fields) {
- assert( _objdata == 0 ); /* partial implementation for now... */
-
- JSObjBuilder b;
-
- int N = fields.size();
- int n = 0;
- JSElemIter i(from);
- bool gotId = false;
- while( i.more() ) {
- Element e = i.next();
- const char *fname = e.fieldName();
- if( fields.count(fname) ) {
- b.append(e);
- ++n;
- gotId = gotId || strcmp(fname, "_id")==0;
- if( n == N && gotId )
- break;
- } else if( strcmp(fname, "_id")==0 ) {
- b.append(e);
- gotId = true;
- if( n == N && gotId )
- break;
- }
- }
-
- if( n ) {
- _objdata = b.decouple(_objsize);
- iFree = true;
- }
-
- return n;
-}
-
-/*-- test things ----------------------------------------------------*/
-
-#pragma pack(push)
-#pragma pack(1)
-
-struct MaxKeyData {
- MaxKeyData() { totsize=7; maxkey=MaxKey; name=0; eoo=EOO; }
- int totsize;
- char maxkey;
- char name;
- char eoo;
-} maxkeydata;
-JSObj maxKey((const char *) &maxkeydata);
-
-struct JSObj0 {
- JSObj0() { totsize = 5; eoo = EOO; }
- int totsize;
- char eoo;
-} js0;
-
-Element::Element() {
- data = &js0.eoo;
- fieldNameSize = 0;
- totalSize = -1;
-}
-
-struct JSObj1 js1;
-
-struct JSObj2 {
- JSObj2() {
- totsize=sizeof(JSObj2);
- s = String; strcpy_s(sname, 7, "abcdef"); slen = 10;
- strcpy_s(sval, 10, "123456789"); eoo = EOO;
- }
- unsigned totsize;
- char s;
- char sname[7];
- unsigned slen;
- char sval[10];
- char eoo;
-} js2;
-
-struct JSUnitTest {
- JSUnitTest() {
- JSObj j1((const char *) &js1);
- JSObj j2((const char *) &js2);
- JSMatcher m(j2);
- assert( m.matches(j1) );
- js2.sval[0] = 'z';
- assert( !m.matches(j1) );
- JSMatcher n(j1);
- assert( n.matches(j1) );
- assert( !n.matches(j2) );
-
- JSObj j0((const char *) &js0);
- JSMatcher p(j0);
- assert( p.matches(j1) );
- assert( p.matches(j2) );
- }
-} jsunittest;
-
-#pragma pack(pop)
-
-struct RXTest {
- RXTest() {
- /*
- static const boost::regex e("(\\d{4}[- ]){3}\\d{4}");
- static const boost::regex b(".....");
- cout << "regex result: " << regex_match("hello", e) << endl;
- cout << "regex result: " << regex_match("abcoo", b) << endl;
- */
- pcrecpp::RE re1(")({a}h.*o");
- pcrecpp::RE re("h.llo");
- assert( re.FullMatch("hello") );
- assert( !re1.FullMatch("hello") );
-
-
- pcrecpp::RE_Options options;
- options.set_utf8(true);
- pcrecpp::RE part("dwi", options);
- assert( part.PartialMatch("dwight") );
- }
-} rxtest;
-
+// jsobj.cpp + +#include "stdafx.h" +#include "jsobj.h" +#include "../util/goodies.h" +#include "javajs.h" + +#if defined(_WIN32) + +#include <hash_map> +using namespace stdext; + +typedef const char * MyStr; +struct less_str { + bool operator()(const MyStr & x, const MyStr & y) const { + if ( strcmp(x, y) > 0) + return true; + + return false; + } +}; + +typedef hash_map<const char*, int, hash_compare<const char *, less_str> > strhashmap; + +#else + +#include <ext/hash_map> +using namespace __gnu_cxx; + +typedef const char * MyStr; +struct eq_str { + bool operator()(const MyStr & x, const MyStr & y) const { + if ( strcmp(x, y) == 0) + return true; + + return false; + } +}; + +typedef hash_map<const char*, int, hash<const char *>, eq_str > strhashmap; + +#endif + +#include "minilex.h" + +MiniLex minilex; + +class Where { +public: + Where() { codeCopy = 0; } + ~Where() { + JavaJS->scopeFree(scope); + delete codeCopy; + scope = 0; func = 0; codeCopy = 0; + } + jlong scope, func; + strhashmap fields; +// map<string,int> fields; + bool fullObject; + int nFields; + char *codeCopy; + + void setFunc(const char *code) { + codeCopy = new char[strlen(code)+1]; + strcpy(codeCopy,code); + func = JavaJS->functionCreate( code ); + minilex.grabVariables(codeCopy, fields); + // if user references db, eg db.foo.save(obj), + // we make sure we have the whole thing. + fullObject = fields.count("fullObject") + + fields.count("db") > 0; + nFields = fields.size(); + } + + void buildSubset(JSObj& src, JSObjBuilder& dst) { + JSElemIter it(src); + int n = 0; + if( !it.more() ) return; + while( 1 ) { + Element e = it.next(); + if( e.eoo() ) + break; + if( //n == 0 && + fields.find(e.fieldName()) != fields.end() + //fields.count(e.fieldName()) + ) { + dst.append(e); + if( ++n >= nFields ) + break; + } + } + } +}; + +JSMatcher::~JSMatcher() { + for( int i = 0; i < nBuilders; i++ ) + delete builders[i]; + delete where; +} + +Element nullElement; + +string Element::toString() { + stringstream s; + switch( type() ) { + case EOO: + return "EOO"; + case Date: + s << fieldName() << ": Date(" << hex << date() << ')'; break; + case Number: + s << fieldName() << ": " << number(); break; + case Bool: + s << fieldName() << ": " << boolean() ? "true" : "false"; break; + case Object: + case Array: + s << fieldName() << ": " << embeddedObject().toString(); break; + case Undefined: + s << fieldName() << ": undefined"; break; + case jstNULL: + s << fieldName() << ": null"; break; + case MaxKey: + s << fieldName() << ": MaxKey"; break; + case Code: + s << fieldName() << ": "; + if( valuestrsize() > 80 ) + s << string(valuestr()).substr(0, 70) << "..."; + else { + s << valuestr(); + } + break; + case String: + s << fieldName() << ": "; + if( valuestrsize() > 80 ) + s << '"' << string(valuestr()).substr(0, 70) << "...\""; + else { + s << '"' << valuestr() << '"'; + } + break; + case jstOID: + s << fieldName() << " : ObjId("; + s << hex << oid().a << hex << oid().b << ')'; + break; + default: + s << fieldName() << ": ?type=" << type(); + break; + } + return s.str(); +} + +int Element::size() { + if( totalSize >= 0 ) + return totalSize; + + int x = 1; + switch( type() ) { + case EOO: + case Undefined: + case jstNULL: + case MaxKey: + break; + case Bool: + x = 2; + break; + case Date: + case Number: + x = 9; + break; + case jstOID: + x = 13; + break; + case Code: + case String: + x = valuestrsize() + 4 + 1; + break; + case DBRef: + x = valuestrsize() + 4 + 12 + 1; + break; + case Object: + case Array: + x = objsize() + 1; + break; + case BinData: + x = valuestrsize() + 4 + 1 + 1/*subtype*/; + break; + case RegEx: + { + const char *p = value(); + int len1 = strlen(p); + p = p + len1 + 1; + x = 1 + len1 + strlen(p) + 2; + } + break; + default: + cout << "Element: bad type " << (int) type() << endl; + assert(false); + } + totalSize = x + fieldNameSize; + + if( !eoo() ) { + const char *next = data + totalSize; + if( *next < 0 || *next > JSTypeMax ) { + // bad type. + cout << "*********************************************\n"; + cout << "Bad data or size in Element::size()" << endl; + cout << "bad type:" << (int) *next << endl; + cout << "totalsize:" << totalSize << " fieldnamesize:" << fieldNameSize << endl; + cout << "lastrec:" << endl; + dumpmemory(data, totalSize + 15); + assert(false); + } + } + + return totalSize; +} + +/* must be same type! */ +inline int compareElementValues(Element& l, Element& r) { + int f; + double x; + switch( l.type() ) { + case EOO: + case Undefined: + case jstNULL: + case MaxKey: + f = l.type() - r.type(); + if( f<0 ) return -1; + return f==0 ? 0 : 1; + case Bool: + return *l.value() - *r.value(); + case Date: + if( l.date() < r.date() ) + return -1; + return l.date() == r.date() ? 0 : 1; + case Number: + x = l.number() - r.number(); + if( x < 0 ) return -1; + return x == 0 ? 0 : 1; + case jstOID: + return memcmp(l.value(), r.value(), 12); + case Code: + case String: + /* todo: utf version */ + return strcmp(l.valuestr(), r.valuestr()); + case Object: + case Array: + case DBRef: + { + int lsz = l.valuesize(); + int rsz = r.valuesize(); + if( lsz - rsz != 0 ) return lsz - rsz; + return memcmp(l.value(), r.value(), lsz); + } + case BinData: + case RegEx: + cout << "compareElementValues: can't compare this type:" << (int) l.type() << endl; + assert(false); + break; + default: + cout << "compareElementValues: bad type " << (int) l.type() << endl; + assert(false); + } + return -1; +} + +/* JSMatcher --------------------------------------*/ + +// If the element is something like: +// a : { $gt : 3 } +// we append +// a : 3 +// else we just append the element. +// +void appendElementHandlingGtLt(JSObjBuilder& b, Element& e) { + if( e.type() == Object ) { + Element fe = e.embeddedObject().firstElement(); + const char *fn = fe.fieldName(); + if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { + b.appendAs(fe, e.fieldName()); + return; + } + } + b.append(e); +} + +int getGtLtOp(Element& e) { + int op = JSMatcher::Equality; + if( e.type() != Object ) + return op; + + Element fe = e.embeddedObject().firstElement(); + const char *fn = fe.fieldName(); + if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { + if( fn[1] == 'g' ) { + if( fn[3] == 0 ) op = JSMatcher::GT; + else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::GTE; + } + else if( fn[1] == 'l' ) { + if( fn[3] == 0 ) op = JSMatcher::LT; + else if( fn[3] == 'e' && fn[4] == 0 ) op = JSMatcher::LTE; + } + } + return op; +} + +#include "pdfile.h" + +JSMatcher::JSMatcher(JSObj &_jsobj) : + where(0), jsobj(_jsobj), nRegex(0) +{ + nBuilders = 0; + + JSElemIter i(jsobj); + n = 0; + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) + break; + + if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) { + // $where: function()... + assert( where == 0 ); + where = new Where(); + const char *code = e.valuestr(); + assert( JavaJS ); + where->scope = JavaJS->scopeCreate(); + JavaJS->scopeSetString(where->scope, "$client", client->name.c_str()); + where->setFunc(code); + continue; + } + + if( e.type() == RegEx ) { + if( nRegex >= 4 ) { + cout << "ERROR: too many regexes in query" << endl; + } + else { + pcrecpp::RE_Options options; + options.set_utf8(true); + const char *flags = e.regexFlags(); + while( flags && *flags ) { + if( *flags == 'i' ) + options.set_caseless(true); + else if( *flags == 'm' ) + options.set_multiline(true); + else if( *flags == 'x' ) + options.set_extended(true); + flags++; + } + regexs[nRegex].re = new pcrecpp::RE(e.regex(), options); + regexs[nRegex].fieldName = e.fieldName(); + nRegex++; + } + continue; + } + + // greater than / less than... + // { a : { $gt: 3 } } + if( e.type() == Object ) { + Element fe = e.embeddedObject().firstElement(); + const char *fn = fe.fieldName(); + if( fn[0] == '$' && fn[1] && fn[2] == 't' ) { + int op = Equality; + if( fn[1] == 'g' ) { + if( fn[3] == 0 ) op = GT; + else if( fn[3] == 'e' && fn[4] == 0 ) op = GTE; + } + else if( fn[1] == 'l' ) { + if( fn[3] == 0 ) op = LT; + else if( fn[3] == 'e' && fn[4] == 0 ) op = LTE; + } + if( op && nBuilders < 8) { + JSObjBuilder *b = new JSObjBuilder(); + builders[nBuilders++] = b; + b->appendAs(fe, e.fieldName()); + toMatch.push_back( b->done().firstElement() ); + compareOp.push_back(op); + n++; + continue; + } + } + } + + { + toMatch.push_back(e); + compareOp.push_back(Equality); + n++; + } + } +} + +inline int JSMatcher::valuesMatch(Element& l, Element& r, int op) { + if( op == 0 ) + return l.valuesEqual(r); + + if( l.type() != r.type() ) + return false; + + int c = compareElementValues(l, r); + int z = 1 << (c+1); + return (op & z); +} + +/* return value + -1 mismatch + 0 missing element + 1 match +*/ +int JSMatcher::matchesDotted(const char *fieldName, Element& toMatch, JSObj& obj, int compareOp, bool *deep, bool isArr) { + { + const char *p = strchr(fieldName, '.'); + if( p ) { + string left(fieldName, p-fieldName); + + Element e = obj.getField(left.c_str()); + if( e.eoo() ) + return 0; + if( e.type() != Object && e.type() != Array ) + return -1; + + JSObj eo = e.embeddedObject(); + return matchesDotted(p+1, toMatch, eo, compareOp, deep, e.type() == Array); + } + } + + Element e = obj.getField(fieldName); + + if( valuesMatch(e, toMatch, compareOp) ) { + return 1; + } + else if( e.type() == Array ) { + JSElemIter ai(e.embeddedObject()); + while( ai.more() ) { + Element z = ai.next(); + if( valuesMatch( z, toMatch, compareOp) ) { + if( deep ) + *deep = true; + return 1; + } + } + } + else if( isArr ) { + JSElemIter ai(obj); + while( ai.more() ) { + Element z = ai.next(); + if( z.type() == Object ) { + JSObj eo = z.embeddedObject(); + int cmp = matchesDotted(fieldName, toMatch, eo, compareOp, deep); + if( cmp > 0 ) { + if( deep ) *deep = true; + return 1; + } + } + } + } + else if( e.eoo() ) { + return 0; + } + return -1; +} + +/* deep means we looked into arrays for a match */ +bool JSMatcher::matches(JSObj& jsobj, bool *deep) { + if( deep ) + *deep = false; + + /* assuming there is usually only one thing to match. if more this + could be slow sometimes. */ + + for( int r = 0; r < nRegex; r++ ) { + RegexMatcher& rm = regexs[r]; + Element e = jsobj.getFieldDotted(rm.fieldName); + if( e.eoo() ) + return false; + { + char buf[64]; + const char *p = buf; + if( e.type() == String ) + p = e.valuestr(); + else if( e.type() == Number ) { + sprintf(buf, "%f", e.number()); + } + else if( e.type() == Date ) { + unsigned long long d = e.date(); + time_t t = (d/1000); + time_t_to_String(t, buf); + } + else + return false; + if( !rm.re->PartialMatch(p) ) + return false; + } + } + + // check normal non-regex cases: + for( int i = 0; i < n; i++ ) { + Element& m = toMatch[i]; + + int cmp = matchesDotted(toMatch[i].fieldName(), toMatch[i], jsobj, compareOp[i], deep); + + /* missing is ok iff we were looking for null */ + if( cmp < 0 ) + return false; + if( cmp == 0 && (m.type() != jstNULL && m.type() != Undefined ) ) + return false; + } + +/* + Element e = jsobj.getFieldDotted(m.fieldName(), arrayElName); + if( !e.eoo() ) { + if( valuesMatch(e, m, compareOp[i]) ) { + goto ok; + } + else if( e.type() == Array ) { + JSElemIter ai(e.embeddedObject()); + while( ai.more() ) { + Element z = ai.next(); + if( valuesMatch( z, m, compareOp[i]) ) { + if( deep ) + *deep = true; + goto ok; + } + } + } + return false; + } +*/ + + /* missing. that is ok iff we were looking for null */ +// if( m.type() == jstNULL || m.type() == Undefined ) +// ; +////// else +// return false; +//ok: +// ; +// } + + if( where ) { + if( where->func == 0 ) + return false; // didn't compile + if( jsobj.objsize() < 200 || where->fullObject ) { + JavaJS->scopeSetObject(where->scope, "obj", &jsobj); + } else { + JSObjBuilder b; + where->buildSubset(jsobj, b); + JSObj temp = b.done(); + JavaJS->scopeSetObject(where->scope, "obj", &temp); + } + if( JavaJS->invoke(where->scope, where->func) ) + return false; + return JavaJS->scopeGetBoolean(where->scope, "return") != 0; + } + + return true; +} + +/* JSObj ------------------------------------------------------------*/ + +string JSObj::toString() const { + stringstream s; + s << "{ "; + JSElemIter i(*this); + Element e = i.next(); + if( !e.eoo() ) + while( 1 ) { + s << e.toString(); + e = i.next(); + if( e.eoo() ) + break; + s << ", "; + } + s << " }"; + return s.str(); +} + +/* well ordered compare */ +int JSObj::woCompare(const JSObj& r) const { + assert( _objdata ); + if( isEmpty() ) + return r.isEmpty() ? 0 : -1; + if( r.isEmpty() ) + return 1; + + JSElemIter i(*this); + JSElemIter j(r); + while( 1 ) { + // so far, equal... + + Element l = i.next(); + Element r = j.next(); + + if( l == r ) { + if( l.eoo() ) + return 0; + continue; + } + + int x = (int) l.type() - (int) r.type(); + if( x != 0 ) + return x; + x = strcmp(l.fieldName(), r.fieldName()); + if( x != 0 ) + return x; + x = compareElementValues(l, r); + assert(x != 0); + return x; + } + return -1; +} + +/* return has eoo() true if no match + supports "." notation to reach into embedded objects +*/ +Element JSObj::getFieldDotted(const char *name) { + { + const char *p = strchr(name, '.'); + if( p ) { + string left(name, p-name); + JSObj sub = getObjectField(left.c_str()); + return sub.isEmpty() ? nullElement : sub.getFieldDotted(p+1); + } + } + + JSElemIter i(*this); + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) + break; + if( strcmp(e.fieldName(), name) == 0 ) + return e; + } + return nullElement; +} + +Element JSObj::getField(const char *name) { + JSElemIter i(*this); + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) + break; + if( strcmp(e.fieldName(), name) == 0 ) + return e; + } + return nullElement; +} + +/* makes a new JSObj with the fields specified in pattern. + fields returned in the order they appear in pattern. + if any field missing, you get back an empty object overall. + + n^2 implementation bad if pattern and object have lots + of fields - normally pattern doesn't so should be fine. +*/ +JSObj JSObj::extractFields(JSObj pattern, JSObjBuilder& b) { + JSElemIter i(pattern); + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) + break; + Element x = getField(e.fieldName()); + if( x.eoo() ) + return JSObj(); + b.append(x); + } + return b.done(); +} + +const char * JSObj::getStringField(const char *name) { + Element e = getField(name); + return e.type() == String ? e.valuestr() : 0; +} + +JSObj JSObj::getObjectField(const char *name) { + Element e = getField(name); + JSType t = e.type(); + return t == Object || t == Array ? e.embeddedObject() : JSObj(); +} + +int JSObj::getFieldNames(set<string>& fields) { + int n = 0; + JSElemIter i(*this); + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) + break; + fields.insert(e.fieldName()); + n++; + } + return n; +} + +/* note: addFields always adds _id even if not specified + returns n added not counting _id unless requested. +*/ +int JSObj::addFields(JSObj& from, set<string>& fields) { + assert( _objdata == 0 ); /* partial implementation for now... */ + + JSObjBuilder b; + + int N = fields.size(); + int n = 0; + JSElemIter i(from); + bool gotId = false; + while( i.more() ) { + Element e = i.next(); + const char *fname = e.fieldName(); + if( fields.count(fname) ) { + b.append(e); + ++n; + gotId = gotId || strcmp(fname, "_id")==0; + if( n == N && gotId ) + break; + } else if( strcmp(fname, "_id")==0 ) { + b.append(e); + gotId = true; + if( n == N && gotId ) + break; + } + } + + if( n ) { + _objdata = b.decouple(_objsize); + iFree = true; + } + + return n; +} + +/*-- test things ----------------------------------------------------*/ + +#pragma pack(push) +#pragma pack(1) + +struct MaxKeyData { + MaxKeyData() { totsize=7; maxkey=MaxKey; name=0; eoo=EOO; } + int totsize; + char maxkey; + char name; + char eoo; +} maxkeydata; +JSObj maxKey((const char *) &maxkeydata); + +struct JSObj0 { + JSObj0() { totsize = 5; eoo = EOO; } + int totsize; + char eoo; +} js0; + +Element::Element() { + data = &js0.eoo; + fieldNameSize = 0; + totalSize = -1; +} + +struct JSObj1 js1; + +struct JSObj2 { + JSObj2() { + totsize=sizeof(JSObj2); + s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; + strcpy_s(sval, 10, "123456789"); eoo = EOO; + } + unsigned totsize; + char s; + char sname[7]; + unsigned slen; + char sval[10]; + char eoo; +} js2; + +struct JSUnitTest { + JSUnitTest() { + JSObj j1((const char *) &js1); + JSObj j2((const char *) &js2); + JSMatcher m(j2); + assert( m.matches(j1) ); + js2.sval[0] = 'z'; + assert( !m.matches(j1) ); + JSMatcher n(j1); + assert( n.matches(j1) ); + assert( !n.matches(j2) ); + + JSObj j0((const char *) &js0); + JSMatcher p(j0); + assert( p.matches(j1) ); + assert( p.matches(j2) ); + } +} jsunittest; + +#pragma pack(pop) + +struct RXTest { + RXTest() { + /* + static const boost::regex e("(\\d{4}[- ]){3}\\d{4}"); + static const boost::regex b("....."); + cout << "regex result: " << regex_match("hello", e) << endl; + cout << "regex result: " << regex_match("abcoo", b) << endl; + */ + pcrecpp::RE re1(")({a}h.*o"); + pcrecpp::RE re("h.llo"); + assert( re.FullMatch("hello") ); + assert( !re1.FullMatch("hello") ); + + + pcrecpp::RE_Options options; + options.set_utf8(true); + pcrecpp::RE part("dwi", options); + assert( part.PartialMatch("dwight") ); + } +} rxtest; + diff --git a/db/jsobj.h b/db/jsobj.h index f441064a97e..faa4d9f159a 100644 --- a/db/jsobj.h +++ b/db/jsobj.h @@ -1,514 +1,514 @@ -// jsobj.h
-
-#pragma once
-
-#include "../stdafx.h"
-#include "../util/builder.h"
-//#include "javajs.h"
-
-#include <set>
-
-class JSObj;
-class Record;
-class JSObjBuilder;
-
-#pragma pack(push)
-#pragma pack(1)
-
-/* BinData = binary data types.
- EOO = end of object
-*/
-enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5,
- Undefined=6, jstOID=7, Bool=8, Date=9 , jstNULL=10, RegEx=11 ,
- DBRef=12, Code=13, JSTypeMax=13, MaxKey=127 };
-
-/* subtypes of BinData.
- bdtCustom and above are ones that the JS compiler understands, but are
- opaque to the database.
-*/
-enum BinDataType { Function=1, ByteArray=2, bdtCustom=128 };
-
-/* Object id's are optional for JSObjects.
- When present they should be the first object member added.
-*/
-struct OID {
- long long a;
- unsigned b;
- bool operator==(const OID& r) { return a==r.a&&b==r.b; }
- void out(){ cout << hex << a << hex << b << endl; };
-};
-
-/* marshalled js object format:
-
- <unsigned totalSize> {<byte JSType><cstring FieldName><Data>}* EOO
- totalSize includes itself.
-
- Data:
- Bool: <byte>
- EOO: nothing follows
- Undefined: nothing follows
- OID: an OID object
- Number: <double>
- String: <unsigned32 strsizewithnull><cstring>
- Date: <8bytes>
- Regex: <cstring regex><cstring options>
- Object: a nested object, leading with its entire size, which terminates with EOO.
- Array: same as object
- BinData:
- <int len>
- <byte subtype>
- <byte[len] data>
-*/
-
-/* db operation message format
-
- unsigned opid; // arbitary; will be echoed back
- byte operation;
-
- dbInsert:
- int reserved;
- string collection;
- a series of JSObjects terminated with a null object (i.e., just EOO)
- dbUpdate: see query.h
- dbDelete: see query.h
- dbQuery: see query.h
-*/
-
-#pragma pack(pop)
-
-/* <type><fieldName ><value>
- -------- size() ------------
- -fieldNameSize-
- value()
- type()
-*/
-class Element {
- friend class JSElemIter;
- friend class JSObj;
-public:
- string toString();
- JSType type() { return (JSType) *data; }
- bool eoo() { return type() == EOO; }
- int size();
-
- // wrap this element up as a singleton object.
- JSObj wrap();
-
- const char * fieldName() {
- if( eoo() ) return ""; // no fieldname for it.
- return data + 1;
- }
-
- // raw data be careful:
- const char * value() const { return (data + fieldNameSize + 1); }
- int valuesize() { return size() - fieldNameSize - 1; }
-
- bool boolean() { return *value() ? true : false; }
-
- unsigned long long date() { return *((unsigned long long*) value()); }
- double& number() { return *((double *) value()); }
- OID& oid() { return *((OID*) value()); }
-
- // for strings
- int valuestrsize() {
- return *((int *) value());
- }
-
- // for objects the size *includes* the size of the size field
- int objsize() {
- return *((int *) value());
- }
-
- // for strings. also gives you start of the real data for an embedded object
- const char * valuestr() { return value() + 4; }
-
- JSObj embeddedObject();
-
- const char *regex() { assert(type() == RegEx); return value(); }
- const char *regexFlags() {
- const char *p = regex();
- return p + strlen(p) + 1;
- }
-
- /* like operator== but doesn't check the fieldname,
- just the value.
- */
- bool valuesEqual(Element& r) {
- bool match= valuesize() == r.valuesize() &&
- memcmp(value(),r.value(),valuesize()) == 0;
- return match;
- // todo: make "0" == 0.0, undefined==null
- }
-
- bool operator==(Element& r) {
- if( strcmp(fieldName(), r.fieldName()) != 0 )
- return false;
- return valuesEqual(r);
-/*
- int sz = size();
- return sz == r.size() &&
- memcmp(data, r.data, sz) == 0;
-*/
- }
-
- const char * rawdata() { return data; }
-
- Element();
-
-private:
- Element(const char *d) : data(d) {
- fieldNameSize = eoo() ? 0 : strlen(fieldName()) + 1;
- totalSize = -1;
- }
- const char *data;
- int fieldNameSize;
- int totalSize;
-};
-
-class JSObj {
- friend class JSElemIter;
-public:
-explicit
- JSObj(const char *msgdata, bool ifree = false) : iFree(ifree) {
- _objdata = msgdata;
- _objsize = *((int*) _objdata);
- }
- JSObj(Record *r);
- JSObj() : _objsize(0), _objdata(0), iFree(false) { }
-
- ~JSObj() { if( iFree ) { free((void*)_objdata); _objdata=0; } }
-
- void iWillFree() {
- assert(!iFree); iFree = true;
- }
-
- string toString() const;
- /* note: addFields always adds _id even if not specified */
- int addFields(JSObj& from, set<string>& fields); /* returns n added */
- int getFieldNames(set<string>& fields);
-
- /* return has eoo() true if no match
- supports "." notation to reach into embedded objects
- */
- Element getFieldDotted(const char *name);
-
- Element getField(const char *name); /* return has eoo() true if no match */
-
- const char * getStringField(const char *name);
- JSObj getObjectField(const char *name);
-
- /* makes a new JSObj with the fields specified in pattern.
- fields returned in the order they appear in pattern.
- if any field missing, you get back an empty object overall.
- */
- JSObj extractFields(JSObj pattern, JSObjBuilder& b);
-
- const char *objdata() const { return _objdata; }
- int objsize() const { return _objsize; } // includes the embedded size field
- bool isEmpty() const { return objsize() <= 5; }
-
- /* this is broken if elements aren't in the same order. */
- bool operator<(const JSObj& r) const { return woCompare(r) < 0; }
-
- /* -1: l<r. 0:l==r. 1:l>r
- wo='well ordered'. fields must be in same order in each object.
- */
- int woCompare(const JSObj& r) const;
- bool woEqual(const JSObj& r) const {
- return _objsize==r._objsize && memcmp(_objdata,r._objdata,_objsize)==0;
- }
- bool operator==(const JSObj& r) const {
- return this->woEqual(r);*this == r;
- }
-
- Element firstElement() {
- return Element(objdata() + 4);
- }
- Element findElement(const char *name);
-
- OID* getOID() {
- Element e = firstElement();
- if( e.type() != jstOID )
- return 0;
- return &e.oid();
- }
-
- JSObj(const JSObj& r) {
- _objsize = r._objsize;
- _objdata = r._objdata;
- iFree = r.iFree;
- if( iFree ) {
- ((JSObj&)r)._objdata = 0;
- ((JSObj&)r).iFree = false;
- }
-
- assert( _objsize == 0 || _objdata );
-
- }
- JSObj& operator=(JSObj& r) {
- if( iFree ) free((void*)_objdata);
- _objsize = r._objsize;
- _objdata = r._objdata;
- iFree = r.iFree;
- /* kind of like auto_ptrs here. note we leave objsize as it was
- so you'll get notified if you try to use the object, instead of just
- thinking it is empty.
- */
- if( iFree ) { r._objdata = 0; r.iFree = false; }
- return *this;
- }
-
- /* makes a copy of the object. Normally, a jsobj points to data "owned"
- by something else. this is a useful way to get your own copy of the buffer
- data (which is freed when the new jsobj destructs).
- */
- JSObj copy() const;
-
- int hash() const {
- unsigned x = 0;
- const char *p = _objdata;
- for( int i = 0; i < _objsize; i++ )
- x = x * 131 + p[i];
- return (x & 0x7fffffff) | 0x8000000; // must be > 0
- }
-
- bool iFree;
-private:
- int _objsize;
- const char *_objdata;
-};
-
-class JSObjBuilder {
-public:
- JSObjBuilder() { b.skip(4); /*leave room for size field*/ }
-
- /* add all the fields from the object specified to this object */
- void appendElements(JSObj x);
-
- void append(Element& e) { b.append((void*) e.rawdata(), e.size()); }
-
- /* append an element but with a new name */
- void appendAs(Element& e, const char *as) {
- b.append((char) e.type());
- b.append(as);
- b.append((void *) e.value(), e.valuesize());
- }
-
- /* add a subobject as a member */
- void append(const char *fieldName, JSObj subObj) {
- b.append((char) Object);
- b.append(fieldName);
- b.append((void *) subObj.objdata(), subObj.objsize());
- }
-
- void appendBool(const char *fieldName, int val) {
- b.append((char) Bool);
- b.append(fieldName);
- b.append((char) (val?1:0));
- }
- void append(const char *fieldName, double n) {
- b.append((char) Number);
- b.append(fieldName);
- b.append(n);
- }
- void appendDate(const char *fieldName, unsigned long long dt) {
- b.append((char) Date);
- b.append(fieldName);
- b.append(dt);
- }
- void append(const char *fieldName, const char *str) {
- b.append((char) String);
- b.append(fieldName);
- b.append((int) strlen(str)+1);
- b.append(str);
- }
-
- JSObj doneAndDecouple() {
- int l;
- return JSObj(decouple(l), true);
- }
-
- /* this version, jsobjbuilder still frees the jsobj
- when the builder goes out of scope. use it this way
- by default, that's simplest.
- */
- JSObj done() {
- return JSObj(_done());
- }
-
- /* assume ownership of the buffer - you must then free it (with free()) */
- char* decouple(int& l) {
- char *x = _done();
- assert( x );
- l = b.len();
- b.decouple();
- return x;
- }
- void decouple() { b.decouple(); } // post done() call version. be sure jsobj frees...
-
-private:
- char* _done() {
- b.append((char) EOO);
- char *data = b.buf();
- *((int*)data) = b.len();
- return data;
- }
-
- BufBuilder b;
-};
-
-class JSElemIter {
-public:
- JSElemIter(const JSObj& jso) {
- pos = jso.objdata() + 4;
- theend = jso.objdata() + jso.objsize();
- }
- bool more() { return pos < theend; }
- Element next() {
- Element e(pos);
- pos += e.size();
- return e;
- }
-private:
- const char *pos;
- const char *theend;
-};
-
-#include <pcrecpp.h>
-
-class RegexMatcher {
-public:
- const char *fieldName;
- pcrecpp::RE *re;
- RegexMatcher() { re = 0; }
- ~RegexMatcher() { delete re; }
-};
-
-// SQL where clause equivalent
-class Where;
-
-/* For when a js object is a query pattern.
-
- e.g.
- db.foo.find( { a : 3 } );
-
- { a : 3 } is the pattern object.
-
- GT/LT:
- { a : { $gt
-
-*/
-class JSMatcher {
- int matchesDotted(
- const char *fieldName,
- Element& toMatch, JSObj& obj,
- int compareOp, bool *deep, bool isArr = false);
-
-public:
- enum {
- Equality = 0,
- LT = 0x1,
- LTE = 0x3,
- GTE = 0x6,
- GT = 0x4
- };
-
- static int opDirection(int op) {
- return op <= LTE ? -1 : 1;
- }
-
- JSMatcher(JSObj& pattern);
-
- ~JSMatcher();
-
- /* deep means we looked into arrays for a match */
- bool matches(JSObj& j, bool *deep = 0);
-
- int getN() { return n; }
-
-private:
- int valuesMatch(Element& l, Element& r, int op);
-
- Where *where;
- JSObj& jsobj;
- vector<Element> toMatch;
- vector<int> compareOp;
- int n;
-
- RegexMatcher regexs[4];
- int nRegex;
-
- // so we delete the mem when we're done:
- JSObjBuilder *builders[8];
- int nBuilders;
-};
-
-extern JSObj maxKey;
-
-/*- just for testing -- */
-
-#pragma pack(push)
-#pragma pack(1)
-struct JSObj1 {
- JSObj1() {
- totsize=sizeof(JSObj1);
- n = Number; strcpy_s(nname, 5, "abcd"); N = 3.1;
- s = String; strcpy_s(sname, 7, "abcdef"); slen = 10;
- strcpy_s(sval, 10, "123456789"); eoo = EOO;
- }
- unsigned totsize;
-
- char n;
- char nname[5];
- double N;
-
- char s;
- char sname[7];
- unsigned slen;
- char sval[10];
-
- char eoo;
-};
-#pragma pack(pop)
-extern JSObj1 js1;
-
-inline JSObj Element::embeddedObject() {
- assert( type()==Object || type()==Array );
- return JSObj(value());
-}
-
-inline JSObj JSObj::copy() const {
- if( _objsize == 0 )
- return *this;
- char *p = (char*) malloc(_objsize);
- memcpy(p, _objdata, _objsize);
- return JSObj(p, true);
-}
-
-// wrap this element up as a singleton object.
-inline JSObj Element::wrap() {
- JSObjBuilder b;
- b.append(*this);
- return b.doneAndDecouple();
-}
-
-inline Element JSObj::findElement(const char *name) {
- JSElemIter it(*this);
- while( it.more() ) {
- Element e = it.next();
- if( strcmp(name, e.fieldName()) == 0 )
- return e;
- }
- return Element();
-}
-
-/* add all the fields from the object specified to this object */
-inline void JSObjBuilder::appendElements(JSObj x) {
- JSElemIter it(x);
- while( it.more() ) {
- Element e = it.next();
- if( e.eoo() ) break;
- append(e);
- }
-}
-
+// jsobj.h + +#pragma once + +#include "../stdafx.h" +#include "../util/builder.h" +//#include "javajs.h" + +#include <set> + +class JSObj; +class Record; +class JSObjBuilder; + +#pragma pack(push) +#pragma pack(1) + +/* BinData = binary data types. + EOO = end of object +*/ +enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5, + Undefined=6, jstOID=7, Bool=8, Date=9 , jstNULL=10, RegEx=11 , + DBRef=12, Code=13, JSTypeMax=13, MaxKey=127 }; + +/* subtypes of BinData. + bdtCustom and above are ones that the JS compiler understands, but are + opaque to the database. +*/ +enum BinDataType { Function=1, ByteArray=2, bdtCustom=128 }; + +/* Object id's are optional for JSObjects. + When present they should be the first object member added. +*/ +struct OID { + long long a; + unsigned b; + bool operator==(const OID& r) { return a==r.a&&b==r.b; } + void out(){ cout << hex << a << hex << b << endl; }; +}; + +/* marshalled js object format: + + <unsigned totalSize> {<byte JSType><cstring FieldName><Data>}* EOO + totalSize includes itself. + + Data: + Bool: <byte> + EOO: nothing follows + Undefined: nothing follows + OID: an OID object + Number: <double> + String: <unsigned32 strsizewithnull><cstring> + Date: <8bytes> + Regex: <cstring regex><cstring options> + Object: a nested object, leading with its entire size, which terminates with EOO. + Array: same as object + BinData: + <int len> + <byte subtype> + <byte[len] data> +*/ + +/* db operation message format + + unsigned opid; // arbitary; will be echoed back + byte operation; + + dbInsert: + int reserved; + string collection; + a series of JSObjects terminated with a null object (i.e., just EOO) + dbUpdate: see query.h + dbDelete: see query.h + dbQuery: see query.h +*/ + +#pragma pack(pop) + +/* <type><fieldName ><value> + -------- size() ------------ + -fieldNameSize- + value() + type() +*/ +class Element { + friend class JSElemIter; + friend class JSObj; +public: + string toString(); + JSType type() { return (JSType) *data; } + bool eoo() { return type() == EOO; } + int size(); + + // wrap this element up as a singleton object. + JSObj wrap(); + + const char * fieldName() { + if( eoo() ) return ""; // no fieldname for it. + return data + 1; + } + + // raw data be careful: + const char * value() const { return (data + fieldNameSize + 1); } + int valuesize() { return size() - fieldNameSize - 1; } + + bool boolean() { return *value() ? true : false; } + + unsigned long long date() { return *((unsigned long long*) value()); } + double& number() { return *((double *) value()); } + OID& oid() { return *((OID*) value()); } + + // for strings + int valuestrsize() { + return *((int *) value()); + } + + // for objects the size *includes* the size of the size field + int objsize() { + return *((int *) value()); + } + + // for strings. also gives you start of the real data for an embedded object + const char * valuestr() { return value() + 4; } + + JSObj embeddedObject(); + + const char *regex() { assert(type() == RegEx); return value(); } + const char *regexFlags() { + const char *p = regex(); + return p + strlen(p) + 1; + } + + /* like operator== but doesn't check the fieldname, + just the value. + */ + bool valuesEqual(Element& r) { + bool match= valuesize() == r.valuesize() && + memcmp(value(),r.value(),valuesize()) == 0; + return match; + // todo: make "0" == 0.0, undefined==null + } + + bool operator==(Element& r) { + if( strcmp(fieldName(), r.fieldName()) != 0 ) + return false; + return valuesEqual(r); +/* + int sz = size(); + return sz == r.size() && + memcmp(data, r.data, sz) == 0; +*/ + } + + const char * rawdata() { return data; } + + Element(); + +private: + Element(const char *d) : data(d) { + fieldNameSize = eoo() ? 0 : strlen(fieldName()) + 1; + totalSize = -1; + } + const char *data; + int fieldNameSize; + int totalSize; +}; + +class JSObj { + friend class JSElemIter; +public: +explicit + JSObj(const char *msgdata, bool ifree = false) : iFree(ifree) { + _objdata = msgdata; + _objsize = *((int*) _objdata); + } + JSObj(Record *r); + JSObj() : _objsize(0), _objdata(0), iFree(false) { } + + ~JSObj() { if( iFree ) { free((void*)_objdata); _objdata=0; } } + + void iWillFree() { + assert(!iFree); iFree = true; + } + + string toString() const; + /* note: addFields always adds _id even if not specified */ + int addFields(JSObj& from, set<string>& fields); /* returns n added */ + int getFieldNames(set<string>& fields); + + /* return has eoo() true if no match + supports "." notation to reach into embedded objects + */ + Element getFieldDotted(const char *name); + + Element getField(const char *name); /* return has eoo() true if no match */ + + const char * getStringField(const char *name); + JSObj getObjectField(const char *name); + + /* makes a new JSObj with the fields specified in pattern. + fields returned in the order they appear in pattern. + if any field missing, you get back an empty object overall. + */ + JSObj extractFields(JSObj pattern, JSObjBuilder& b); + + const char *objdata() const { return _objdata; } + int objsize() const { return _objsize; } // includes the embedded size field + bool isEmpty() const { return objsize() <= 5; } + + /* this is broken if elements aren't in the same order. */ + bool operator<(const JSObj& r) const { return woCompare(r) < 0; } + + /* -1: l<r. 0:l==r. 1:l>r + wo='well ordered'. fields must be in same order in each object. + */ + int woCompare(const JSObj& r) const; + bool woEqual(const JSObj& r) const { + return _objsize==r._objsize && memcmp(_objdata,r._objdata,_objsize)==0; + } + bool operator==(const JSObj& r) const { + return this->woEqual(r);*this == r; + } + + Element firstElement() { + return Element(objdata() + 4); + } + Element findElement(const char *name); + + OID* getOID() { + Element e = firstElement(); + if( e.type() != jstOID ) + return 0; + return &e.oid(); + } + + JSObj(const JSObj& r) { + _objsize = r._objsize; + _objdata = r._objdata; + iFree = r.iFree; + if( iFree ) { + ((JSObj&)r)._objdata = 0; + ((JSObj&)r).iFree = false; + } + + assert( _objsize == 0 || _objdata ); + + } + JSObj& operator=(JSObj& r) { + if( iFree ) free((void*)_objdata); + _objsize = r._objsize; + _objdata = r._objdata; + iFree = r.iFree; + /* kind of like auto_ptrs here. note we leave objsize as it was + so you'll get notified if you try to use the object, instead of just + thinking it is empty. + */ + if( iFree ) { r._objdata = 0; r.iFree = false; } + return *this; + } + + /* makes a copy of the object. Normally, a jsobj points to data "owned" + by something else. this is a useful way to get your own copy of the buffer + data (which is freed when the new jsobj destructs). + */ + JSObj copy() const; + + int hash() const { + unsigned x = 0; + const char *p = _objdata; + for( int i = 0; i < _objsize; i++ ) + x = x * 131 + p[i]; + return (x & 0x7fffffff) | 0x8000000; // must be > 0 + } + + bool iFree; +private: + int _objsize; + const char *_objdata; +}; + +class JSObjBuilder { +public: + JSObjBuilder() { b.skip(4); /*leave room for size field*/ } + + /* add all the fields from the object specified to this object */ + void appendElements(JSObj x); + + void append(Element& e) { b.append((void*) e.rawdata(), e.size()); } + + /* append an element but with a new name */ + void appendAs(Element& e, const char *as) { + b.append((char) e.type()); + b.append(as); + b.append((void *) e.value(), e.valuesize()); + } + + /* add a subobject as a member */ + void append(const char *fieldName, JSObj subObj) { + b.append((char) Object); + b.append(fieldName); + b.append((void *) subObj.objdata(), subObj.objsize()); + } + + void appendBool(const char *fieldName, int val) { + b.append((char) Bool); + b.append(fieldName); + b.append((char) (val?1:0)); + } + void append(const char *fieldName, double n) { + b.append((char) Number); + b.append(fieldName); + b.append(n); + } + void appendDate(const char *fieldName, unsigned long long dt) { + b.append((char) Date); + b.append(fieldName); + b.append(dt); + } + void append(const char *fieldName, const char *str) { + b.append((char) String); + b.append(fieldName); + b.append((int) strlen(str)+1); + b.append(str); + } + + JSObj doneAndDecouple() { + int l; + return JSObj(decouple(l), true); + } + + /* this version, jsobjbuilder still frees the jsobj + when the builder goes out of scope. use it this way + by default, that's simplest. + */ + JSObj done() { + return JSObj(_done()); + } + + /* assume ownership of the buffer - you must then free it (with free()) */ + char* decouple(int& l) { + char *x = _done(); + assert( x ); + l = b.len(); + b.decouple(); + return x; + } + void decouple() { b.decouple(); } // post done() call version. be sure jsobj frees... + +private: + char* _done() { + b.append((char) EOO); + char *data = b.buf(); + *((int*)data) = b.len(); + return data; + } + + BufBuilder b; +}; + +class JSElemIter { +public: + JSElemIter(const JSObj& jso) { + pos = jso.objdata() + 4; + theend = jso.objdata() + jso.objsize(); + } + bool more() { return pos < theend; } + Element next() { + Element e(pos); + pos += e.size(); + return e; + } +private: + const char *pos; + const char *theend; +}; + +#include <pcrecpp.h> + +class RegexMatcher { +public: + const char *fieldName; + pcrecpp::RE *re; + RegexMatcher() { re = 0; } + ~RegexMatcher() { delete re; } +}; + +// SQL where clause equivalent +class Where; + +/* For when a js object is a query pattern. + + e.g. + db.foo.find( { a : 3 } ); + + { a : 3 } is the pattern object. + + GT/LT: + { a : { $gt + +*/ +class JSMatcher { + int matchesDotted( + const char *fieldName, + Element& toMatch, JSObj& obj, + int compareOp, bool *deep, bool isArr = false); + +public: + enum { + Equality = 0, + LT = 0x1, + LTE = 0x3, + GTE = 0x6, + GT = 0x4 + }; + + static int opDirection(int op) { + return op <= LTE ? -1 : 1; + } + + JSMatcher(JSObj& pattern); + + ~JSMatcher(); + + /* deep means we looked into arrays for a match */ + bool matches(JSObj& j, bool *deep = 0); + + int getN() { return n; } + +private: + int valuesMatch(Element& l, Element& r, int op); + + Where *where; + JSObj& jsobj; + vector<Element> toMatch; + vector<int> compareOp; + int n; + + RegexMatcher regexs[4]; + int nRegex; + + // so we delete the mem when we're done: + JSObjBuilder *builders[8]; + int nBuilders; +}; + +extern JSObj maxKey; + +/*- just for testing -- */ + +#pragma pack(push) +#pragma pack(1) +struct JSObj1 { + JSObj1() { + totsize=sizeof(JSObj1); + n = Number; strcpy_s(nname, 5, "abcd"); N = 3.1; + s = String; strcpy_s(sname, 7, "abcdef"); slen = 10; + strcpy_s(sval, 10, "123456789"); eoo = EOO; + } + unsigned totsize; + + char n; + char nname[5]; + double N; + + char s; + char sname[7]; + unsigned slen; + char sval[10]; + + char eoo; +}; +#pragma pack(pop) +extern JSObj1 js1; + +inline JSObj Element::embeddedObject() { + assert( type()==Object || type()==Array ); + return JSObj(value()); +} + +inline JSObj JSObj::copy() const { + if( _objsize == 0 ) + return *this; + char *p = (char*) malloc(_objsize); + memcpy(p, _objdata, _objsize); + return JSObj(p, true); +} + +// wrap this element up as a singleton object. +inline JSObj Element::wrap() { + JSObjBuilder b; + b.append(*this); + return b.doneAndDecouple(); +} + +inline Element JSObj::findElement(const char *name) { + JSElemIter it(*this); + while( it.more() ) { + Element e = it.next(); + if( strcmp(name, e.fieldName()) == 0 ) + return e; + } + return Element(); +} + +/* add all the fields from the object specified to this object */ +inline void JSObjBuilder::appendElements(JSObj x) { + JSElemIter it(x); + while( it.more() ) { + Element e = it.next(); + if( e.eoo() ) break; + append(e); + } +} + diff --git a/db/minilex.h b/db/minilex.h index 64c98fb9ff8..b63228f4822 100644 --- a/db/minilex.h +++ b/db/minilex.h @@ -1,97 +1,97 @@ -// minilex.h
-// mini js lexical analyzer. idea is to be dumb and fast.
-
-struct MiniLex {
- strhashmap reserved;
- bool ic[256]; // ic=Identifier Character
- bool starter[256];
-
- // dm: very dumb about comments and escaped quotes -- but we are faster then at least,
- // albeit returning too much (which is ok for jsbobj current usage).
- void grabVariables(char *code /*modified and must stay in scope*/, strhashmap& vars) {
- char *p = code;
- char last = 0;
- while( *p ) {
- if( starter[*p] ) {
- char *q = p+1;
- while( *q && ic[*q] ) q++;
- const char *identifier = p;
- bool done = *q == 0;
- *q = 0;
- if( !reserved.count(identifier) ) {
- // we try to be smart about 'obj' but have to be careful as obj.obj
- // can happen; this is so that nFields is right for simplistic where cases
- // so we can stop scanning in jsobj when we find the field of interest.
- if( strcmp(identifier,"obj")==0 && p>code && p[-1] != '.' )
- ;
- else
- vars[identifier] = 1;
- }
- if( done )
- break;
- p = q + 1;
- continue;
- }
-
- if( *p == '\'' ) {
- p++;
- while( *p && *p != '\'' ) p++;
- }
- else if( *p == '"' ) {
- p++;
- while( *p && *p != '"' ) p++;
- }
- p++;
- }
- }
-
- MiniLex() {
- strhashmap atest;
- atest["foo"] = 3;
- assert( atest.count("bar") == 0 );
- assert( atest.count("foo") == 1 );
- assert( atest["foo"] == 3 );
-
- for( int i = 0; i < 256; i++ ) {
- ic[i] = starter[i] = false;
- }
- for( int i = 'a'; i <= 'z'; i++ )
- ic[i] = starter[i] = true;
- for( int i = 'A'; i <= 'Z'; i++ )
- ic[i] = starter[i] = true;
- for( int i = '0'; i <= '9'; i++ )
- ic[i] = true;
- for( int i = 128; i < 256; i++ )
- ic[i] = starter[i] = true;
- ic['$'] = starter['$'] = true;
- ic['_'] = starter['_'] = true;
-
- reserved["break"] = true;
- reserved["case"] = true;
- reserved["catch"] = true;
- reserved["continue"] = true;
- reserved["default"] = true;
- reserved["delete"] = true;
- reserved["do"] = true;
- reserved["else"] = true;
- reserved["finally"] = true;
- reserved["for"] = true;
- reserved["function"] = true;
- reserved["if"] = true;
- reserved["in"] = true;
- reserved["instanceof"] = true;
- reserved["new"] = true;
- reserved["return"] = true;
- reserved["switch"] = true;
- reserved["this"] = true;
- reserved["throw"] = true;
- reserved["try"] = true;
- reserved["typeof"] = true;
- reserved["var"] = true;
- reserved["void"] = true;
- reserved["while"] = true;
- reserved["with "] = true;
- }
-};
-
-
+// minilex.h +// mini js lexical analyzer. idea is to be dumb and fast. + +struct MiniLex { + strhashmap reserved; + bool ic[256]; // ic=Identifier Character + bool starter[256]; + + // dm: very dumb about comments and escaped quotes -- but we are faster then at least, + // albeit returning too much (which is ok for jsbobj current usage). + void grabVariables(char *code /*modified and must stay in scope*/, strhashmap& vars) { + char *p = code; + char last = 0; + while( *p ) { + if( starter[*p] ) { + char *q = p+1; + while( *q && ic[*q] ) q++; + const char *identifier = p; + bool done = *q == 0; + *q = 0; + if( !reserved.count(identifier) ) { + // we try to be smart about 'obj' but have to be careful as obj.obj + // can happen; this is so that nFields is right for simplistic where cases + // so we can stop scanning in jsobj when we find the field of interest. + if( strcmp(identifier,"obj")==0 && p>code && p[-1] != '.' ) + ; + else + vars[identifier] = 1; + } + if( done ) + break; + p = q + 1; + continue; + } + + if( *p == '\'' ) { + p++; + while( *p && *p != '\'' ) p++; + } + else if( *p == '"' ) { + p++; + while( *p && *p != '"' ) p++; + } + p++; + } + } + + MiniLex() { + strhashmap atest; + atest["foo"] = 3; + assert( atest.count("bar") == 0 ); + assert( atest.count("foo") == 1 ); + assert( atest["foo"] == 3 ); + + for( int i = 0; i < 256; i++ ) { + ic[i] = starter[i] = false; + } + for( int i = 'a'; i <= 'z'; i++ ) + ic[i] = starter[i] = true; + for( int i = 'A'; i <= 'Z'; i++ ) + ic[i] = starter[i] = true; + for( int i = '0'; i <= '9'; i++ ) + ic[i] = true; + for( int i = 128; i < 256; i++ ) + ic[i] = starter[i] = true; + ic['$'] = starter['$'] = true; + ic['_'] = starter['_'] = true; + + reserved["break"] = true; + reserved["case"] = true; + reserved["catch"] = true; + reserved["continue"] = true; + reserved["default"] = true; + reserved["delete"] = true; + reserved["do"] = true; + reserved["else"] = true; + reserved["finally"] = true; + reserved["for"] = true; + reserved["function"] = true; + reserved["if"] = true; + reserved["in"] = true; + reserved["instanceof"] = true; + reserved["new"] = true; + reserved["return"] = true; + reserved["switch"] = true; + reserved["this"] = true; + reserved["throw"] = true; + reserved["try"] = true; + reserved["typeof"] = true; + reserved["var"] = true; + reserved["void"] = true; + reserved["while"] = true; + reserved["with "] = true; + } +}; + + diff --git a/db/namespace.h b/db/namespace.h index ed9613d58f4..6caefc633f3 100644 --- a/db/namespace.h +++ b/db/namespace.h @@ -1,225 +1,225 @@ -// namespace.h
-
-#pragma once
-
-#include "../util/hashtab.h"
-#include "../util/mmap.h"
-
-class Cursor;
-
-#pragma pack(push)
-#pragma pack(1)
-
-class Namespace {
-public:
- Namespace(const char *ns) {
- *this = ns;
- }
- Namespace& operator=(const char *ns) {
- memset(buf, 0, 128); /* this is just to keep stuff clean in the files for easy dumping and reading */
- strcpy_s(buf, 128, ns); return *this;
- }
-
- void kill() {
- buf[0] = 0x7f;
- }
-
- bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; }
- int hash() const {
- unsigned x = 0;
- const char *p = buf;
- while( *p ) {
- x = x * 131 + *p;
- p++;
- }
- return (x & 0x7fffffff) | 0x8000000; // must be > 0
- }
-
- char buf[128];
-};
-
-const int Buckets = 19;
-const int MaxBucket = 18;
-const int MaxIndexes = 10;
-
-class IndexDetails {
-public:
- DiskLoc head; /* btree head */
- /* index info object.
- { name:"nameofindex", ns:"parentnsname", key: {keypattobject} }
- */
- DiskLoc info;
-
- /* pull out the relevant key objects from obj, so we
- can index them. Note that the set is multiple elements
- only when it's a "multikey" array.
- keys will be left empty if key not found in the object.
- */
- void getKeysFromObject(JSObj& obj, set<JSObj>& keys);
-
- // returns name of this index's storage area
- // client.table.$index
- string indexNamespace() {
- JSObj io = info.obj();
- string s;
- s.reserve(128);
- s = io.getStringField("ns");
- assert( !s.empty() );
- s += ".$";
- s += io.getStringField("name");
- return s;
- }
-};
-
-extern int bucketSizes[];
-
-class NamespaceDetails {
-public:
- NamespaceDetails() {
- datasize = nrecords = 0;
- lastExtentSize = 0;
- nIndexes = 0;
- capped = 0;
- max = 0x7fffffff;
- memset(reserved, 0, sizeof(reserved));
- }
- DiskLoc firstExtent;
- DiskLoc lastExtent;
- DiskLoc deletedList[Buckets];
- long long datasize;
- long long nrecords;
- int lastExtentSize;
- int nIndexes;
- IndexDetails indexes[MaxIndexes];
- int capped;
- int max; // max # of objects for a capped table.
- char reserved[256-16-4-4-8*MaxIndexes-8-8];
-
- //returns offset in indexes[]
- int findIndexByName(const char *name) {
- for( int i = 0; i < nIndexes; i++ ) {
- if( strcmp(indexes[i].info.obj().getStringField("name"),name) == 0 )
- return i;
- }
- return -1;
- }
-
- /* return which "deleted bucket" for this size object */
- static int bucket(int n) {
- for( int i = 0; i < Buckets; i++ )
- if( bucketSizes[i] > n )
- return i;
- return Buckets-1;
- }
-
- DiskLoc alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc);
- void addDeletedRec(DeletedRecord *d, DiskLoc dloc);
- void dumpDeleted(set<DiskLoc> *extents = 0);
-private:
- DiskLoc __stdAlloc(int len);
- DiskLoc _alloc(const char *ns, int len);
- void compact();
-};
-
-#pragma pack(pop)
-
-class NamespaceIndex {
- friend class NamespaceCursor;
-public:
- NamespaceIndex() { }
-
- void init(const char *dir, const char *client) {
- string path = dir;
- path += client;
- path += ".ns";
- const int LEN = 16 * 1024 * 1024;
- void *p = f.map(path.c_str(), LEN);
- if( p == 0 ) {
- problem() << "couldn't open namespace.idx " << path.c_str() << " terminating" << endl;
- exit(-3);
- }
- ht = new HashTable<Namespace,NamespaceDetails>(p, LEN, "namespace index");
- }
-
- void add(const char *ns, DiskLoc& loc) {
- Namespace n(ns);
- NamespaceDetails details;
- details.lastExtent = details.firstExtent = loc;
- ht->put(n, details);
- }
-
- NamespaceDetails* details(const char *ns) {
- Namespace n(ns);
- return ht->get(n);
- }
-
- void kill(const char *ns) {
- Namespace n(ns);
- ht->kill(n);
- }
-
- bool find(const char *ns, DiskLoc& loc) {
- NamespaceDetails *l = details(ns);
- if( l ) {
- loc = l->firstExtent;
- return true;
- }
- return false;
- }
-
-private:
- MemoryMappedFile f;
- HashTable<Namespace,NamespaceDetails> *ht;
-};
-
-extern const char *dbpath;
-
-/*
-class NamespaceIndexMgr {
-public:
- NamespaceIndexMgr() { }
- NamespaceIndex* get(const char *client) {
- map<string,NamespaceIndex*>::iterator it = m.find(client);
- if( it != m.end() )
- return it->second;
- NamespaceIndex *ni = new NamespaceIndex();
- ni->init(dbpath, client);
- m[client] = ni;
- return ni;
- }
-private:
- map<string,NamespaceIndex*> m;
-};
-
-extern NamespaceIndexMgr namespaceIndexMgr;
-*/
-
-// "client.a.b.c" -> "client"
-inline void nsToClient(const char *ns, char *client) {
- const char *p = ns;
- char *q = client;
- while( *p != '.' ) {
- if( *p == 0 ) {
- assert(false);
- *client = 0;
- return;
- }
- *q++ = *p++;
- }
- *q = 0;
- assert(q-client<256);
-}
-
-/*
-inline NamespaceIndex* nsindex(const char *ns) {
- char client[256];
- nsToClient(ns, client);
- return namespaceIndexMgr.get(client);
-}
-
-inline NamespaceDetails* nsdetails(const char *ns) {
- return nsindex(ns)->details(ns);
-}
-*/
-
-//auto_ptr<Cursor> makeNamespaceCursor();
+// namespace.h + +#pragma once + +#include "../util/hashtab.h" +#include "../util/mmap.h" + +class Cursor; + +#pragma pack(push) +#pragma pack(1) + +class Namespace { +public: + Namespace(const char *ns) { + *this = ns; + } + Namespace& operator=(const char *ns) { + memset(buf, 0, 128); /* this is just to keep stuff clean in the files for easy dumping and reading */ + strcpy_s(buf, 128, ns); return *this; + } + + void kill() { + buf[0] = 0x7f; + } + + bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; } + int hash() const { + unsigned x = 0; + const char *p = buf; + while( *p ) { + x = x * 131 + *p; + p++; + } + return (x & 0x7fffffff) | 0x8000000; // must be > 0 + } + + char buf[128]; +}; + +const int Buckets = 19; +const int MaxBucket = 18; +const int MaxIndexes = 10; + +class IndexDetails { +public: + DiskLoc head; /* btree head */ + /* index info object. + { name:"nameofindex", ns:"parentnsname", key: {keypattobject} } + */ + DiskLoc info; + + /* pull out the relevant key objects from obj, so we + can index them. Note that the set is multiple elements + only when it's a "multikey" array. + keys will be left empty if key not found in the object. + */ + void getKeysFromObject(JSObj& obj, set<JSObj>& keys); + + // returns name of this index's storage area + // client.table.$index + string indexNamespace() { + JSObj io = info.obj(); + string s; + s.reserve(128); + s = io.getStringField("ns"); + assert( !s.empty() ); + s += ".$"; + s += io.getStringField("name"); + return s; + } +}; + +extern int bucketSizes[]; + +class NamespaceDetails { +public: + NamespaceDetails() { + datasize = nrecords = 0; + lastExtentSize = 0; + nIndexes = 0; + capped = 0; + max = 0x7fffffff; + memset(reserved, 0, sizeof(reserved)); + } + DiskLoc firstExtent; + DiskLoc lastExtent; + DiskLoc deletedList[Buckets]; + long long datasize; + long long nrecords; + int lastExtentSize; + int nIndexes; + IndexDetails indexes[MaxIndexes]; + int capped; + int max; // max # of objects for a capped table. + char reserved[256-16-4-4-8*MaxIndexes-8-8]; + + //returns offset in indexes[] + int findIndexByName(const char *name) { + for( int i = 0; i < nIndexes; i++ ) { + if( strcmp(indexes[i].info.obj().getStringField("name"),name) == 0 ) + return i; + } + return -1; + } + + /* return which "deleted bucket" for this size object */ + static int bucket(int n) { + for( int i = 0; i < Buckets; i++ ) + if( bucketSizes[i] > n ) + return i; + return Buckets-1; + } + + DiskLoc alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc); + void addDeletedRec(DeletedRecord *d, DiskLoc dloc); + void dumpDeleted(set<DiskLoc> *extents = 0); +private: + DiskLoc __stdAlloc(int len); + DiskLoc _alloc(const char *ns, int len); + void compact(); +}; + +#pragma pack(pop) + +class NamespaceIndex { + friend class NamespaceCursor; +public: + NamespaceIndex() { } + + void init(const char *dir, const char *client) { + string path = dir; + path += client; + path += ".ns"; + const int LEN = 16 * 1024 * 1024; + void *p = f.map(path.c_str(), LEN); + if( p == 0 ) { + problem() << "couldn't open namespace.idx " << path.c_str() << " terminating" << endl; + exit(-3); + } + ht = new HashTable<Namespace,NamespaceDetails>(p, LEN, "namespace index"); + } + + void add(const char *ns, DiskLoc& loc) { + Namespace n(ns); + NamespaceDetails details; + details.lastExtent = details.firstExtent = loc; + ht->put(n, details); + } + + NamespaceDetails* details(const char *ns) { + Namespace n(ns); + return ht->get(n); + } + + void kill(const char *ns) { + Namespace n(ns); + ht->kill(n); + } + + bool find(const char *ns, DiskLoc& loc) { + NamespaceDetails *l = details(ns); + if( l ) { + loc = l->firstExtent; + return true; + } + return false; + } + +private: + MemoryMappedFile f; + HashTable<Namespace,NamespaceDetails> *ht; +}; + +extern const char *dbpath; + +/* +class NamespaceIndexMgr { +public: + NamespaceIndexMgr() { } + NamespaceIndex* get(const char *client) { + map<string,NamespaceIndex*>::iterator it = m.find(client); + if( it != m.end() ) + return it->second; + NamespaceIndex *ni = new NamespaceIndex(); + ni->init(dbpath, client); + m[client] = ni; + return ni; + } +private: + map<string,NamespaceIndex*> m; +}; + +extern NamespaceIndexMgr namespaceIndexMgr; +*/ + +// "client.a.b.c" -> "client" +inline void nsToClient(const char *ns, char *client) { + const char *p = ns; + char *q = client; + while( *p != '.' ) { + if( *p == 0 ) { + assert(false); + *client = 0; + return; + } + *q++ = *p++; + } + *q = 0; + assert(q-client<256); +} + +/* +inline NamespaceIndex* nsindex(const char *ns) { + char client[256]; + nsToClient(ns, client); + return namespaceIndexMgr.get(client); +} + +inline NamespaceDetails* nsdetails(const char *ns) { + return nsindex(ns)->details(ns); +} +*/ + +//auto_ptr<Cursor> makeNamespaceCursor(); diff --git a/db/objwrappers.h b/db/objwrappers.h index afd3a5b479d..8868e9e7d4e 100644 --- a/db/objwrappers.h +++ b/db/objwrappers.h @@ -1,9 +1,9 @@ -// objwrappers.h
-
-#pragma once
-
-#include "../stdafx.h"
-#include "jsobj.h"
-
-
-
+// objwrappers.h + +#pragma once + +#include "../stdafx.h" +#include "jsobj.h" + + + diff --git a/db/pdfile.cpp b/db/pdfile.cpp index c063d91b1c1..ddf74adf615 100644 --- a/db/pdfile.cpp +++ b/db/pdfile.cpp @@ -1,969 +1,969 @@ -// pdfile.cpp
-
-/*
-todo:
-_ manage deleted records. bucket?
-_ use deleted on inserts!
-_ quantize allocations
-_ table scans must be sequential, not next/prev pointers
-_ regex support
-*/
-
-#include "stdafx.h"
-#include "pdfile.h"
-#include "db.h"
-#include "../util/mmap.h"
-#include "../util/hashtab.h"
-#include "objwrappers.h"
-#include "btree.h"
-#include <algorithm>
-#include <list>
-
-const char *dbpath = "/data/db/";
-
-DataFileMgr theDataFileMgr;
-map<string,Client*> clients;
-Client *client;
-const char *curNs = "";
-int MAGIC = 0x1000;
-int curOp = -2;
-int callDepth = 0;
-
-extern int otherTraceLevel;
-
-void sayDbContext(const char *errmsg) {
- if( errmsg ) {
- cout << errmsg << '\n';
- problem() << errmsg << endl;
- }
- cout << " client: " << (client ? client->name.c_str() : "null");
- cout << " op:" << curOp << ' ' << callDepth << endl;
- if( client )
- cout << " ns: " << curNs << endl;
-}
-
-JSObj::JSObj(Record *r) {
- _objdata = r->data;
- _objsize = *((int*) _objdata);
- if( _objsize > r->netLength() ) {
- cout << "About to assert fail _objsize <= r->netLength()" << endl;
- cout << " _objsize: " << _objsize << endl;
- cout << " netLength(): " << r->netLength() << endl;
- cout << " extentOfs: " << r->extentOfs << endl;
- cout << " nextOfs: " << r->nextOfs << endl;
- cout << " prevOfs: " << r->prevOfs << endl;
- assert( _objsize <= r->netLength() );
- }
- iFree = false;
-}
-
-/*---------------------------------------------------------------------*/
-
-int bucketSizes[] = {
- 32, 64, 128, 256, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000,
- 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000,
- 0x400000, 0x800000
-};
-
-//NamespaceIndexMgr namespaceIndexMgr;
-
-void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) {
- DEBUGGING cout << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl;
- int b = bucket(d->lengthWithHeaders);
- DiskLoc& list = deletedList[b];
- DiskLoc oldHead = list;
- list = dloc;
- d->nextDeleted = oldHead;
-}
-
-/* lenToAlloc is WITH header
-*/
-DiskLoc NamespaceDetails::alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc) {
- lenToAlloc = (lenToAlloc + 3) & 0xfffffffc;
- DiskLoc loc = _alloc(ns, lenToAlloc);
- if( loc.isNull() )
- return loc;
-
- DeletedRecord *r = loc.drec();
-
- /* note we want to grab from the front so our next pointers on disk tend
- to go in a forward direction which is important for performance. */
- int regionlen = r->lengthWithHeaders;
- extentLoc.set(loc.a(), r->extentOfs);
-
- DEBUGGING cout << "TEMP: alloc() returns " << loc.toString() << ' ' << ns << " lentoalloc:" << lenToAlloc << " ext:" << extentLoc.toString() << endl;
-
- int left = regionlen - lenToAlloc;
- if( left < 24 || (left < (lenToAlloc >> 3) && capped == 0) ) {
- // you get the whole thing.
- return loc;
- }
-
- /* split off some for further use. */
- r->lengthWithHeaders = lenToAlloc;
- DiskLoc newDelLoc = loc;
- newDelLoc.inc(lenToAlloc);
- DeletedRecord *newDel = newDelLoc.drec();
- newDel->extentOfs = r->extentOfs;
- newDel->lengthWithHeaders = left;
- newDel->nextDeleted.Null();
-
- addDeletedRec(newDel, newDelLoc);
-
- return loc;
-}
-
-/* for non-capped collections.
- returned item is out of the deleted list upon return
-*/
-DiskLoc NamespaceDetails::__stdAlloc(int len) {
- DiskLoc *prev;
- DiskLoc *bestprev = 0;
- DiskLoc bestmatch;
- int bestmatchlen = 0x7fffffff;
- int b = bucket(len);
- DiskLoc cur = deletedList[b]; prev = &deletedList[b];
- int extra = 5; // look for a better fit, a little.
- int chain = 0;
- while( 1 ) {
- {
- int a = cur.a();
- if( a < -1 || a >= 100000 ) {
- problem() << "Assertion failure - a() out of range in _alloc() " << a << endl;
- cout << "Assertion failure - a() out of range in _alloc() " << a << '\n';
- sayDbContext();
- if( cur == *prev )
- prev->Null();
- cur.Null();
- }
- }
- if( cur.isNull() ) {
- // move to next bucket. if we were doing "extra", just break
- if( bestmatchlen < 0x7fffffff )
- break;
- b++;
- if( b > MaxBucket ) {
- // out of space. alloc a new extent.
- return DiskLoc();
- }
- cur = deletedList[b]; prev = &deletedList[b];
- continue;
- }
- DeletedRecord *r = cur.drec();
- if( r->lengthWithHeaders >= len &&
- r->lengthWithHeaders < bestmatchlen ) {
- bestmatchlen = r->lengthWithHeaders;
- bestmatch = cur;
- bestprev = prev;
- }
- if( bestmatchlen < 0x7fffffff && --extra <= 0 )
- break;
- if( ++chain > 30 && b < MaxBucket ) {
- // too slow, force move to next bucket to grab a big chunk
- b++;
- chain = 0;
- cur.Null();
- }
- else {
- cur = r->nextDeleted; prev = &r->nextDeleted;
- }
- }
-
- /* unlink ourself from the deleted list */
- *bestprev = bestmatch.drec()->nextDeleted;
-
- return bestmatch;
-}
-
-void NamespaceDetails::dumpDeleted(set<DiskLoc> *extents) {
-// cout << "DUMP deleted chains" << endl;
- for( int i = 0; i < Buckets; i++ ) {
-// cout << " bucket " << i << endl;
- DiskLoc dl = deletedList[i];
- while( !dl.isNull() ) {
- DeletedRecord *r = dl.drec();
- DiskLoc extLoc(dl.a(), r->extentOfs);
- if( extents == 0 || extents->count(extLoc) <= 0 ) {
- cout << " bucket " << i << endl;
- cout << " " << dl.toString() << " ext:" << extLoc.toString();
- if( extents && extents->count(extLoc) <= 0 )
- cout << '?';
- cout << " len:" << r->lengthWithHeaders << endl;
- }
- dl = r->nextDeleted;
- }
- }
-// cout << endl;
-}
-
-/* combine adjacent deleted records
-
- this is O(n^2) but we call it for capped tables where typically n==1 or 2!
- (or 3...there will be a little unused sliver at the end of the extent.)
-*/
-void NamespaceDetails::compact() {
- assert(capped);
- list<DiskLoc> drecs;
-
- for( int i = 0; i < Buckets; i++ ) {
- DiskLoc dl = deletedList[i];
- deletedList[i].Null();
- while( !dl.isNull() ) {
- DeletedRecord *r = dl.drec();
- drecs.push_back(dl);
- dl = r->nextDeleted;
- }
- }
-
- drecs.sort();
-
- list<DiskLoc>::iterator j = drecs.begin();
- assert( j != drecs.end() );
- DiskLoc a = *j;
- while( 1 ) {
- j++;
- if( j == drecs.end() ) {
- DEBUGGING cout << "TEMP: compact adddelrec\n";
- addDeletedRec(a.drec(), a);
- break;
- }
- DiskLoc b = *j;
- while( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) {
- // a & b are adjacent. merge.
- a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders;
- j++;
- if( j == drecs.end() ) {
- DEBUGGING cout << "temp: compact adddelrec2\n";
- addDeletedRec(a.drec(), a);
- return;
- }
- b = *j;
- }
- DEBUGGING cout << "temp: compact adddelrec3\n";
- addDeletedRec(a.drec(), a);
- a = b;
- }
-}
-
-DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
- if( !capped )
- return __stdAlloc(len);
-
- assert( len < 400000000 );
- int passes = 0;
- DiskLoc loc;
-
- // delete records until we have room and the max # objects limit achieved.
- while( 1 ) {
- if( nrecords < max ) {
- loc = __stdAlloc(len);
- if( !loc.isNull() )
- break;
- }
-
- DiskLoc fr = firstExtent.ext()->firstRecord;
- if( fr.isNull() ) {
- cout << "couldn't make room for new record in capped ns\n";
- cout << " ns:" << ns;
- cout << "\n len: " << len << endl;
- assert(false);
- return DiskLoc();
- }
-
- theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true);
- compact();
- assert( ++passes < 5000 );
- }
-
- return loc;
-}
-
-/*
-class NamespaceCursor : public Cursor {
-public:
- virtual bool ok() { return i >= 0; }
- virtual Record* _current() { assert(false); return 0; }
- virtual DiskLoc currLoc() { assert(false); return DiskLoc(); }
-
- virtual JSObj current() {
- NamespaceDetails &d = namespaceIndex.ht->nodes[i].value;
- JSObjBuilder b;
- b.append("name", namespaceIndex.ht->nodes[i].k.buf);
- return b.done();
- }
-
- virtual bool advance() {
- while( 1 ) {
- i++;
- if( i >= namespaceIndex.ht->n )
- break;
- if( namespaceIndex.ht->nodes[i].inUse() )
- return true;
- }
- i = -1000000;
- return false;
- }
-
- NamespaceCursor() {
- i = -1;
- advance();
- }
-private:
- int i;
-};
-
-auto_ptr<Cursor> makeNamespaceCursor() {
- return auto_ptr<Cursor>(new NamespaceCursor());
-}*/
-
-void newNamespace(const char *ns) {
- cout << "New namespace: " << ns << endl;
- if( strstr(ns, "system.namespaces") == 0 ) {
- JSObjBuilder b;
- b.append("name", ns);
- JSObj j = b.done();
- char client[256];
- nsToClient(ns, client);
- string s = client;
- s += ".system.namespaces";
- theDataFileMgr.insert(s.c_str(), j.objdata(), j.objsize(), true);
- }
-}
-
-int initialExtentSize(int len) {
- long long sz = len * 16;
- if( len < 1000 ) sz = len * 64;
- if( sz > 1000000000 )
- sz = 1000000000;
- int z = ((int)sz) & 0xffffff00;
- assert( z > len );
- cout << "initialExtentSize(" << len << ") returns " << z << endl;
- return z;
-}
-
-// { ..., capped: true, size: ..., max: ... }
-bool userCreateNS(const char *ns, JSObj& j) {
- if( nsdetails(ns) )
- return false;
-
- cout << j.toString() << endl;
-
- newNamespace(ns);
-
- int ies = initialExtentSize(128);
- Element e = j.findElement("size");
- if( e.type() == Number ) {
- ies = (int) e.number();
- ies += 256;
- ies &= 0xffffff00;
- if( ies > 1024 * 1024 * 1024 + 256 ) return false;
- }
-
- client->newestFile()->newExtent(ns, ies);
- NamespaceDetails *d = nsdetails(ns);
- assert(d);
-
- e = j.findElement("capped");
- if( e.type() == Bool && e.boolean() ) {
- d->capped = 1;
- e = j.findElement("max");
- if( e.type() == Number ) {
- int mx = (int) e.number();
- if( mx > 0 )
- d->max = mx;
- }
- }
-
- return true;
-}
-
-/*---------------------------------------------------------------------*/
-
-void PhysicalDataFile::open(int fn, const char *filename) {
- int length;
-
- if( fn <= 4 ) {
- length = (64*1024*1024) << fn;
- if( strstr(filename, "alleyinsider") && length < 1024 * 1024 * 1024 )
- length = 1024 * 1024 * 1024;
- } else
- length = 0x7ff00000;
-
- assert( length >= 64*1024*1024 && length % 4096 == 0 );
-
- assert(fn == fileNo);
- header = (PDFHeader *) mmf.map(filename, length);
- assert(header);
- header->init(fileNo, length);
-}
-
-/* prev - previous extent for this namespace. null=this is the first one. */
-Extent* PhysicalDataFile::newExtent(const char *ns, int approxSize, int loops) {
- assert( approxSize >= 0 && approxSize <= 0x7ff00000 );
-
- int ExtentSize = approxSize <= header->unusedLength ? approxSize : header->unusedLength;
- DiskLoc loc;
- if( ExtentSize <= 0 ) {
- if( loops > 8 ) {
- assert(false);
- return 0;
- }
- cout << "INFO: newExtent(): file full, adding a new file " << ns << endl;
- return client->addAFile()->newExtent(ns, approxSize, loops+1);
- }
- int offset = header->unused.getOfs();
- header->unused.setOfs( fileNo, offset + ExtentSize );
- header->unusedLength -= ExtentSize;
- loc.setOfs(fileNo, offset);
- Extent *e = _getExtent(loc);
- DiskLoc emptyLoc = e->init(ns, ExtentSize, fileNo, offset);
-
- DiskLoc oldExtentLoc;
- NamespaceIndex *ni = nsindex(ns);
- NamespaceDetails *details = ni->details(ns);
- if( details ) {
- assert( !details->firstExtent.isNull() );
- e->xprev = details->lastExtent;
- details->lastExtent.ext()->xnext = loc;
- details->lastExtent = loc;
- }
- else {
- ni->add(ns, loc);
- details = ni->details(ns);
- }
-
- details->lastExtentSize = approxSize;
- DEBUGGING cout << "temp: newextent adddelrec " << ns << endl;
- details->addDeletedRec(emptyLoc.drec(), emptyLoc);
-
- cout << "new extent size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset << dec;
- cout << " emptyLoc:" << hex << emptyLoc.getOfs() << dec;
- cout << ' ' << ns << endl;
- return e;
-}
-
-/*---------------------------------------------------------------------*/
-
-/* assumes already zeroed -- insufficient for block 'reuse' perhaps */
-DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset) {
- magic = 0x41424344;
- myLoc.setOfs(_fileNo, _offset);
- xnext.Null(); xprev.Null();
- ns = nsname;
- length = _length;
- firstRecord.Null(); lastRecord.Null();
-
- DiskLoc emptyLoc = myLoc;
- emptyLoc.inc( (extentData-(char*)this) );
-
- DeletedRecord *empty1 = (DeletedRecord *) extentData;
- DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);
- assert( empty == empty1 );
- empty->lengthWithHeaders = _length - (extentData - (char *) this);
- empty->extentOfs = myLoc.getOfs();
- return emptyLoc;
-}
-
-/*
-Record* Extent::newRecord(int len) {
- if( firstEmptyRegion.isNull() )
- return 0;
-
- assert(len > 0);
- int newRecSize = len + Record::HeaderSize;
- DiskLoc newRecordLoc = firstEmptyRegion;
- Record *r = getRecord(newRecordLoc);
- int left = r->netLength() - len;
- if( left < 0 ) {
- //
- firstEmptyRegion.Null();
- return 0;
- }
-
- DiskLoc nextEmpty = r->next.getNextEmpty(firstEmptyRegion);
- r->lengthWithHeaders = newRecSize;
- r->next.markAsFirstOrLastInExtent(this); // we're now last in the extent
- if( !lastRecord.isNull() ) {
- assert(getRecord(lastRecord)->next.lastInExtent()); // it was the last one
- getRecord(lastRecord)->next.set(newRecordLoc); // until now
- r->prev.set(lastRecord);
- }
- else {
- r->prev.markAsFirstOrLastInExtent(this); // we are the first in the extent
- assert( firstRecord.isNull() );
- firstRecord = newRecordLoc;
- }
- lastRecord = newRecordLoc;
-
- if( left < Record::HeaderSize + 32 ) {
- firstEmptyRegion.Null();
- }
- else {
- firstEmptyRegion.inc(newRecSize);
- Record *empty = getRecord(firstEmptyRegion);
- empty->next.set(nextEmpty); // not for empty records, unless in-use records, next and prev can be null.
- empty->prev.Null();
- empty->lengthWithHeaders = left;
- }
-
- return r;
-}
-*/
-
-/*---------------------------------------------------------------------*/
-
-auto_ptr<Cursor> DataFileMgr::findAll(const char *ns) {
- DiskLoc loc;
- bool found = nsindex(ns)->find(ns, loc);
- if( !found ) {
- // cout << "info: findAll() namespace does not exist: " << ns << endl;
- return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
- }
-
- Extent *e = getExtent(loc);
-
- DEBUGGING {
- cout << "temp: listing extents for " << ns << endl;
- DiskLoc tmp = loc;
- set<DiskLoc> extents;
-
- while( 1 ) {
- Extent *f = getExtent(tmp);
- cout << "extent: " << tmp.toString() << endl;
- extents.insert(tmp);
- tmp = f->xnext;
- if( tmp.isNull() )
- break;
- f = f->getNextExtent();
- }
-
- cout << endl;
- nsdetails(ns)->dumpDeleted(&extents);
- }
-
- while( e->firstRecord.isNull() && !e->xnext.isNull() ) {
- OCCASIONALLY cout << "info DFM::findAll(): extent " << loc.toString() << " was empty, skipping ahead" << endl;
- // find a nonempty extent
- // it might be nice to free the whole extent here! but have to clean up free recs then.
- e = e->getNextExtent();
- }
- return auto_ptr<Cursor>(new BasicCursor( e->firstRecord ));
-}
-
-/* get a table scan cursor, but can be forward or reverse direction */
-auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order) {
- Element el = order.findElement("$natural");
- if( el.type() != Number || el.number() >= 0 )
- return DataFileMgr::findAll(ns);
-
- // "reverse natural order" // "reverse natural order"
- NamespaceDetails *d = nsdetails(ns);
- if( !d )
- return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
- Extent *e = d->lastExtent.ext();
- while( e->lastRecord.isNull() && !e->xprev.isNull() ) {
- OCCASIONALLY cout << " findTableScan: extent empty, skipping ahead" << endl;
- e = e->getPrevExtent();
- }
- return auto_ptr<Cursor>(new ReverseCursor( e->lastRecord ));
-}
-
-void aboutToDelete(const DiskLoc& dl);
-
-/* pull out the relevant key objects from obj, so we
- can index them. Note that the set is multiple elements
- only when it's a "multikey" array.
- keys will be left empty if key not found in the object.
-*/
-void IndexDetails::getKeysFromObject(JSObj& obj, set<JSObj>& keys) {
- JSObj keyPattern = info.obj().getObjectField("key");
- JSObjBuilder b;
- JSObj key = obj.extractFields(keyPattern, b);
- if( key.isEmpty() )
- return;
- Element f = key.firstElement();
- if( f.type() != Array ) {
- b.decouple();
- key.iWillFree();
- keys.insert(key);
- return;
- }
- JSObj arr = f.embeddedObject();
-// cout << arr.toString() << endl;
- JSElemIter i(arr);
- while( i.more() ) {
- Element e = i.next();
- if( e.eoo() ) break;
- JSObjBuilder b;
-
- b.appendAs(e, f.fieldName());
- JSObj o = b.doneAndDecouple();
- assert( o.objdata() );
- keys.insert(o);
- }
-}
-
-int nUnindexes = 0;
-
-void _unindexRecord(const char *ns, IndexDetails& id, JSObj& obj, const DiskLoc& dl) {
- set<JSObj> keys;
- id.getKeysFromObject(obj, keys);
- for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {
- JSObj j = *i;
-// cout << "UNINDEX: j:" << j.toString() << " head:" << id.head.toString() << dl.toString() << endl;
- if( otherTraceLevel >= 5 ) {
- cout << "_unindexRecord() " << obj.toString();
- cout << "\n unindex:" << j.toString() << endl;
- }
- nUnindexes++;
- bool ok = false;
- try {
- ok = id.head.btree()->unindex(id.head, id, j, dl);
- }
- catch(AssertionException) {
- cout << " caught assertion _unindexRecord " << id.indexNamespace() << '\n';
- problem() << "Assertion failure: _unindex failed " << id.indexNamespace() << endl;
- cout << "Assertion failure: _unindex failed" << '\n';
- cout << " obj:" << obj.toString() << '\n';
- cout << " key:" << j.toString() << '\n';
- cout << " dl:" << dl.toString() << endl;
- sayDbContext();
- }
-
- if( !ok ) {
- cout << "unindex failed (key too big?) " << id.indexNamespace() << '\n';
- }
- }
-}
-
-void unindexRecord(const char *ns, NamespaceDetails *d, Record *todelete, const DiskLoc& dl) {
- if( d->nIndexes == 0 ) return;
- JSObj obj(todelete);
- for( int i = 0; i < d->nIndexes; i++ ) {
- _unindexRecord(ns, d->indexes[i], obj, dl);
- }
-}
-
-void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK)
-{
- int tempextofs = todelete->extentOfs;
-
- NamespaceDetails* d = nsdetails(ns);
- if( d->capped && !cappedOK ) {
- cout << "failing remove on a capped ns " << ns << endl;
- return;
- }
-
- /* check if any cursors point to us. if so, advance them. */
- aboutToDelete(dl);
-
- unindexRecord(ns, d, todelete, dl);
-
- /* remove ourself from the record next/prev chain */
- {
- if( todelete->prevOfs != DiskLoc::NullOfs )
- todelete->getPrev(dl).rec()->nextOfs = todelete->nextOfs;
- if( todelete->nextOfs != DiskLoc::NullOfs )
- todelete->getNext(dl).rec()->prevOfs = todelete->prevOfs;
- }
-
- /* remove ourself from extent pointers */
- {
- Extent *e = todelete->myExtent(dl);
- if( e->firstRecord == dl ) {
- if( todelete->nextOfs == DiskLoc::NullOfs )
- e->firstRecord.Null();
- else
- e->firstRecord.setOfs(dl.a(), todelete->nextOfs);
- }
- if( e->lastRecord == dl ) {
- if( todelete->prevOfs == DiskLoc::NullOfs )
- e->lastRecord.Null();
- else
- e->lastRecord.setOfs(dl.a(), todelete->prevOfs);
- }
- }
-
- /* add to the free list */
- {
- d->nrecords--;
- d->datasize -= todelete->netLength();
-/// DEBUGGING << "temp: dddelrec deleterecord " << ns << endl;
-// if( todelete->extentOfs == 0xaca500 ) {
-// cout << "break\n";
-// }
-/*
-TEMP: add deleted rec 0:aca5b0 aca500
-temp: adddelrec deleterecord admin.blog.posts
-TEMP: add deleted rec 0:b9e750 b6a500
-temp: adddelrec deleterecord admin.blog.posts
-*/
-
- d->addDeletedRec((DeletedRecord*)todelete, dl);
- }
-}
-
-void setDifference(set<JSObj>& l, set<JSObj>& r, vector<JSObj*> &diff) {
- set<JSObj>::iterator i = l.begin();
- set<JSObj>::iterator j = r.begin();
- while( 1 ) {
- if( i == l.end() )
- break;
- while( j != r.end() && *j < *i )
- j++;
- if( j == r.end() || !i->woEqual(*j) ) {
- const JSObj *jo = &*i;
- diff.push_back( (JSObj *) jo );
- }
- i++;
- }
-}
-
-/** Note: as written so far, if the object shrinks a lot, we don't free up space. */
-void DataFileMgr::update(
- const char *ns,
- Record *toupdate, const DiskLoc& dl,
- const char *buf, int len, stringstream& ss)
-{
- NamespaceDetails *d = nsdetails(ns);
-
- if( toupdate->netLength() < len ) {
- if( d && d->capped ) {
- ss << " failing a growing update on a capped ns " << ns << endl;
- return;
- }
-
- // doesn't fit.
- if( client->profile )
- ss << " moved ";
- deleteRecord(ns, toupdate, dl);
- insert(ns, buf, len);
- return;
- }
-
- /* has any index keys changed? */
- {
- NamespaceDetails *d = nsdetails(ns);
- if( d->nIndexes ) {
- JSObj newObj(buf);
- JSObj oldObj = dl.obj();
- for( int i = 0; i < d->nIndexes; i++ ) {
- IndexDetails& idx = d->indexes[i];
- JSObj idxKey = idx.info.obj().getObjectField("key");
-
- set<JSObj> oldkeys;
- set<JSObj> newkeys;
- idx.getKeysFromObject(oldObj, oldkeys);
- idx.getKeysFromObject(newObj, newkeys);
- vector<JSObj*> removed;
- setDifference(oldkeys, newkeys, removed);
- string idxns = idx.indexNamespace();
- for( unsigned i = 0; i < removed.size(); i++ ) {
- try {
- idx.head.btree()->unindex(idx.head, idx, *removed[i], dl);
- }
- catch(AssertionException) {
- ss << " exception update unindex ";
- cout << " caught assertion update unindex " << idxns.c_str() << '\n';
- problem() << " caught assertion update unindex " << idxns.c_str() << endl;
- }
- }
- vector<JSObj*> added;
- setDifference(newkeys, oldkeys, added);
- assert( !dl.isNull() );
- for( unsigned i = 0; i < added.size(); i++ ) {
- try {
- idx.head.btree()->insert(
- idx.head,
- dl, *added[i], false, idx, true);
- }
- catch(AssertionException) {
- ss << " exception update index ";
- cout << " caught assertion update index " << idxns.c_str() << '\n';
- problem() << " caught assertion update index " << idxns.c_str() << endl;
- }
- }
- if( client->profile )
- ss << "<br>" << added.size() << " key updates ";
-
- }
- }
- }
-
- // update in place
- memcpy(toupdate->data, buf, len);
-}
-
-int followupExtentSize(int len, int lastExtentLen) {
- int x = initialExtentSize(len);
- int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.2);
- int sz = y > x ? y : x;
- sz = ((int)sz) & 0xffffff00;
- assert( sz > len );
- return sz;
-}
-
-int deb=0;
-
-/* add keys to indexes for a new record */
-void _indexRecord(IndexDetails& idx, JSObj& obj, DiskLoc newRecordLoc) {
-
- set<JSObj> keys;
- idx.getKeysFromObject(obj, keys);
- for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) {
- assert( !newRecordLoc.isNull() );
- try {
-// DEBUGGING << "temp index: " << newRecordLoc.toString() << obj.toString() << endl;
- idx.head.btree()->insert(idx.head, newRecordLoc,
- (JSObj&) *i, false, idx, true);
- }
- catch(AssertionException) {
- cout << " caught assertion _indexRecord " << idx.indexNamespace() << '\n';
- problem() << " caught assertion _indexRecord " << idx.indexNamespace() << endl;
- }
- }
-}
-
-/* note there are faster ways to build an index in bulk, that can be
- done eventually */
-void addExistingToIndex(const char *ns, IndexDetails& idx) {
- cout << "Adding all existing records for " << ns << " to new index" << endl;
- int n = 0;
- auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
- while( c->ok() ) {
- JSObj js = c->current();
- _indexRecord(idx, js, c->currLoc());
- c->advance();
- n++;
- };
- cout << " indexing complete for " << n << " records" << endl;
-}
-
-/* add keys to indexes for a new record */
-void indexRecord(NamespaceDetails *d, const void *buf, int len, DiskLoc newRecordLoc) {
- JSObj obj((const char *)buf);
- for( int i = 0; i < d->nIndexes; i++ ) {
- _indexRecord(d->indexes[i], obj, newRecordLoc);
- }
-}
-
-DiskLoc DataFileMgr::insert(const char *ns, const void *buf, int len, bool god) {
- bool addIndex = false;
- const char *sys = strstr(ns, "system.");
- if( sys ) {
- if( sys == ns ) {
- cout << "ERROR: attempt to insert for invalid client 'system': " << ns << endl;
- return DiskLoc();
- }
- if( strstr(ns, ".system.") ) {
- if( strstr(ns, ".system.indexes") )
- addIndex = true;
- else if( !god ) {
- cout << "ERROR: attempt to insert in system namespace " << ns << endl;
- return DiskLoc();
- }
- }
- }
-
- NamespaceDetails *d = nsdetails(ns);
- if( d == 0 ) {
- newNamespace(ns);
- client->newestFile()->newExtent(ns, initialExtentSize(len));
- d = nsdetails(ns);
- }
-
- NamespaceDetails *tableToIndex = 0;
-
- const char *tabletoidxns = 0;
- if( addIndex ) {
- JSObj io((const char *) buf);
- const char *name = io.getStringField("name"); // name of the index
- tabletoidxns = io.getStringField("ns"); // table it indexes
- JSObj key = io.getObjectField("key");
- if( name == 0 || *name == 0 || tabletoidxns == 0 || key.isEmpty() || key.objsize() > 2048 ) {
- cout << "user warning: bad add index attempt name:" << (name?name:"") << " ns:" <<
- (tabletoidxns?tabletoidxns:"") << endl;
- return DiskLoc();
- }
- tableToIndex = nsdetails(tabletoidxns);
- if( tableToIndex == 0 ) {
- cout << "user warning: bad add index attempt, no such table(ns):" << tabletoidxns << endl;
- return DiskLoc();
- }
- if( tableToIndex->nIndexes >= MaxIndexes ) {
- cout << "user warning: bad add index attempt, too many indexes for:" << tabletoidxns << endl;
- return DiskLoc();
- }
- if( tableToIndex->findIndexByName(name) >= 0 ) {
- //cout << "INFO: index:" << name << " already exists for:" << tabletoidxns << endl;
- return DiskLoc();
- }
- //indexFullNS = tabletoidxns;
- //indexFullNS += ".$";
- //indexFullNS += name; // client.table.$index -- note this doesn't contain jsobjs, it contains BtreeBuckets.
- }
-
- DiskLoc extentLoc;
- int lenWHdr = len + Record::HeaderSize;
- DiskLoc loc = d->alloc(ns, lenWHdr, extentLoc);
- if( loc.isNull() ) {
- // out of space
- if( d->capped == 0 ) { // size capped doesn't grow
- cout << "allocating new extent for " << ns << endl;
- client->newestFile()->newExtent(ns, followupExtentSize(len, d->lastExtentSize));
- loc = d->alloc(ns, lenWHdr, extentLoc);
- }
- if( loc.isNull() ) {
- cout << "out of space in datafile. capped:" << d->capped << endl;
- assert(d->capped);
- return DiskLoc();
- }
- }
-
- Record *r = loc.rec();
- assert( r->lengthWithHeaders >= lenWHdr );
- memcpy(r->data, buf, len);
- Extent *e = r->myExtent(loc);
- if( e->lastRecord.isNull() ) {
- e->firstRecord = e->lastRecord = loc;
- r->prevOfs = r->nextOfs = DiskLoc::NullOfs;
- }
- else {
- Record *oldlast = e->lastRecord.rec();
- r->prevOfs = e->lastRecord.getOfs();
- r->nextOfs = DiskLoc::NullOfs;
- oldlast->nextOfs = loc.getOfs();
- e->lastRecord = loc;
- }
-
- d->nrecords++;
- d->datasize += r->netLength();
-
- if( tableToIndex ) {
- IndexDetails& idxinfo = tableToIndex->indexes[tableToIndex->nIndexes];
- idxinfo.info = loc;
- idxinfo.head = BtreeBucket::addHead(idxinfo);
- tableToIndex->nIndexes++;
- /* todo: index existing records here */
- addExistingToIndex(tabletoidxns, idxinfo);
- }
-
- /* add this record to our indexes */
- if( d->nIndexes )
- indexRecord(d, buf, len, loc);
-
-// cout << " inserted at loc:" << hex << loc.getOfs() << " lenwhdr:" << hex << lenWHdr << dec << ' ' << ns << endl;
- return loc;
-}
-
-void DataFileMgr::init(const char *dir) {
-/* string path = dir;
- path += "temp.dat";
- temp.open(path.c_str(), 64 * 1024 * 1024);
-*/
-}
-
-void pdfileInit() {
-// namespaceIndex.init(dbpath);
- theDataFileMgr.init(dbpath);
-}
+// pdfile.cpp + +/* +todo: +_ manage deleted records. bucket? +_ use deleted on inserts! +_ quantize allocations +_ table scans must be sequential, not next/prev pointers +_ regex support +*/ + +#include "stdafx.h" +#include "pdfile.h" +#include "db.h" +#include "../util/mmap.h" +#include "../util/hashtab.h" +#include "objwrappers.h" +#include "btree.h" +#include <algorithm> +#include <list> + +const char *dbpath = "/data/db/"; + +DataFileMgr theDataFileMgr; +map<string,Client*> clients; +Client *client; +const char *curNs = ""; +int MAGIC = 0x1000; +int curOp = -2; +int callDepth = 0; + +extern int otherTraceLevel; + +void sayDbContext(const char *errmsg) { + if( errmsg ) { + cout << errmsg << '\n'; + problem() << errmsg << endl; + } + cout << " client: " << (client ? client->name.c_str() : "null"); + cout << " op:" << curOp << ' ' << callDepth << endl; + if( client ) + cout << " ns: " << curNs << endl; +} + +JSObj::JSObj(Record *r) { + _objdata = r->data; + _objsize = *((int*) _objdata); + if( _objsize > r->netLength() ) { + cout << "About to assert fail _objsize <= r->netLength()" << endl; + cout << " _objsize: " << _objsize << endl; + cout << " netLength(): " << r->netLength() << endl; + cout << " extentOfs: " << r->extentOfs << endl; + cout << " nextOfs: " << r->nextOfs << endl; + cout << " prevOfs: " << r->prevOfs << endl; + assert( _objsize <= r->netLength() ); + } + iFree = false; +} + +/*---------------------------------------------------------------------*/ + +int bucketSizes[] = { + 32, 64, 128, 256, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, + 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, + 0x400000, 0x800000 +}; + +//NamespaceIndexMgr namespaceIndexMgr; + +void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) { + DEBUGGING cout << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl; + int b = bucket(d->lengthWithHeaders); + DiskLoc& list = deletedList[b]; + DiskLoc oldHead = list; + list = dloc; + d->nextDeleted = oldHead; +} + +/* lenToAlloc is WITH header +*/ +DiskLoc NamespaceDetails::alloc(const char *ns, int lenToAlloc, DiskLoc& extentLoc) { + lenToAlloc = (lenToAlloc + 3) & 0xfffffffc; + DiskLoc loc = _alloc(ns, lenToAlloc); + if( loc.isNull() ) + return loc; + + DeletedRecord *r = loc.drec(); + + /* note we want to grab from the front so our next pointers on disk tend + to go in a forward direction which is important for performance. */ + int regionlen = r->lengthWithHeaders; + extentLoc.set(loc.a(), r->extentOfs); + + DEBUGGING cout << "TEMP: alloc() returns " << loc.toString() << ' ' << ns << " lentoalloc:" << lenToAlloc << " ext:" << extentLoc.toString() << endl; + + int left = regionlen - lenToAlloc; + if( left < 24 || (left < (lenToAlloc >> 3) && capped == 0) ) { + // you get the whole thing. + return loc; + } + + /* split off some for further use. */ + r->lengthWithHeaders = lenToAlloc; + DiskLoc newDelLoc = loc; + newDelLoc.inc(lenToAlloc); + DeletedRecord *newDel = newDelLoc.drec(); + newDel->extentOfs = r->extentOfs; + newDel->lengthWithHeaders = left; + newDel->nextDeleted.Null(); + + addDeletedRec(newDel, newDelLoc); + + return loc; +} + +/* for non-capped collections. + returned item is out of the deleted list upon return +*/ +DiskLoc NamespaceDetails::__stdAlloc(int len) { + DiskLoc *prev; + DiskLoc *bestprev = 0; + DiskLoc bestmatch; + int bestmatchlen = 0x7fffffff; + int b = bucket(len); + DiskLoc cur = deletedList[b]; prev = &deletedList[b]; + int extra = 5; // look for a better fit, a little. + int chain = 0; + while( 1 ) { + { + int a = cur.a(); + if( a < -1 || a >= 100000 ) { + problem() << "Assertion failure - a() out of range in _alloc() " << a << endl; + cout << "Assertion failure - a() out of range in _alloc() " << a << '\n'; + sayDbContext(); + if( cur == *prev ) + prev->Null(); + cur.Null(); + } + } + if( cur.isNull() ) { + // move to next bucket. if we were doing "extra", just break + if( bestmatchlen < 0x7fffffff ) + break; + b++; + if( b > MaxBucket ) { + // out of space. alloc a new extent. + return DiskLoc(); + } + cur = deletedList[b]; prev = &deletedList[b]; + continue; + } + DeletedRecord *r = cur.drec(); + if( r->lengthWithHeaders >= len && + r->lengthWithHeaders < bestmatchlen ) { + bestmatchlen = r->lengthWithHeaders; + bestmatch = cur; + bestprev = prev; + } + if( bestmatchlen < 0x7fffffff && --extra <= 0 ) + break; + if( ++chain > 30 && b < MaxBucket ) { + // too slow, force move to next bucket to grab a big chunk + b++; + chain = 0; + cur.Null(); + } + else { + cur = r->nextDeleted; prev = &r->nextDeleted; + } + } + + /* unlink ourself from the deleted list */ + *bestprev = bestmatch.drec()->nextDeleted; + + return bestmatch; +} + +void NamespaceDetails::dumpDeleted(set<DiskLoc> *extents) { +// cout << "DUMP deleted chains" << endl; + for( int i = 0; i < Buckets; i++ ) { +// cout << " bucket " << i << endl; + DiskLoc dl = deletedList[i]; + while( !dl.isNull() ) { + DeletedRecord *r = dl.drec(); + DiskLoc extLoc(dl.a(), r->extentOfs); + if( extents == 0 || extents->count(extLoc) <= 0 ) { + cout << " bucket " << i << endl; + cout << " " << dl.toString() << " ext:" << extLoc.toString(); + if( extents && extents->count(extLoc) <= 0 ) + cout << '?'; + cout << " len:" << r->lengthWithHeaders << endl; + } + dl = r->nextDeleted; + } + } +// cout << endl; +} + +/* combine adjacent deleted records + + this is O(n^2) but we call it for capped tables where typically n==1 or 2! + (or 3...there will be a little unused sliver at the end of the extent.) +*/ +void NamespaceDetails::compact() { + assert(capped); + list<DiskLoc> drecs; + + for( int i = 0; i < Buckets; i++ ) { + DiskLoc dl = deletedList[i]; + deletedList[i].Null(); + while( !dl.isNull() ) { + DeletedRecord *r = dl.drec(); + drecs.push_back(dl); + dl = r->nextDeleted; + } + } + + drecs.sort(); + + list<DiskLoc>::iterator j = drecs.begin(); + assert( j != drecs.end() ); + DiskLoc a = *j; + while( 1 ) { + j++; + if( j == drecs.end() ) { + DEBUGGING cout << "TEMP: compact adddelrec\n"; + addDeletedRec(a.drec(), a); + break; + } + DiskLoc b = *j; + while( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) { + // a & b are adjacent. merge. + a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders; + j++; + if( j == drecs.end() ) { + DEBUGGING cout << "temp: compact adddelrec2\n"; + addDeletedRec(a.drec(), a); + return; + } + b = *j; + } + DEBUGGING cout << "temp: compact adddelrec3\n"; + addDeletedRec(a.drec(), a); + a = b; + } +} + +DiskLoc NamespaceDetails::_alloc(const char *ns, int len) { + if( !capped ) + return __stdAlloc(len); + + assert( len < 400000000 ); + int passes = 0; + DiskLoc loc; + + // delete records until we have room and the max # objects limit achieved. + while( 1 ) { + if( nrecords < max ) { + loc = __stdAlloc(len); + if( !loc.isNull() ) + break; + } + + DiskLoc fr = firstExtent.ext()->firstRecord; + if( fr.isNull() ) { + cout << "couldn't make room for new record in capped ns\n"; + cout << " ns:" << ns; + cout << "\n len: " << len << endl; + assert(false); + return DiskLoc(); + } + + theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true); + compact(); + assert( ++passes < 5000 ); + } + + return loc; +} + +/* +class NamespaceCursor : public Cursor { +public: + virtual bool ok() { return i >= 0; } + virtual Record* _current() { assert(false); return 0; } + virtual DiskLoc currLoc() { assert(false); return DiskLoc(); } + + virtual JSObj current() { + NamespaceDetails &d = namespaceIndex.ht->nodes[i].value; + JSObjBuilder b; + b.append("name", namespaceIndex.ht->nodes[i].k.buf); + return b.done(); + } + + virtual bool advance() { + while( 1 ) { + i++; + if( i >= namespaceIndex.ht->n ) + break; + if( namespaceIndex.ht->nodes[i].inUse() ) + return true; + } + i = -1000000; + return false; + } + + NamespaceCursor() { + i = -1; + advance(); + } +private: + int i; +}; + +auto_ptr<Cursor> makeNamespaceCursor() { + return auto_ptr<Cursor>(new NamespaceCursor()); +}*/ + +void newNamespace(const char *ns) { + cout << "New namespace: " << ns << endl; + if( strstr(ns, "system.namespaces") == 0 ) { + JSObjBuilder b; + b.append("name", ns); + JSObj j = b.done(); + char client[256]; + nsToClient(ns, client); + string s = client; + s += ".system.namespaces"; + theDataFileMgr.insert(s.c_str(), j.objdata(), j.objsize(), true); + } +} + +int initialExtentSize(int len) { + long long sz = len * 16; + if( len < 1000 ) sz = len * 64; + if( sz > 1000000000 ) + sz = 1000000000; + int z = ((int)sz) & 0xffffff00; + assert( z > len ); + cout << "initialExtentSize(" << len << ") returns " << z << endl; + return z; +} + +// { ..., capped: true, size: ..., max: ... } +bool userCreateNS(const char *ns, JSObj& j) { + if( nsdetails(ns) ) + return false; + + cout << j.toString() << endl; + + newNamespace(ns); + + int ies = initialExtentSize(128); + Element e = j.findElement("size"); + if( e.type() == Number ) { + ies = (int) e.number(); + ies += 256; + ies &= 0xffffff00; + if( ies > 1024 * 1024 * 1024 + 256 ) return false; + } + + client->newestFile()->newExtent(ns, ies); + NamespaceDetails *d = nsdetails(ns); + assert(d); + + e = j.findElement("capped"); + if( e.type() == Bool && e.boolean() ) { + d->capped = 1; + e = j.findElement("max"); + if( e.type() == Number ) { + int mx = (int) e.number(); + if( mx > 0 ) + d->max = mx; + } + } + + return true; +} + +/*---------------------------------------------------------------------*/ + +void PhysicalDataFile::open(int fn, const char *filename) { + int length; + + if( fn <= 4 ) { + length = (64*1024*1024) << fn; + if( strstr(filename, "alleyinsider") && length < 1024 * 1024 * 1024 ) + length = 1024 * 1024 * 1024; + } else + length = 0x7ff00000; + + assert( length >= 64*1024*1024 && length % 4096 == 0 ); + + assert(fn == fileNo); + header = (PDFHeader *) mmf.map(filename, length); + assert(header); + header->init(fileNo, length); +} + +/* prev - previous extent for this namespace. null=this is the first one. */ +Extent* PhysicalDataFile::newExtent(const char *ns, int approxSize, int loops) { + assert( approxSize >= 0 && approxSize <= 0x7ff00000 ); + + int ExtentSize = approxSize <= header->unusedLength ? approxSize : header->unusedLength; + DiskLoc loc; + if( ExtentSize <= 0 ) { + if( loops > 8 ) { + assert(false); + return 0; + } + cout << "INFO: newExtent(): file full, adding a new file " << ns << endl; + return client->addAFile()->newExtent(ns, approxSize, loops+1); + } + int offset = header->unused.getOfs(); + header->unused.setOfs( fileNo, offset + ExtentSize ); + header->unusedLength -= ExtentSize; + loc.setOfs(fileNo, offset); + Extent *e = _getExtent(loc); + DiskLoc emptyLoc = e->init(ns, ExtentSize, fileNo, offset); + + DiskLoc oldExtentLoc; + NamespaceIndex *ni = nsindex(ns); + NamespaceDetails *details = ni->details(ns); + if( details ) { + assert( !details->firstExtent.isNull() ); + e->xprev = details->lastExtent; + details->lastExtent.ext()->xnext = loc; + details->lastExtent = loc; + } + else { + ni->add(ns, loc); + details = ni->details(ns); + } + + details->lastExtentSize = approxSize; + DEBUGGING cout << "temp: newextent adddelrec " << ns << endl; + details->addDeletedRec(emptyLoc.drec(), emptyLoc); + + cout << "new extent size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset << dec; + cout << " emptyLoc:" << hex << emptyLoc.getOfs() << dec; + cout << ' ' << ns << endl; + return e; +} + +/*---------------------------------------------------------------------*/ + +/* assumes already zeroed -- insufficient for block 'reuse' perhaps */ +DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset) { + magic = 0x41424344; + myLoc.setOfs(_fileNo, _offset); + xnext.Null(); xprev.Null(); + ns = nsname; + length = _length; + firstRecord.Null(); lastRecord.Null(); + + DiskLoc emptyLoc = myLoc; + emptyLoc.inc( (extentData-(char*)this) ); + + DeletedRecord *empty1 = (DeletedRecord *) extentData; + DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc); + assert( empty == empty1 ); + empty->lengthWithHeaders = _length - (extentData - (char *) this); + empty->extentOfs = myLoc.getOfs(); + return emptyLoc; +} + +/* +Record* Extent::newRecord(int len) { + if( firstEmptyRegion.isNull() ) + return 0; + + assert(len > 0); + int newRecSize = len + Record::HeaderSize; + DiskLoc newRecordLoc = firstEmptyRegion; + Record *r = getRecord(newRecordLoc); + int left = r->netLength() - len; + if( left < 0 ) { + // + firstEmptyRegion.Null(); + return 0; + } + + DiskLoc nextEmpty = r->next.getNextEmpty(firstEmptyRegion); + r->lengthWithHeaders = newRecSize; + r->next.markAsFirstOrLastInExtent(this); // we're now last in the extent + if( !lastRecord.isNull() ) { + assert(getRecord(lastRecord)->next.lastInExtent()); // it was the last one + getRecord(lastRecord)->next.set(newRecordLoc); // until now + r->prev.set(lastRecord); + } + else { + r->prev.markAsFirstOrLastInExtent(this); // we are the first in the extent + assert( firstRecord.isNull() ); + firstRecord = newRecordLoc; + } + lastRecord = newRecordLoc; + + if( left < Record::HeaderSize + 32 ) { + firstEmptyRegion.Null(); + } + else { + firstEmptyRegion.inc(newRecSize); + Record *empty = getRecord(firstEmptyRegion); + empty->next.set(nextEmpty); // not for empty records, unless in-use records, next and prev can be null. + empty->prev.Null(); + empty->lengthWithHeaders = left; + } + + return r; +} +*/ + +/*---------------------------------------------------------------------*/ + +auto_ptr<Cursor> DataFileMgr::findAll(const char *ns) { + DiskLoc loc; + bool found = nsindex(ns)->find(ns, loc); + if( !found ) { + // cout << "info: findAll() namespace does not exist: " << ns << endl; + return auto_ptr<Cursor>(new BasicCursor(DiskLoc())); + } + + Extent *e = getExtent(loc); + + DEBUGGING { + cout << "temp: listing extents for " << ns << endl; + DiskLoc tmp = loc; + set<DiskLoc> extents; + + while( 1 ) { + Extent *f = getExtent(tmp); + cout << "extent: " << tmp.toString() << endl; + extents.insert(tmp); + tmp = f->xnext; + if( tmp.isNull() ) + break; + f = f->getNextExtent(); + } + + cout << endl; + nsdetails(ns)->dumpDeleted(&extents); + } + + while( e->firstRecord.isNull() && !e->xnext.isNull() ) { + OCCASIONALLY cout << "info DFM::findAll(): extent " << loc.toString() << " was empty, skipping ahead" << endl; + // find a nonempty extent + // it might be nice to free the whole extent here! but have to clean up free recs then. + e = e->getNextExtent(); + } + return auto_ptr<Cursor>(new BasicCursor( e->firstRecord )); +} + +/* get a table scan cursor, but can be forward or reverse direction */ +auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order) { + Element el = order.findElement("$natural"); + if( el.type() != Number || el.number() >= 0 ) + return DataFileMgr::findAll(ns); + + // "reverse natural order" // "reverse natural order" + NamespaceDetails *d = nsdetails(ns); + if( !d ) + return auto_ptr<Cursor>(new BasicCursor(DiskLoc())); + Extent *e = d->lastExtent.ext(); + while( e->lastRecord.isNull() && !e->xprev.isNull() ) { + OCCASIONALLY cout << " findTableScan: extent empty, skipping ahead" << endl; + e = e->getPrevExtent(); + } + return auto_ptr<Cursor>(new ReverseCursor( e->lastRecord )); +} + +void aboutToDelete(const DiskLoc& dl); + +/* pull out the relevant key objects from obj, so we + can index them. Note that the set is multiple elements + only when it's a "multikey" array. + keys will be left empty if key not found in the object. +*/ +void IndexDetails::getKeysFromObject(JSObj& obj, set<JSObj>& keys) { + JSObj keyPattern = info.obj().getObjectField("key"); + JSObjBuilder b; + JSObj key = obj.extractFields(keyPattern, b); + if( key.isEmpty() ) + return; + Element f = key.firstElement(); + if( f.type() != Array ) { + b.decouple(); + key.iWillFree(); + keys.insert(key); + return; + } + JSObj arr = f.embeddedObject(); +// cout << arr.toString() << endl; + JSElemIter i(arr); + while( i.more() ) { + Element e = i.next(); + if( e.eoo() ) break; + JSObjBuilder b; + + b.appendAs(e, f.fieldName()); + JSObj o = b.doneAndDecouple(); + assert( o.objdata() ); + keys.insert(o); + } +} + +int nUnindexes = 0; + +void _unindexRecord(const char *ns, IndexDetails& id, JSObj& obj, const DiskLoc& dl) { + set<JSObj> keys; + id.getKeysFromObject(obj, keys); + for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) { + JSObj j = *i; +// cout << "UNINDEX: j:" << j.toString() << " head:" << id.head.toString() << dl.toString() << endl; + if( otherTraceLevel >= 5 ) { + cout << "_unindexRecord() " << obj.toString(); + cout << "\n unindex:" << j.toString() << endl; + } + nUnindexes++; + bool ok = false; + try { + ok = id.head.btree()->unindex(id.head, id, j, dl); + } + catch(AssertionException) { + cout << " caught assertion _unindexRecord " << id.indexNamespace() << '\n'; + problem() << "Assertion failure: _unindex failed " << id.indexNamespace() << endl; + cout << "Assertion failure: _unindex failed" << '\n'; + cout << " obj:" << obj.toString() << '\n'; + cout << " key:" << j.toString() << '\n'; + cout << " dl:" << dl.toString() << endl; + sayDbContext(); + } + + if( !ok ) { + cout << "unindex failed (key too big?) " << id.indexNamespace() << '\n'; + } + } +} + +void unindexRecord(const char *ns, NamespaceDetails *d, Record *todelete, const DiskLoc& dl) { + if( d->nIndexes == 0 ) return; + JSObj obj(todelete); + for( int i = 0; i < d->nIndexes; i++ ) { + _unindexRecord(ns, d->indexes[i], obj, dl); + } +} + +void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK) +{ + int tempextofs = todelete->extentOfs; + + NamespaceDetails* d = nsdetails(ns); + if( d->capped && !cappedOK ) { + cout << "failing remove on a capped ns " << ns << endl; + return; + } + + /* check if any cursors point to us. if so, advance them. */ + aboutToDelete(dl); + + unindexRecord(ns, d, todelete, dl); + + /* remove ourself from the record next/prev chain */ + { + if( todelete->prevOfs != DiskLoc::NullOfs ) + todelete->getPrev(dl).rec()->nextOfs = todelete->nextOfs; + if( todelete->nextOfs != DiskLoc::NullOfs ) + todelete->getNext(dl).rec()->prevOfs = todelete->prevOfs; + } + + /* remove ourself from extent pointers */ + { + Extent *e = todelete->myExtent(dl); + if( e->firstRecord == dl ) { + if( todelete->nextOfs == DiskLoc::NullOfs ) + e->firstRecord.Null(); + else + e->firstRecord.setOfs(dl.a(), todelete->nextOfs); + } + if( e->lastRecord == dl ) { + if( todelete->prevOfs == DiskLoc::NullOfs ) + e->lastRecord.Null(); + else + e->lastRecord.setOfs(dl.a(), todelete->prevOfs); + } + } + + /* add to the free list */ + { + d->nrecords--; + d->datasize -= todelete->netLength(); +/// DEBUGGING << "temp: dddelrec deleterecord " << ns << endl; +// if( todelete->extentOfs == 0xaca500 ) { +// cout << "break\n"; +// } +/* +TEMP: add deleted rec 0:aca5b0 aca500 +temp: adddelrec deleterecord admin.blog.posts +TEMP: add deleted rec 0:b9e750 b6a500 +temp: adddelrec deleterecord admin.blog.posts +*/ + + d->addDeletedRec((DeletedRecord*)todelete, dl); + } +} + +void setDifference(set<JSObj>& l, set<JSObj>& r, vector<JSObj*> &diff) { + set<JSObj>::iterator i = l.begin(); + set<JSObj>::iterator j = r.begin(); + while( 1 ) { + if( i == l.end() ) + break; + while( j != r.end() && *j < *i ) + j++; + if( j == r.end() || !i->woEqual(*j) ) { + const JSObj *jo = &*i; + diff.push_back( (JSObj *) jo ); + } + i++; + } +} + +/** Note: as written so far, if the object shrinks a lot, we don't free up space. */ +void DataFileMgr::update( + const char *ns, + Record *toupdate, const DiskLoc& dl, + const char *buf, int len, stringstream& ss) +{ + NamespaceDetails *d = nsdetails(ns); + + if( toupdate->netLength() < len ) { + if( d && d->capped ) { + ss << " failing a growing update on a capped ns " << ns << endl; + return; + } + + // doesn't fit. + if( client->profile ) + ss << " moved "; + deleteRecord(ns, toupdate, dl); + insert(ns, buf, len); + return; + } + + /* has any index keys changed? */ + { + NamespaceDetails *d = nsdetails(ns); + if( d->nIndexes ) { + JSObj newObj(buf); + JSObj oldObj = dl.obj(); + for( int i = 0; i < d->nIndexes; i++ ) { + IndexDetails& idx = d->indexes[i]; + JSObj idxKey = idx.info.obj().getObjectField("key"); + + set<JSObj> oldkeys; + set<JSObj> newkeys; + idx.getKeysFromObject(oldObj, oldkeys); + idx.getKeysFromObject(newObj, newkeys); + vector<JSObj*> removed; + setDifference(oldkeys, newkeys, removed); + string idxns = idx.indexNamespace(); + for( unsigned i = 0; i < removed.size(); i++ ) { + try { + idx.head.btree()->unindex(idx.head, idx, *removed[i], dl); + } + catch(AssertionException) { + ss << " exception update unindex "; + cout << " caught assertion update unindex " << idxns.c_str() << '\n'; + problem() << " caught assertion update unindex " << idxns.c_str() << endl; + } + } + vector<JSObj*> added; + setDifference(newkeys, oldkeys, added); + assert( !dl.isNull() ); + for( unsigned i = 0; i < added.size(); i++ ) { + try { + idx.head.btree()->insert( + idx.head, + dl, *added[i], false, idx, true); + } + catch(AssertionException) { + ss << " exception update index "; + cout << " caught assertion update index " << idxns.c_str() << '\n'; + problem() << " caught assertion update index " << idxns.c_str() << endl; + } + } + if( client->profile ) + ss << "<br>" << added.size() << " key updates "; + + } + } + } + + // update in place + memcpy(toupdate->data, buf, len); +} + +int followupExtentSize(int len, int lastExtentLen) { + int x = initialExtentSize(len); + int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.2); + int sz = y > x ? y : x; + sz = ((int)sz) & 0xffffff00; + assert( sz > len ); + return sz; +} + +int deb=0; + +/* add keys to indexes for a new record */ +void _indexRecord(IndexDetails& idx, JSObj& obj, DiskLoc newRecordLoc) { + + set<JSObj> keys; + idx.getKeysFromObject(obj, keys); + for( set<JSObj>::iterator i=keys.begin(); i != keys.end(); i++ ) { + assert( !newRecordLoc.isNull() ); + try { +// DEBUGGING << "temp index: " << newRecordLoc.toString() << obj.toString() << endl; + idx.head.btree()->insert(idx.head, newRecordLoc, + (JSObj&) *i, false, idx, true); + } + catch(AssertionException) { + cout << " caught assertion _indexRecord " << idx.indexNamespace() << '\n'; + problem() << " caught assertion _indexRecord " << idx.indexNamespace() << endl; + } + } +} + +/* note there are faster ways to build an index in bulk, that can be + done eventually */ +void addExistingToIndex(const char *ns, IndexDetails& idx) { + cout << "Adding all existing records for " << ns << " to new index" << endl; + int n = 0; + auto_ptr<Cursor> c = theDataFileMgr.findAll(ns); + while( c->ok() ) { + JSObj js = c->current(); + _indexRecord(idx, js, c->currLoc()); + c->advance(); + n++; + }; + cout << " indexing complete for " << n << " records" << endl; +} + +/* add keys to indexes for a new record */ +void indexRecord(NamespaceDetails *d, const void *buf, int len, DiskLoc newRecordLoc) { + JSObj obj((const char *)buf); + for( int i = 0; i < d->nIndexes; i++ ) { + _indexRecord(d->indexes[i], obj, newRecordLoc); + } +} + +DiskLoc DataFileMgr::insert(const char *ns, const void *buf, int len, bool god) { + bool addIndex = false; + const char *sys = strstr(ns, "system."); + if( sys ) { + if( sys == ns ) { + cout << "ERROR: attempt to insert for invalid client 'system': " << ns << endl; + return DiskLoc(); + } + if( strstr(ns, ".system.") ) { + if( strstr(ns, ".system.indexes") ) + addIndex = true; + else if( !god ) { + cout << "ERROR: attempt to insert in system namespace " << ns << endl; + return DiskLoc(); + } + } + } + + NamespaceDetails *d = nsdetails(ns); + if( d == 0 ) { + newNamespace(ns); + client->newestFile()->newExtent(ns, initialExtentSize(len)); + d = nsdetails(ns); + } + + NamespaceDetails *tableToIndex = 0; + + const char *tabletoidxns = 0; + if( addIndex ) { + JSObj io((const char *) buf); + const char *name = io.getStringField("name"); // name of the index + tabletoidxns = io.getStringField("ns"); // table it indexes + JSObj key = io.getObjectField("key"); + if( name == 0 || *name == 0 || tabletoidxns == 0 || key.isEmpty() || key.objsize() > 2048 ) { + cout << "user warning: bad add index attempt name:" << (name?name:"") << " ns:" << + (tabletoidxns?tabletoidxns:"") << endl; + return DiskLoc(); + } + tableToIndex = nsdetails(tabletoidxns); + if( tableToIndex == 0 ) { + cout << "user warning: bad add index attempt, no such table(ns):" << tabletoidxns << endl; + return DiskLoc(); + } + if( tableToIndex->nIndexes >= MaxIndexes ) { + cout << "user warning: bad add index attempt, too many indexes for:" << tabletoidxns << endl; + return DiskLoc(); + } + if( tableToIndex->findIndexByName(name) >= 0 ) { + //cout << "INFO: index:" << name << " already exists for:" << tabletoidxns << endl; + return DiskLoc(); + } + //indexFullNS = tabletoidxns; + //indexFullNS += ".$"; + //indexFullNS += name; // client.table.$index -- note this doesn't contain jsobjs, it contains BtreeBuckets. + } + + DiskLoc extentLoc; + int lenWHdr = len + Record::HeaderSize; + DiskLoc loc = d->alloc(ns, lenWHdr, extentLoc); + if( loc.isNull() ) { + // out of space + if( d->capped == 0 ) { // size capped doesn't grow + cout << "allocating new extent for " << ns << endl; + client->newestFile()->newExtent(ns, followupExtentSize(len, d->lastExtentSize)); + loc = d->alloc(ns, lenWHdr, extentLoc); + } + if( loc.isNull() ) { + cout << "out of space in datafile. capped:" << d->capped << endl; + assert(d->capped); + return DiskLoc(); + } + } + + Record *r = loc.rec(); + assert( r->lengthWithHeaders >= lenWHdr ); + memcpy(r->data, buf, len); + Extent *e = r->myExtent(loc); + if( e->lastRecord.isNull() ) { + e->firstRecord = e->lastRecord = loc; + r->prevOfs = r->nextOfs = DiskLoc::NullOfs; + } + else { + Record *oldlast = e->lastRecord.rec(); + r->prevOfs = e->lastRecord.getOfs(); + r->nextOfs = DiskLoc::NullOfs; + oldlast->nextOfs = loc.getOfs(); + e->lastRecord = loc; + } + + d->nrecords++; + d->datasize += r->netLength(); + + if( tableToIndex ) { + IndexDetails& idxinfo = tableToIndex->indexes[tableToIndex->nIndexes]; + idxinfo.info = loc; + idxinfo.head = BtreeBucket::addHead(idxinfo); + tableToIndex->nIndexes++; + /* todo: index existing records here */ + addExistingToIndex(tabletoidxns, idxinfo); + } + + /* add this record to our indexes */ + if( d->nIndexes ) + indexRecord(d, buf, len, loc); + +// cout << " inserted at loc:" << hex << loc.getOfs() << " lenwhdr:" << hex << lenWHdr << dec << ' ' << ns << endl; + return loc; +} + +void DataFileMgr::init(const char *dir) { +/* string path = dir; + path += "temp.dat"; + temp.open(path.c_str(), 64 * 1024 * 1024); +*/ +} + +void pdfileInit() { +// namespaceIndex.init(dbpath); + theDataFileMgr.init(dbpath); +} diff --git a/db/pdfile.h b/db/pdfile.h index bb94089d917..f88f11d704f 100644 --- a/db/pdfile.h +++ b/db/pdfile.h @@ -1,456 +1,456 @@ -/* pdfile.h
-
- Files:
- client.ns - namespace index
- client.1 - data files
- client.2
- ...
-*/
-
-#pragma once
-
-// see version, versionMinor, below.
-const int VERSION = 4;
-
-#include "../stdafx.h"
-#include "../util/mmap.h"
-#include "storage.h"
-#include "jsobj.h"
-#include "namespace.h"
-
-class PDFHeader;
-class Extent;
-class Record;
-class Cursor;
-
-/*---------------------------------------------------------------------*/
-
-class PDFHeader;
-class PhysicalDataFile {
- friend class DataFileMgr;
- friend class BasicCursor;
-public:
- PhysicalDataFile(int fn) : fileNo(fn) { }
- void open(int fileNo, const char *filename);
-
-
- Extent* newExtent(const char *ns, int approxSize, int loops = 0);
-private:
- Extent* getExtent(DiskLoc loc);
- Extent* _getExtent(DiskLoc loc);
- Record* recordAt(DiskLoc dl);
-
- MemoryMappedFile mmf;
- PDFHeader *header;
-int __unUsEd;
-// int length;
- int fileNo;
-};
-
-class DataFileMgr {
- friend class BasicCursor;
-public:
- void init(const char *);
-
- void update(
- const char *ns,
- Record *toupdate, const DiskLoc& dl,
- const char *buf, int len, stringstream& profiling);
- DiskLoc insert(const char *ns, const void *buf, int len, bool god = false);
- void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false);
- static auto_ptr<Cursor> findAll(const char *ns);
-
- static Extent* getExtent(const DiskLoc& dl);
- static Record* getRecord(const DiskLoc& dl);
-private:
- vector<PhysicalDataFile *> files;
-};
-
-extern DataFileMgr theDataFileMgr;
-
-#pragma pack(push)
-#pragma pack(1)
-
-class DeletedRecord {
-public:
- int lengthWithHeaders;
- int extentOfs;
- DiskLoc nextDeleted;
-};
-
-class Record {
-public:
- enum { HeaderSize = 16 };
- int lengthWithHeaders;
- int extentOfs;
- int nextOfs;
- int prevOfs;
- char data[4];
- int netLength() { return lengthWithHeaders - HeaderSize; }
- //void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }
-
- /* use this when a record is deleted. basically a union with next/prev fields */
- DeletedRecord& asDeleted() { return *((DeletedRecord*) this); }
-
- Extent* myExtent(const DiskLoc& myLoc) {
- return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs));
- }
- /* get the next record in the namespace, traversing extents as necessary */
- DiskLoc getNext(const DiskLoc& myLoc);
- DiskLoc getPrev(const DiskLoc& myLoc);
-};
-
-/* extents are regions where all the records within the region
- belong to the same namespace.
-*/
-class Extent {
-public:
- unsigned magic;
- DiskLoc myLoc;
- DiskLoc xnext, xprev; /* next/prev extent for this namespace */
- Namespace ns; /* which namespace this extent is for. this is just for troubleshooting really */
- int length; /* size of the extent, including these fields */
- // DiskLoc firstEmptyRegion;
- DiskLoc firstRecord, lastRecord;
- char extentData[4];
-
- bool validates() {
- return !(firstRecord.isNull() ^ lastRecord.isNull()) &&
- length >= 0 && !myLoc.isNull();
- }
-
- void dump(iostream& s) {
- s << " loc:" << myLoc.toString() << " xnext:" << xnext.toString() << " xprev:" << xprev.toString() << '\n';
- s << " ns:" << ns.buf << '\n';
- s << " size:" << length << " firstRecord:" << firstRecord.toString() << " lastRecord:" << lastRecord.toString() << '\n';
- }
-
- /* assumes already zeroed -- insufficient for block 'reuse' perhaps
- Returns a DeletedRecord location which is the data in the extent ready for us.
- Caller will need to add that to the freelist structure in namespacedetail.
- */
- DiskLoc init(const char *nsname, int _length, int _fileNo, int _offset);
-
- void assertOk() { assert(magic == 0x41424344); }
-
- Record* newRecord(int len);
-
- Record* getRecord(DiskLoc dl) {
- assert( !dl.isNull() );
- assert( dl.sameFile(myLoc) );
- int x = dl.getOfs() - myLoc.getOfs();
- assert( x > 0 );
- return (Record *) (((char *) this) + x);
- }
-
- Extent* getNextExtent() { return xnext.isNull() ? 0 : DataFileMgr::getExtent(xnext); }
- Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }
-};
-
-/*
- ----------------------
- Header
- ----------------------
- Extent (for a particular namespace)
- Record
- ...
- Record (some chained for unused space)
- ----------------------
- more Extents...
- ----------------------
-*/
-
-/* data file header */
-class PDFHeader {
-public:
- int version;
- int versionMinor;
- int fileLength;
- DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */
- int unusedLength;
- char reserved[8192 - 4*4 - 8];
-
- char data[4];
-
- static int headerSize() { return sizeof(PDFHeader) - 4; }
-
- bool uninitialized() { if( version == 0 ) return true; assert(version == VERSION); return false; }
-
- Record* getRecord(DiskLoc dl) {
- int ofs = dl.getOfs();
- assert( ofs >= headerSize() );
- return (Record*) (((char *) this) + ofs);
- }
-
- void init(int fileno, int filelength) {
- if( uninitialized() ) {
- assert(filelength > 32768 );
- assert( headerSize() == 8192 );
- fileLength = filelength;
- version = VERSION;
- versionMinor = 0;
- unused.setOfs( fileno, headerSize() );
- assert( (data-(char*)this) == headerSize() );
- unusedLength = fileLength - headerSize() - 16;
- memcpy(data+unusedLength, " \nthe end\n", 16);
- }
- }
-};
-
-#pragma pack(pop)
-
-inline Extent* PhysicalDataFile::_getExtent(DiskLoc loc) {
- loc.assertOk();
- Extent *e = (Extent *) (((char *)header) + loc.getOfs());
- return e;
-}
-
-inline Extent* PhysicalDataFile::getExtent(DiskLoc loc) {
- Extent *e = _getExtent(loc);
- e->assertOk();
- return e;
-}
-
-class Cursor {
-public:
- virtual bool ok() = 0;
- bool eof() { return !ok(); }
- virtual Record* _current() = 0;
- virtual JSObj current() = 0;
- virtual DiskLoc currLoc() = 0;
- virtual bool advance() = 0; /*true=ok*/
-
- /* optional to implement. if implemented, means 'this' is a prototype */
- virtual Cursor* clone() { return 0; }
-
- virtual bool tempStopOnMiss() { return false; }
-
- /* called after every query block is iterated -- i.e. between getMore() blocks
- so you can note where we are, if necessary.
- */
- virtual void noteLocation() { }
-
- /* called before query getmore block is iterated */
- virtual void checkLocation() { }
-
- virtual const char * toString() { return "abstract?"; }
-
- /* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */
- set<DiskLoc> dups;
- bool dup(DiskLoc loc) {
- /* to save mem only call this when there is risk of dups (e.g. when 'deep'/multikey) */
- if( dups.count(loc) > 0 )
- return true;
- dups.insert(loc);
- return false;
- }
-};
-
-class BasicCursor : public Cursor {
-public:
- bool ok() { return !curr.isNull(); }
- Record* _current() {
- assert( ok() );
- return curr.rec();
- }
- JSObj current() {
- Record *r = _current();
- JSObj j(r);
- return j;
- }
- virtual DiskLoc currLoc() { return curr; }
-
- bool advance() {
- if( eof() )
- return false;
- Record *r = _current();
- curr = r->getNext(curr);
- return ok();
- }
-
- BasicCursor(DiskLoc dl) : curr(dl) { }
- BasicCursor() { }
- virtual const char * toString() { return "BasicCursor"; }
-
- DiskLoc curr;
-};
-
-class ReverseCursor : public BasicCursor {
-public:
- bool advance() {
- if( eof() )
- return false;
- Record *r = _current();
- curr = r->getPrev(curr);
- return ok();
- }
-
- ReverseCursor(DiskLoc dl) : BasicCursor(dl) { }
- ReverseCursor() { }
- virtual const char * toString() { return "ReverseCursor"; }
-};
-
-inline Record* PhysicalDataFile::recordAt(DiskLoc dl) { return header->getRecord(dl); }
-
-void sayDbContext(const char *msg = 0);
-
-inline DiskLoc Record::getNext(const DiskLoc& myLoc) {
- if( nextOfs != DiskLoc::NullOfs ) {
- /* defensive */
- if( nextOfs >= 0 && nextOfs < 10 ) {
- sayDbContext("Assertion failure - Record::getNext() referencing a deleted record?");
- return DiskLoc();
- }
-
- return DiskLoc(myLoc.a(), nextOfs);
- }
- Extent *e = myExtent(myLoc);
- while( 1 ) {
- if( e->xnext.isNull() )
- return DiskLoc(); // end of table.
- e = e->xnext.ext();
- if( !e->firstRecord.isNull() )
- break;
- // entire extent could be empty, keep looking
- }
- return e->firstRecord;
-}
-inline DiskLoc Record::getPrev(const DiskLoc& myLoc) {
- if( prevOfs != DiskLoc::NullOfs )
- return DiskLoc(myLoc.a(), prevOfs);
- Extent *e = myExtent(myLoc);
- if( e->xprev.isNull() )
- return DiskLoc();
- return e->xprev.ext()->lastRecord;
-}
-
-inline Record* DiskLoc::rec() const {
- return DataFileMgr::getRecord(*this);
-}
-inline JSObj DiskLoc::obj() const {
- return JSObj(rec());
-}
-inline DeletedRecord* DiskLoc::drec() const {
- assert( fileNo != -1 );
- return (DeletedRecord*) rec();
-}
-inline Extent* DiskLoc::ext() const {
- return DataFileMgr::getExtent(*this);
-}
-
-inline BtreeBucket* DiskLoc::btree() const {
- assert( fileNo != -1 );
- return (BtreeBucket*) rec()->data;
-}
-
-inline Bucket* DiskLoc::bucket() const {
- assert( fileNo != -1 );
- return (Bucket*) rec()->data;
-}
-
-/*---------------------------------------------------------------------*/
-
-// customer, or rather a customer's database -- i guess down the line
-// there might be more than one for a cust, we'll see.
-class Client {
-public:
- Client(const char *nm) : name(nm) {
- namespaceIndex = new NamespaceIndex();
- namespaceIndex->init(dbpath, nm);
- profile = 0;
- profileName = name + ".system.profile";
- }
-
- PhysicalDataFile* getFile(int n) {
- assert(this);
-
- if( n < 0 || n >= 10000 ) {
- cout << "getFile(): n=" << n << endl;
- assert( n >= 0 && n < 10000 );
- }
-#if defined(_DEBUG)
- if( n > 100 )
- cout << "getFile(): n=" << n << "?" << endl;
-#endif
- while( n >= (int) files.size() )
- files.push_back(0);
- PhysicalDataFile* p = files[n];
- if( p == 0 ) {
- p = new PhysicalDataFile(n);
- files[n] = p;
- stringstream out;
- out << dbpath << name << '.' << n;
- p->open(n, out.str().c_str());
- }
- return p;
- }
-
- PhysicalDataFile* addAFile() {
- int n = (int) files.size();
- return getFile(n);
- }
-
- PhysicalDataFile* newestFile() {
- int n = (int) files.size();
- if( n > 0 ) n--;
- return getFile(n);
- }
-
- vector<PhysicalDataFile*> files;
- string name; // "alleyinsider"
- NamespaceIndex *namespaceIndex;
- int profile; // 0=off.
- string profileName; // "alleyinsider.system.profile"
-};
-
-// tempish...move to TLS or pass all the way down as a parm
-extern map<string,Client*> clients;
-extern Client *client;
-extern const char *curNs;
-inline void setClient(const char *ns) {
- char cl[256];
- curNs = ns;
- nsToClient(ns, cl);
- map<string,Client*>::iterator it = clients.find(cl);
- if( it != clients.end() ) {
- client = it->second;
- return;
- }
- Client *c = new Client(cl);
- clients[cl] = c;
- client = c;
-}
-
-inline NamespaceIndex* nsindex(const char *ns) {
-#if defined(_DEBUG)
- char buf[256];
- nsToClient(ns, buf);
- if( client->name != buf ) {
- cout << "ERROR: attempt to write to wrong database client\n";
- cout << " ns:" << ns << '\n';
- cout << " client->name:" << client->name << endl;
- assert( client->name == buf );
- }
-#endif
- return client->namespaceIndex;
-}
-
-inline NamespaceDetails* nsdetails(const char *ns) {
- return nsindex(ns)->details(ns);
-}
-
-inline PhysicalDataFile& DiskLoc::pdf() const {
- assert( fileNo != -1 );
- return *client->getFile(fileNo);
-}
-
-inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) {
- assert( dl.a() != -1 );
- return client->getFile(dl.a())->getExtent(dl);
-}
-
-inline Record* DataFileMgr::getRecord(const DiskLoc& dl) {
- assert( dl.a() != -1 );
- return client->getFile(dl.a())->recordAt(dl);
-}
-
+/* pdfile.h + + Files: + client.ns - namespace index + client.1 - data files + client.2 + ... +*/ + +#pragma once + +// see version, versionMinor, below. +const int VERSION = 4; + +#include "../stdafx.h" +#include "../util/mmap.h" +#include "storage.h" +#include "jsobj.h" +#include "namespace.h" + +class PDFHeader; +class Extent; +class Record; +class Cursor; + +/*---------------------------------------------------------------------*/ + +class PDFHeader; +class PhysicalDataFile { + friend class DataFileMgr; + friend class BasicCursor; +public: + PhysicalDataFile(int fn) : fileNo(fn) { } + void open(int fileNo, const char *filename); + + + Extent* newExtent(const char *ns, int approxSize, int loops = 0); +private: + Extent* getExtent(DiskLoc loc); + Extent* _getExtent(DiskLoc loc); + Record* recordAt(DiskLoc dl); + + MemoryMappedFile mmf; + PDFHeader *header; +int __unUsEd; +// int length; + int fileNo; +}; + +class DataFileMgr { + friend class BasicCursor; +public: + void init(const char *); + + void update( + const char *ns, + Record *toupdate, const DiskLoc& dl, + const char *buf, int len, stringstream& profiling); + DiskLoc insert(const char *ns, const void *buf, int len, bool god = false); + void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false); + static auto_ptr<Cursor> findAll(const char *ns); + + static Extent* getExtent(const DiskLoc& dl); + static Record* getRecord(const DiskLoc& dl); +private: + vector<PhysicalDataFile *> files; +}; + +extern DataFileMgr theDataFileMgr; + +#pragma pack(push) +#pragma pack(1) + +class DeletedRecord { +public: + int lengthWithHeaders; + int extentOfs; + DiskLoc nextDeleted; +}; + +class Record { +public: + enum { HeaderSize = 16 }; + int lengthWithHeaders; + int extentOfs; + int nextOfs; + int prevOfs; + char data[4]; + int netLength() { return lengthWithHeaders - HeaderSize; } + //void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; } + + /* use this when a record is deleted. basically a union with next/prev fields */ + DeletedRecord& asDeleted() { return *((DeletedRecord*) this); } + + Extent* myExtent(const DiskLoc& myLoc) { + return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs)); + } + /* get the next record in the namespace, traversing extents as necessary */ + DiskLoc getNext(const DiskLoc& myLoc); + DiskLoc getPrev(const DiskLoc& myLoc); +}; + +/* extents are regions where all the records within the region + belong to the same namespace. +*/ +class Extent { +public: + unsigned magic; + DiskLoc myLoc; + DiskLoc xnext, xprev; /* next/prev extent for this namespace */ + Namespace ns; /* which namespace this extent is for. this is just for troubleshooting really */ + int length; /* size of the extent, including these fields */ + // DiskLoc firstEmptyRegion; + DiskLoc firstRecord, lastRecord; + char extentData[4]; + + bool validates() { + return !(firstRecord.isNull() ^ lastRecord.isNull()) && + length >= 0 && !myLoc.isNull(); + } + + void dump(iostream& s) { + s << " loc:" << myLoc.toString() << " xnext:" << xnext.toString() << " xprev:" << xprev.toString() << '\n'; + s << " ns:" << ns.buf << '\n'; + s << " size:" << length << " firstRecord:" << firstRecord.toString() << " lastRecord:" << lastRecord.toString() << '\n'; + } + + /* assumes already zeroed -- insufficient for block 'reuse' perhaps + Returns a DeletedRecord location which is the data in the extent ready for us. + Caller will need to add that to the freelist structure in namespacedetail. + */ + DiskLoc init(const char *nsname, int _length, int _fileNo, int _offset); + + void assertOk() { assert(magic == 0x41424344); } + + Record* newRecord(int len); + + Record* getRecord(DiskLoc dl) { + assert( !dl.isNull() ); + assert( dl.sameFile(myLoc) ); + int x = dl.getOfs() - myLoc.getOfs(); + assert( x > 0 ); + return (Record *) (((char *) this) + x); + } + + Extent* getNextExtent() { return xnext.isNull() ? 0 : DataFileMgr::getExtent(xnext); } + Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); } +}; + +/* + ---------------------- + Header + ---------------------- + Extent (for a particular namespace) + Record + ... + Record (some chained for unused space) + ---------------------- + more Extents... + ---------------------- +*/ + +/* data file header */ +class PDFHeader { +public: + int version; + int versionMinor; + int fileLength; + DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */ + int unusedLength; + char reserved[8192 - 4*4 - 8]; + + char data[4]; + + static int headerSize() { return sizeof(PDFHeader) - 4; } + + bool uninitialized() { if( version == 0 ) return true; assert(version == VERSION); return false; } + + Record* getRecord(DiskLoc dl) { + int ofs = dl.getOfs(); + assert( ofs >= headerSize() ); + return (Record*) (((char *) this) + ofs); + } + + void init(int fileno, int filelength) { + if( uninitialized() ) { + assert(filelength > 32768 ); + assert( headerSize() == 8192 ); + fileLength = filelength; + version = VERSION; + versionMinor = 0; + unused.setOfs( fileno, headerSize() ); + assert( (data-(char*)this) == headerSize() ); + unusedLength = fileLength - headerSize() - 16; + memcpy(data+unusedLength, " \nthe end\n", 16); + } + } +}; + +#pragma pack(pop) + +inline Extent* PhysicalDataFile::_getExtent(DiskLoc loc) { + loc.assertOk(); + Extent *e = (Extent *) (((char *)header) + loc.getOfs()); + return e; +} + +inline Extent* PhysicalDataFile::getExtent(DiskLoc loc) { + Extent *e = _getExtent(loc); + e->assertOk(); + return e; +} + +class Cursor { +public: + virtual bool ok() = 0; + bool eof() { return !ok(); } + virtual Record* _current() = 0; + virtual JSObj current() = 0; + virtual DiskLoc currLoc() = 0; + virtual bool advance() = 0; /*true=ok*/ + + /* optional to implement. if implemented, means 'this' is a prototype */ + virtual Cursor* clone() { return 0; } + + virtual bool tempStopOnMiss() { return false; } + + /* called after every query block is iterated -- i.e. between getMore() blocks + so you can note where we are, if necessary. + */ + virtual void noteLocation() { } + + /* called before query getmore block is iterated */ + virtual void checkLocation() { } + + virtual const char * toString() { return "abstract?"; } + + /* used for multikey index traversal to avoid sending back dups. see JSMatcher::matches() */ + set<DiskLoc> dups; + bool dup(DiskLoc loc) { + /* to save mem only call this when there is risk of dups (e.g. when 'deep'/multikey) */ + if( dups.count(loc) > 0 ) + return true; + dups.insert(loc); + return false; + } +}; + +class BasicCursor : public Cursor { +public: + bool ok() { return !curr.isNull(); } + Record* _current() { + assert( ok() ); + return curr.rec(); + } + JSObj current() { + Record *r = _current(); + JSObj j(r); + return j; + } + virtual DiskLoc currLoc() { return curr; } + + bool advance() { + if( eof() ) + return false; + Record *r = _current(); + curr = r->getNext(curr); + return ok(); + } + + BasicCursor(DiskLoc dl) : curr(dl) { } + BasicCursor() { } + virtual const char * toString() { return "BasicCursor"; } + + DiskLoc curr; +}; + +class ReverseCursor : public BasicCursor { +public: + bool advance() { + if( eof() ) + return false; + Record *r = _current(); + curr = r->getPrev(curr); + return ok(); + } + + ReverseCursor(DiskLoc dl) : BasicCursor(dl) { } + ReverseCursor() { } + virtual const char * toString() { return "ReverseCursor"; } +}; + +inline Record* PhysicalDataFile::recordAt(DiskLoc dl) { return header->getRecord(dl); } + +void sayDbContext(const char *msg = 0); + +inline DiskLoc Record::getNext(const DiskLoc& myLoc) { + if( nextOfs != DiskLoc::NullOfs ) { + /* defensive */ + if( nextOfs >= 0 && nextOfs < 10 ) { + sayDbContext("Assertion failure - Record::getNext() referencing a deleted record?"); + return DiskLoc(); + } + + return DiskLoc(myLoc.a(), nextOfs); + } + Extent *e = myExtent(myLoc); + while( 1 ) { + if( e->xnext.isNull() ) + return DiskLoc(); // end of table. + e = e->xnext.ext(); + if( !e->firstRecord.isNull() ) + break; + // entire extent could be empty, keep looking + } + return e->firstRecord; +} +inline DiskLoc Record::getPrev(const DiskLoc& myLoc) { + if( prevOfs != DiskLoc::NullOfs ) + return DiskLoc(myLoc.a(), prevOfs); + Extent *e = myExtent(myLoc); + if( e->xprev.isNull() ) + return DiskLoc(); + return e->xprev.ext()->lastRecord; +} + +inline Record* DiskLoc::rec() const { + return DataFileMgr::getRecord(*this); +} +inline JSObj DiskLoc::obj() const { + return JSObj(rec()); +} +inline DeletedRecord* DiskLoc::drec() const { + assert( fileNo != -1 ); + return (DeletedRecord*) rec(); +} +inline Extent* DiskLoc::ext() const { + return DataFileMgr::getExtent(*this); +} + +inline BtreeBucket* DiskLoc::btree() const { + assert( fileNo != -1 ); + return (BtreeBucket*) rec()->data; +} + +inline Bucket* DiskLoc::bucket() const { + assert( fileNo != -1 ); + return (Bucket*) rec()->data; +} + +/*---------------------------------------------------------------------*/ + +// customer, or rather a customer's database -- i guess down the line +// there might be more than one for a cust, we'll see. +class Client { +public: + Client(const char *nm) : name(nm) { + namespaceIndex = new NamespaceIndex(); + namespaceIndex->init(dbpath, nm); + profile = 0; + profileName = name + ".system.profile"; + } + + PhysicalDataFile* getFile(int n) { + assert(this); + + if( n < 0 || n >= 10000 ) { + cout << "getFile(): n=" << n << endl; + assert( n >= 0 && n < 10000 ); + } +#if defined(_DEBUG) + if( n > 100 ) + cout << "getFile(): n=" << n << "?" << endl; +#endif + while( n >= (int) files.size() ) + files.push_back(0); + PhysicalDataFile* p = files[n]; + if( p == 0 ) { + p = new PhysicalDataFile(n); + files[n] = p; + stringstream out; + out << dbpath << name << '.' << n; + p->open(n, out.str().c_str()); + } + return p; + } + + PhysicalDataFile* addAFile() { + int n = (int) files.size(); + return getFile(n); + } + + PhysicalDataFile* newestFile() { + int n = (int) files.size(); + if( n > 0 ) n--; + return getFile(n); + } + + vector<PhysicalDataFile*> files; + string name; // "alleyinsider" + NamespaceIndex *namespaceIndex; + int profile; // 0=off. + string profileName; // "alleyinsider.system.profile" +}; + +// tempish...move to TLS or pass all the way down as a parm +extern map<string,Client*> clients; +extern Client *client; +extern const char *curNs; +inline void setClient(const char *ns) { + char cl[256]; + curNs = ns; + nsToClient(ns, cl); + map<string,Client*>::iterator it = clients.find(cl); + if( it != clients.end() ) { + client = it->second; + return; + } + Client *c = new Client(cl); + clients[cl] = c; + client = c; +} + +inline NamespaceIndex* nsindex(const char *ns) { +#if defined(_DEBUG) + char buf[256]; + nsToClient(ns, buf); + if( client->name != buf ) { + cout << "ERROR: attempt to write to wrong database client\n"; + cout << " ns:" << ns << '\n'; + cout << " client->name:" << client->name << endl; + assert( client->name == buf ); + } +#endif + return client->namespaceIndex; +} + +inline NamespaceDetails* nsdetails(const char *ns) { + return nsindex(ns)->details(ns); +} + +inline PhysicalDataFile& DiskLoc::pdf() const { + assert( fileNo != -1 ); + return *client->getFile(fileNo); +} + +inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) { + assert( dl.a() != -1 ); + return client->getFile(dl.a())->getExtent(dl); +} + +inline Record* DataFileMgr::getRecord(const DiskLoc& dl) { + assert( dl.a() != -1 ); + return client->getFile(dl.a())->recordAt(dl); +} + diff --git a/db/query.cpp b/db/query.cpp index 80f4bbf316c..bd19ea653dd 100644 --- a/db/query.cpp +++ b/db/query.cpp @@ -1,854 +1,854 @@ -// query.cpp
-
-#include "stdafx.h"
-#include "query.h"
-#include "pdfile.h"
-#include "jsobj.h"
-#include "../util/builder.h"
-#include <time.h>
-#include "introspect.h"
-#include "btree.h"
-#include "../util/lruishmap.h"
-#include "javajs.h"
-
-//ns->query->DiskLoc
-LRUishMap<JSObj,DiskLoc,5> lrutest(123);
-
-int nextCursorId = 1;
-
-#pragma pack(push)
-#pragma pack(1)
-struct EmptyObject {
- EmptyObject() { len = 5; jstype = EOO; }
- int len;
- char jstype;
-} emptyObject;
-#pragma pack(pop)
-
-JSObj emptyObj((char *) &emptyObject);
-
-int getGtLtOp(Element& e);
-void appendElementHandlingGtLt(JSObjBuilder& b, Element& e);
-
-/* todo: _ cache query plans
- _ use index on partial match with the query
-
- parameters
- query - the query, e.g., { name: 'joe' }
- order - order by spec, e.g., { name: 1 } 1=ASC, -1=DESC
-
-*/
-auto_ptr<Cursor> getIndexCursor(const char *ns, JSObj& query, JSObj& order) {
- NamespaceDetails *d = nsdetails(ns);
- if( d == 0 ) return auto_ptr<Cursor>();
-
- // queryFields, e.g. { 'name' }
- set<string> queryFields;
- query.getFieldNames(queryFields);
-
- if( !order.isEmpty() ) {
- set<string> orderFields;
- order.getFieldNames(orderFields);
- // order by
- for(int i = 0; i < d->nIndexes; i++ ) {
- JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }
- assert( strcmp(ns, idxInfo.getStringField("ns")) == 0 );
- JSObj idxKey = idxInfo.getObjectField("key");
- set<string> keyFields;
- idxKey.getFieldNames(keyFields);
- if( keyFields == orderFields ) {
- bool reverse =
- order.firstElement().type() == Number &&
- order.firstElement().number() < 0;
- JSObjBuilder b;
- /* todo: start with the right key, not just beginning of index, when query is also
- specified!
- */
- return auto_ptr<Cursor>(new BtreeCursor(d->indexes[i].head, reverse ? maxKey : emptyObj, reverse ? -1 : 1, false));
- }
- }
- }
-
- // regular query without order by
- for(int i = 0; i < d->nIndexes; i++ ) {
- JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: }
- JSObj idxKey = idxInfo.getObjectField("key");
- set<string> keyFields;
- idxKey.getFieldNames(keyFields);
- if( keyFields == queryFields ) {
- JSObjBuilder b;
- JSObj q = query.extractFields(idxKey, b);
- /* regexp: only supported if form is /^text/ */
- JSObjBuilder b2;
- JSElemIter it(q);
- bool first = true;
- while( it.more() ) {
- Element e = it.next();
- if( e.eoo() )
- break;
-
- // GT/LT
- if( e.type() == Object ) {
- int op = getGtLtOp(e);
- if( op ) {
- if( !first || !it.next().eoo() ) {
- // compound keys with GT/LT not supported yet via index.
- goto fail;
- }
- int direction = - JSMatcher::opDirection(op);
- return auto_ptr<Cursor>( new BtreeCursor(
- d->indexes[i].head,
- direction == 1 ? emptyObj : maxKey,
- direction,
- true) );
- }
- }
-
- first = false;
- if( e.type() == RegEx ) {
- if( *e.regexFlags() )
- goto fail;
- const char *re = e.regex();
- const char *p = re;
- if( *p++ != '^' ) goto fail;
- while( *p ) {
- if( *p == ' ' || (*p>='0'&&*p<='9') || (*p>='@'&&*p<='Z') || (*p>='a'&&*p<='z') )
- ;
- else
- goto fail;
- p++;
- }
- if( it.more() && !it.next().eoo() ) // we must be the last part of the key (for now until we are smarter)
- goto fail;
- // ok!
- b2.append(e.fieldName(), re+1);
- break;
- }
- else {
- b2.append(e);
- //appendElementHandlingGtLt(b2, e);
- }
- }
- JSObj q2 = b2.done();
- return auto_ptr<Cursor>(
- new BtreeCursor(d->indexes[i].head, q2, 1, true));
- }
- }
-
-fail:
- return auto_ptr<Cursor>();
-}
-
-void deleteObjects(const char *ns, JSObj pattern, bool justOne) {
-// cout << "TEMP delete ns:" << ns << " queryobjsize:" <<
-// pattern.objsize() << endl;
-
- if( strstr(ns, ".system.") ) {
- if( strstr(ns, ".system.namespaces") ){
- cout << "WARNING: delete on system namespace " << ns << endl;
- }
- else if( strstr(ns, ".system.indexes") ) {
- cout << "WARNING: delete on system namespace " << ns << endl;
- }
- else {
- cout << "ERROR: attempt to delete in system namespace " << ns << endl;
- return;
- }
- }
-
- JSMatcher matcher(pattern);
- JSObj order;
- auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);
- if( c.get() == 0 )
- c = theDataFileMgr.findAll(ns);
-
- Cursor &tempDebug = *c;
- int temp = 0;
- int tempd = 0;
-
-DiskLoc _tempDelLoc;
-
- while( c->ok() ) {
- temp++;
-
- Record *r = c->_current();
- DiskLoc rloc = c->currLoc();
- c->advance(); // must advance before deleting as the next ptr will die
- JSObj js(r);
- //cout << "TEMP: " << js.toString() << endl;
- bool deep;
- if( !matcher.matches(js, &deep) ) {
- if( c->tempStopOnMiss() )
- break;
- }
- else {
- assert( !deep || !c->dup(rloc) ); // can't be a dup, we deleted it!
-// cout << " found match to delete" << endl;
- if( !justOne )
- c->noteLocation();
-_tempDelLoc = rloc;
- theDataFileMgr.deleteRecord(ns, r, rloc);
- tempd = temp;
- if( justOne )
- return;
- c->checkLocation();
- }
- }
-}
-
-struct Mod {
- const char *fieldName;
- double n;
-};
-
-void applyMods(vector<Mod>& mods, JSObj obj) {
- for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ ) {
- Mod& m = *i;
- Element e = obj.findElement(m.fieldName);
- if( e.type() == Number ) {
- e.number() += m.n;
- }
- }
-}
-
-/* get special operations like $inc
- { $inc: { a:1, b:1 } }
-*/
-void getMods(vector<Mod>& mods, JSObj from) {
- JSElemIter it(from);
- while( it.more() ) {
- Element e = it.next();
- if( strcmp(e.fieldName(), "$inc") == 0 && e.type() == Object ) {
- JSObj j = e.embeddedObject();
- JSElemIter jt(j);
- while( jt.more() ) {
- Element f = jt.next();
- if( f.eoo() )
- break;
- Mod m;
- m.fieldName = f.fieldName();
- if( f.type() == Number ) {
- m.n = f.number();
- mods.push_back(m);
- }
- }
- }
- }
-}
-
-/*
-todo:
- smart requery find record immediately
-*/
-void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss) {
-//cout << "TEMP BAD";
-//lrutest.find(updateobj);
-
- int profile = client->profile;
-
- // cout << "update ns:" << ns << " objsize:" << updateobj.objsize() << " queryobjsize:" <<
- // pattern.objsize();
-
- if( strstr(ns, ".system.") ) {
- cout << "\nERROR: attempt to update in system namespace " << ns << endl;
- ss << " can't update system namespace ";
- return;
- }
-
- int nscanned = 0;
- {
- JSMatcher matcher(pattern);
- JSObj order;
- auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order);
- if( c.get() == 0 )
- c = theDataFileMgr.findAll(ns);
- while( c->ok() ) {
- Record *r = c->_current();
- nscanned++;
- JSObj js(r);
- if( !matcher.matches(js) ) {
- if( c->tempStopOnMiss() )
- break;
- }
- else {
- /* note: we only update one row and quit. if you do multiple later,
- be careful or multikeys in arrays could break things badly. best
- to only allow updating a single row with a multikey lookup.
- */
-
- if( profile )
- ss << " nscanned:" << nscanned;
-
- /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
- regular ones at the moment. */
- if( updateobj.firstElement().fieldName()[0] == '$' ) {
- vector<Mod> mods;
- getMods(mods, updateobj);
- applyMods(mods, c->currLoc().obj());
- if( profile )
- ss << " fastmod ";
- return;
- }
-
- theDataFileMgr.update(ns, r, c->currLoc(), updateobj.objdata(), updateobj.objsize(), ss);
- return;
- }
- c->advance();
- }
- }
-
- if( profile )
- ss << " nscanned:" << nscanned;
-
- if( upsert ) {
- if( updateobj.firstElement().fieldName()[0] == '$' ) {
- /* upsert of an $inc. build a default */
- vector<Mod> mods;
- getMods(mods, updateobj);
- JSObjBuilder b;
- b.appendElements(pattern);
- for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ )
- b.append(i->fieldName, i->n);
- JSObj obj = b.done();
- theDataFileMgr.insert(ns, (void*) obj.objdata(), obj.objsize());
- if( profile )
- ss << " fastmodinsert ";
- return;
- }
- if( profile )
- ss << " upsert ";
- theDataFileMgr.insert(ns, (void*) updateobj.objdata(), updateobj.objsize());
- }
-}
-
-int queryTraceLevel = 0;
-int otherTraceLevel = 0;
-
-int initialExtentSize(int len);
-
-void clean(const char *ns, NamespaceDetails *d) {
- for( int i = 0; i < Buckets; i++ )
- d->deletedList[i].Null();
-}
-
-string validateNS(const char *ns, NamespaceDetails *d) {
- bool valid = true;
- stringstream ss;
- ss << "\nvalidate\n";
- if( d->capped )
- ss << " capped:" << d->capped << " max:" << d->max << '\n';
-
- ss << " firstExtent:" << d->firstExtent.toString() << " lastExtent:" << d->lastExtent.toString() << '\n';
- ss << " datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
-
- try {
-
- {
- ss << " first extent:\n";
- d->firstExtent.ext()->dump(ss);
- valid = valid && d->firstExtent.ext()->validates();
- }
-
- auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
- int n = 0;
- long long len = 0;
- long long nlen = 0;
- while( c->ok() ) {
- n++;
- Record *r = c->_current();
- len += r->lengthWithHeaders;
- nlen += r->netLength();
- c->advance();
- }
- ss << " " << n << " objects found, nobj:" << d->nrecords << "\n";
- ss << " " << len << " bytes record data w/headers\n";
- ss << " " << nlen << " bytes record data wout/headers\n";
-
- ss << " deletedList: ";
- for( int i = 0; i < Buckets; i++ ) {
- ss << (d->deletedList[i].isNull() ? '0' : '1');
- }
- ss << endl;
- int ndel = 0;
- long long delSize = 0;
- for( int i = 0; i < Buckets; i++ ) {
- DiskLoc loc = d->deletedList[i];
- while( !loc.isNull() ) {
- ndel++;
- DeletedRecord *d = loc.drec();
- delSize += d->lengthWithHeaders;
- loc = d->nextDeleted;
- }
- }
- ss << " deleted: n: " << ndel << " size: " << delSize << endl;
-
- int idxn = 0;
- try {
- ss << " nIndexes:" << d->nIndexes << endl;
- for( ; idxn < d->nIndexes; idxn++ ) {
- ss << " " << d->indexes[idxn].indexNamespace() << " keys:" <<
- d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl;
- }
- }
- catch(...) {
- ss << "\n exception during index validate idxn:" << idxn << endl;
- }
-
- }
- catch(AssertionException) {
- ss << "\n exception during validate\n" << endl;
- }
-
- if( !valid )
- ss << " ns corrupt, requires dbchk\n";
-
- return ss.str();
-}
-
-bool userCreateNS(const char *ns, JSObj& j);
-
-const int edebug=1;
-
-bool dbEval(JSObj& cmd, JSObjBuilder& result) {
- Element e = cmd.firstElement();
- assert( e.type() == Code );
- const char *code = e.valuestr();
- if ( ! JavaJS )
- JavaJS = new JavaJSImpl();
-
- jlong f = JavaJS->functionCreate(code);
- if( f == 0 ) {
- result.append("errmsg", "compile failed");
- return false;
- }
-
- Scope s;
- s.setString("$client", client->name.c_str());
- Element args = cmd.findElement("args");
- if( args.type() == Array ) {
- JSObj eo = args.embeddedObject();
- if( edebug ) {
- cout << "args:" << eo.toString() << endl;
- cout << "code:\n" << code << endl;
- }
- s.setObject("args", eo);
- }
-
- int res = s.invoke(f);
- if( res ) {
- result.append("errno", (double) res);
- result.append("errmsg", "invoke failed");
- return false;
- }
-
- int type = s.type("return");
- if( type == Object || type == Array )
- result.append("retval", s.getObject("return"));
- else if( type == Number )
- result.append("retval", s.getNumber("return"));
- else if( type == String )
- result.append("retval", s.getString("return").c_str());
- else if( type == Bool ) {
- result.appendBool("retval", s.getBoolean("return"));
- }
-
- return true;
-}
-
-// e.g.
-// system.cmd$.find( { queryTraceLevel: 2 } );
-//
-// returns true if ran a cmd
-//
-inline bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) {
-
- const char *p = strchr(ns, '.');
- if( !p ) return false;
- if( strcmp(p, ".$cmd") != 0 ) return false;
-
- bool ok = false;
- bool valid = false;
-
- //cout << jsobj.toString() << endl;
-
- Element e;
- e = jsobj.firstElement();
-
- if( e.eoo() ) goto done;
- if( e.type() == Code ) {
- valid = true;
- ok = dbEval(jsobj, anObjBuilderForYa);
- }
- else if( e.type() == Number ) {
- if( strcmp(e.fieldName(), "profile") == 0 ) {
- anObjBuilderForYa.append("was", (double) client->profile);
- int p = (int) e.number();
- valid = true;
- if( p == -1 )
- ok = true;
- else if( p >= 0 && p <= 2 ) {
- ok = true;
- client->profile = p;
- }
- else {
- ok = false;
- }
- }
- else {
- if( strncmp(ns, "admin", p-ns) != 0 ) // admin only
- return false;
- if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) {
- valid = ok = true;
- queryTraceLevel = (int) e.number();
- } else if( strcmp(e.fieldName(),"traceAll") == 0 ) {
- valid = ok = true;
- queryTraceLevel = (int) e.number();
- otherTraceLevel = (int) e.number();
- }
- }
- }
- else if( e.type() == String ) {
- string us(ns, p-ns);
-
- if( strcmp( e.fieldName(), "create") == 0 ) {
- valid = true;
- string ns = us + '.' + e.valuestr();
- ok = userCreateNS(ns.c_str(), jsobj);
- }
- else if( strcmp( e.fieldName(), "clean") == 0 ) {
- valid = true;
- string dropNs = us + '.' + e.valuestr();
- NamespaceDetails *d = nsdetails(dropNs.c_str());
- cout << "CMD: clean " << dropNs << endl;
- if( d ) {
- ok = true;
- anObjBuilderForYa.append("ns", dropNs.c_str());
- clean(dropNs.c_str(), d);
- }
- else {
- anObjBuilderForYa.append("errmsg", "ns not found");
- }
- }
- else if( strcmp( e.fieldName(), "drop") == 0 ) {
- valid = true;
- string dropNs = us + '.' + e.valuestr();
- NamespaceDetails *d = nsdetails(dropNs.c_str());
- cout << "CMD: clean " << dropNs << endl;
- if( d ) {
- ok = true;
- anObjBuilderForYa.append("ns", dropNs.c_str());
- client->namespaceIndex->kill(dropNs.c_str());
- }
- else {
- anObjBuilderForYa.append("errmsg", "ns not found");
- }
- }
- else if( strcmp( e.fieldName(), "validate") == 0 ) {
- valid = true;
- string toValidateNs = us + '.' + e.valuestr();
- NamespaceDetails *d = nsdetails(toValidateNs.c_str());
- cout << "CMD: validate " << toValidateNs << endl;
- if( d ) {
- ok = true;
- anObjBuilderForYa.append("ns", toValidateNs.c_str());
- string s = validateNS(toValidateNs.c_str(), d);
- anObjBuilderForYa.append("result", s.c_str());
- }
- else {
- anObjBuilderForYa.append("errmsg", "ns not found");
- }
- }
- else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) {
- valid = true;
- /* note: temp implementation. space not reclaimed! */
- string toDeleteNs = us + '.' + e.valuestr();
- NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
- cout << "CMD: deleteIndexes " << toDeleteNs << endl;
- if( d ) {
- Element f = jsobj.findElement("index");
- if( !f.eoo() ) {
- // delete a specific index
- if( f.type() == String ) {
- const char *idxName = f.valuestr();
- if( *idxName == '*' && idxName[1] == 0 ) {
- ok = true;
- cout << " d->nIndexes was " << d->nIndexes << endl;
- anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);
- anObjBuilderForYa.append("msg", "all indexes deleted for collection");
- cout << " alpha implementation, space not reclaimed" << endl;
- d->nIndexes = 0;
- }
- else {
- // delete just one index
- int x = d->findIndexByName(idxName);
- if( x >= 0 ) {
- cout << " d->nIndexes was " << d->nIndexes << endl;
- anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes);
- d->nIndexes--;
- for( int i = x; i < d->nIndexes; i++ )
- d->indexes[i] = d->indexes[i+1];
- ok=true;
- cout << " alpha implementation, space not reclaimed" << endl;
- } else {
- cout << "deleteIndexes: " << idxName << " not found" << endl;
- }
- }
- }
- }
- }
- else {
- anObjBuilderForYa.append("errmsg", "ns not found");
- }
- }
- }
-
-done:
- if( !valid )
- anObjBuilderForYa.append("errmsg", "no such cmd");
- anObjBuilderForYa.append("ok", ok?1.0:0.0);
- JSObj x = anObjBuilderForYa.done();
- b.append((void*) x.objdata(), x.objsize());
- return true;
-}
-
-bool runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) {
- try {
- return _runCommands(ns, jsobj, ss, b, anObjBuilderForYa);
- }
- catch( AssertionException ) {
- ;
- }
- ss << " assertion ";
- anObjBuilderForYa.append("errmsg", "db assertion failure");
- anObjBuilderForYa.append("ok", 0.0);
- JSObj x = anObjBuilderForYa.done();
- b.append((void*) x.objdata(), x.objsize());
- return true;
-}
-
-int nCaught = 0;
-
-void killCursors(int n, long long *ids) {
- int k = 0;
- for( int i = 0; i < n; i++ ) {
- if( ClientCursor::erase(ids[i]) )
- k++;
- }
- cout << "killCursors: found " << k << " of " << n << endl;
-}
-
-auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order);
-
-QueryResult* runQuery(const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj,
- auto_ptr< set<string> > filter, stringstream& ss)
-{
- bool wantMore = true;
- int ntoreturn = _ntoreturn;
- if( _ntoreturn < 0 ) {
- ntoreturn = -_ntoreturn;
- wantMore = false;
- }
- ss << "query " << ns << " ntoreturn:" << ntoreturn;
- if( ntoskip )
- ss << " ntoskip:" << ntoskip;
- if( client->profile )
- ss << "<br>query: " << jsobj.toString() << ' ';
-
- int n = 0;
- BufBuilder b(32768);
- JSObjBuilder cmdResBuf;
- long long cursorid = 0;
-
- b.skip(sizeof(QueryResult));
-
- /* we assume you are using findOne() for running a cmd... */
- if( ntoreturn == 1 && runCommands(ns, jsobj, ss, b, cmdResBuf) ) {
- n = 1;
- }
- else {
-
- JSObj query = jsobj.getObjectField("query");
- JSObj order = jsobj.getObjectField("orderby");
- if( query.isEmpty() && order.isEmpty() )
- query = jsobj;
-
- auto_ptr<JSMatcher> matcher(new JSMatcher(query));
- JSMatcher &debug1 = *matcher;
- assert( debug1.getN() < 5000 );
-
- int nscanned = 0;
- auto_ptr<Cursor> c = getSpecialCursor(ns);
-
- /*try*/{
-
- if( c.get() == 0 ) {
- c = getIndexCursor(ns, query, order);
- }
- if( c.get() == 0 ) {
- //c = theDataFileMgr.findAll(ns);
- c = findTableScan(ns, order);
- }
-
- while( c->ok() ) {
- JSObj js = c->current();
- if( queryTraceLevel >= 50 )
- cout << " checking against:\n " << js.toString() << endl;
- nscanned++;
- bool deep;
-
-JSMatcher &debug = *matcher;
-assert( debug.getN() < 5000 );
-
- if( !matcher->matches(js, &deep) ) {
- if( c->tempStopOnMiss() )
- break;
- }
- else if( !deep || !c->dup(c->currLoc()) ) { // i.e., check for dups on deep items only
- // got a match.
- if( ntoskip > 0 ) {
- ntoskip--;
- }
- else {
- bool ok = true;
- assert( js.objsize() >= 0 ); //defensive for segfaults
- if( filter.get() ) {
- // we just want certain fields from the object.
- JSObj x;
- ok = x.addFields(js, *filter) > 0;
- if( ok )
- b.append((void*) x.objdata(), x.objsize());
- }
- else {
- b.append((void*) js.objdata(), js.objsize());
- }
- if( ok ) {
- n++;
- if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
- (ntoreturn==0 && b.len()>1*1024*1024) ) {
- /* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
- if( wantMore && ntoreturn != 1 ) {
- c->advance();
- if( c->ok() ) {
- // more...so save a cursor
- ClientCursor *cc = new ClientCursor();
- cc->c = c;
- cursorid = allocCursorId();
- cc->cursorid = cursorid;
- cc->matcher = matcher;
- cc->ns = ns;
- cc->pos = n;
- ClientCursor::add(cc);
- cc->updateLocation();
- cc->filter = filter;
- }
- }
- break;
- }
- }
- }
- }
- c->advance();
- }
-
- if( client->profile )
- ss << " nscanned:" << nscanned << ' ';
- }
- /*catch( AssertionException e ) {
- if( n )
- throw e;
- if( nCaught++ >= 1000 ) {
- cout << "Too many query exceptions, terminating" << endl;
- exit(-8);
- }
- cout << " Assertion running query, returning an empty result" << endl;
- }*/
- }
-
- QueryResult *qr = (QueryResult *) b.buf();
- qr->_data[0] = 0;
- qr->_data[1] = 0;
- qr->_data[2] = 0;
- qr->_data[3] = 0;
- qr->len = b.len();
- ss << " reslen:" << b.len();
- // qr->channel = 0;
- qr->operation = opReply;
- qr->cursorId = cursorid;
- qr->startingFrom = 0;
- qr->nReturned = n;
- b.decouple();
-
- ss << " nreturned:" << n;
- return qr;
-}
-
-QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid) {
-
-// cout << "getMore ns:" << ns << " ntoreturn:" << ntoreturn << " cursorid:" <<
-// cursorid << endl;
-
- BufBuilder b(32768);
-
- ClientCursor *cc = ClientCursor::find(cursorid);
-
- b.skip(sizeof(QueryResult));
-
- int start = 0;
- int n = 0;
-
- if( cc ) {
- start = cc->pos;
- Cursor *c = cc->c.get();
- while( 1 ) {
- if( !c->ok() ) {
-done:
- // done! kill cursor.
- bool ok = ClientCursor::erase(cursorid);
- assert(ok);
- cursorid = 0;
- cc = 0;
- break;
- }
- JSObj js = c->current();
- bool deep;
- if( !cc->matcher->matches(js, &deep) ) {
- if( c->tempStopOnMiss() )
- goto done;
- }
- else if( !deep || !c->dup(c->currLoc()) ) {
- bool ok = true;
- if( cc->filter.get() ) {
- JSObj x;
- ok = x.addFields(js, *cc->filter) > 0;
- if( ok )
- b.append((void*) x.objdata(), x.objsize());
- }
- else {
- b.append((void*) js.objdata(), js.objsize());
- }
- if( ok ) {
- n++;
- if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) ||
- (ntoreturn==0 && b.len()>1*1024*1024) ) {
- c->advance();
- cc->pos += n;
- cc->updateLocation();
- break;
- }
- }
- }
- c->advance();
- }
- }
-
- QueryResult *qr = (QueryResult *) b.buf();
- qr->cursorId = cursorid;
- qr->startingFrom = start;
- qr->len = b.len();
- // qr->reserved = 0;
- qr->operation = opReply;
- qr->nReturned = n;
- b.decouple();
-
- return qr;
-}
+// query.cpp + +#include "stdafx.h" +#include "query.h" +#include "pdfile.h" +#include "jsobj.h" +#include "../util/builder.h" +#include <time.h> +#include "introspect.h" +#include "btree.h" +#include "../util/lruishmap.h" +#include "javajs.h" + +//ns->query->DiskLoc +LRUishMap<JSObj,DiskLoc,5> lrutest(123); + +int nextCursorId = 1; + +#pragma pack(push) +#pragma pack(1) +struct EmptyObject { + EmptyObject() { len = 5; jstype = EOO; } + int len; + char jstype; +} emptyObject; +#pragma pack(pop) + +JSObj emptyObj((char *) &emptyObject); + +int getGtLtOp(Element& e); +void appendElementHandlingGtLt(JSObjBuilder& b, Element& e); + +/* todo: _ cache query plans + _ use index on partial match with the query + + parameters + query - the query, e.g., { name: 'joe' } + order - order by spec, e.g., { name: 1 } 1=ASC, -1=DESC + +*/ +auto_ptr<Cursor> getIndexCursor(const char *ns, JSObj& query, JSObj& order) { + NamespaceDetails *d = nsdetails(ns); + if( d == 0 ) return auto_ptr<Cursor>(); + + // queryFields, e.g. { 'name' } + set<string> queryFields; + query.getFieldNames(queryFields); + + if( !order.isEmpty() ) { + set<string> orderFields; + order.getFieldNames(orderFields); + // order by + for(int i = 0; i < d->nIndexes; i++ ) { + JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: } + assert( strcmp(ns, idxInfo.getStringField("ns")) == 0 ); + JSObj idxKey = idxInfo.getObjectField("key"); + set<string> keyFields; + idxKey.getFieldNames(keyFields); + if( keyFields == orderFields ) { + bool reverse = + order.firstElement().type() == Number && + order.firstElement().number() < 0; + JSObjBuilder b; + /* todo: start with the right key, not just beginning of index, when query is also + specified! + */ + return auto_ptr<Cursor>(new BtreeCursor(d->indexes[i].head, reverse ? maxKey : emptyObj, reverse ? -1 : 1, false)); + } + } + } + + // regular query without order by + for(int i = 0; i < d->nIndexes; i++ ) { + JSObj idxInfo = d->indexes[i].info.obj(); // { name:, ns:, key: } + JSObj idxKey = idxInfo.getObjectField("key"); + set<string> keyFields; + idxKey.getFieldNames(keyFields); + if( keyFields == queryFields ) { + JSObjBuilder b; + JSObj q = query.extractFields(idxKey, b); + /* regexp: only supported if form is /^text/ */ + JSObjBuilder b2; + JSElemIter it(q); + bool first = true; + while( it.more() ) { + Element e = it.next(); + if( e.eoo() ) + break; + + // GT/LT + if( e.type() == Object ) { + int op = getGtLtOp(e); + if( op ) { + if( !first || !it.next().eoo() ) { + // compound keys with GT/LT not supported yet via index. + goto fail; + } + int direction = - JSMatcher::opDirection(op); + return auto_ptr<Cursor>( new BtreeCursor( + d->indexes[i].head, + direction == 1 ? emptyObj : maxKey, + direction, + true) ); + } + } + + first = false; + if( e.type() == RegEx ) { + if( *e.regexFlags() ) + goto fail; + const char *re = e.regex(); + const char *p = re; + if( *p++ != '^' ) goto fail; + while( *p ) { + if( *p == ' ' || (*p>='0'&&*p<='9') || (*p>='@'&&*p<='Z') || (*p>='a'&&*p<='z') ) + ; + else + goto fail; + p++; + } + if( it.more() && !it.next().eoo() ) // we must be the last part of the key (for now until we are smarter) + goto fail; + // ok! + b2.append(e.fieldName(), re+1); + break; + } + else { + b2.append(e); + //appendElementHandlingGtLt(b2, e); + } + } + JSObj q2 = b2.done(); + return auto_ptr<Cursor>( + new BtreeCursor(d->indexes[i].head, q2, 1, true)); + } + } + +fail: + return auto_ptr<Cursor>(); +} + +void deleteObjects(const char *ns, JSObj pattern, bool justOne) { +// cout << "TEMP delete ns:" << ns << " queryobjsize:" << +// pattern.objsize() << endl; + + if( strstr(ns, ".system.") ) { + if( strstr(ns, ".system.namespaces") ){ + cout << "WARNING: delete on system namespace " << ns << endl; + } + else if( strstr(ns, ".system.indexes") ) { + cout << "WARNING: delete on system namespace " << ns << endl; + } + else { + cout << "ERROR: attempt to delete in system namespace " << ns << endl; + return; + } + } + + JSMatcher matcher(pattern); + JSObj order; + auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order); + if( c.get() == 0 ) + c = theDataFileMgr.findAll(ns); + + Cursor &tempDebug = *c; + int temp = 0; + int tempd = 0; + +DiskLoc _tempDelLoc; + + while( c->ok() ) { + temp++; + + Record *r = c->_current(); + DiskLoc rloc = c->currLoc(); + c->advance(); // must advance before deleting as the next ptr will die + JSObj js(r); + //cout << "TEMP: " << js.toString() << endl; + bool deep; + if( !matcher.matches(js, &deep) ) { + if( c->tempStopOnMiss() ) + break; + } + else { + assert( !deep || !c->dup(rloc) ); // can't be a dup, we deleted it! +// cout << " found match to delete" << endl; + if( !justOne ) + c->noteLocation(); +_tempDelLoc = rloc; + theDataFileMgr.deleteRecord(ns, r, rloc); + tempd = temp; + if( justOne ) + return; + c->checkLocation(); + } + } +} + +struct Mod { + const char *fieldName; + double n; +}; + +void applyMods(vector<Mod>& mods, JSObj obj) { + for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ ) { + Mod& m = *i; + Element e = obj.findElement(m.fieldName); + if( e.type() == Number ) { + e.number() += m.n; + } + } +} + +/* get special operations like $inc + { $inc: { a:1, b:1 } } +*/ +void getMods(vector<Mod>& mods, JSObj from) { + JSElemIter it(from); + while( it.more() ) { + Element e = it.next(); + if( strcmp(e.fieldName(), "$inc") == 0 && e.type() == Object ) { + JSObj j = e.embeddedObject(); + JSElemIter jt(j); + while( jt.more() ) { + Element f = jt.next(); + if( f.eoo() ) + break; + Mod m; + m.fieldName = f.fieldName(); + if( f.type() == Number ) { + m.n = f.number(); + mods.push_back(m); + } + } + } + } +} + +/* +todo: + smart requery find record immediately +*/ +void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss) { +//cout << "TEMP BAD"; +//lrutest.find(updateobj); + + int profile = client->profile; + + // cout << "update ns:" << ns << " objsize:" << updateobj.objsize() << " queryobjsize:" << + // pattern.objsize(); + + if( strstr(ns, ".system.") ) { + cout << "\nERROR: attempt to update in system namespace " << ns << endl; + ss << " can't update system namespace "; + return; + } + + int nscanned = 0; + { + JSMatcher matcher(pattern); + JSObj order; + auto_ptr<Cursor> c = getIndexCursor(ns, pattern, order); + if( c.get() == 0 ) + c = theDataFileMgr.findAll(ns); + while( c->ok() ) { + Record *r = c->_current(); + nscanned++; + JSObj js(r); + if( !matcher.matches(js) ) { + if( c->tempStopOnMiss() ) + break; + } + else { + /* note: we only update one row and quit. if you do multiple later, + be careful or multikeys in arrays could break things badly. best + to only allow updating a single row with a multikey lookup. + */ + + if( profile ) + ss << " nscanned:" << nscanned; + + /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some + regular ones at the moment. */ + if( updateobj.firstElement().fieldName()[0] == '$' ) { + vector<Mod> mods; + getMods(mods, updateobj); + applyMods(mods, c->currLoc().obj()); + if( profile ) + ss << " fastmod "; + return; + } + + theDataFileMgr.update(ns, r, c->currLoc(), updateobj.objdata(), updateobj.objsize(), ss); + return; + } + c->advance(); + } + } + + if( profile ) + ss << " nscanned:" << nscanned; + + if( upsert ) { + if( updateobj.firstElement().fieldName()[0] == '$' ) { + /* upsert of an $inc. build a default */ + vector<Mod> mods; + getMods(mods, updateobj); + JSObjBuilder b; + b.appendElements(pattern); + for( vector<Mod>::iterator i = mods.begin(); i != mods.end(); i++ ) + b.append(i->fieldName, i->n); + JSObj obj = b.done(); + theDataFileMgr.insert(ns, (void*) obj.objdata(), obj.objsize()); + if( profile ) + ss << " fastmodinsert "; + return; + } + if( profile ) + ss << " upsert "; + theDataFileMgr.insert(ns, (void*) updateobj.objdata(), updateobj.objsize()); + } +} + +int queryTraceLevel = 0; +int otherTraceLevel = 0; + +int initialExtentSize(int len); + +void clean(const char *ns, NamespaceDetails *d) { + for( int i = 0; i < Buckets; i++ ) + d->deletedList[i].Null(); +} + +string validateNS(const char *ns, NamespaceDetails *d) { + bool valid = true; + stringstream ss; + ss << "\nvalidate\n"; + if( d->capped ) + ss << " capped:" << d->capped << " max:" << d->max << '\n'; + + ss << " firstExtent:" << d->firstExtent.toString() << " lastExtent:" << d->lastExtent.toString() << '\n'; + ss << " datasize?:" << d->datasize << " nrecords?:" << d->nrecords << " lastExtentSize:" << d->lastExtentSize << '\n'; + + try { + + { + ss << " first extent:\n"; + d->firstExtent.ext()->dump(ss); + valid = valid && d->firstExtent.ext()->validates(); + } + + auto_ptr<Cursor> c = theDataFileMgr.findAll(ns); + int n = 0; + long long len = 0; + long long nlen = 0; + while( c->ok() ) { + n++; + Record *r = c->_current(); + len += r->lengthWithHeaders; + nlen += r->netLength(); + c->advance(); + } + ss << " " << n << " objects found, nobj:" << d->nrecords << "\n"; + ss << " " << len << " bytes record data w/headers\n"; + ss << " " << nlen << " bytes record data wout/headers\n"; + + ss << " deletedList: "; + for( int i = 0; i < Buckets; i++ ) { + ss << (d->deletedList[i].isNull() ? '0' : '1'); + } + ss << endl; + int ndel = 0; + long long delSize = 0; + for( int i = 0; i < Buckets; i++ ) { + DiskLoc loc = d->deletedList[i]; + while( !loc.isNull() ) { + ndel++; + DeletedRecord *d = loc.drec(); + delSize += d->lengthWithHeaders; + loc = d->nextDeleted; + } + } + ss << " deleted: n: " << ndel << " size: " << delSize << endl; + + int idxn = 0; + try { + ss << " nIndexes:" << d->nIndexes << endl; + for( ; idxn < d->nIndexes; idxn++ ) { + ss << " " << d->indexes[idxn].indexNamespace() << " keys:" << + d->indexes[idxn].head.btree()->fullValidate(d->indexes[idxn].head) << endl; + } + } + catch(...) { + ss << "\n exception during index validate idxn:" << idxn << endl; + } + + } + catch(AssertionException) { + ss << "\n exception during validate\n" << endl; + } + + if( !valid ) + ss << " ns corrupt, requires dbchk\n"; + + return ss.str(); +} + +bool userCreateNS(const char *ns, JSObj& j); + +const int edebug=1; + +bool dbEval(JSObj& cmd, JSObjBuilder& result) { + Element e = cmd.firstElement(); + assert( e.type() == Code ); + const char *code = e.valuestr(); + if ( ! JavaJS ) + JavaJS = new JavaJSImpl(); + + jlong f = JavaJS->functionCreate(code); + if( f == 0 ) { + result.append("errmsg", "compile failed"); + return false; + } + + Scope s; + s.setString("$client", client->name.c_str()); + Element args = cmd.findElement("args"); + if( args.type() == Array ) { + JSObj eo = args.embeddedObject(); + if( edebug ) { + cout << "args:" << eo.toString() << endl; + cout << "code:\n" << code << endl; + } + s.setObject("args", eo); + } + + int res = s.invoke(f); + if( res ) { + result.append("errno", (double) res); + result.append("errmsg", "invoke failed"); + return false; + } + + int type = s.type("return"); + if( type == Object || type == Array ) + result.append("retval", s.getObject("return")); + else if( type == Number ) + result.append("retval", s.getNumber("return")); + else if( type == String ) + result.append("retval", s.getString("return").c_str()); + else if( type == Bool ) { + result.appendBool("retval", s.getBoolean("return")); + } + + return true; +} + +// e.g. +// system.cmd$.find( { queryTraceLevel: 2 } ); +// +// returns true if ran a cmd +// +inline bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { + + const char *p = strchr(ns, '.'); + if( !p ) return false; + if( strcmp(p, ".$cmd") != 0 ) return false; + + bool ok = false; + bool valid = false; + + //cout << jsobj.toString() << endl; + + Element e; + e = jsobj.firstElement(); + + if( e.eoo() ) goto done; + if( e.type() == Code ) { + valid = true; + ok = dbEval(jsobj, anObjBuilderForYa); + } + else if( e.type() == Number ) { + if( strcmp(e.fieldName(), "profile") == 0 ) { + anObjBuilderForYa.append("was", (double) client->profile); + int p = (int) e.number(); + valid = true; + if( p == -1 ) + ok = true; + else if( p >= 0 && p <= 2 ) { + ok = true; + client->profile = p; + } + else { + ok = false; + } + } + else { + if( strncmp(ns, "admin", p-ns) != 0 ) // admin only + return false; + if( strcmp(e.fieldName(),"queryTraceLevel") == 0 ) { + valid = ok = true; + queryTraceLevel = (int) e.number(); + } else if( strcmp(e.fieldName(),"traceAll") == 0 ) { + valid = ok = true; + queryTraceLevel = (int) e.number(); + otherTraceLevel = (int) e.number(); + } + } + } + else if( e.type() == String ) { + string us(ns, p-ns); + + if( strcmp( e.fieldName(), "create") == 0 ) { + valid = true; + string ns = us + '.' + e.valuestr(); + ok = userCreateNS(ns.c_str(), jsobj); + } + else if( strcmp( e.fieldName(), "clean") == 0 ) { + valid = true; + string dropNs = us + '.' + e.valuestr(); + NamespaceDetails *d = nsdetails(dropNs.c_str()); + cout << "CMD: clean " << dropNs << endl; + if( d ) { + ok = true; + anObjBuilderForYa.append("ns", dropNs.c_str()); + clean(dropNs.c_str(), d); + } + else { + anObjBuilderForYa.append("errmsg", "ns not found"); + } + } + else if( strcmp( e.fieldName(), "drop") == 0 ) { + valid = true; + string dropNs = us + '.' + e.valuestr(); + NamespaceDetails *d = nsdetails(dropNs.c_str()); + cout << "CMD: clean " << dropNs << endl; + if( d ) { + ok = true; + anObjBuilderForYa.append("ns", dropNs.c_str()); + client->namespaceIndex->kill(dropNs.c_str()); + } + else { + anObjBuilderForYa.append("errmsg", "ns not found"); + } + } + else if( strcmp( e.fieldName(), "validate") == 0 ) { + valid = true; + string toValidateNs = us + '.' + e.valuestr(); + NamespaceDetails *d = nsdetails(toValidateNs.c_str()); + cout << "CMD: validate " << toValidateNs << endl; + if( d ) { + ok = true; + anObjBuilderForYa.append("ns", toValidateNs.c_str()); + string s = validateNS(toValidateNs.c_str(), d); + anObjBuilderForYa.append("result", s.c_str()); + } + else { + anObjBuilderForYa.append("errmsg", "ns not found"); + } + } + else if( strcmp(e.fieldName(),"deleteIndexes") == 0 ) { + valid = true; + /* note: temp implementation. space not reclaimed! */ + string toDeleteNs = us + '.' + e.valuestr(); + NamespaceDetails *d = nsdetails(toDeleteNs.c_str()); + cout << "CMD: deleteIndexes " << toDeleteNs << endl; + if( d ) { + Element f = jsobj.findElement("index"); + if( !f.eoo() ) { + // delete a specific index + if( f.type() == String ) { + const char *idxName = f.valuestr(); + if( *idxName == '*' && idxName[1] == 0 ) { + ok = true; + cout << " d->nIndexes was " << d->nIndexes << endl; + anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes); + anObjBuilderForYa.append("msg", "all indexes deleted for collection"); + cout << " alpha implementation, space not reclaimed" << endl; + d->nIndexes = 0; + } + else { + // delete just one index + int x = d->findIndexByName(idxName); + if( x >= 0 ) { + cout << " d->nIndexes was " << d->nIndexes << endl; + anObjBuilderForYa.append("nIndexesWas", (double)d->nIndexes); + d->nIndexes--; + for( int i = x; i < d->nIndexes; i++ ) + d->indexes[i] = d->indexes[i+1]; + ok=true; + cout << " alpha implementation, space not reclaimed" << endl; + } else { + cout << "deleteIndexes: " << idxName << " not found" << endl; + } + } + } + } + } + else { + anObjBuilderForYa.append("errmsg", "ns not found"); + } + } + } + +done: + if( !valid ) + anObjBuilderForYa.append("errmsg", "no such cmd"); + anObjBuilderForYa.append("ok", ok?1.0:0.0); + JSObj x = anObjBuilderForYa.done(); + b.append((void*) x.objdata(), x.objsize()); + return true; +} + +bool runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b, JSObjBuilder& anObjBuilderForYa) { + try { + return _runCommands(ns, jsobj, ss, b, anObjBuilderForYa); + } + catch( AssertionException ) { + ; + } + ss << " assertion "; + anObjBuilderForYa.append("errmsg", "db assertion failure"); + anObjBuilderForYa.append("ok", 0.0); + JSObj x = anObjBuilderForYa.done(); + b.append((void*) x.objdata(), x.objsize()); + return true; +} + +int nCaught = 0; + +void killCursors(int n, long long *ids) { + int k = 0; + for( int i = 0; i < n; i++ ) { + if( ClientCursor::erase(ids[i]) ) + k++; + } + cout << "killCursors: found " << k << " of " << n << endl; +} + +auto_ptr<Cursor> findTableScan(const char *ns, JSObj& order); + +QueryResult* runQuery(const char *ns, int ntoskip, int _ntoreturn, JSObj jsobj, + auto_ptr< set<string> > filter, stringstream& ss) +{ + bool wantMore = true; + int ntoreturn = _ntoreturn; + if( _ntoreturn < 0 ) { + ntoreturn = -_ntoreturn; + wantMore = false; + } + ss << "query " << ns << " ntoreturn:" << ntoreturn; + if( ntoskip ) + ss << " ntoskip:" << ntoskip; + if( client->profile ) + ss << "<br>query: " << jsobj.toString() << ' '; + + int n = 0; + BufBuilder b(32768); + JSObjBuilder cmdResBuf; + long long cursorid = 0; + + b.skip(sizeof(QueryResult)); + + /* we assume you are using findOne() for running a cmd... */ + if( ntoreturn == 1 && runCommands(ns, jsobj, ss, b, cmdResBuf) ) { + n = 1; + } + else { + + JSObj query = jsobj.getObjectField("query"); + JSObj order = jsobj.getObjectField("orderby"); + if( query.isEmpty() && order.isEmpty() ) + query = jsobj; + + auto_ptr<JSMatcher> matcher(new JSMatcher(query)); + JSMatcher &debug1 = *matcher; + assert( debug1.getN() < 5000 ); + + int nscanned = 0; + auto_ptr<Cursor> c = getSpecialCursor(ns); + + /*try*/{ + + if( c.get() == 0 ) { + c = getIndexCursor(ns, query, order); + } + if( c.get() == 0 ) { + //c = theDataFileMgr.findAll(ns); + c = findTableScan(ns, order); + } + + while( c->ok() ) { + JSObj js = c->current(); + if( queryTraceLevel >= 50 ) + cout << " checking against:\n " << js.toString() << endl; + nscanned++; + bool deep; + +JSMatcher &debug = *matcher; +assert( debug.getN() < 5000 ); + + if( !matcher->matches(js, &deep) ) { + if( c->tempStopOnMiss() ) + break; + } + else if( !deep || !c->dup(c->currLoc()) ) { // i.e., check for dups on deep items only + // got a match. + if( ntoskip > 0 ) { + ntoskip--; + } + else { + bool ok = true; + assert( js.objsize() >= 0 ); //defensive for segfaults + if( filter.get() ) { + // we just want certain fields from the object. + JSObj x; + ok = x.addFields(js, *filter) > 0; + if( ok ) + b.append((void*) x.objdata(), x.objsize()); + } + else { + b.append((void*) js.objdata(), js.objsize()); + } + if( ok ) { + n++; + if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) || + (ntoreturn==0 && b.len()>1*1024*1024) ) { + /* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */ + if( wantMore && ntoreturn != 1 ) { + c->advance(); + if( c->ok() ) { + // more...so save a cursor + ClientCursor *cc = new ClientCursor(); + cc->c = c; + cursorid = allocCursorId(); + cc->cursorid = cursorid; + cc->matcher = matcher; + cc->ns = ns; + cc->pos = n; + ClientCursor::add(cc); + cc->updateLocation(); + cc->filter = filter; + } + } + break; + } + } + } + } + c->advance(); + } + + if( client->profile ) + ss << " nscanned:" << nscanned << ' '; + } + /*catch( AssertionException e ) { + if( n ) + throw e; + if( nCaught++ >= 1000 ) { + cout << "Too many query exceptions, terminating" << endl; + exit(-8); + } + cout << " Assertion running query, returning an empty result" << endl; + }*/ + } + + QueryResult *qr = (QueryResult *) b.buf(); + qr->_data[0] = 0; + qr->_data[1] = 0; + qr->_data[2] = 0; + qr->_data[3] = 0; + qr->len = b.len(); + ss << " reslen:" << b.len(); + // qr->channel = 0; + qr->operation = opReply; + qr->cursorId = cursorid; + qr->startingFrom = 0; + qr->nReturned = n; + b.decouple(); + + ss << " nreturned:" << n; + return qr; +} + +QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid) { + +// cout << "getMore ns:" << ns << " ntoreturn:" << ntoreturn << " cursorid:" << +// cursorid << endl; + + BufBuilder b(32768); + + ClientCursor *cc = ClientCursor::find(cursorid); + + b.skip(sizeof(QueryResult)); + + int start = 0; + int n = 0; + + if( cc ) { + start = cc->pos; + Cursor *c = cc->c.get(); + while( 1 ) { + if( !c->ok() ) { +done: + // done! kill cursor. + bool ok = ClientCursor::erase(cursorid); + assert(ok); + cursorid = 0; + cc = 0; + break; + } + JSObj js = c->current(); + bool deep; + if( !cc->matcher->matches(js, &deep) ) { + if( c->tempStopOnMiss() ) + goto done; + } + else if( !deep || !c->dup(c->currLoc()) ) { + bool ok = true; + if( cc->filter.get() ) { + JSObj x; + ok = x.addFields(js, *cc->filter) > 0; + if( ok ) + b.append((void*) x.objdata(), x.objsize()); + } + else { + b.append((void*) js.objdata(), js.objsize()); + } + if( ok ) { + n++; + if( (ntoreturn>0 && (n >= ntoreturn || b.len() > 16*1024*1024)) || + (ntoreturn==0 && b.len()>1*1024*1024) ) { + c->advance(); + cc->pos += n; + cc->updateLocation(); + break; + } + } + } + c->advance(); + } + } + + QueryResult *qr = (QueryResult *) b.buf(); + qr->cursorId = cursorid; + qr->startingFrom = start; + qr->len = b.len(); + // qr->reserved = 0; + qr->operation = opReply; + qr->nReturned = n; + b.decouple(); + + return qr; +} diff --git a/db/query.h b/db/query.h index 44180f8c98c..b0f7a503fcb 100644 --- a/db/query.h +++ b/db/query.h @@ -1,135 +1,135 @@ -// query.h
-
-#pragma once
-
-#include "../stdafx.h"
-#include "../grid/message.h"
-#include "jsobj.h"
-#include "storage.h"
-
-/* requests:
-
- dbDelete
- int reserved=0;
- string collection;
- int flags=0; // 1=DeleteSingle
- JSObject query;
- dbUpdate:
- int reserved;
- string collection;
- int flags; // 1=upsert
- JSObject query;
- JSObject objectToUpdate;
- objectToUpdate may include { $inc: <field> }.
- dbQuery:
- int reserved;
- string collection;
- int nToSkip;
- int nToReturn; // how many you want back as the beginning of the cursor data
- JSObject query;
- [JSObject fieldsToReturn]
- dbGetMore:
- int reserved;
- string collection; // redundant, might use for security.
- int nToReturn;
- int64 cursorID;
- dbKillCursors=2007:
- int reserved;
- int n;
- int64 cursorIDs[n];
-
- Note that on Update, there is only one object, which is different
- from insert where you can pass a list of objects to insert in the db.
- Note that the update field layout is very similar layout to Query.
-*/
-
-/* db response format
-
- Query or GetMore:
- int reserved;
- int64 cursorID;
- int startingFrom;
- int nReturned; // 0=infinity
- list of marshalled JSObjects;
-*/
-
-#pragma pack(push)
-#pragma pack(1)
-
-struct QueryResult : public MsgData {
- long long cursorId;
- int startingFrom;
- int nReturned;
- const char *data() { return (char *) (((int *)&nReturned)+1); }
-};
-
-#pragma pack(pop)
-
-QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid);
-
-// caller must free() returned QueryResult.
-QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn,
- JSObj j, auto_ptr< set<string> > fieldFilter,
- stringstream&);
-
-void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss);
-void deleteObjects(const char *ns, JSObj pattern, bool justOne);
-
-class ClientCursor;
-typedef map<long long, ClientCursor*> CCMap;
-extern CCMap clientCursors; /* cursorid -> ClientCursor */
-
-/* Cursor -- and its derived classes -- are our internal cursors.
-
- ClientCursor is a wrapper that represents a cursorid from our client
- application's perspective.
-*/
-class Cursor;
-class ClientCursor {
- friend class CursInspector;
-public:
- ClientCursor() {
- cursorid=0; pos=0; nextAtThisLocation=0;
-#if defined(_WIN32)
- cout << "clientcursor() " << cursorid << endl;
-#endif
- }
- ~ClientCursor();
- long long cursorid;
- string ns;
- auto_ptr<JSMatcher> matcher;
- auto_ptr<Cursor> c;
- int pos;
- DiskLoc lastLoc;
- auto_ptr< set<string> > filter;
-
- /* report to us that a new clientcursor exists so we can track it. You still
- do the initial updateLocation() yourself.
- */
- static void add(ClientCursor*);
-
- static bool erase(long long cursorid);
-
- static ClientCursor* find(long long id) {
- CCMap::iterator it = clientCursors.find(id);
- if( it == clientCursors.end() ) {
- cout << "ClientCursor::find(): cursor not found in map " << id << endl;
- return 0;
- }
- return it->second;
- }
-
- /* call when cursor's location changes so that we can update the
- cursorsbylocation map. if you are locked and internally iterating, only
- need to call when you are ready to "unlock".
- */
- void updateLocation();
-
-private:
- void addToByLocation(DiskLoc cl);
- static void cleanupByLocation(DiskLoc loc, long long cursorid);
-public:
- ClientCursor *nextAtThisLocation;
-};
-
-long long allocCursorId();
+// query.h + +#pragma once + +#include "../stdafx.h" +#include "../grid/message.h" +#include "jsobj.h" +#include "storage.h" + +/* requests: + + dbDelete + int reserved=0; + string collection; + int flags=0; // 1=DeleteSingle + JSObject query; + dbUpdate: + int reserved; + string collection; + int flags; // 1=upsert + JSObject query; + JSObject objectToUpdate; + objectToUpdate may include { $inc: <field> }. + dbQuery: + int reserved; + string collection; + int nToSkip; + int nToReturn; // how many you want back as the beginning of the cursor data + JSObject query; + [JSObject fieldsToReturn] + dbGetMore: + int reserved; + string collection; // redundant, might use for security. + int nToReturn; + int64 cursorID; + dbKillCursors=2007: + int reserved; + int n; + int64 cursorIDs[n]; + + Note that on Update, there is only one object, which is different + from insert where you can pass a list of objects to insert in the db. + Note that the update field layout is very similar layout to Query. +*/ + +/* db response format + + Query or GetMore: + int reserved; + int64 cursorID; + int startingFrom; + int nReturned; // 0=infinity + list of marshalled JSObjects; +*/ + +#pragma pack(push) +#pragma pack(1) + +struct QueryResult : public MsgData { + long long cursorId; + int startingFrom; + int nReturned; + const char *data() { return (char *) (((int *)&nReturned)+1); } +}; + +#pragma pack(pop) + +QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid); + +// caller must free() returned QueryResult. +QueryResult* runQuery(const char *ns, int ntoskip, int ntoreturn, + JSObj j, auto_ptr< set<string> > fieldFilter, + stringstream&); + +void updateObjects(const char *ns, JSObj updateobj, JSObj pattern, bool upsert, stringstream& ss); +void deleteObjects(const char *ns, JSObj pattern, bool justOne); + +class ClientCursor; +typedef map<long long, ClientCursor*> CCMap; +extern CCMap clientCursors; /* cursorid -> ClientCursor */ + +/* Cursor -- and its derived classes -- are our internal cursors. + + ClientCursor is a wrapper that represents a cursorid from our client + application's perspective. +*/ +class Cursor; +class ClientCursor { + friend class CursInspector; +public: + ClientCursor() { + cursorid=0; pos=0; nextAtThisLocation=0; +#if defined(_WIN32) + cout << "clientcursor() " << cursorid << endl; +#endif + } + ~ClientCursor(); + long long cursorid; + string ns; + auto_ptr<JSMatcher> matcher; + auto_ptr<Cursor> c; + int pos; + DiskLoc lastLoc; + auto_ptr< set<string> > filter; + + /* report to us that a new clientcursor exists so we can track it. You still + do the initial updateLocation() yourself. + */ + static void add(ClientCursor*); + + static bool erase(long long cursorid); + + static ClientCursor* find(long long id) { + CCMap::iterator it = clientCursors.find(id); + if( it == clientCursors.end() ) { + cout << "ClientCursor::find(): cursor not found in map " << id << endl; + return 0; + } + return it->second; + } + + /* call when cursor's location changes so that we can update the + cursorsbylocation map. if you are locked and internally iterating, only + need to call when you are ready to "unlock". + */ + void updateLocation(); + +private: + void addToByLocation(DiskLoc cl); + static void cleanupByLocation(DiskLoc loc, long long cursorid); +public: + ClientCursor *nextAtThisLocation; +}; + +long long allocCursorId(); diff --git a/db/resource.h b/db/resource.h index 6fafd161abb..3041137c21a 100644 --- a/db/resource.h +++ b/db/resource.h @@ -1,14 +1,14 @@ -//{{NO_DEPENDENCIES}}
-// Microsoft Visual C++ generated include file.
-// Used by db.rc
-
-// Next default values for new objects
-//
-#ifdef APSTUDIO_INVOKED
-#ifndef APSTUDIO_READONLY_SYMBOLS
-#define _APS_NEXT_RESOURCE_VALUE 101
-#define _APS_NEXT_COMMAND_VALUE 40001
-#define _APS_NEXT_CONTROL_VALUE 1001
-#define _APS_NEXT_SYMED_VALUE 101
-#endif
-#endif
+//{{NO_DEPENDENCIES}} +// Microsoft Visual C++ generated include file. +// Used by db.rc + +// Next default values for new objects +// +#ifdef APSTUDIO_INVOKED +#ifndef APSTUDIO_READONLY_SYMBOLS +#define _APS_NEXT_RESOURCE_VALUE 101 +#define _APS_NEXT_COMMAND_VALUE 40001 +#define _APS_NEXT_CONTROL_VALUE 1001 +#define _APS_NEXT_SYMED_VALUE 101 +#endif +#endif diff --git a/db/storage.h b/db/storage.h index 78d1dbf1a5e..fbeed51091b 100644 --- a/db/storage.h +++ b/db/storage.h @@ -1,90 +1,90 @@ -/* storage.h
-
- Storage subsystem management.
- Lays out our datafiles on disk, manages disk space.
-*/
-
-#pragma once
-
-#pragma pack(push)
-#pragma pack(1)
-
-class Record;
-class DeletedRecord;
-class Extent;
-class BtreeBucket;
-class JSObj;
-class PhysicalDataFile;
-class Bucket;
-
-class DiskLoc {
- int fileNo; /* this will be volume, file #, etc. */
- int ofs;
-public:
- enum { NullOfs = -1 };
- int a() const { return fileNo; }
- DiskLoc(int a, int b) : fileNo(a), ofs(b) {
- assert(ofs!=0);
- }
- DiskLoc() { fileNo = -1; ofs = NullOfs; }
-
- DiskLoc(const DiskLoc& l) { fileNo=l.fileNo; ofs=l.ofs; }
-
- bool isNull() const { return ofs == NullOfs; }
- void Null() { fileNo = -1; ofs = NullOfs; }
- void assertOk() { assert(!isNull()); }
-
- string toString() const {
- if( isNull() )
- return "null";
- stringstream ss;
- ss << hex << fileNo << ':' << ofs;
- return ss.str();
- }
-
- int& GETOFS() { return ofs; }
- int getOfs() const { return ofs; }
- void set(int a, int b) { fileNo=a; ofs=b; }
- void setOfs(int _fileNo, int _ofs) {
- fileNo = _fileNo;
- ofs = _ofs;
- }
-
- void inc(int amt) {
- assert( !isNull() );
- ofs += amt;
- }
-
- bool sameFile(DiskLoc b) { return fileNo == b.fileNo; }
-
- bool operator==(const DiskLoc& b) const { return fileNo==b.fileNo && ofs == b.ofs; }
- bool operator!=(const DiskLoc& b) const { return !(*this==b); }
- const DiskLoc& operator=(const DiskLoc& b) {
- fileNo=b.fileNo; ofs = b.ofs;
- assert(ofs!=0);
- return *this;
- }
- int compare(const DiskLoc& b) const {
- int x = fileNo - b.fileNo;
- if( x )
- return x;
- if( ofs == b.ofs ) return 0;
- return ofs < b.ofs ? -1 : 1;
- }
- bool operator<(const DiskLoc& b) const {
- if( fileNo == b.fileNo )
- return ofs < b.ofs;
- return fileNo < b.fileNo;
- }
-
- JSObj obj() const;
- Record* rec() const;
- DeletedRecord* drec() const;
- Extent* ext() const;
- BtreeBucket* btree() const;
- Bucket* bucket() const;
-
- PhysicalDataFile& pdf() const;
-};
-
-#pragma pack(pop)
+/* storage.h + + Storage subsystem management. + Lays out our datafiles on disk, manages disk space. +*/ + +#pragma once + +#pragma pack(push) +#pragma pack(1) + +class Record; +class DeletedRecord; +class Extent; +class BtreeBucket; +class JSObj; +class PhysicalDataFile; +class Bucket; + +class DiskLoc { + int fileNo; /* this will be volume, file #, etc. */ + int ofs; +public: + enum { NullOfs = -1 }; + int a() const { return fileNo; } + DiskLoc(int a, int b) : fileNo(a), ofs(b) { + assert(ofs!=0); + } + DiskLoc() { fileNo = -1; ofs = NullOfs; } + + DiskLoc(const DiskLoc& l) { fileNo=l.fileNo; ofs=l.ofs; } + + bool isNull() const { return ofs == NullOfs; } + void Null() { fileNo = -1; ofs = NullOfs; } + void assertOk() { assert(!isNull()); } + + string toString() const { + if( isNull() ) + return "null"; + stringstream ss; + ss << hex << fileNo << ':' << ofs; + return ss.str(); + } + + int& GETOFS() { return ofs; } + int getOfs() const { return ofs; } + void set(int a, int b) { fileNo=a; ofs=b; } + void setOfs(int _fileNo, int _ofs) { + fileNo = _fileNo; + ofs = _ofs; + } + + void inc(int amt) { + assert( !isNull() ); + ofs += amt; + } + + bool sameFile(DiskLoc b) { return fileNo == b.fileNo; } + + bool operator==(const DiskLoc& b) const { return fileNo==b.fileNo && ofs == b.ofs; } + bool operator!=(const DiskLoc& b) const { return !(*this==b); } + const DiskLoc& operator=(const DiskLoc& b) { + fileNo=b.fileNo; ofs = b.ofs; + assert(ofs!=0); + return *this; + } + int compare(const DiskLoc& b) const { + int x = fileNo - b.fileNo; + if( x ) + return x; + if( ofs == b.ofs ) return 0; + return ofs < b.ofs ? -1 : 1; + } + bool operator<(const DiskLoc& b) const { + if( fileNo == b.fileNo ) + return ofs < b.ofs; + return fileNo < b.fileNo; + } + + JSObj obj() const; + Record* rec() const; + DeletedRecord* drec() const; + Extent* ext() const; + BtreeBucket* btree() const; + Bucket* bucket() const; + + PhysicalDataFile& pdf() const; +}; + +#pragma pack(pop) diff --git a/grid/message.cpp b/grid/message.cpp index 0a10667e49f..caf2cd7e8a8 100644 --- a/grid/message.cpp +++ b/grid/message.cpp @@ -1,196 +1,196 @@ -/* message
-
- todo: authenticate; encrypt?
-*/
-
-#include "stdafx.h"
-#include "message.h"
-#include <time.h>
-#include "../util/goodies.h"
-
-// if you want trace output:
-#define mmm(x)
-
-/* listener ------------------------------------------------------------------- */
-
-void Listener::listen() {
- SockAddr me(port);
- int sock = socket(AF_INET, SOCK_STREAM, 0);
- if( sock == INVALID_SOCKET ) {
- cout << "ERROR: listen(): invalid socket? " << errno << endl;
- return;
- }
- prebindOptions( sock );
- if( bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) {
- cout << "listen(): bind() failed errno:" << errno << endl;
- if( errno == 98 )
- cout << "98 == addr already in use" << endl;
- closesocket(sock);
- return;
- }
-
- if( ::listen(sock, 128) != 0 ) {
- cout << "listen(): listen() failed " << errno << endl;
- closesocket(sock);
- return;
- }
-
- SockAddr from;
- while( 1 ) {
- int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize);
- if( s < 0 ) {
- cout << "Listener: accept() returns " << s << " errno:" << errno << endl;
- continue;
- }
- disableNagle(s);
- cout << "Listener: connection accepted from " << from.toString() << endl;
- accepted( new MessagingPort(s, from) );
- }
-}
-
-/* messagingport -------------------------------------------------------------- */
-
-MSGID NextMsgId;
-struct MsgStart {
- MsgStart() {
- NextMsgId = (((unsigned) time(0)) << 16) ^ curTimeMillis();
- assert(MsgDataHeaderSize == 16);
- }
-} msgstart;
-
-MessagingPort::MessagingPort(int _sock, SockAddr& _far) : sock(_sock), farEnd(_far) { }
-
-MessagingPort::MessagingPort() {
- sock = -1;
-}
-
-void MessagingPort::shutdown() {
- if( sock >= 0 ) {
- closesocket(sock);
- sock = -1;
- }
-}
-
-MessagingPort::~MessagingPort() {
- shutdown();
-}
-
-bool MessagingPort::connect(SockAddr& _far)
-{
- farEnd = _far;
-
- sock = socket(AF_INET, SOCK_STREAM, 0);
- if( sock == INVALID_SOCKET ) {
- cout << "ERROR: connect(): invalid socket? " << errno << endl;
- return false;
- }
- if( ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize) ) {
- cout << "ERROR: connect(): connect() failed" << errno << endl;
- closesocket(sock); sock = -1;
- return false;
- }
- disableNagle(sock);
- return true;
-}
-
-bool MessagingPort::recv(Message& m) {
- mmm( cout << "* recv() sock:" << this->sock << endl; )
- int len = -1;
-
- char *lenbuf = (char *) &len;
- int lft = 4;
- while( 1 ) {
- int x = ::recv(sock, lenbuf, lft, 0);
- if( x == 0 ) {
- cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;
- m.reset();
- return false;
- }
- if( x < 0 ) {
- cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString()<<endl;
- m.reset();
- return false;
- }
- lft -= x;
- if( lft == 0 )
- break;
- lenbuf += x;
- cout << "MessagingPort::recv(): got " << x << " bytes wanted 4, lft=" << lft << endl;
- assert( lft > 0 );
- }
-
-// assert( x == 4 );
-
- assert( len >= 0 && len <= 16000000 );
-
- int z = (len+1023)&0xfffffc00; assert(z>=len);
- MsgData *md = (MsgData *) malloc(z);
- md->len = len;
-
- if ( len <= 0 ){
- cout << "got a length of 0, something is wrong" << endl;
- return false;
- }
-
- char *p = (char *) &md->id;
- int left = len -4;
- while( 1 ) {
- int x = ::recv(sock, p, left, 0);
- if( x == 0 ) {
- cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl;
- m.reset();
- return false;
- }
- if( x < 0 ) {
- cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString() << endl;
- m.reset();
- return false;
- }
- left -= x;
- p += x;
- if( left <= 0 )
- break;
- }
-
- m.setData(md, true);
- return true;
-}
-
-void MessagingPort::reply(Message& received, Message& response) {
- say(received.from, response, received.data->id);
-}
-
-bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) {
- mmm( cout << "*call()" << endl; )
- MSGID old = toSend.data->id;
- say(to, toSend);
- while( 1 ) {
- bool ok = recv(response);
- if( !ok )
- return false;
- //cout << "got response: " << response.data->responseTo << endl;
- if( response.data->responseTo == toSend.data->id )
- break;
- cout << "********************" << endl;
- cout << "ERROR: MessagingPort::call() wrong id got:" << response.data->responseTo << " expect:" << toSend.data->id << endl;
- cout << " old:" << old << endl;
- cout << " response msgid:" << response.data->id << endl;
- cout << " response len: " << response.data->len << endl;
- assert(false);
- response.reset();
- }
- mmm( cout << "*call() end" << endl; )
- return true;
-}
-
-void MessagingPort::say(SockAddr& to, Message& toSend, int responseTo) {
- mmm( cout << "* say() sock:" << this->sock << " thr:" << GetCurrentThreadId() << endl; )
- MSGID msgid = NextMsgId;
- ++NextMsgId;
- toSend.data->id = msgid;
- toSend.data->responseTo = responseTo;
- int x = ::send(sock, (char *) toSend.data, toSend.data->len, 0);
- if( x <= 0 ) {
- cout << "MessagingPort::say: send() error " << errno << ' ' << farEnd.toString() << endl;
- }
-}
+/* message + + todo: authenticate; encrypt? +*/ + +#include "stdafx.h" +#include "message.h" +#include <time.h> +#include "../util/goodies.h" + +// if you want trace output: +#define mmm(x) + +/* listener ------------------------------------------------------------------- */ + +void Listener::listen() { + SockAddr me(port); + int sock = socket(AF_INET, SOCK_STREAM, 0); + if( sock == INVALID_SOCKET ) { + cout << "ERROR: listen(): invalid socket? " << errno << endl; + return; + } + prebindOptions( sock ); + if( bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) { + cout << "listen(): bind() failed errno:" << errno << endl; + if( errno == 98 ) + cout << "98 == addr already in use" << endl; + closesocket(sock); + return; + } + + if( ::listen(sock, 128) != 0 ) { + cout << "listen(): listen() failed " << errno << endl; + closesocket(sock); + return; + } + + SockAddr from; + while( 1 ) { + int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize); + if( s < 0 ) { + cout << "Listener: accept() returns " << s << " errno:" << errno << endl; + continue; + } + disableNagle(s); + cout << "Listener: connection accepted from " << from.toString() << endl; + accepted( new MessagingPort(s, from) ); + } +} + +/* messagingport -------------------------------------------------------------- */ + +MSGID NextMsgId; +struct MsgStart { + MsgStart() { + NextMsgId = (((unsigned) time(0)) << 16) ^ curTimeMillis(); + assert(MsgDataHeaderSize == 16); + } +} msgstart; + +MessagingPort::MessagingPort(int _sock, SockAddr& _far) : sock(_sock), farEnd(_far) { } + +MessagingPort::MessagingPort() { + sock = -1; +} + +void MessagingPort::shutdown() { + if( sock >= 0 ) { + closesocket(sock); + sock = -1; + } +} + +MessagingPort::~MessagingPort() { + shutdown(); +} + +bool MessagingPort::connect(SockAddr& _far) +{ + farEnd = _far; + + sock = socket(AF_INET, SOCK_STREAM, 0); + if( sock == INVALID_SOCKET ) { + cout << "ERROR: connect(): invalid socket? " << errno << endl; + return false; + } + if( ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize) ) { + cout << "ERROR: connect(): connect() failed" << errno << endl; + closesocket(sock); sock = -1; + return false; + } + disableNagle(sock); + return true; +} + +bool MessagingPort::recv(Message& m) { + mmm( cout << "* recv() sock:" << this->sock << endl; ) + int len = -1; + + char *lenbuf = (char *) &len; + int lft = 4; + while( 1 ) { + int x = ::recv(sock, lenbuf, lft, 0); + if( x == 0 ) { + cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl; + m.reset(); + return false; + } + if( x < 0 ) { + cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString()<<endl; + m.reset(); + return false; + } + lft -= x; + if( lft == 0 ) + break; + lenbuf += x; + cout << "MessagingPort::recv(): got " << x << " bytes wanted 4, lft=" << lft << endl; + assert( lft > 0 ); + } + +// assert( x == 4 ); + + assert( len >= 0 && len <= 16000000 ); + + int z = (len+1023)&0xfffffc00; assert(z>=len); + MsgData *md = (MsgData *) malloc(z); + md->len = len; + + if ( len <= 0 ){ + cout << "got a length of 0, something is wrong" << endl; + return false; + } + + char *p = (char *) &md->id; + int left = len -4; + while( 1 ) { + int x = ::recv(sock, p, left, 0); + if( x == 0 ) { + cout << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl; + m.reset(); + return false; + } + if( x < 0 ) { + cout << "MessagingPort::recv(): recv() error " << errno << ' ' << farEnd.toString() << endl; + m.reset(); + return false; + } + left -= x; + p += x; + if( left <= 0 ) + break; + } + + m.setData(md, true); + return true; +} + +void MessagingPort::reply(Message& received, Message& response) { + say(received.from, response, received.data->id); +} + +bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) { + mmm( cout << "*call()" << endl; ) + MSGID old = toSend.data->id; + say(to, toSend); + while( 1 ) { + bool ok = recv(response); + if( !ok ) + return false; + //cout << "got response: " << response.data->responseTo << endl; + if( response.data->responseTo == toSend.data->id ) + break; + cout << "********************" << endl; + cout << "ERROR: MessagingPort::call() wrong id got:" << response.data->responseTo << " expect:" << toSend.data->id << endl; + cout << " old:" << old << endl; + cout << " response msgid:" << response.data->id << endl; + cout << " response len: " << response.data->len << endl; + assert(false); + response.reset(); + } + mmm( cout << "*call() end" << endl; ) + return true; +} + +void MessagingPort::say(SockAddr& to, Message& toSend, int responseTo) { + mmm( cout << "* say() sock:" << this->sock << " thr:" << GetCurrentThreadId() << endl; ) + MSGID msgid = NextMsgId; + ++NextMsgId; + toSend.data->id = msgid; + toSend.data->responseTo = responseTo; + int x = ::send(sock, (char *) toSend.data, toSend.data->len, 0); + if( x <= 0 ) { + cout << "MessagingPort::say: send() error " << errno << ' ' << farEnd.toString() << endl; + } +} diff --git a/grid/message.h b/grid/message.h index 1395fa51e0a..604e9a67343 100644 --- a/grid/message.h +++ b/grid/message.h @@ -1,129 +1,129 @@ -// message.h
-
-#pragma once
-
-#include "../util/sock.h"
-
-class Message;
-class MessagingPort;
-typedef WrappingInt MSGID;
-const int DBPort = 27017;
-
-class Listener {
-public:
- Listener(int p) : port(p) { }
- void listen(); // never returns (start a thread)
-
- /* spawn a thread, etc., then return */
- virtual void accepted(MessagingPort *mp) = 0;
-private:
- int port;
-};
-
-class AbstractMessagingPort {
-public:
- virtual void reply(Message& received, Message& response) = 0;
-};
-
-class MessagingPort : public AbstractMessagingPort {
-public:
- MessagingPort(int sock, SockAddr& farEnd);
- MessagingPort();
- ~MessagingPort();
-
- void shutdown();
-
- bool connect(SockAddr& farEnd);
-
- /* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's.
- also, the Message data will go out of scope on the subsequent recv call.
- */
- bool recv(Message& m);
- void reply(Message& received, Message& response);
- bool call(SockAddr& to, Message& toSend, Message& response);
- void say(SockAddr& to, Message& toSend, int responseTo = -1);
-
-private:
- int sock;
- SockAddr farEnd;
-};
-
-#pragma pack(push)
-#pragma pack(1)
-
-enum Operations {
- opReply = 1, /* reply. responseTo is set. */
-
- dbMsg = 1000, /* generic msg command followed by a string */
-
- dbUpdate = 2001, /* update object */
- dbInsert = 2002,
-// dbGetByOID = 2003,
- dbQuery = 2004,
- dbGetMore = 2005,
- dbDelete = 2006,
- dbKillCursors = 2007
-};
-
-struct MsgData {
- int len; /* len of the msg, including this field */
- MSGID id; /* request/reply id's match... */
- int responseTo; /* id of the message we are responding to */
- int operation;
- char _data[4];
-
- int dataLen();
-};
-const int MsgDataHeaderSize = sizeof(MsgData) - 4;
-inline int MsgData::dataLen() { return len - MsgDataHeaderSize; }
-
-#pragma pack(pop)
-
-class Message {
-public:
- Message() { data = 0; freeIt = false; }
- Message( void * _data , bool _freeIt ){ data = (MsgData*)_data; freeIt = _freeIt; };
- ~Message() { reset(); }
-
- SockAddr from;
- MsgData *data;
-
- Message& operator=(Message& r) {
- assert( data == 0 );
- data = r.data;
- assert( r.freeIt );
- r.freeIt = false;
- r.data = 0;
- freeIt = true;
- return *this;
- }
-
- void reset() {
- if( freeIt && data )
- free(data);
- data = 0; freeIt = false;
- }
-
- void setData(MsgData *d, bool _freeIt) {
- assert( data == 0 );
- freeIt = _freeIt;
- data = d;
- }
- void setData(int operation, const char *msgtxt) {
- setData(operation, msgtxt, strlen(msgtxt)+1);
- }
- void setData(int operation, const char *msgdata, int len) {
- assert(data == 0);
- int dataLen = len + sizeof(MsgData) - 4;
- MsgData *d = (MsgData *) malloc(dataLen);
- memcpy(d->_data, msgdata, len);
- d->len = dataLen;
- d->operation = operation;
- freeIt= true;
- data = d;
- }
-
-private:
- bool freeIt;
-};
-
+// message.h + +#pragma once + +#include "../util/sock.h" + +class Message; +class MessagingPort; +typedef WrappingInt MSGID; +const int DBPort = 27017; + +class Listener { +public: + Listener(int p) : port(p) { } + void listen(); // never returns (start a thread) + + /* spawn a thread, etc., then return */ + virtual void accepted(MessagingPort *mp) = 0; +private: + int port; +}; + +class AbstractMessagingPort { +public: + virtual void reply(Message& received, Message& response) = 0; +}; + +class MessagingPort : public AbstractMessagingPort { +public: + MessagingPort(int sock, SockAddr& farEnd); + MessagingPort(); + ~MessagingPort(); + + void shutdown(); + + bool connect(SockAddr& farEnd); + + /* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's. + also, the Message data will go out of scope on the subsequent recv call. + */ + bool recv(Message& m); + void reply(Message& received, Message& response); + bool call(SockAddr& to, Message& toSend, Message& response); + void say(SockAddr& to, Message& toSend, int responseTo = -1); + +private: + int sock; + SockAddr farEnd; +}; + +#pragma pack(push) +#pragma pack(1) + +enum Operations { + opReply = 1, /* reply. responseTo is set. */ + + dbMsg = 1000, /* generic msg command followed by a string */ + + dbUpdate = 2001, /* update object */ + dbInsert = 2002, +// dbGetByOID = 2003, + dbQuery = 2004, + dbGetMore = 2005, + dbDelete = 2006, + dbKillCursors = 2007 +}; + +struct MsgData { + int len; /* len of the msg, including this field */ + MSGID id; /* request/reply id's match... */ + int responseTo; /* id of the message we are responding to */ + int operation; + char _data[4]; + + int dataLen(); +}; +const int MsgDataHeaderSize = sizeof(MsgData) - 4; +inline int MsgData::dataLen() { return len - MsgDataHeaderSize; } + +#pragma pack(pop) + +class Message { +public: + Message() { data = 0; freeIt = false; } + Message( void * _data , bool _freeIt ){ data = (MsgData*)_data; freeIt = _freeIt; }; + ~Message() { reset(); } + + SockAddr from; + MsgData *data; + + Message& operator=(Message& r) { + assert( data == 0 ); + data = r.data; + assert( r.freeIt ); + r.freeIt = false; + r.data = 0; + freeIt = true; + return *this; + } + + void reset() { + if( freeIt && data ) + free(data); + data = 0; freeIt = false; + } + + void setData(MsgData *d, bool _freeIt) { + assert( data == 0 ); + freeIt = _freeIt; + data = d; + } + void setData(int operation, const char *msgtxt) { + setData(operation, msgtxt, strlen(msgtxt)+1); + } + void setData(int operation, const char *msgdata, int len) { + assert(data == 0); + int dataLen = len + sizeof(MsgData) - 4; + MsgData *d = (MsgData *) malloc(dataLen); + memcpy(d->_data, msgdata, len); + d->len = dataLen; + d->operation = operation; + freeIt= true; + data = d; + } + +private: + bool freeIt; +}; + diff --git a/grid/protocol.h b/grid/protocol.h index f98c6cde1f4..e7d0298ec9b 100644 --- a/grid/protocol.h +++ b/grid/protocol.h @@ -1,261 +1,261 @@ -// protocol.h
-
-#pragma once
-
-NOT USED
-
-#include "boost/thread/mutex.hpp"
-#include "boost/thread/condition.hpp"
-#include "../util/sock.h"
-#include "../util/goodies.h"
-
-typedef WrappingInt MSGID;
-
-struct Fragment;
-
-#if 0
-#define ptrace(x)
-#else
-#define ptrace(x) { cout << curTimeMillis() % 10000; x }
-#endif
-
-#if 1
-#define etrace(x)
-#else
-#define etrace(x) { cout << curTimeMillis() % 10000; x }
-#endif
-
-class F; // fragment
-class MR; // message. R=receiver side.
-class CR; // connection receiver side
-class MS; // message S=sender side.
-class CS; // connection sender side
-class ProtocolConnection; // connection overall
-
-/* ip:port:channel
- We have one receiver thread per process (per ip address destination), and then
- multiplex messages out to multiple connections(generally one per thread) which
- each have a 'channel'.
-*/
-class EndPoint {
-public:
- EndPoint() : channel(0) { }
- int channel;
- SockAddr sa;
- bool operator<(const EndPoint& r) const {
- if( channel != r.channel )
- return channel < r.channel;
- return sa < r.sa;
- }
- string toString() {
- stringstream out;
- out << sa.toString() << ':';
- if( channel == -2 ) out << "ANYCHANNEL";
- else if( channel == -1 ) out << "AUTOASSIGNCHANNEL";
- else out << channel;
- return out.str();
- }
-};
-
-/* the double underscore stuff here is the actual implementation glue.
- wanted to keep that stuff clean and separate. so put your implementation
- of these in protoimpl.h.
-*/
-
-void __sendRESET(ProtocolConnection *pc);
-
-// sender ->
-void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *, bool retran=false); // transmit a fragment
-void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, MSGID msgid, int fragNo); // transmit the REQUEST ACK msg
-
-// receiver ->
-void __sendACK(ProtocolConnection *pc, MSGID msgid); // transmit ACK
-void __sendMISSING(ProtocolConnection *pc, EndPoint& to, MSGID msgid, vector<short>& ids); // transmit MISSING
-
-// -> receiver
-F* __recv(UDPConnection& c, SockAddr& from); /* recv from socket a fragment and pass back */
-
-class F {
-public:
- F(Fragment *f);
- ~F();
- int __num(); //frag #
- int __len();
- MSGID __msgid();
- int __channel();
- bool __isREQUESTACK(); // if true, this is just a request for acknowledgement not real data
- int __firstFragMsgLen(); // only works on first fragment
-
- // sender side:
- bool __isACK(); // if true, this is an ack of a message
- bool __isMISSING(); // if true, list of frags to retransmit
- short* __getMissing(int& n); // get the missing fragno list
-
- Fragment *internals;
- enum { NORMAL, ACK, MISSING, REQUESTACK, RESET } op;
-};
-
-class MR {
-public:
- MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from);
- ~MR() { freeFrags(); }
- void freeFrags() {
- for( unsigned i = 0; i < f.size(); i++ )
- delete f[i];
- }
- bool got(F *f, EndPoint& from); // received a fragment
- bool gotFirst() { return f[0] != 0; }
- ProtocolConnection& pc;
- void removeFromReceivingList();
- bool complete();
- const MSGID msgid;
- int n() { return f.size(); }
-public:
- int messageLenExpected;
- int nExpected, nReceived;
- void reportMissings(bool reportAll);
- vector<F*> f;
- vector<unsigned> reportTimes;
- EndPoint from;
-};
-
-/* this is for knowing what is already received. we might get dup packets later and need
- to ignore them. */
-class MsgTracker {
-public:
- std::list<MSGID> recentlyReceivedList;
- std::set<MSGID> recentlyReceived;
- MSGID lastFullyReceived;
-
- void reset() {
- recentlyReceivedList.clear();
- recentlyReceived.clear();
- lastFullyReceived = 0;
- }
-
- void got(MSGID m) {
- unsigned sz = recentlyReceived.size();
- if( sz > 256 ) {
- recentlyReceived.erase(recentlyReceivedList.front());
- recentlyReceivedList.pop_front();
- }
- recentlyReceivedList.push_back(m);
- recentlyReceived.insert(m);
- if( m > lastFullyReceived || sz == 0 )
- lastFullyReceived = m;
- }
-};
-
-class CR {
- friend class MR;
-public:
- ~CR() { ptrace( cout << ".warning: ~CR() not implemented" << endl; ) }
- CR(ProtocolConnection& _pc) : pc(_pc) { }
- MR* recv();
-public:
- MR* getPendingMsg(F *fr);
- bool oldMessageId(int channel, MSGID m);
- void queueReceived(MR*);
-
- ProtocolConnection& pc;
- boost::condition receivedSome;
- vector<MR*> received; /* ready to dequeue and use */
- map<int,MR*> pendingMessages; /* partly received msgs */
- MsgTracker oldMsgTracker;
-};
-
-/* -- sender side ------------------------------------------------*/
-
-class CS {
-public:
- CS(ProtocolConnection& _pc);
-
- ProtocolConnection& pc;
- vector<MS*> pendingSend;
- boost::condition msgSent;
- void resetIt();
-
- double delayMax;
- double delay;
- void delayGotMissing() {
- double delayOld = delay;
- if( delay == 0 )
- delay = 2.0;
- else
- delay = delay * 1.25;
- if( delay > delayMax ) delay = delayMax;
- if( delay != delayOld )
- cout << ".DELAY INCREASED TO " << delay << endl;
- }
- void delaySentMsg() {
- if( delay != 0.0 ) {
- delay = delay * 0.5;
- if( delay<0.5 ) delay = 0;
- cout << ".DELAY DECREASED TO " << delay << endl;
- }
- }
-};
-
-typedef map<EndPoint,ProtocolConnection*> EndPointToPC;
-extern EndPointToPC pcMap; /* the *far* endpoint -> pc */
-
-/* -- overall Connection object ----------------------------------*/
-
-#pragma warning( disable: 4355 )
-
-class ProtocolConnection {
-public:
- string toString();
-
- ProtocolConnection(ProtocolConnection& par, EndPoint& to);
- ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd);
- ~ProtocolConnection();
-
- void shutdown();
- bool acceptAnyChannel() const;
- UDPConnection& udpConnection;
- /* note the channel for myEnd might be "any" for the any pc -
- so you can't use that channel for sending. Use MS/MR
- for that.
- */
- EndPoint myEnd;
- EndPoint farEnd;
-
- /* if this was instantiated automatically for an acceptAnyChannel(),
- keep a ptr back to it and queue received msgs there.
- */
- ProtocolConnection *parent;
-
- CR cr;
- CS cs;
- bool first; // true if yet to send first message on this conn
-
-private:
- void init();
-};
-
-/* -- sender side ------------------------------------------------*/
-
-class MS {
-public:
- MS(ProtocolConnection *_pc, EndPoint &_to, MSGID _msgid) :
- pc(_pc), to(_to), msgid(_msgid), complainInterval(50) { }
- ~MS() {
- for( unsigned i = 0; i < fragments.size(); i++ )
- delete fragments[i];
- }
-
- /* init fragments, then call this */
- void send();
-
- vector<F*> fragments;
-
- /* request retrainsmissions. */
- bool complain(unsigned now);
-
- ProtocolConnection* pc;
- EndPoint to;
- const MSGID msgid;
- unsigned lastComplainTime;
- unsigned complainInterval;
-};
+// protocol.h + +#pragma once + +NOT USED + +#include "boost/thread/mutex.hpp" +#include "boost/thread/condition.hpp" +#include "../util/sock.h" +#include "../util/goodies.h" + +typedef WrappingInt MSGID; + +struct Fragment; + +#if 0 +#define ptrace(x) +#else +#define ptrace(x) { cout << curTimeMillis() % 10000; x } +#endif + +#if 1 +#define etrace(x) +#else +#define etrace(x) { cout << curTimeMillis() % 10000; x } +#endif + +class F; // fragment +class MR; // message. R=receiver side. +class CR; // connection receiver side +class MS; // message S=sender side. +class CS; // connection sender side +class ProtocolConnection; // connection overall + +/* ip:port:channel + We have one receiver thread per process (per ip address destination), and then + multiplex messages out to multiple connections(generally one per thread) which + each have a 'channel'. +*/ +class EndPoint { +public: + EndPoint() : channel(0) { } + int channel; + SockAddr sa; + bool operator<(const EndPoint& r) const { + if( channel != r.channel ) + return channel < r.channel; + return sa < r.sa; + } + string toString() { + stringstream out; + out << sa.toString() << ':'; + if( channel == -2 ) out << "ANYCHANNEL"; + else if( channel == -1 ) out << "AUTOASSIGNCHANNEL"; + else out << channel; + return out.str(); + } +}; + +/* the double underscore stuff here is the actual implementation glue. + wanted to keep that stuff clean and separate. so put your implementation + of these in protoimpl.h. +*/ + +void __sendRESET(ProtocolConnection *pc); + +// sender -> +void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *, bool retran=false); // transmit a fragment +void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, MSGID msgid, int fragNo); // transmit the REQUEST ACK msg + +// receiver -> +void __sendACK(ProtocolConnection *pc, MSGID msgid); // transmit ACK +void __sendMISSING(ProtocolConnection *pc, EndPoint& to, MSGID msgid, vector<short>& ids); // transmit MISSING + +// -> receiver +F* __recv(UDPConnection& c, SockAddr& from); /* recv from socket a fragment and pass back */ + +class F { +public: + F(Fragment *f); + ~F(); + int __num(); //frag # + int __len(); + MSGID __msgid(); + int __channel(); + bool __isREQUESTACK(); // if true, this is just a request for acknowledgement not real data + int __firstFragMsgLen(); // only works on first fragment + + // sender side: + bool __isACK(); // if true, this is an ack of a message + bool __isMISSING(); // if true, list of frags to retransmit + short* __getMissing(int& n); // get the missing fragno list + + Fragment *internals; + enum { NORMAL, ACK, MISSING, REQUESTACK, RESET } op; +}; + +class MR { +public: + MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from); + ~MR() { freeFrags(); } + void freeFrags() { + for( unsigned i = 0; i < f.size(); i++ ) + delete f[i]; + } + bool got(F *f, EndPoint& from); // received a fragment + bool gotFirst() { return f[0] != 0; } + ProtocolConnection& pc; + void removeFromReceivingList(); + bool complete(); + const MSGID msgid; + int n() { return f.size(); } +public: + int messageLenExpected; + int nExpected, nReceived; + void reportMissings(bool reportAll); + vector<F*> f; + vector<unsigned> reportTimes; + EndPoint from; +}; + +/* this is for knowing what is already received. we might get dup packets later and need + to ignore them. */ +class MsgTracker { +public: + std::list<MSGID> recentlyReceivedList; + std::set<MSGID> recentlyReceived; + MSGID lastFullyReceived; + + void reset() { + recentlyReceivedList.clear(); + recentlyReceived.clear(); + lastFullyReceived = 0; + } + + void got(MSGID m) { + unsigned sz = recentlyReceived.size(); + if( sz > 256 ) { + recentlyReceived.erase(recentlyReceivedList.front()); + recentlyReceivedList.pop_front(); + } + recentlyReceivedList.push_back(m); + recentlyReceived.insert(m); + if( m > lastFullyReceived || sz == 0 ) + lastFullyReceived = m; + } +}; + +class CR { + friend class MR; +public: + ~CR() { ptrace( cout << ".warning: ~CR() not implemented" << endl; ) } + CR(ProtocolConnection& _pc) : pc(_pc) { } + MR* recv(); +public: + MR* getPendingMsg(F *fr); + bool oldMessageId(int channel, MSGID m); + void queueReceived(MR*); + + ProtocolConnection& pc; + boost::condition receivedSome; + vector<MR*> received; /* ready to dequeue and use */ + map<int,MR*> pendingMessages; /* partly received msgs */ + MsgTracker oldMsgTracker; +}; + +/* -- sender side ------------------------------------------------*/ + +class CS { +public: + CS(ProtocolConnection& _pc); + + ProtocolConnection& pc; + vector<MS*> pendingSend; + boost::condition msgSent; + void resetIt(); + + double delayMax; + double delay; + void delayGotMissing() { + double delayOld = delay; + if( delay == 0 ) + delay = 2.0; + else + delay = delay * 1.25; + if( delay > delayMax ) delay = delayMax; + if( delay != delayOld ) + cout << ".DELAY INCREASED TO " << delay << endl; + } + void delaySentMsg() { + if( delay != 0.0 ) { + delay = delay * 0.5; + if( delay<0.5 ) delay = 0; + cout << ".DELAY DECREASED TO " << delay << endl; + } + } +}; + +typedef map<EndPoint,ProtocolConnection*> EndPointToPC; +extern EndPointToPC pcMap; /* the *far* endpoint -> pc */ + +/* -- overall Connection object ----------------------------------*/ + +#pragma warning( disable: 4355 ) + +class ProtocolConnection { +public: + string toString(); + + ProtocolConnection(ProtocolConnection& par, EndPoint& to); + ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd); + ~ProtocolConnection(); + + void shutdown(); + bool acceptAnyChannel() const; + UDPConnection& udpConnection; + /* note the channel for myEnd might be "any" for the any pc - + so you can't use that channel for sending. Use MS/MR + for that. + */ + EndPoint myEnd; + EndPoint farEnd; + + /* if this was instantiated automatically for an acceptAnyChannel(), + keep a ptr back to it and queue received msgs there. + */ + ProtocolConnection *parent; + + CR cr; + CS cs; + bool first; // true if yet to send first message on this conn + +private: + void init(); +}; + +/* -- sender side ------------------------------------------------*/ + +class MS { +public: + MS(ProtocolConnection *_pc, EndPoint &_to, MSGID _msgid) : + pc(_pc), to(_to), msgid(_msgid), complainInterval(50) { } + ~MS() { + for( unsigned i = 0; i < fragments.size(); i++ ) + delete fragments[i]; + } + + /* init fragments, then call this */ + void send(); + + vector<F*> fragments; + + /* request retrainsmissions. */ + bool complain(unsigned now); + + ProtocolConnection* pc; + EndPoint to; + const MSGID msgid; + unsigned lastComplainTime; + unsigned complainInterval; +}; diff --git a/grid/protoimpl.h b/grid/protoimpl.h index 5ad19a47250..9ca5277e527 100644 --- a/grid/protoimpl.h +++ b/grid/protoimpl.h @@ -1,242 +1,242 @@ -// protoimpl.h
-
-#pragma once
-
-/* packet dumping level of detail. */
-const bool dumpPackets = false; // this must be true to get anything at all
-const bool dumpIP = false; // output the ip address
-const bool dumpBytesDetailed = false; // more data output
-
-#include "message.h"
-
-extern boost::mutex coutmutex;
-
-const int FragMax = 1480;
-const int FragHeader = 10;
-const int MSS = FragMax - FragHeader;
-
-#pragma pack(push)
-#pragma pack(1)
-
-struct Fragment {
- enum { MinFragmentLen = FragHeader + 1 };
- MSGID msgId;
- short channel;
- short fragmentLen;
- short fragmentNo;
- char data[16];
- int fragmentDataLen() { return fragmentLen - FragHeader; }
- char* fragmentData() { return data; }
-
- bool ok(int nRead) {
- if( nRead < MinFragmentLen || fragmentLen > nRead || fragmentLen < MinFragmentLen ) {
- ptrace( cout << ".recv: fragment bad. fragmentLen:" << fragmentLen << " nRead:" << nRead << endl; )
- return false;
- }
- if( fragmentNo == 0 && fragmentLen < MinFragmentLen + MsgDataHeaderSize ) {
- ptrace( cout << ".recv: bad first fragment. fragmentLen:" << fragmentLen << endl; )
- return false;
- }
- return true;
- }
-
- MsgData* startOfMsgData() { assert(fragmentNo == 0); return (MsgData *) data; }
-};
-#pragma pack(pop)
-
-inline void DUMP(Fragment& f, SockAddr& t, const char *tabs) {
- if( !dumpPackets )
- return;
- cout << tabs << curTimeMillis() % 10000 << ' ';
- short s = f.fragmentNo;
- if( s == -32768 )
- cout << "ACK M:" << f.msgId % 1000;
- else if( s == -32767 )
- cout << "MISSING";
- else if( s == -32766 )
- cout << "RESET ch:" << f.channel;
- else if( s < 0 )
- cout << "REQUESTACK";
- else
- cout << '#' << s << ' ' << f.fragmentLen << " M:" << f.msgId % 1000;
- cout << ' ';
- if( dumpIP )
- cout << t.toString();
-}
-
-inline void DUMPDATA(Fragment& f, const char *tabs) {
- if( !dumpPackets )
- return;
- if( f.fragmentNo >= 0 ) {
- cout << '\n' << tabs;
- int x = f.fragmentDataLen();
- if( dumpBytesDetailed ) {
- char *p = (char *) &f;
- cout << hex << *((unsigned*)p) << ' '; p+=4;
- cout << *((short*)p) << ' '; p+=2;
- cout << *((short*)p) << ' '; p+=2;
- cout << *((short*)p) << '|'; p+=2;
- if( x < 16 ) cout << "???";
- else {
- for( int i = 0; i < 4; i++ ) { // MSGDATA
- cout << *((unsigned*)p);
- cout << (i < 3 ? ' ' : '|');
- p += 4;
- }
- cout << '\n' << tabs;
- x -= 16;
- if( x > 32 ) x = 32;
- while( x-- > 0 ) { cout << (unsigned) (unsigned char) *p++ << ' '; }
- }
- }
- else {
- char *p = f.data;
- if( f.fragmentNo == 0 ) {
- p += 16; x -= 16;
- }
- if( x > 28 ) x = 28;
- for( int i = 0; i < x; i++ ) {
- if( *p == 0 ) cout << (char) 0xb0;
- else cout << (*p >= 32 ? *p : '.');
- p++;
- }
- }
- }
- cout << dec << endl;
-}
-
-inline void SEND(UDPConnection& c, Fragment &f, SockAddr& to, const char *extra="") {
- lock lk(coutmutex);
- DUMP(f, to, "\t\t\t\t\t>");
- c.sendto((char *) &f, f.fragmentLen, to);
- if( dumpPackets )
- cout << extra;
- DUMPDATA(f, "\t\t\t\t\t ");
-}
-
-// sender ->
-inline void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *f, bool retran) {
- assert( f->internals->channel == to.channel );
- ptrace( cout << ".sendfrag " << f->__num() << ' ' << retran << endl; )
- SEND(pc->udpConnection, *f->internals, to.sa, retran ? " retran" : "");
-}
-
-inline void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to,
- MSGID msgid, int fragNo) {
- Fragment f;
- f.msgId = msgid;
- f.channel = to.channel; assert( f.channel >= 0 );
- f.fragmentNo = ((short) -fragNo) -1;
- f.fragmentLen = FragHeader;
- ptrace( cout << ".requesting ack, fragno=" << f.fragmentNo << " msg:" << f.msgId << ' ' << to.toString() << endl; )
- SEND(pc->udpConnection, f, to.sa);
-}
-
-// receiver ->
-inline void __sendACK(ProtocolConnection *pc, MSGID msgid) {
- ptrace( cout << "...__sendACK() to:" << pc->farEnd.toString() << " msg:" << msgid << endl; )
- Fragment f;
- f.msgId = msgid;
- f.channel = pc->farEnd.channel; assert( f.channel >= 0 );
- f.fragmentNo = -32768;
- f.fragmentLen = FragHeader;
- SEND(pc->udpConnection, f, pc->farEnd.sa);
-}
-
-/* this is to clear old state for the channel in terms of what msgids are
- already sent.
-*/
-inline void __sendRESET(ProtocolConnection *pc) {
- Fragment f;
- f.msgId = -1;
- f.channel = pc->farEnd.channel; assert( f.channel >= 0 );
- f.fragmentNo = -32766;
- f.fragmentLen = FragHeader;
- ptrace( cout << "...__sendRESET() to:" << pc->farEnd.toString() << endl; )
- SEND(pc->udpConnection, f, pc->farEnd.sa);
-}
-
-inline void __sendMISSING(ProtocolConnection *pc, EndPoint& to,
- MSGID msgid, vector<short>& ids) {
- int n = ids.size();
- ptrace( cout << "..sendMISSING n:" << n << " firstmissing:" << ids[0] << " last:" << ids[ids.size()-1] << to.toString() << endl; )
- if( n > 256 ) {
- ptrace( cout << "\t..sendMISSING limiting to 256 ids" << endl; )
- n = 256;
- }
- Fragment *f = (Fragment*) malloc(FragHeader + n*2);
- f->msgId = msgid;
- f->channel = to.channel; assert( f->channel >= 0 );
- f->fragmentNo = -32767;
- f->fragmentLen = FragHeader + n*2;
- short *s = (short *) f->data;
- for( int i = 0; i < n; i++ )
- *s++ = ids[i];
-// ptrace( cout << "...sendMISSING fraglen:" << f->fragmentLen << endl; )
- SEND(pc->udpConnection, *f, to.sa);
- free(f);
-}
-
-// -> receiver
-inline F* __recv(UDPConnection& c, SockAddr& from) {
- Fragment *f = (Fragment *) malloc(MaxMTU);
- int n;
- while( 1 ) {
-// n = c.recvfrom((char*) f, c.mtu(), from);
- n = c.recvfrom((char*) f, MaxMTU, from);
-// cout << "recvfrom returned " << n << endl;
- if( n >= 0 )
- break;
- if( !goingAway ) {
- cout << ".recvfrom returned error " << getLastError() << " socket:" << c.sock << endl;
- cout << "sleeping 2 seconds " << endl;
- sleepsecs(2);
- }
- }
- assert( f->fragmentLen == n );
- if( f->fragmentNo > 0 ) {
- // don't waste tons of space if the maxmtu is 16k but we get 1480
- unsigned newsz = (f->fragmentLen + 255) & 0xffffff00;
- if( newsz < MaxMTU )
- f = (Fragment *) realloc(f, newsz);
- }
- {
- lock lk(coutmutex);
- DUMP(*f, from, "\t\t\t\t\t\t\t\t\t\t<");
- DUMPDATA(*f, "\t\t\t\t\t\t\t\t\t\t ");
- }
- return new F(f);
-}
-
-inline F::F(Fragment *f) : internals(f), op(NORMAL) {
- if( internals->fragmentNo < 0 ) {
- if( internals->fragmentNo == -32768 ) {
- op = ACK;
- ptrace( cout << ".got ACK msg:" << internals->msgId << endl; )
- } else if( internals->fragmentNo == -32767 ) {
- op = MISSING;
- ptrace( cout << ".got MISSING" << endl; )
- } else if( internals->fragmentNo == -32766 ) {
- op = RESET;
- } else {
- op = REQUESTACK;
- internals->fragmentNo = -(internals->fragmentNo+1);
- ptrace( cout << ".got REQUESTACK frag:" << internals->fragmentNo << " msg:" << internals->msgId << endl; )
- }
- }
-}
-inline F::~F() { free(internals); internals=0; }
-inline int F::__num() { return internals->fragmentNo; }
-inline int F::__len() { return internals->fragmentLen; }
-inline MSGID F::__msgid() { return internals->msgId; }
-inline int F::__channel() { return internals->channel; }
-inline bool F::__isREQUESTACK() { return op == REQUESTACK; }
-inline bool F::__isACK() { return op == ACK; }
-inline bool F::__isMISSING() { return op == MISSING; }
-inline short* F::__getMissing(int& n) {
- n = internals->fragmentDataLen() / 2;
- return (short *) internals->fragmentData();
-}
-inline int F::__firstFragMsgLen() {
- return internals->startOfMsgData()->len;
-}
+// protoimpl.h + +#pragma once + +/* packet dumping level of detail. */ +const bool dumpPackets = false; // this must be true to get anything at all +const bool dumpIP = false; // output the ip address +const bool dumpBytesDetailed = false; // more data output + +#include "message.h" + +extern boost::mutex coutmutex; + +const int FragMax = 1480; +const int FragHeader = 10; +const int MSS = FragMax - FragHeader; + +#pragma pack(push) +#pragma pack(1) + +struct Fragment { + enum { MinFragmentLen = FragHeader + 1 }; + MSGID msgId; + short channel; + short fragmentLen; + short fragmentNo; + char data[16]; + int fragmentDataLen() { return fragmentLen - FragHeader; } + char* fragmentData() { return data; } + + bool ok(int nRead) { + if( nRead < MinFragmentLen || fragmentLen > nRead || fragmentLen < MinFragmentLen ) { + ptrace( cout << ".recv: fragment bad. fragmentLen:" << fragmentLen << " nRead:" << nRead << endl; ) + return false; + } + if( fragmentNo == 0 && fragmentLen < MinFragmentLen + MsgDataHeaderSize ) { + ptrace( cout << ".recv: bad first fragment. fragmentLen:" << fragmentLen << endl; ) + return false; + } + return true; + } + + MsgData* startOfMsgData() { assert(fragmentNo == 0); return (MsgData *) data; } +}; +#pragma pack(pop) + +inline void DUMP(Fragment& f, SockAddr& t, const char *tabs) { + if( !dumpPackets ) + return; + cout << tabs << curTimeMillis() % 10000 << ' '; + short s = f.fragmentNo; + if( s == -32768 ) + cout << "ACK M:" << f.msgId % 1000; + else if( s == -32767 ) + cout << "MISSING"; + else if( s == -32766 ) + cout << "RESET ch:" << f.channel; + else if( s < 0 ) + cout << "REQUESTACK"; + else + cout << '#' << s << ' ' << f.fragmentLen << " M:" << f.msgId % 1000; + cout << ' '; + if( dumpIP ) + cout << t.toString(); +} + +inline void DUMPDATA(Fragment& f, const char *tabs) { + if( !dumpPackets ) + return; + if( f.fragmentNo >= 0 ) { + cout << '\n' << tabs; + int x = f.fragmentDataLen(); + if( dumpBytesDetailed ) { + char *p = (char *) &f; + cout << hex << *((unsigned*)p) << ' '; p+=4; + cout << *((short*)p) << ' '; p+=2; + cout << *((short*)p) << ' '; p+=2; + cout << *((short*)p) << '|'; p+=2; + if( x < 16 ) cout << "???"; + else { + for( int i = 0; i < 4; i++ ) { // MSGDATA + cout << *((unsigned*)p); + cout << (i < 3 ? ' ' : '|'); + p += 4; + } + cout << '\n' << tabs; + x -= 16; + if( x > 32 ) x = 32; + while( x-- > 0 ) { cout << (unsigned) (unsigned char) *p++ << ' '; } + } + } + else { + char *p = f.data; + if( f.fragmentNo == 0 ) { + p += 16; x -= 16; + } + if( x > 28 ) x = 28; + for( int i = 0; i < x; i++ ) { + if( *p == 0 ) cout << (char) 0xb0; + else cout << (*p >= 32 ? *p : '.'); + p++; + } + } + } + cout << dec << endl; +} + +inline void SEND(UDPConnection& c, Fragment &f, SockAddr& to, const char *extra="") { + lock lk(coutmutex); + DUMP(f, to, "\t\t\t\t\t>"); + c.sendto((char *) &f, f.fragmentLen, to); + if( dumpPackets ) + cout << extra; + DUMPDATA(f, "\t\t\t\t\t "); +} + +// sender -> +inline void __sendFrag(ProtocolConnection *pc, EndPoint& to, F *f, bool retran) { + assert( f->internals->channel == to.channel ); + ptrace( cout << ".sendfrag " << f->__num() << ' ' << retran << endl; ) + SEND(pc->udpConnection, *f->internals, to.sa, retran ? " retran" : ""); +} + +inline void __sendREQUESTACK(ProtocolConnection *pc, EndPoint& to, + MSGID msgid, int fragNo) { + Fragment f; + f.msgId = msgid; + f.channel = to.channel; assert( f.channel >= 0 ); + f.fragmentNo = ((short) -fragNo) -1; + f.fragmentLen = FragHeader; + ptrace( cout << ".requesting ack, fragno=" << f.fragmentNo << " msg:" << f.msgId << ' ' << to.toString() << endl; ) + SEND(pc->udpConnection, f, to.sa); +} + +// receiver -> +inline void __sendACK(ProtocolConnection *pc, MSGID msgid) { + ptrace( cout << "...__sendACK() to:" << pc->farEnd.toString() << " msg:" << msgid << endl; ) + Fragment f; + f.msgId = msgid; + f.channel = pc->farEnd.channel; assert( f.channel >= 0 ); + f.fragmentNo = -32768; + f.fragmentLen = FragHeader; + SEND(pc->udpConnection, f, pc->farEnd.sa); +} + +/* this is to clear old state for the channel in terms of what msgids are + already sent. +*/ +inline void __sendRESET(ProtocolConnection *pc) { + Fragment f; + f.msgId = -1; + f.channel = pc->farEnd.channel; assert( f.channel >= 0 ); + f.fragmentNo = -32766; + f.fragmentLen = FragHeader; + ptrace( cout << "...__sendRESET() to:" << pc->farEnd.toString() << endl; ) + SEND(pc->udpConnection, f, pc->farEnd.sa); +} + +inline void __sendMISSING(ProtocolConnection *pc, EndPoint& to, + MSGID msgid, vector<short>& ids) { + int n = ids.size(); + ptrace( cout << "..sendMISSING n:" << n << " firstmissing:" << ids[0] << " last:" << ids[ids.size()-1] << to.toString() << endl; ) + if( n > 256 ) { + ptrace( cout << "\t..sendMISSING limiting to 256 ids" << endl; ) + n = 256; + } + Fragment *f = (Fragment*) malloc(FragHeader + n*2); + f->msgId = msgid; + f->channel = to.channel; assert( f->channel >= 0 ); + f->fragmentNo = -32767; + f->fragmentLen = FragHeader + n*2; + short *s = (short *) f->data; + for( int i = 0; i < n; i++ ) + *s++ = ids[i]; +// ptrace( cout << "...sendMISSING fraglen:" << f->fragmentLen << endl; ) + SEND(pc->udpConnection, *f, to.sa); + free(f); +} + +// -> receiver +inline F* __recv(UDPConnection& c, SockAddr& from) { + Fragment *f = (Fragment *) malloc(MaxMTU); + int n; + while( 1 ) { +// n = c.recvfrom((char*) f, c.mtu(), from); + n = c.recvfrom((char*) f, MaxMTU, from); +// cout << "recvfrom returned " << n << endl; + if( n >= 0 ) + break; + if( !goingAway ) { + cout << ".recvfrom returned error " << getLastError() << " socket:" << c.sock << endl; + cout << "sleeping 2 seconds " << endl; + sleepsecs(2); + } + } + assert( f->fragmentLen == n ); + if( f->fragmentNo > 0 ) { + // don't waste tons of space if the maxmtu is 16k but we get 1480 + unsigned newsz = (f->fragmentLen + 255) & 0xffffff00; + if( newsz < MaxMTU ) + f = (Fragment *) realloc(f, newsz); + } + { + lock lk(coutmutex); + DUMP(*f, from, "\t\t\t\t\t\t\t\t\t\t<"); + DUMPDATA(*f, "\t\t\t\t\t\t\t\t\t\t "); + } + return new F(f); +} + +inline F::F(Fragment *f) : internals(f), op(NORMAL) { + if( internals->fragmentNo < 0 ) { + if( internals->fragmentNo == -32768 ) { + op = ACK; + ptrace( cout << ".got ACK msg:" << internals->msgId << endl; ) + } else if( internals->fragmentNo == -32767 ) { + op = MISSING; + ptrace( cout << ".got MISSING" << endl; ) + } else if( internals->fragmentNo == -32766 ) { + op = RESET; + } else { + op = REQUESTACK; + internals->fragmentNo = -(internals->fragmentNo+1); + ptrace( cout << ".got REQUESTACK frag:" << internals->fragmentNo << " msg:" << internals->msgId << endl; ) + } + } +} +inline F::~F() { free(internals); internals=0; } +inline int F::__num() { return internals->fragmentNo; } +inline int F::__len() { return internals->fragmentLen; } +inline MSGID F::__msgid() { return internals->msgId; } +inline int F::__channel() { return internals->channel; } +inline bool F::__isREQUESTACK() { return op == REQUESTACK; } +inline bool F::__isACK() { return op == ACK; } +inline bool F::__isMISSING() { return op == MISSING; } +inline short* F::__getMissing(int& n) { + n = internals->fragmentDataLen() / 2; + return (short *) internals->fragmentData(); +} +inline int F::__firstFragMsgLen() { + return internals->startOfMsgData()->len; +} diff --git a/grid/protorecv.cpp b/grid/protorecv.cpp index 0639939ff08..49200d8ab56 100644 --- a/grid/protorecv.cpp +++ b/grid/protorecv.cpp @@ -1,386 +1,386 @@ -// protorecv.cpp
-
-#include "stdafx.h"
-#include "protocol.h"
-#include "boost/thread.hpp"
-#include "../util/goodies.h"
-#include "../util/sock.h"
-#include "protoimpl.h"
-#include "../db/introspect.h"
-
-boost::mutex biglock;
-boost::mutex coutmutex;
-boost::mutex threadStarterMutex;
-boost::condition threadActivate; // creating a new thread, grabbing threadUseThisOne
-ProtocolConnection *threadUseThisOne = 0;
-void receiverThread();
-
-map<SockAddr,ProtocolConnection*> firstPCForThisAddress;
-
-/* todo: eventually support multiple listeners on diff ports. not
- bothering yet.
-*/
-ProtocolConnection *any = 0;
-
-EndPointToPC pcMap;
-
-class GeneralInspector : public SingleResultObjCursor {
- Cursor* clone() { return new GeneralInspector(*this); }
- void fill() {
- b.append("version", "1.0.0.1");
- b.append("versionDesc", "none");
- b.append("nConnections", pcMap.size());
- }
-public:
- GeneralInspector() { reg("intr.general"); }
-} _geninspectorproto;
-
-#include <signal.h>
-
-/* our version of netstat */
-void sighandler(int x) {
- cout << "ProtocolConnections:" << endl;
- lock lk(biglock);
- EndPointToPC::iterator it = pcMap.begin();
- while( it != pcMap.end() ) {
- cout << " conn " << it->second->toString() << endl;
- it++;
- }
- cout << "any: ";
- if( any ) cout << any->toString();
- cout << "\ndone" << endl;
-}
-
-struct SetSignal {
- SetSignal() {
-#if !defined(_WIN32)
- signal(SIGUSR2, sighandler);
-#endif
- }
-} setSignal;
-
-string ProtocolConnection::toString() {
- stringstream out;
- out << myEnd.toString() << " <> " <<
- farEnd.toString() << " rcvd.size:" << cr.received.size() <<
- " pndngMsgs.size:" << cr.pendingMessages.size() <<
- " pndngSnd.size:" << cs.pendingSend.size();
- return out.str();
-}
-
-ProtocolConnection::~ProtocolConnection() {
- cout << ".warning: ~ProtocolConnection() not implemented (leaks mem etc)" << endl;
- if( any == this )
- any = 0;
-}
-
-void ProtocolConnection::shutdown() {
- ptrace( cout << ".shutdown()" << endl; )
- if( acceptAnyChannel() || first )
- return;
- ptrace( cout << ". to:" << to.toString() << endl; )
- __sendRESET(this);
-}
-
-inline ProtocolConnection::ProtocolConnection(ProtocolConnection& par, EndPoint& _to) :
- udpConnection(par.udpConnection), myEnd(par.myEnd), cs(*this), cr(*this)
-{
- parent = ∥
- farEnd = _to;
- first = true;
- // todo: LOCK
-
- assert(pcMap.count(farEnd) == 0);
-// pcMap[myEnd] = this;
-}
-
-inline bool ProtocolConnection::acceptAnyChannel() const {
- return myEnd.channel == MessagingPort::ANYCHANNEL;
-}
-
-inline void ProtocolConnection::init() {
- first = true;
- lock lk(biglock);
- lock tslk(threadStarterMutex);
-
- if( acceptAnyChannel() ) {
- assert( any == 0 );
- any = this;
- }
- else {
- pcMap[farEnd] = this;
- }
-
- if( firstPCForThisAddress.count(myEnd.sa) == 0 ) {
- firstPCForThisAddress[myEnd.sa] = this;
- // need a receiver thread. one per port # we listen on. shared by channels.
- boost::thread receiver(receiverThread);
- threadUseThisOne = this;
- threadActivate.notify_one();
- return;
- }
-}
-
-ProtocolConnection::ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd) :
- udpConnection(c), myEnd(e), cs(*this), cr(*this)
-{
- parent = this;
- if( _farEnd ) {
- farEnd.channel = myEnd.channel;
- farEnd.sa = *_farEnd;
- }
- init();
-}
-
-/* find message for fragment */
-MR* CR::getPendingMsg(F *fr) {
- MR *m;
- map<int,MR*>::iterator i = pendingMessages.find(fr->__msgid());
- if( i == pendingMessages.end() ) {
- if( pendingMessages.size() > 20 ) {
- cout << ".warning: pendingMessages.size()>20, ignoring msg until we dequeue" << endl;
- return 0;
- }
- m = new MR(&pc, fr->__msgid(), pc.farEnd);
- pendingMessages[fr->__msgid()] = m;
- }
- else
- m = i->second;
- return m;
-/*
- MR*& m = pendingMessages[fr->__msgid()];
- if( m == 0 )
- m = new MR(&pc, fr->__msgid(), fromAddr);
- return m;
-*/
-}
-
-void MR::removeFromReceivingList() {
- pc.cr.pendingMessages.erase(msgid);
-}
-
-MR::MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from) :
- pc(*_pc), msgid(_msgid), from(_from), nReceived(0)
-{
- messageLenExpected = nExpected = -1;
- f.push_back(0);
-}
-
-void MR::reportMissings(bool reportAll) {
- vector<short> missing;
- unsigned t = curTimeMillis();
- for( unsigned i = 0; i < f.size(); i++ ) {
- if( f[i] == 0 ) {
- int diff = tdiff(reportTimes[i],t);
- assert( diff >= 0 || reportTimes[i] == 0 );
- if( diff > 100 || reportTimes[i]==0 ||
- (reportAll && diff > 1) ) {
- reportTimes[i] = t;
- assert( i < 25000 );
- missing.push_back(i);
- }
- }
- }
- if( !missing.empty() )
- __sendMISSING(&pc, from, msgid, missing);
-}
-
-inline bool MR::complete() {
- if( nReceived == nExpected ) {
- assert( nExpected == (int) f.size() );
- assert( f[0] != 0 );
- return true;
- }
- return false;
-/*
- if( nExpected > (int) f.size() || nExpected == -1 )
- return false;
- for( unsigned i = 0; i < f.size(); i++ )
- if( f[i] == 0 )
- return false;
- return true;
-*/
-}
-
-/* true=msg complete */
-bool MR::got(F *frag, EndPoint& fromAddr) {
- MSGID msgid = frag->__msgid();
- int i = frag->__num();
- if( i == 544 && !frag->__isREQUESTACK() ) {
- cout << "************ GOT LAST FRAGMENT #544" << endl;
- }
- if( nExpected < 0 && i == 0 ) {
- messageLenExpected = frag->__firstFragMsgLen();
- if( messageLenExpected == frag->__len()-FragHeader )
- nExpected = 1;
- else {
- int mss = frag->__len()-FragHeader;
- assert( messageLenExpected > mss );
- nExpected = (messageLenExpected + mss - 1) / mss;
- ptrace( cout << ".got first frag, expect:" << nExpected << "packets, expectedLen:" << messageLenExpected << endl; )
- }
- }
- if( i >= (int) f.size() )
- f.resize(i+1, 0);
- if( frag->__isREQUESTACK() ) {
- ptrace( cout << "...got(): processing a REQUESTACK" << endl; )
- /* we're simply seeing if we got the data, and then acking. */
- delete frag;
- frag = 0;
- reportTimes.resize(f.size(), 0);
- if( complete() ) {
- __sendACK(&pc, msgid);
- return true;
- }
- reportMissings(true);
- return false;
- }
- else if( f[i] != 0 ) {
- ptrace( cout << "\t.dup packet i:" << i << ' ' << from.toString() << endl; )
- delete frag;
- return false;
- }
- if( frag ) {
- f[i] = frag;
- nReceived++;
- }
- reportTimes.resize(f.size(), 0);
-
- if( !complete() )
- return false;
- __sendACK(&pc, msgid);
- return true;
-
-/*
- if( f[0] == 0 || f[i] == 0 ) {
-// reportMissings(frag == 0);
- return false;
- }
- if( i+1 < nExpected ) {
-//cout << "TEMP COMMENT" << endl;
-// if( i > 0 && f[i-1] == 0 )
-// reportMissings(frag == 0);
- return false;
- }
- // last fragment received
- if( !complete() ) {
-// reportMissings(frag == 0);
- return false;
- }
- __sendACK(&pc, fromAddr, msgid);
- return frag != 0;
-*/
-}
-
-MR* CR::recv() {
- MR *x;
- {
- lock lk(biglock);
- while( received.empty() )
- receivedSome.wait(lk);
- x = received.back();
- received.pop_back();
- }
- return x;
-}
-
-// this is how we tell protosend.cpp we got these
-void gotACK(F*, ProtocolConnection *);
-void gotMISSING(F*, ProtocolConnection *);
-
-void receiverThread() {
- ProtocolConnection *startingConn; // this thread manages many; this is just initiator or the parent for acceptany
- UDPConnection *uc;
- {
- lock lk(threadStarterMutex);
- while( 1 ) {
- if( threadUseThisOne != 0 ) {
- uc = &threadUseThisOne->udpConnection;
- startingConn = threadUseThisOne;
- threadUseThisOne = 0;
- break;
- }
- threadActivate.wait(lk);
- }
- }
-
- cout << "\n.Activating a new receiverThread\n " << startingConn->toString() << '\n' << endl;
-
- EndPoint fromAddr;
- while( 1 ) {
- F *f = __recv(*uc, fromAddr.sa);
- lock l(biglock);
- fromAddr.channel = f->__channel();
- ptrace( cout << "..__recv() from:" << fromAddr.toString() << " frag:" << f->__num() << endl; )
- assert( fromAddr.channel >= 0 );
- EndPointToPC::iterator it = pcMap.find(fromAddr);
- ProtocolConnection *mypc;
- if( it == pcMap.end() ) {
- if( !startingConn->acceptAnyChannel() ) {
- cout << ".WARNING: got packet from an unknown endpoint:" << fromAddr.toString() << endl;
- cout << ". this may be ok if you just restarted" << endl;
- delete f;
- continue;
- }
- cout << ".New connection accepted from " << fromAddr.toString() << endl;
- mypc = new ProtocolConnection(*startingConn, fromAddr);
- pcMap[fromAddr] = mypc;
- }
- else
- mypc = it->second;
-
- assert( fromAddr.channel == mypc->farEnd.channel );
- MsgTracker& track = mypc->cr.oldMsgTracker;
-
- if( f->op != F::NORMAL ) {
- if( f->__isACK() ) {
- gotACK(f, mypc);
- delete f;
- continue;
- }
- if( f->__isMISSING() ) {
- gotMISSING(f, mypc);
- delete f;
- continue;
- }
- if( f->op == F::RESET ) {
- ptrace( cout << ".got RESET" << endl; )
- track.reset();
- mypc->cs.resetIt();
- delete f;
- continue;
- }
- }
-
- if( track.recentlyReceived.count(f->__msgid()) ) {
- // already done with it. ignore, other than acking.
- if( f->__isREQUESTACK() )
- __sendACK(mypc, f->__msgid());
- else {
- ptrace( cout << ".ignoring packet about msg already received msg:" << f->__msgid() << " op:" << f->op << endl; )
- }
- delete f;
- continue;
- }
-
- if( f->__msgid() <= track.lastFullyReceived && !track.recentlyReceivedList.empty() ) {
- // reconnect on an old channel?
- ptrace( cout << ".warning: strange msgid:" << f->__msgid() << " received, last:" << track->lastFullyReceived << " conn:" << fromAddr.toString() << endl; )
- }
-
- MR *m = mypc->cr.getPendingMsg(f); /* todo: optimize for single fragment case? */
- if( m == 0 ) {
- ptrace( cout << "..getPendingMsg() returns 0" << endl; )
- delete f;
- continue;
- }
- if( m->got(f, fromAddr) ) {
- track.got(m->msgid);
- m->removeFromReceivingList();
- {
- mypc->parent->cr.received.push_back(m);
- mypc->parent->cr.receivedSome.notify_one();
- }
- }
- }
-}
+// protorecv.cpp + +#include "stdafx.h" +#include "protocol.h" +#include "boost/thread.hpp" +#include "../util/goodies.h" +#include "../util/sock.h" +#include "protoimpl.h" +#include "../db/introspect.h" + +boost::mutex biglock; +boost::mutex coutmutex; +boost::mutex threadStarterMutex; +boost::condition threadActivate; // creating a new thread, grabbing threadUseThisOne +ProtocolConnection *threadUseThisOne = 0; +void receiverThread(); + +map<SockAddr,ProtocolConnection*> firstPCForThisAddress; + +/* todo: eventually support multiple listeners on diff ports. not + bothering yet. +*/ +ProtocolConnection *any = 0; + +EndPointToPC pcMap; + +class GeneralInspector : public SingleResultObjCursor { + Cursor* clone() { return new GeneralInspector(*this); } + void fill() { + b.append("version", "1.0.0.1"); + b.append("versionDesc", "none"); + b.append("nConnections", pcMap.size()); + } +public: + GeneralInspector() { reg("intr.general"); } +} _geninspectorproto; + +#include <signal.h> + +/* our version of netstat */ +void sighandler(int x) { + cout << "ProtocolConnections:" << endl; + lock lk(biglock); + EndPointToPC::iterator it = pcMap.begin(); + while( it != pcMap.end() ) { + cout << " conn " << it->second->toString() << endl; + it++; + } + cout << "any: "; + if( any ) cout << any->toString(); + cout << "\ndone" << endl; +} + +struct SetSignal { + SetSignal() { +#if !defined(_WIN32) + signal(SIGUSR2, sighandler); +#endif + } +} setSignal; + +string ProtocolConnection::toString() { + stringstream out; + out << myEnd.toString() << " <> " << + farEnd.toString() << " rcvd.size:" << cr.received.size() << + " pndngMsgs.size:" << cr.pendingMessages.size() << + " pndngSnd.size:" << cs.pendingSend.size(); + return out.str(); +} + +ProtocolConnection::~ProtocolConnection() { + cout << ".warning: ~ProtocolConnection() not implemented (leaks mem etc)" << endl; + if( any == this ) + any = 0; +} + +void ProtocolConnection::shutdown() { + ptrace( cout << ".shutdown()" << endl; ) + if( acceptAnyChannel() || first ) + return; + ptrace( cout << ". to:" << to.toString() << endl; ) + __sendRESET(this); +} + +inline ProtocolConnection::ProtocolConnection(ProtocolConnection& par, EndPoint& _to) : + udpConnection(par.udpConnection), myEnd(par.myEnd), cs(*this), cr(*this) +{ + parent = ∥ + farEnd = _to; + first = true; + // todo: LOCK + + assert(pcMap.count(farEnd) == 0); +// pcMap[myEnd] = this; +} + +inline bool ProtocolConnection::acceptAnyChannel() const { + return myEnd.channel == MessagingPort::ANYCHANNEL; +} + +inline void ProtocolConnection::init() { + first = true; + lock lk(biglock); + lock tslk(threadStarterMutex); + + if( acceptAnyChannel() ) { + assert( any == 0 ); + any = this; + } + else { + pcMap[farEnd] = this; + } + + if( firstPCForThisAddress.count(myEnd.sa) == 0 ) { + firstPCForThisAddress[myEnd.sa] = this; + // need a receiver thread. one per port # we listen on. shared by channels. + boost::thread receiver(receiverThread); + threadUseThisOne = this; + threadActivate.notify_one(); + return; + } +} + +ProtocolConnection::ProtocolConnection(UDPConnection& c, EndPoint& e, SockAddr *_farEnd) : + udpConnection(c), myEnd(e), cs(*this), cr(*this) +{ + parent = this; + if( _farEnd ) { + farEnd.channel = myEnd.channel; + farEnd.sa = *_farEnd; + } + init(); +} + +/* find message for fragment */ +MR* CR::getPendingMsg(F *fr) { + MR *m; + map<int,MR*>::iterator i = pendingMessages.find(fr->__msgid()); + if( i == pendingMessages.end() ) { + if( pendingMessages.size() > 20 ) { + cout << ".warning: pendingMessages.size()>20, ignoring msg until we dequeue" << endl; + return 0; + } + m = new MR(&pc, fr->__msgid(), pc.farEnd); + pendingMessages[fr->__msgid()] = m; + } + else + m = i->second; + return m; +/* + MR*& m = pendingMessages[fr->__msgid()]; + if( m == 0 ) + m = new MR(&pc, fr->__msgid(), fromAddr); + return m; +*/ +} + +void MR::removeFromReceivingList() { + pc.cr.pendingMessages.erase(msgid); +} + +MR::MR(ProtocolConnection *_pc, MSGID _msgid, EndPoint& _from) : + pc(*_pc), msgid(_msgid), from(_from), nReceived(0) +{ + messageLenExpected = nExpected = -1; + f.push_back(0); +} + +void MR::reportMissings(bool reportAll) { + vector<short> missing; + unsigned t = curTimeMillis(); + for( unsigned i = 0; i < f.size(); i++ ) { + if( f[i] == 0 ) { + int diff = tdiff(reportTimes[i],t); + assert( diff >= 0 || reportTimes[i] == 0 ); + if( diff > 100 || reportTimes[i]==0 || + (reportAll && diff > 1) ) { + reportTimes[i] = t; + assert( i < 25000 ); + missing.push_back(i); + } + } + } + if( !missing.empty() ) + __sendMISSING(&pc, from, msgid, missing); +} + +inline bool MR::complete() { + if( nReceived == nExpected ) { + assert( nExpected == (int) f.size() ); + assert( f[0] != 0 ); + return true; + } + return false; +/* + if( nExpected > (int) f.size() || nExpected == -1 ) + return false; + for( unsigned i = 0; i < f.size(); i++ ) + if( f[i] == 0 ) + return false; + return true; +*/ +} + +/* true=msg complete */ +bool MR::got(F *frag, EndPoint& fromAddr) { + MSGID msgid = frag->__msgid(); + int i = frag->__num(); + if( i == 544 && !frag->__isREQUESTACK() ) { + cout << "************ GOT LAST FRAGMENT #544" << endl; + } + if( nExpected < 0 && i == 0 ) { + messageLenExpected = frag->__firstFragMsgLen(); + if( messageLenExpected == frag->__len()-FragHeader ) + nExpected = 1; + else { + int mss = frag->__len()-FragHeader; + assert( messageLenExpected > mss ); + nExpected = (messageLenExpected + mss - 1) / mss; + ptrace( cout << ".got first frag, expect:" << nExpected << "packets, expectedLen:" << messageLenExpected << endl; ) + } + } + if( i >= (int) f.size() ) + f.resize(i+1, 0); + if( frag->__isREQUESTACK() ) { + ptrace( cout << "...got(): processing a REQUESTACK" << endl; ) + /* we're simply seeing if we got the data, and then acking. */ + delete frag; + frag = 0; + reportTimes.resize(f.size(), 0); + if( complete() ) { + __sendACK(&pc, msgid); + return true; + } + reportMissings(true); + return false; + } + else if( f[i] != 0 ) { + ptrace( cout << "\t.dup packet i:" << i << ' ' << from.toString() << endl; ) + delete frag; + return false; + } + if( frag ) { + f[i] = frag; + nReceived++; + } + reportTimes.resize(f.size(), 0); + + if( !complete() ) + return false; + __sendACK(&pc, msgid); + return true; + +/* + if( f[0] == 0 || f[i] == 0 ) { +// reportMissings(frag == 0); + return false; + } + if( i+1 < nExpected ) { +//cout << "TEMP COMMENT" << endl; +// if( i > 0 && f[i-1] == 0 ) +// reportMissings(frag == 0); + return false; + } + // last fragment received + if( !complete() ) { +// reportMissings(frag == 0); + return false; + } + __sendACK(&pc, fromAddr, msgid); + return frag != 0; +*/ +} + +MR* CR::recv() { + MR *x; + { + lock lk(biglock); + while( received.empty() ) + receivedSome.wait(lk); + x = received.back(); + received.pop_back(); + } + return x; +} + +// this is how we tell protosend.cpp we got these +void gotACK(F*, ProtocolConnection *); +void gotMISSING(F*, ProtocolConnection *); + +void receiverThread() { + ProtocolConnection *startingConn; // this thread manages many; this is just initiator or the parent for acceptany + UDPConnection *uc; + { + lock lk(threadStarterMutex); + while( 1 ) { + if( threadUseThisOne != 0 ) { + uc = &threadUseThisOne->udpConnection; + startingConn = threadUseThisOne; + threadUseThisOne = 0; + break; + } + threadActivate.wait(lk); + } + } + + cout << "\n.Activating a new receiverThread\n " << startingConn->toString() << '\n' << endl; + + EndPoint fromAddr; + while( 1 ) { + F *f = __recv(*uc, fromAddr.sa); + lock l(biglock); + fromAddr.channel = f->__channel(); + ptrace( cout << "..__recv() from:" << fromAddr.toString() << " frag:" << f->__num() << endl; ) + assert( fromAddr.channel >= 0 ); + EndPointToPC::iterator it = pcMap.find(fromAddr); + ProtocolConnection *mypc; + if( it == pcMap.end() ) { + if( !startingConn->acceptAnyChannel() ) { + cout << ".WARNING: got packet from an unknown endpoint:" << fromAddr.toString() << endl; + cout << ". this may be ok if you just restarted" << endl; + delete f; + continue; + } + cout << ".New connection accepted from " << fromAddr.toString() << endl; + mypc = new ProtocolConnection(*startingConn, fromAddr); + pcMap[fromAddr] = mypc; + } + else + mypc = it->second; + + assert( fromAddr.channel == mypc->farEnd.channel ); + MsgTracker& track = mypc->cr.oldMsgTracker; + + if( f->op != F::NORMAL ) { + if( f->__isACK() ) { + gotACK(f, mypc); + delete f; + continue; + } + if( f->__isMISSING() ) { + gotMISSING(f, mypc); + delete f; + continue; + } + if( f->op == F::RESET ) { + ptrace( cout << ".got RESET" << endl; ) + track.reset(); + mypc->cs.resetIt(); + delete f; + continue; + } + } + + if( track.recentlyReceived.count(f->__msgid()) ) { + // already done with it. ignore, other than acking. + if( f->__isREQUESTACK() ) + __sendACK(mypc, f->__msgid()); + else { + ptrace( cout << ".ignoring packet about msg already received msg:" << f->__msgid() << " op:" << f->op << endl; ) + } + delete f; + continue; + } + + if( f->__msgid() <= track.lastFullyReceived && !track.recentlyReceivedList.empty() ) { + // reconnect on an old channel? + ptrace( cout << ".warning: strange msgid:" << f->__msgid() << " received, last:" << track->lastFullyReceived << " conn:" << fromAddr.toString() << endl; ) + } + + MR *m = mypc->cr.getPendingMsg(f); /* todo: optimize for single fragment case? */ + if( m == 0 ) { + ptrace( cout << "..getPendingMsg() returns 0" << endl; ) + delete f; + continue; + } + if( m->got(f, fromAddr) ) { + track.got(m->msgid); + m->removeFromReceivingList(); + { + mypc->parent->cr.received.push_back(m); + mypc->parent->cr.receivedSome.notify_one(); + } + } + } +} diff --git a/grid/protosend.cpp b/grid/protosend.cpp index 2d22450ced1..dfd31036127 100644 --- a/grid/protosend.cpp +++ b/grid/protosend.cpp @@ -1,188 +1,188 @@ -// protosend.cpp
-
-/* todo: redo locking! */
-
-#include "stdafx.h"
-#include "protocol.h"
-#include "boost/thread.hpp"
-#include "../util/goodies.h"
-#include "../util/sock.h"
-#include "protoimpl.h"
-using namespace boost;
-typedef boost::mutex::scoped_lock lock;
-
-/* todo: granularity? */
-extern boost::mutex biglock;
-
-void senderComplainThread();
-
-boost::thread sender(senderComplainThread);
-
-bool erasePending(ProtocolConnection *pc, int id);
-
-inline bool MS::complain(unsigned now) {
- if( tdiff(lastComplainTime, now) < (int) complainInterval )
- return false;
- if( complainInterval > 4000 ) {
- etrace( cout << ".no ack of send after " << complainInterval/1000 << " seconds " << to.toString() << endl; )
- if( complainInterval > 35000 ) {
- etrace( cout << ".GIVING UP on sending message" << endl; )
- erasePending(pc, msgid);
- return true;
- }
- }
- complainInterval *= 2;
- lastComplainTime = now;
- __sendREQUESTACK(pc, to, msgid, fragments.size()-1);
- return false;
-}
-
-// where's my ack?
-void senderComplainThread() {
- sleepmillis(200);
-
- while( 1 ) {
- sleepmillis(50);
- {
- lock lk(biglock);
- unsigned now = curTimeMillis();
- /* todo: more efficient data structure here. */
- for( EndPointToPC::iterator i = pcMap.begin(); i != pcMap.end(); i++ ) {
- ProtocolConnection *pc = i->second;
- for( vector<MS*>::iterator j = pc->cs.pendingSend.begin(); j < pc->cs.pendingSend.end(); j++ )
- if( (*j)->complain(now) )
- break;
- }
- }
- }
-}
-
-inline MS* _slocked_getMSForFrag(F* fr, ProtocolConnection *pc) {
- int id = fr->__msgid();
- vector<MS*>::iterator it;
- for( it = pc->cs.pendingSend.begin(); it<pc->cs.pendingSend.end(); it++ ) {
- MS *m = *it;
- if( m->msgid == id )
- return m;
- }
- return 0;
-}
-
-// received a MISSING message from the other end. retransmit.
-void gotMISSING(F* fr, ProtocolConnection *pc) {
- pc->cs.delayGotMissing();
-
- ptrace( cout << ".gotMISSING() msgid:" << fr->__msgid() << endl; )
- MS *m = _slocked_getMSForFrag(fr, pc);
- if( m ) {
- {
- int df = tdiff(m->lastComplainTime, curTimeMillis()) * 2;
- if( df < 10 )
- df = 10;
- if( df > 2000 )
- df = 2000;
- m->complainInterval = (unsigned) df;
- cout << "TEMP: set complainInterval to " << m->complainInterval << endl;
- }
-
- int n;
- short* s = fr->__getMissing(n);
- assert( n > 0 && n < 5000 );
- for( int i = 0; i < n; i++ ) {
- // ptrace( cout << ".. resending frag #" << s[i] << ' ' << m->to.toString() << endl; )
- __sendFrag(pc, m->to, m->fragments[s[i]], true);
- if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {
- ptrace( cout << "SLEEP" << endl; )
- sleepmillis((int) pc->cs.delay);
- }
- }
- return;
- }
- ptrace( cout << "\t.warning: gotMISSING for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; )
-}
-
-/* done sending a msg, clean up and notify sender to unblock */
-bool erasePending(ProtocolConnection *pc, int id) {
- vector<MS*>::iterator it;
- CS& cs = pc->cs;
- for( it = cs.pendingSend.begin(); it < cs.pendingSend.end(); it++ ) {
- MS *m = *it;
- if( m->msgid == id ) {
- cs.pendingSend.erase(it);
- ptrace( cout << "..gotACK/erase: found pendingSend msg:" << id << endl; )
- delete m;
- cs.msgSent.notify_one();
- return true;
- }
- }
- return false;
-}
-
-// received an ACK message. so we can discard our saved copy we were trying to send, we
-// are done with it.
-void gotACK(F* fr, ProtocolConnection *pc) {
- if( erasePending(pc, fr->__msgid()) )
- return;
- ptrace( cout << ".warning: got ack for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; )
-}
-
-void MS::send() {
- /* flow control */
- lock lk(biglock);
- ptrace( cout << "..MS::send() pending=" << pc->cs.pendingSend.size() << endl; )
-
- if( pc->acceptAnyChannel() ) {
- EndPointToPC::iterator it = pcMap.find(to);
- if( it == pcMap.end() ) {
- cout << ".ERROR: can't find ProtocolConnection object for " << to.toString() << endl;
- assert(false);
- return;
- }
- /* switch to the child protocolconnection */
- pc = it->second;
- }
- else {
- assert(pc->myEnd.channel == to.channel);
- }
-
- if( pc->first ) {
- pc->first = false;
- if( pc->myEnd.channel >= 0 )
- __sendRESET(pc);
- assert( pc->farEnd.channel == to.channel);
- if( pc->farEnd.sa.isLocalHost() )
- pc->cs.delayMax = 1.0;
- }
-
- // not locked here, probably ok to call size()
- if( pc->cs.pendingSend.size() >= 1 ) {
- cout << ".waiting for queued sends to complete " << pc->cs.pendingSend.size() << endl;
- while( pc->cs.pendingSend.size() >= 1 )
- pc->cs.msgSent.wait(lk);
- cout << ".waitend" << endl;
- }
-
- lastComplainTime = curTimeMillis();
- pc->cs.pendingSend.push_back(this);
- /* todo: pace */
- for( unsigned i = 0; i < fragments.size(); i++ ) {
- __sendFrag(pc, to, fragments[i]);
- if( i % 64 == 0 && pc->cs.delay >= 1.0 ) {
- ptrace( cout << ".sleep " << pc->cs.delay << endl; )
- sleepmillis((int) pc->cs.delay);
- }
- }
-
- pc->cs.delaySentMsg();
-}
-
-CS::CS(ProtocolConnection& _pc) :
- pc(_pc), delay(0), delayMax(10)
-{
-}
-
-void CS::resetIt() {
- for( vector<MS*>::iterator i = pendingSend.begin(); i < pendingSend.end(); i++ )
- delete (*i);
- pendingSend.clear();
-}
+// protosend.cpp + +/* todo: redo locking! */ + +#include "stdafx.h" +#include "protocol.h" +#include "boost/thread.hpp" +#include "../util/goodies.h" +#include "../util/sock.h" +#include "protoimpl.h" +using namespace boost; +typedef boost::mutex::scoped_lock lock; + +/* todo: granularity? */ +extern boost::mutex biglock; + +void senderComplainThread(); + +boost::thread sender(senderComplainThread); + +bool erasePending(ProtocolConnection *pc, int id); + +inline bool MS::complain(unsigned now) { + if( tdiff(lastComplainTime, now) < (int) complainInterval ) + return false; + if( complainInterval > 4000 ) { + etrace( cout << ".no ack of send after " << complainInterval/1000 << " seconds " << to.toString() << endl; ) + if( complainInterval > 35000 ) { + etrace( cout << ".GIVING UP on sending message" << endl; ) + erasePending(pc, msgid); + return true; + } + } + complainInterval *= 2; + lastComplainTime = now; + __sendREQUESTACK(pc, to, msgid, fragments.size()-1); + return false; +} + +// where's my ack? +void senderComplainThread() { + sleepmillis(200); + + while( 1 ) { + sleepmillis(50); + { + lock lk(biglock); + unsigned now = curTimeMillis(); + /* todo: more efficient data structure here. */ + for( EndPointToPC::iterator i = pcMap.begin(); i != pcMap.end(); i++ ) { + ProtocolConnection *pc = i->second; + for( vector<MS*>::iterator j = pc->cs.pendingSend.begin(); j < pc->cs.pendingSend.end(); j++ ) + if( (*j)->complain(now) ) + break; + } + } + } +} + +inline MS* _slocked_getMSForFrag(F* fr, ProtocolConnection *pc) { + int id = fr->__msgid(); + vector<MS*>::iterator it; + for( it = pc->cs.pendingSend.begin(); it<pc->cs.pendingSend.end(); it++ ) { + MS *m = *it; + if( m->msgid == id ) + return m; + } + return 0; +} + +// received a MISSING message from the other end. retransmit. +void gotMISSING(F* fr, ProtocolConnection *pc) { + pc->cs.delayGotMissing(); + + ptrace( cout << ".gotMISSING() msgid:" << fr->__msgid() << endl; ) + MS *m = _slocked_getMSForFrag(fr, pc); + if( m ) { + { + int df = tdiff(m->lastComplainTime, curTimeMillis()) * 2; + if( df < 10 ) + df = 10; + if( df > 2000 ) + df = 2000; + m->complainInterval = (unsigned) df; + cout << "TEMP: set complainInterval to " << m->complainInterval << endl; + } + + int n; + short* s = fr->__getMissing(n); + assert( n > 0 && n < 5000 ); + for( int i = 0; i < n; i++ ) { + // ptrace( cout << ".. resending frag #" << s[i] << ' ' << m->to.toString() << endl; ) + __sendFrag(pc, m->to, m->fragments[s[i]], true); + if( i % 64 == 0 && pc->cs.delay >= 1.0 ) { + ptrace( cout << "SLEEP" << endl; ) + sleepmillis((int) pc->cs.delay); + } + } + return; + } + ptrace( cout << "\t.warning: gotMISSING for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; ) +} + +/* done sending a msg, clean up and notify sender to unblock */ +bool erasePending(ProtocolConnection *pc, int id) { + vector<MS*>::iterator it; + CS& cs = pc->cs; + for( it = cs.pendingSend.begin(); it < cs.pendingSend.end(); it++ ) { + MS *m = *it; + if( m->msgid == id ) { + cs.pendingSend.erase(it); + ptrace( cout << "..gotACK/erase: found pendingSend msg:" << id << endl; ) + delete m; + cs.msgSent.notify_one(); + return true; + } + } + return false; +} + +// received an ACK message. so we can discard our saved copy we were trying to send, we +// are done with it. +void gotACK(F* fr, ProtocolConnection *pc) { + if( erasePending(pc, fr->__msgid()) ) + return; + ptrace( cout << ".warning: got ack for an unknown msg id:" << fr->__msgid() << ' ' << pc->farEnd.toString() << endl; ) +} + +void MS::send() { + /* flow control */ + lock lk(biglock); + ptrace( cout << "..MS::send() pending=" << pc->cs.pendingSend.size() << endl; ) + + if( pc->acceptAnyChannel() ) { + EndPointToPC::iterator it = pcMap.find(to); + if( it == pcMap.end() ) { + cout << ".ERROR: can't find ProtocolConnection object for " << to.toString() << endl; + assert(false); + return; + } + /* switch to the child protocolconnection */ + pc = it->second; + } + else { + assert(pc->myEnd.channel == to.channel); + } + + if( pc->first ) { + pc->first = false; + if( pc->myEnd.channel >= 0 ) + __sendRESET(pc); + assert( pc->farEnd.channel == to.channel); + if( pc->farEnd.sa.isLocalHost() ) + pc->cs.delayMax = 1.0; + } + + // not locked here, probably ok to call size() + if( pc->cs.pendingSend.size() >= 1 ) { + cout << ".waiting for queued sends to complete " << pc->cs.pendingSend.size() << endl; + while( pc->cs.pendingSend.size() >= 1 ) + pc->cs.msgSent.wait(lk); + cout << ".waitend" << endl; + } + + lastComplainTime = curTimeMillis(); + pc->cs.pendingSend.push_back(this); + /* todo: pace */ + for( unsigned i = 0; i < fragments.size(); i++ ) { + __sendFrag(pc, to, fragments[i]); + if( i % 64 == 0 && pc->cs.delay >= 1.0 ) { + ptrace( cout << ".sleep " << pc->cs.delay << endl; ) + sleepmillis((int) pc->cs.delay); + } + } + + pc->cs.delaySentMsg(); +} + +CS::CS(ProtocolConnection& _pc) : + pc(_pc), delay(0), delayMax(10) +{ +} + +void CS::resetIt() { + for( vector<MS*>::iterator i = pendingSend.begin(); i < pendingSend.end(); i++ ) + delete (*i); + pendingSend.clear(); +} diff --git a/stdafx.cpp b/stdafx.cpp index 0d4d4346506..c61f9d1ab85 100644 --- a/stdafx.cpp +++ b/stdafx.cpp @@ -1,33 +1,33 @@ -// stdafx.cpp : source file that includes just the standard includes
-// db.pch will be the pre-compiled header
-// stdafx.obj will contain the pre-compiled type information
-
-#include "stdafx.h"
-
-// TODO: reference any additional headers you need in STDAFX.H
-// and not in this file
-
-struct MyAsserts {
- MyAsserts() {
-
- }
-} myassertsstdafx;
-
-#undef assert
-
-#undef yassert
-#include "assert.h"
-
-void sayDbContext(const char *p = 0);
-
-void wasserted(const char *msg, const char *file, unsigned line) {
- problem() << "Assertion failure " << msg << ' ' << file << ' ' << line << endl;
- cout << "Assertion failure " << msg << endl;
- cout << ' ' << file << ' ' << line << endl;
- sayDbContext();
-}
-
-void asserted(const char *msg, const char *file, unsigned line) {
- wasserted(msg, file, line);
- throw AssertionException();
-}
+// stdafx.cpp : source file that includes just the standard includes +// db.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "stdafx.h" + +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file + +struct MyAsserts { + MyAsserts() { + + } +} myassertsstdafx; + +#undef assert + +#undef yassert +#include "assert.h" + +void sayDbContext(const char *p = 0); + +void wasserted(const char *msg, const char *file, unsigned line) { + problem() << "Assertion failure " << msg << ' ' << file << ' ' << line << endl; + cout << "Assertion failure " << msg << endl; + cout << ' ' << file << ' ' << line << endl; + sayDbContext(); +} + +void asserted(const char *msg, const char *file, unsigned line) { + wasserted(msg, file, line); + throw AssertionException(); +} @@ -1,111 +1,111 @@ -// stdafx.h : include file for standard system include files,
-// or project specific include files that are used frequently, but
-// are changed infrequently
-//
-
-#pragma once
-
-#if defined(_WIN32)
-const bool debug=true;
-#else
-const bool debug=false;
-#endif
-
-#include "targetver.h"
-
-//#include "assert.h"
-
-// you can catch these
-class AssertionException {
-public:
- AssertionException() { }
-};
-
-void asserted(const char *msg, const char *file, unsigned line);
-void wasserted(const char *msg, const char *file, unsigned line);
-
-#ifdef assert
-#undef assert
-#endif
-
-#define assert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )
-
-#define xassert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) )
-
-#define yassert 1
-
-/* warning only - keeps going */
-#define wassert(_Expression) (void)( (!!(_Expression)) || (wasserted(#_Expression, __FILE__, __LINE__), 0) )
-
-#include <stdio.h>
-#include <sstream>
-#include <signal.h>
-
-typedef char _TCHAR;
-
-#include <iostream>
-#include <fstream>
-using namespace std;
-
-#include "time.h"
-#include <map>
-#include <string>
-#include <vector>
-#include <set>
-
-#if !defined(_WIN32)
-typedef int HANDLE;
-inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); }
-#else
-typedef void *HANDLE;
-#endif
-
-//#if defined(CHAR)
-//#error CHAR already defined?
-//#endif
-
-//#if defined(_WIN32_WINNT)
-//typedef wchar_t CHAR;
-//#else
-// more to be done...linux unicode is 32 bit.
-//typedef unsigned short CHAR; // 16 bit unicode
-//#endif
-
-#define null (0)
-
-#include <vector>
-
-// for debugging
-typedef struct _Ints { int i[100]; } *Ints;
-typedef struct _Chars { char c[200]; } *Chars;
-
-typedef char CHARS[400];
-
-typedef struct _OWS {
- int size;
- char type;
- char string[400];
-} *OWS;
-
-extern ofstream problems;
-
-class Client;
-extern Client *client;
-extern const char *curNs;
-
-// not threadsafe
-inline ofstream& problem() {
- time_t t;
- time(&t);
- string now(ctime(&t),0,20);
- problems << now;
- if( client )
- problems << curNs << ' ';
- return problems;
-}
-
-#define DEBUGGING if( 0 )
-
-extern unsigned occasion;
-
-#define OCCASIONALLY if( ++occasion % 16 == 0 )
+// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#pragma once + +#if defined(_WIN32) +const bool debug=true; +#else +const bool debug=false; +#endif + +#include "targetver.h" + +//#include "assert.h" + +// you can catch these +class AssertionException { +public: + AssertionException() { } +}; + +void asserted(const char *msg, const char *file, unsigned line); +void wasserted(const char *msg, const char *file, unsigned line); + +#ifdef assert +#undef assert +#endif + +#define assert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) ) + +#define xassert(_Expression) (void)( (!!(_Expression)) || (asserted(#_Expression, __FILE__, __LINE__), 0) ) + +#define yassert 1 + +/* warning only - keeps going */ +#define wassert(_Expression) (void)( (!!(_Expression)) || (wasserted(#_Expression, __FILE__, __LINE__), 0) ) + +#include <stdio.h> +#include <sstream> +#include <signal.h> + +typedef char _TCHAR; + +#include <iostream> +#include <fstream> +using namespace std; + +#include "time.h" +#include <map> +#include <string> +#include <vector> +#include <set> + +#if !defined(_WIN32) +typedef int HANDLE; +inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); } +#else +typedef void *HANDLE; +#endif + +//#if defined(CHAR) +//#error CHAR already defined? +//#endif + +//#if defined(_WIN32_WINNT) +//typedef wchar_t CHAR; +//#else +// more to be done...linux unicode is 32 bit. +//typedef unsigned short CHAR; // 16 bit unicode +//#endif + +#define null (0) + +#include <vector> + +// for debugging +typedef struct _Ints { int i[100]; } *Ints; +typedef struct _Chars { char c[200]; } *Chars; + +typedef char CHARS[400]; + +typedef struct _OWS { + int size; + char type; + char string[400]; +} *OWS; + +extern ofstream problems; + +class Client; +extern Client *client; +extern const char *curNs; + +// not threadsafe +inline ofstream& problem() { + time_t t; + time(&t); + string now(ctime(&t),0,20); + problems << now; + if( client ) + problems << curNs << ' '; + return problems; +} + +#define DEBUGGING if( 0 ) + +extern unsigned occasion; + +#define OCCASIONALLY if( ++occasion % 16 == 0 ) diff --git a/targetver.h b/targetver.h index 3965af59bcd..11ff16bf5aa 100644 --- a/targetver.h +++ b/targetver.h @@ -1,5 +1,5 @@ -#pragma once
-#ifndef _WIN32_WINNT // Allow use of features specific to Windows Vista or later.
-#define _WIN32_WINNT 0x0600 // Change this to the appropriate value to target other versions of Windows.
-#endif
-
+#pragma once +#ifndef _WIN32_WINNT // Allow use of features specific to Windows Vista or later. +#define _WIN32_WINNT 0x0600 // Change this to the appropriate value to target other versions of Windows. +#endif + diff --git a/util/builder.h b/util/builder.h index ddc49ec2c91..67b8c38d87f 100644 --- a/util/builder.h +++ b/util/builder.h @@ -1,67 +1,67 @@ -/* builder.h
-
-*/
-
-#pragma once
-
-#include "../stdafx.h"
-
-class BufBuilder {
-public:
- BufBuilder(int initsize = 512) : size(initsize) {
- data = (char *) malloc(size); assert(data);
- l = 0;
- }
- ~BufBuilder() { kill(); }
-
- void kill() {
- if( data ) {
- free(data);
- data = 0;
- }
- }
-
- /* leave room for some stuff later */
- void skip(int n) { grow(n); }
-
- /* note this may be deallocated (realloced) if you keep writing. */
- char* buf() { return data; }
-
- /* assume ownership of the buffer - you must then free it */
- void decouple() { data = 0; }
-
- template<class T> void append(T j) { *((T*)grow(sizeof(T))) = j; }
- void append(short j) { append<short>(j); }
- void append(int j) { append<int>(j); }
- void append(unsigned j) { append<unsigned>(j); }
- void append(bool j) { append<bool>(j); }
- void append(double j) { append<double>(j); }
-
- void append(void *src, int len) { memcpy(grow(len), src, len); }
-
- void append(const char *str) {
- append((void*) str, strlen(str)+1);
- }
-
- int len() { return l; }
-
-private:
- /* returns the pre-grow write position */
- char* grow(int by) {
- int oldlen = l;
- l += by;
- if( l > size ) {
- int a = size * 2;
- if( l > a )
- a = l + 16 * 1024;
- assert( a < 64 * 1024 * 1024 );
- data = (char *) realloc(data, a);
- size= a;
- }
- return data + oldlen;
- }
-
- char *data;
- int l;
- int size;
-};
+/* builder.h + +*/ + +#pragma once + +#include "../stdafx.h" + +class BufBuilder { +public: + BufBuilder(int initsize = 512) : size(initsize) { + data = (char *) malloc(size); assert(data); + l = 0; + } + ~BufBuilder() { kill(); } + + void kill() { + if( data ) { + free(data); + data = 0; + } + } + + /* leave room for some stuff later */ + void skip(int n) { grow(n); } + + /* note this may be deallocated (realloced) if you keep writing. */ + char* buf() { return data; } + + /* assume ownership of the buffer - you must then free it */ + void decouple() { data = 0; } + + template<class T> void append(T j) { *((T*)grow(sizeof(T))) = j; } + void append(short j) { append<short>(j); } + void append(int j) { append<int>(j); } + void append(unsigned j) { append<unsigned>(j); } + void append(bool j) { append<bool>(j); } + void append(double j) { append<double>(j); } + + void append(void *src, int len) { memcpy(grow(len), src, len); } + + void append(const char *str) { + append((void*) str, strlen(str)+1); + } + + int len() { return l; } + +private: + /* returns the pre-grow write position */ + char* grow(int by) { + int oldlen = l; + l += by; + if( l > size ) { + int a = size * 2; + if( l > a ) + a = l + 16 * 1024; + assert( a < 64 * 1024 * 1024 ); + data = (char *) realloc(data, a); + size= a; + } + return data + oldlen; + } + + char *data; + int l; + int size; +}; diff --git a/util/goodies.h b/util/goodies.h index 79933a9a2b8..94ca721aa8b 100644 --- a/util/goodies.h +++ b/util/goodies.h @@ -1,128 +1,128 @@ -// goodies.h
-// miscellaneous junk
-
-#pragma once
-
-#include "../stdafx.h"
-
-#if !defined(_WIN32)
-#include <pthread.h>
-inline pthread_t GetCurrentThreadId() { return pthread_self(); }
-#endif
-
-/* set to TRUE if we are exiting */
-extern bool goingAway;
-
-bool isPrime(int n);
-int nextPrime(int n);
-
-inline void dumpmemory(const char *data, int len) {
- if( len > 1024 )
- len = 1024;
- try {
- const char *q = data;
- const char *p = q;
- while( len > 0 ) {
- for( int i = 0; i < 16; i++ ) {
- if( *p >= 32 && *p <= 126 )
- cout << *p;
- else
- cout << '.';
- p++;
- }
- cout << " ";
- p -= 16;
- for( int i = 0; i < 16; i++ )
- cout << (unsigned) ((unsigned char)*p++) << ' ';
- cout << endl;
- len -= 16;
- }
- } catch(...) {
- }
-}
-
-#undef yassert
-#include <boost/thread/thread.hpp>
-#include <boost/thread/xtime.hpp>
-#undef assert
-#define assert xassert
-#define yassert 1
-
-struct WrappingInt {
- WrappingInt() { x = 0; }
- WrappingInt(unsigned z) : x(z) { }
- unsigned x;
- operator unsigned() const { return x; }
- WrappingInt& operator++() { x++; return *this; }
- static int diff(unsigned a, unsigned b) { return a-b; }
- bool operator<=(WrappingInt r) {
- // platform dependent
- int df = (r.x - x);
- return df >= 0;
- }
- bool operator>(WrappingInt r) { return !(r<=*this); }
-};
-
-#include <ctime>
-
-inline void time_t_to_String(time_t t, char *buf) {
-#if defined(_WIN32)
- ctime_s(buf, 64, &t);
-#else
- ctime_r(&t, buf);
-#endif
- buf[24] = 0; // don't want the \n
-}
-
-inline void sleepsecs(int s) {
- boost::xtime xt;
- boost::xtime_get(&xt, boost::TIME_UTC);
- xt.sec += s;
- boost::thread::sleep(xt);
-}
-inline void sleepmillis(int s) {
- boost::xtime xt;
- boost::xtime_get(&xt, boost::TIME_UTC);
- xt.nsec += s * 1000000;
- boost::thread::sleep(xt);
-}
-// note this wraps
-inline int tdiff(unsigned told, unsigned tnew) {
- return WrappingInt::diff(tnew, told);
-}
-inline unsigned curTimeMillis() {
- boost::xtime xt;
- boost::xtime_get(&xt, boost::TIME_UTC);
- unsigned t = xt.nsec / 1000000;
- return (xt.sec & 0xfffff) * 1000 + t;
-}
-inline unsigned long long jsTime() {
- boost::xtime xt;
- boost::xtime_get(&xt, boost::TIME_UTC);
- unsigned long long t = xt.nsec / 1000000;
- return ((unsigned long long) xt.sec * 1000) + t;
-}
-// measures up to 1024 seconds. or, 512 seconds with tdiff that is...
-inline unsigned curTimeMicros() {
- boost::xtime xt;
- boost::xtime_get(&xt, boost::TIME_UTC);
- unsigned t = xt.nsec / 1000;
- unsigned secs = xt.sec % 1024;
- t = secs*1000000 + t;
- return t;
-}
-using namespace boost;
-typedef boost::mutex::scoped_lock lock;
-
-// simple scoped timer
-class Timer {
-public:
- Timer() { old = curTimeMicros(); }
- int millis() { return micros() / 1000; }
- int micros() {
- unsigned n = curTimeMicros();
- return tdiff(old, n);
- }
-private:
- unsigned old;
-};
+// goodies.h +// miscellaneous junk + +#pragma once + +#include "../stdafx.h" + +#if !defined(_WIN32) +#include <pthread.h> +inline pthread_t GetCurrentThreadId() { return pthread_self(); } +#endif + +/* set to TRUE if we are exiting */ +extern bool goingAway; + +bool isPrime(int n); +int nextPrime(int n); + +inline void dumpmemory(const char *data, int len) { + if( len > 1024 ) + len = 1024; + try { + const char *q = data; + const char *p = q; + while( len > 0 ) { + for( int i = 0; i < 16; i++ ) { + if( *p >= 32 && *p <= 126 ) + cout << *p; + else + cout << '.'; + p++; + } + cout << " "; + p -= 16; + for( int i = 0; i < 16; i++ ) + cout << (unsigned) ((unsigned char)*p++) << ' '; + cout << endl; + len -= 16; + } + } catch(...) { + } +} + +#undef yassert +#include <boost/thread/thread.hpp> +#include <boost/thread/xtime.hpp> +#undef assert +#define assert xassert +#define yassert 1 + +struct WrappingInt { + WrappingInt() { x = 0; } + WrappingInt(unsigned z) : x(z) { } + unsigned x; + operator unsigned() const { return x; } + WrappingInt& operator++() { x++; return *this; } + static int diff(unsigned a, unsigned b) { return a-b; } + bool operator<=(WrappingInt r) { + // platform dependent + int df = (r.x - x); + return df >= 0; + } + bool operator>(WrappingInt r) { return !(r<=*this); } +}; + +#include <ctime> + +inline void time_t_to_String(time_t t, char *buf) { +#if defined(_WIN32) + ctime_s(buf, 64, &t); +#else + ctime_r(&t, buf); +#endif + buf[24] = 0; // don't want the \n +} + +inline void sleepsecs(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += s; + boost::thread::sleep(xt); +} +inline void sleepmillis(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.nsec += s * 1000000; + boost::thread::sleep(xt); +} +// note this wraps +inline int tdiff(unsigned told, unsigned tnew) { + return WrappingInt::diff(tnew, told); +} +inline unsigned curTimeMillis() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000000; + return (xt.sec & 0xfffff) * 1000 + t; +} +inline unsigned long long jsTime() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned long long t = xt.nsec / 1000000; + return ((unsigned long long) xt.sec * 1000) + t; +} +// measures up to 1024 seconds. or, 512 seconds with tdiff that is... +inline unsigned curTimeMicros() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000; + unsigned secs = xt.sec % 1024; + t = secs*1000000 + t; + return t; +} +using namespace boost; +typedef boost::mutex::scoped_lock lock; + +// simple scoped timer +class Timer { +public: + Timer() { old = curTimeMicros(); } + int millis() { return micros() / 1000; } + int micros() { + unsigned n = curTimeMicros(); + return tdiff(old, n); + } +private: + unsigned old; +}; diff --git a/util/hashtab.h b/util/hashtab.h index 8b8b53b182e..9ea3c2342d5 100644 --- a/util/hashtab.h +++ b/util/hashtab.h @@ -1,110 +1,110 @@ -/* hashtab.h
-
- Simple, fixed size hash table. Darn simple.
-
- Uses a contiguous block of memory, so you can put it in a memory mapped file very easily.
-*/
-
-#pragma once
-
-#include "../stdafx.h"
-#include <map>
-
-#pragma pack(push)
-#pragma pack(1)
-
-/* you should define:
-
- int Key::hash() return > 0 always.
-*/
-
-template <
- class Key,
- class Type
->
-class HashTable {
-public:
- const char *name;
- struct Node {
- int hash;
- Key k;
- Type value;
- bool inUse() { return hash != 0; }
- } *nodes;
- int n;
-
- int _find(const Key& k, bool& found) {
- found = false;
- int h = k.hash();
- int i = h % n;
- int start = i;
- int chain = 0;
- while( 1 ) {
- if( !nodes[i].inUse() ) {
- return i;
- }
- if( nodes[i].hash == h && nodes[i].k == k ) {
- found = true;
- return i;
- }
- chain++;
- i = (i+1) % n;
- if( i == start ) {
- cout << "warning: hashtable is full " << name << endl;
- return -1;
- }
- if( chain == 200 )
- cout << "warning: hashtable long chain " << name << endl;
- }
- }
-
-public:
- /* buf must be all zeroes on initialization. */
- HashTable(void *buf, int buflen, const char *_name) : name(_name) {
- int m = sizeof(Node);
- // cout << "hashtab init, buflen:" << buflen << " m:" << m << endl;
- n = buflen / m;
- if( (n & 1) == 0 )
- n--;
- nodes = (Node *) buf;
- assert(nodes[n-1].hash == 0);
- assert(nodes[0].hash == 0);
-
- assert( sizeof(Node) == 628 );
- //cout << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << endl;
- }
-
- Type* get(const Key& k) {
- bool found;
- int i = _find(k, found);
- if( found )
- return &nodes[i].value;
- return 0;
- }
-
- void kill(const Key& k) {
- bool found;
- int i = _find(k, found);
- if( i >= 0 && found )
- nodes[i].k.kill();
- }
-
- void put(const Key& k, const Type& value) {
- bool found;
- int i = _find(k, found);
- if( i < 0 )
- return;
- if( !found ) {
- nodes[i].k = k;
- nodes[i].hash = k.hash();
- }
- else {
- assert( nodes[i].hash == k.hash() );
- }
- nodes[i].value = value;
- }
-
-};
-
-#pragma pack(pop)
-
+/* hashtab.h + + Simple, fixed size hash table. Darn simple. + + Uses a contiguous block of memory, so you can put it in a memory mapped file very easily. +*/ + +#pragma once + +#include "../stdafx.h" +#include <map> + +#pragma pack(push) +#pragma pack(1) + +/* you should define: + + int Key::hash() return > 0 always. +*/ + +template < + class Key, + class Type +> +class HashTable { +public: + const char *name; + struct Node { + int hash; + Key k; + Type value; + bool inUse() { return hash != 0; } + } *nodes; + int n; + + int _find(const Key& k, bool& found) { + found = false; + int h = k.hash(); + int i = h % n; + int start = i; + int chain = 0; + while( 1 ) { + if( !nodes[i].inUse() ) { + return i; + } + if( nodes[i].hash == h && nodes[i].k == k ) { + found = true; + return i; + } + chain++; + i = (i+1) % n; + if( i == start ) { + cout << "warning: hashtable is full " << name << endl; + return -1; + } + if( chain == 200 ) + cout << "warning: hashtable long chain " << name << endl; + } + } + +public: + /* buf must be all zeroes on initialization. */ + HashTable(void *buf, int buflen, const char *_name) : name(_name) { + int m = sizeof(Node); + // cout << "hashtab init, buflen:" << buflen << " m:" << m << endl; + n = buflen / m; + if( (n & 1) == 0 ) + n--; + nodes = (Node *) buf; + assert(nodes[n-1].hash == 0); + assert(nodes[0].hash == 0); + + assert( sizeof(Node) == 628 ); + //cout << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << endl; + } + + Type* get(const Key& k) { + bool found; + int i = _find(k, found); + if( found ) + return &nodes[i].value; + return 0; + } + + void kill(const Key& k) { + bool found; + int i = _find(k, found); + if( i >= 0 && found ) + nodes[i].k.kill(); + } + + void put(const Key& k, const Type& value) { + bool found; + int i = _find(k, found); + if( i < 0 ) + return; + if( !found ) { + nodes[i].k = k; + nodes[i].hash = k.hash(); + } + else { + assert( nodes[i].hash == k.hash() ); + } + nodes[i].value = value; + } + +}; + +#pragma pack(pop) + diff --git a/util/lruishmap.h b/util/lruishmap.h index 1fd63d577fb..eca27609897 100644 --- a/util/lruishmap.h +++ b/util/lruishmap.h @@ -1,59 +1,59 @@ -// lru-ish map.h
-
-#pragma once
-
-#include "../stdafx.h"
-#include "../util/goodies.h"
-
-/* Your K object must define:
- int hash() - must always return > 0.
- operator==
-*/
-
-template <class K, class V, int MaxChain>
-class LRUishMap {
-public:
- LRUishMap(int _n) {
- n = nextPrime(_n);
- keys = new K[n];
- hashes = new int[n];
- for( int i = 0; i < n; i++ ) hashes[i] = 0;
- }
- ~LRUishMap() {
- delete[] keys;
- delete[] hashes;
- }
-
- int _find(const K& k, bool& found) {
- int h = k.hash();
- assert( h > 0 );
- int j = h % n;
- int first = j;
- for( int i = 0; i < MaxChain; i++ ) {
- if( hashes[j] == h ) {
- if( keys[j] == k ) {
- found = true;
- return j;
- }
- }
- else if( hashes[j] == 0 ) {
- found = false;
- return j;
- }
- }
- found = false;
- return first;
- }
-
- V* find(const K& k) {
- bool found;
- int j = _find(k, found);
- return found ? &values[j] : 0;
- }
-
-private:
- int n;
- K *keys;
- int *hashes;
- V *values;
-};
+// lru-ish map.h + +#pragma once + +#include "../stdafx.h" +#include "../util/goodies.h" + +/* Your K object must define: + int hash() - must always return > 0. + operator== +*/ + +template <class K, class V, int MaxChain> +class LRUishMap { +public: + LRUishMap(int _n) { + n = nextPrime(_n); + keys = new K[n]; + hashes = new int[n]; + for( int i = 0; i < n; i++ ) hashes[i] = 0; + } + ~LRUishMap() { + delete[] keys; + delete[] hashes; + } + + int _find(const K& k, bool& found) { + int h = k.hash(); + assert( h > 0 ); + int j = h % n; + int first = j; + for( int i = 0; i < MaxChain; i++ ) { + if( hashes[j] == h ) { + if( keys[j] == k ) { + found = true; + return j; + } + } + else if( hashes[j] == 0 ) { + found = false; + return j; + } + } + found = false; + return first; + } + + V* find(const K& k) { + bool found; + int j = _find(k, found); + return found ? &values[j] : 0; + } + +private: + int n; + K *keys; + int *hashes; + V *values; +}; diff --git a/util/mmap.cpp b/util/mmap.cpp index 6955d1ca96c..f034b2a3cea 100644 --- a/util/mmap.cpp +++ b/util/mmap.cpp @@ -1,144 +1,144 @@ -// mmap.cpp
-
-#include "stdafx.h"
-#include "mmap.h"
-
-#if defined(_WIN32)
-
-#include "windows.h"
-
-MemoryMappedFile::MemoryMappedFile() {
- fd = 0; maphandle = 0; view = 0;
-}
-
-MemoryMappedFile::~MemoryMappedFile() {
- if( view )
- UnmapViewOfFile(view);
- view = 0;
- if( maphandle )
- CloseHandle(maphandle);
- maphandle = 0;
- if( fd )
- CloseHandle(fd);
- fd = 0;
-}
-
-std::wstring toWideString(const char *s) {
- //const std::basic_string<TCHAR> s) {
- std::basic_ostringstream<TCHAR> buf;
- buf << s;
- return buf.str();
-}
-
-unsigned mapped = 0;
-
-void* MemoryMappedFile::map(const char *filename, int length) {
- std::wstring filenamew = toWideString(filename);
-
- fd = CreateFile(
- filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ,
- NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
- if( fd == INVALID_HANDLE_VALUE ) {
- cout << "CreateFile failed " << filename << endl;
- return 0;
- }
-
-#if defined(_WIN32)
- if( mapped > 500000000 ) {
- cout << "too much mem mapped for win32" << endl;
- if( length > 50000000 )
- length = 50000000;
- }
- mapped += length;
-#endif
-
- maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL);
- if( maphandle == NULL ) {
- cout << "CreateFileMapping failed " << filename << endl;
- return 0;
- }
-
- view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0);
- if( view == 0 )
- cout << "MapViewOfFile failed " << filename << ' ' << GetLastError() << endl;
-
- return view;
-}
-
-void MemoryMappedFile::flush(bool) {
-}
-
-#else
-
-#include <errno.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-MemoryMappedFile::MemoryMappedFile() {
- fd = 0; maphandle = 0; view = 0; len = 0;
-}
-
-MemoryMappedFile::~MemoryMappedFile() {
- if( view )
- munmap(view, len);
- view = 0;
-
- if( fd )
- close(fd);
- fd = 0;
-}
-
-#ifndef O_NOATIME
-#warning NO O_NOATIME
-#define O_NOATIME 0
-#endif
-
-void* MemoryMappedFile::map(const char *filename, int length) {
- len = length;
-
- fd = open(filename, O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);
- if( !fd ) {
- cout << "couldn't open " << filename << ' ' << errno << endl;
- return 0;
- }
-
- /* make sure the file is the full desired length */
- off_t filelen = lseek(fd, 0, SEEK_END);
- if( filelen < length ) {
- cout << "map: file length=" << filelen << " want:" << length << endl;
- if( filelen != 0 ) {
- cout << " failing mapping" << endl;
- return 0;
- }
- cout << " writing file to full length with zeroes..." << endl;
- int z = 8192;
- char buf[z];
- memset(buf, 0, z);
- int left = length;
- while( 1 ) {
- if( left <= z ) {
- write(fd, buf, left);
- break;
- }
- write(fd, buf, z);
- left -= z;
- }
- cout << " done" << endl;
- }
-
- lseek(fd, length, SEEK_SET);
- write(fd, "", 1);
-
- view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
- return view;
-}
-
-void MemoryMappedFile::flush(bool sync) {
- if( msync(view, len, sync ? MS_SYNC : MS_ASYNC) )
- cout << "msync error " << errno << endl;
-}
-
-#endif
-
+// mmap.cpp + +#include "stdafx.h" +#include "mmap.h" + +#if defined(_WIN32) + +#include "windows.h" + +MemoryMappedFile::MemoryMappedFile() { + fd = 0; maphandle = 0; view = 0; +} + +MemoryMappedFile::~MemoryMappedFile() { + if( view ) + UnmapViewOfFile(view); + view = 0; + if( maphandle ) + CloseHandle(maphandle); + maphandle = 0; + if( fd ) + CloseHandle(fd); + fd = 0; +} + +std::wstring toWideString(const char *s) { + //const std::basic_string<TCHAR> s) { + std::basic_ostringstream<TCHAR> buf; + buf << s; + return buf.str(); +} + +unsigned mapped = 0; + +void* MemoryMappedFile::map(const char *filename, int length) { + std::wstring filenamew = toWideString(filename); + + fd = CreateFile( + filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if( fd == INVALID_HANDLE_VALUE ) { + cout << "CreateFile failed " << filename << endl; + return 0; + } + +#if defined(_WIN32) + if( mapped > 500000000 ) { + cout << "too much mem mapped for win32" << endl; + if( length > 50000000 ) + length = 50000000; + } + mapped += length; +#endif + + maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL); + if( maphandle == NULL ) { + cout << "CreateFileMapping failed " << filename << endl; + return 0; + } + + view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0); + if( view == 0 ) + cout << "MapViewOfFile failed " << filename << ' ' << GetLastError() << endl; + + return view; +} + +void MemoryMappedFile::flush(bool) { +} + +#else + +#include <errno.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +MemoryMappedFile::MemoryMappedFile() { + fd = 0; maphandle = 0; view = 0; len = 0; +} + +MemoryMappedFile::~MemoryMappedFile() { + if( view ) + munmap(view, len); + view = 0; + + if( fd ) + close(fd); + fd = 0; +} + +#ifndef O_NOATIME +#warning NO O_NOATIME +#define O_NOATIME 0 +#endif + +void* MemoryMappedFile::map(const char *filename, int length) { + len = length; + + fd = open(filename, O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR); + if( !fd ) { + cout << "couldn't open " << filename << ' ' << errno << endl; + return 0; + } + + /* make sure the file is the full desired length */ + off_t filelen = lseek(fd, 0, SEEK_END); + if( filelen < length ) { + cout << "map: file length=" << filelen << " want:" << length << endl; + if( filelen != 0 ) { + cout << " failing mapping" << endl; + return 0; + } + cout << " writing file to full length with zeroes..." << endl; + int z = 8192; + char buf[z]; + memset(buf, 0, z); + int left = length; + while( 1 ) { + if( left <= z ) { + write(fd, buf, left); + break; + } + write(fd, buf, z); + left -= z; + } + cout << " done" << endl; + } + + lseek(fd, length, SEEK_SET); + write(fd, "", 1); + + view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + return view; +} + +void MemoryMappedFile::flush(bool sync) { + if( msync(view, len, sync ? MS_SYNC : MS_ASYNC) ) + cout << "msync error " << errno << endl; +} + +#endif + diff --git a/util/mmap.h b/util/mmap.h index aae4eed0f32..15f638787d8 100644 --- a/util/mmap.h +++ b/util/mmap.h @@ -1,22 +1,22 @@ -// mmap.h
-
-#pragma once
-
-class MemoryMappedFile {
-public:
- MemoryMappedFile();
- ~MemoryMappedFile();
-
- /* only smart enough right now to deal with files of a fixed length.
- creates if DNE
- */
- void* map(const char *filename, int length);
-
- void flush(bool sync);
-
-private:
- HANDLE fd;
- HANDLE maphandle;
- void *view;
- int len;
-};
+// mmap.h + +#pragma once + +class MemoryMappedFile { +public: + MemoryMappedFile(); + ~MemoryMappedFile(); + + /* only smart enough right now to deal with files of a fixed length. + creates if DNE + */ + void* map(const char *filename, int length); + + void flush(bool sync); + +private: + HANDLE fd; + HANDLE maphandle; + void *view; + int len; +}; diff --git a/util/sock.cpp b/util/sock.cpp index 7b0c20e7ff5..1b86edd4f5e 100644 --- a/util/sock.cpp +++ b/util/sock.cpp @@ -1,161 +1,161 @@ -// sock.cpp
-
-#include "stdafx.h"
-#include "sock.h"
-
-void sendtest() {
- cout << "sendtest\n";
- SockAddr me(27016);
- SockAddr dest("127.0.0.1", 27015);
- UDPConnection c;
- if( c.init(me) ) {
- char buf[256];
- cout << "sendto: ";
- cout << c.sendto(buf, sizeof(buf), dest) << " errno:" << h_errno << endl;
- }
- cout << "end\n";
-}
-
-void listentest() {
- cout << "listentest\n";
- SockAddr me(27015);
- SockAddr sender;
- UDPConnection c;
- if( c.init(me) ) {
- char buf[256];
- cout << "recvfrom: ";
- cout << c.recvfrom(buf, sizeof(buf), sender) << " errno:" << h_errno << endl;
- }
- cout << "end listentest\n";
-}
-
-void xmain();
-struct SockStartupTests {
- SockStartupTests() {
-#if defined(_WIN32)
- WSADATA d;
- if( WSAStartup(MAKEWORD(2,2), &d) != 0 ) {
- cout << "ERROR: wsastartup failed " << errno << endl;
- problem() << "ERROR: wsastartup failed " << errno << endl;
- exit(1);
- }
-#endif
- //cout << "ntohl:" << ntohl(256) << endl;
- //sendtest();
- //listentest();
- }
-} sstests;
-
-#if 0
-void smain() {
-
- WSADATA wsaData;
- SOCKET RecvSocket;
- sockaddr_in RecvAddr;
- int Port = 27015;
- char RecvBuf[1024];
- int BufLen = 1024;
- sockaddr_in SenderAddr;
- int SenderAddrSize = sizeof(SenderAddr);
-
- //-----------------------------------------------
- // Initialize Winsock
- WSAStartup(MAKEWORD(2,2), &wsaData);
-
- //-----------------------------------------------
- // Create a receiver socket to receive datagrams
- RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
- prebindOptions( RecvSocket );
-
- //-----------------------------------------------
- // Bind the socket to any address and the specified port.
- RecvAddr.sin_family = AF_INET;
- RecvAddr.sin_port = htons(Port);
- RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-
- bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));
-
- //-----------------------------------------------
- // Call the recvfrom function to receive datagrams
- // on the bound socket.
- printf("Receiving datagrams...\n");
- recvfrom(RecvSocket,
- RecvBuf,
- BufLen,
- 0,
- (SOCKADDR *)&SenderAddr,
- &SenderAddrSize);
-
- //-----------------------------------------------
- // Close the socket when finished receiving datagrams
- printf("Finished receiving. Closing socket.\n");
- closesocket(RecvSocket);
-
- //-----------------------------------------------
- // Clean up and exit.
- printf("Exiting.\n");
- WSACleanup();
- return;
-}
-
-
-
-
-void xmain() {
-
- WSADATA wsaData;
- SOCKET RecvSocket;
- sockaddr_in RecvAddr;
- int Port = 27015;
- char RecvBuf[1024];
- int BufLen = 1024;
- sockaddr_in SenderAddr;
- int SenderAddrSize = sizeof(SenderAddr);
-
- //-----------------------------------------------
- // Initialize Winsock
- WSAStartup(MAKEWORD(2,2), &wsaData);
-
- //-----------------------------------------------
- // Create a receiver socket to receive datagrams
-
- RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
- prebindOptions( RecvSocket );
-
- //-----------------------------------------------
- // Bind the socket to any address and the specified port.
- RecvAddr.sin_family = AF_INET;
- RecvAddr.sin_port = htons(Port);
- RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);
-
- SockAddr a(Port);
- bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize);
-// bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));
-
- SockAddr b;
-
- //-----------------------------------------------
- // Call the recvfrom function to receive datagrams
- // on the bound socket.
- printf("Receiving datagrams...\n");
- recvfrom(RecvSocket,
- RecvBuf,
- BufLen,
- 0,
- (SOCKADDR *) &b.sa, &b.addressSize);
-// (SOCKADDR *)&SenderAddr,
-// &SenderAddrSize);
-
- //-----------------------------------------------
- // Close the socket when finished receiving datagrams
- printf("Finished receiving. Closing socket.\n");
- closesocket(RecvSocket);
-
- //-----------------------------------------------
- // Clean up and exit.
- printf("Exiting.\n");
- WSACleanup();
- return;
-}
-
-#endif
+// sock.cpp + +#include "stdafx.h" +#include "sock.h" + +void sendtest() { + cout << "sendtest\n"; + SockAddr me(27016); + SockAddr dest("127.0.0.1", 27015); + UDPConnection c; + if( c.init(me) ) { + char buf[256]; + cout << "sendto: "; + cout << c.sendto(buf, sizeof(buf), dest) << " errno:" << h_errno << endl; + } + cout << "end\n"; +} + +void listentest() { + cout << "listentest\n"; + SockAddr me(27015); + SockAddr sender; + UDPConnection c; + if( c.init(me) ) { + char buf[256]; + cout << "recvfrom: "; + cout << c.recvfrom(buf, sizeof(buf), sender) << " errno:" << h_errno << endl; + } + cout << "end listentest\n"; +} + +void xmain(); +struct SockStartupTests { + SockStartupTests() { +#if defined(_WIN32) + WSADATA d; + if( WSAStartup(MAKEWORD(2,2), &d) != 0 ) { + cout << "ERROR: wsastartup failed " << errno << endl; + problem() << "ERROR: wsastartup failed " << errno << endl; + exit(1); + } +#endif + //cout << "ntohl:" << ntohl(256) << endl; + //sendtest(); + //listentest(); + } +} sstests; + +#if 0 +void smain() { + + WSADATA wsaData; + SOCKET RecvSocket; + sockaddr_in RecvAddr; + int Port = 27015; + char RecvBuf[1024]; + int BufLen = 1024; + sockaddr_in SenderAddr; + int SenderAddrSize = sizeof(SenderAddr); + + //----------------------------------------------- + // Initialize Winsock + WSAStartup(MAKEWORD(2,2), &wsaData); + + //----------------------------------------------- + // Create a receiver socket to receive datagrams + RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + prebindOptions( RecvSocket ); + + //----------------------------------------------- + // Bind the socket to any address and the specified port. + RecvAddr.sin_family = AF_INET; + RecvAddr.sin_port = htons(Port); + RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY); + + bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr)); + + //----------------------------------------------- + // Call the recvfrom function to receive datagrams + // on the bound socket. + printf("Receiving datagrams...\n"); + recvfrom(RecvSocket, + RecvBuf, + BufLen, + 0, + (SOCKADDR *)&SenderAddr, + &SenderAddrSize); + + //----------------------------------------------- + // Close the socket when finished receiving datagrams + printf("Finished receiving. Closing socket.\n"); + closesocket(RecvSocket); + + //----------------------------------------------- + // Clean up and exit. + printf("Exiting.\n"); + WSACleanup(); + return; +} + + + + +void xmain() { + + WSADATA wsaData; + SOCKET RecvSocket; + sockaddr_in RecvAddr; + int Port = 27015; + char RecvBuf[1024]; + int BufLen = 1024; + sockaddr_in SenderAddr; + int SenderAddrSize = sizeof(SenderAddr); + + //----------------------------------------------- + // Initialize Winsock + WSAStartup(MAKEWORD(2,2), &wsaData); + + //----------------------------------------------- + // Create a receiver socket to receive datagrams + + RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + prebindOptions( RecvSocket ); + + //----------------------------------------------- + // Bind the socket to any address and the specified port. + RecvAddr.sin_family = AF_INET; + RecvAddr.sin_port = htons(Port); + RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY); + + SockAddr a(Port); + bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize); +// bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr)); + + SockAddr b; + + //----------------------------------------------- + // Call the recvfrom function to receive datagrams + // on the bound socket. + printf("Receiving datagrams...\n"); + recvfrom(RecvSocket, + RecvBuf, + BufLen, + 0, + (SOCKADDR *) &b.sa, &b.addressSize); +// (SOCKADDR *)&SenderAddr, +// &SenderAddrSize); + + //----------------------------------------------- + // Close the socket when finished receiving datagrams + printf("Finished receiving. Closing socket.\n"); + closesocket(RecvSocket); + + //----------------------------------------------- + // Clean up and exit. + printf("Exiting.\n"); + WSACleanup(); + return; +} + +#endif diff --git a/util/sock.h b/util/sock.h index def64035c80..3c00fe6a83f 100644 --- a/util/sock.h +++ b/util/sock.h @@ -1,161 +1,161 @@ -// sock.h
-
-#pragma once
-
-#include <stdio.h>
-#include <sstream>
-#include "goodies.h"
-
-#if defined(_WIN32)
-#include <winsock2.h>
-#include <ws2tcpip.h>
-typedef int socklen_t;
-inline int getLastError() { return WSAGetLastError(); }
-inline void disableNagle(int sock) {
- int x = 1;
- if( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) )
- cout << "ERROR: disableNagle failed" << endl;
-}
-inline void prebindOptions( int sock ){
-}
-#else
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <errno.h>
-inline void closesocket(int s) { close(s); }
-const int INVALID_SOCKET = -1;
-typedef int SOCKET;
-#define h_errno errno
-inline int getLastError() { return errno; }
-inline void disableNagle(int sock) {
- int x = 1;
- - #ifdef SOL_TCP
- int level = SOL_TCP;
- #else
- int level = SOL_SOCKET;
- #endif
+// sock.h + +#pragma once + +#include <stdio.h> +#include <sstream> +#include "goodies.h" + +#if defined(_WIN32) +#include <winsock2.h> +#include <ws2tcpip.h> +typedef int socklen_t; +inline int getLastError() { return WSAGetLastError(); } +inline void disableNagle(int sock) { + int x = 1; + if( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) ) + cout << "ERROR: disableNagle failed" << endl; +} +inline void prebindOptions( int sock ){ +} +#else +#include <sys/socket.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <arpa/inet.h> +#include <errno.h> +inline void closesocket(int s) { close(s); } +const int INVALID_SOCKET = -1; +typedef int SOCKET; +#define h_errno errno +inline int getLastError() { return errno; } +inline void disableNagle(int sock) { + int x = 1; + + #ifdef SOL_TCP + int level = SOL_TCP; + #else + int level = SOL_SOCKET; + #endif if( setsockopt(sock, level, TCP_NODELAY, (char *) &x, sizeof(x)) ) - cout << "ERROR: disableNagle failed" << endl;
+ cout << "ERROR: disableNagle failed" << endl; + +} +inline void prebindOptions( int sock ){ + cout << "doing prebind option" << endl; + int x = 1; + if ( setsockopt( sock , SOL_SOCKET, SO_REUSEADDR, &x, sizeof(x)) < 0 ) + cout << "Failed to set socket opt, SO_REUSEADDR" << endl; +} + + +#endif + +struct SockAddr { + SockAddr() { addressSize = sizeof(sockaddr_in); memset(&sa, 0, sizeof(sa)); } + SockAddr(int sourcePort); /* listener side */ + SockAddr(const char *ip, int port); /* EndPoint (remote) side, or if you want to specify which interface locally */ + + struct sockaddr_in sa; + socklen_t addressSize; + + bool isLocalHost() const { +#if defined(_WIN32) + return sa.sin_addr.S_un.S_addr == 0x100007f; +#else + return sa.sin_addr.s_addr == 0x100007f; +#endif + } + + string toString() { + stringstream out; + out << inet_ntoa(sa.sin_addr) << ':' + << sa.sin_port; + return out.str(); + } + + unsigned getPort() { return sa.sin_port; } + + bool operator==(const SockAddr& r) const { + return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr && + sa.sin_port == r.sa.sin_port; + } + bool operator!=(const SockAddr& r) const { return !(*this == r); } + bool operator<(const SockAddr& r) const { + if( sa.sin_port >= r.sa.sin_port ) + return false; + return sa.sin_addr.s_addr < r.sa.sin_addr.s_addr; + } +}; + +const int MaxMTU = 16384; + +class UDPConnection { +public: + UDPConnection() { sock = 0; } + ~UDPConnection() { if( sock ) { closesocket(sock); sock = 0; } } + bool init(const SockAddr& myAddr); + int recvfrom(char *buf, int len, SockAddr& sender); + int sendto(char *buf, int len, const SockAddr& EndPoint); + int mtu(const SockAddr& sa) { + return sa.isLocalHost() ? 16384 : 1480; + } + + SOCKET sock; +}; + +inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) { + return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize); +} + +inline int UDPConnection::sendto(char *buf, int len, const SockAddr& EndPoint) { + if( 0 && rand() < (RAND_MAX>>4) ) { + cout << " NOTSENT "; + // cout << curTimeMillis() << " .TEST: NOT SENDING PACKET" << endl; + return 0; + } + return ::sendto(sock, buf, len, 0, (sockaddr *) &EndPoint.sa, EndPoint.addressSize); +} + +inline bool UDPConnection::init(const SockAddr& myAddr) { + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if( sock == INVALID_SOCKET ) { + cout << "invalid socket? " << errno << endl; + return false; + } + //cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl; + if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) { + cout << "udp init failed" << endl; + closesocket(sock); + sock = 0; + return false; + } + socklen_t optLen; + int rcvbuf; + if (getsockopt(sock, + SOL_SOCKET, + SO_RCVBUF, + (char*)&rcvbuf, + &optLen) != -1) + cout << "SO_RCVBUF:" << rcvbuf << endl; + return true; +} + +inline SockAddr::SockAddr(int sourcePort) { + memset(sa.sin_zero, 0, sizeof(sa.sin_zero)); + sa.sin_family = AF_INET; + sa.sin_port = htons(sourcePort); + sa.sin_addr.s_addr = htonl(INADDR_ANY); + addressSize = sizeof(sa); +} +inline SockAddr::SockAddr(const char *ip, int port) { + memset(sa.sin_zero, 0, sizeof(sa.sin_zero)); + sa.sin_family = AF_INET; + sa.sin_port = htons(port); + sa.sin_addr.s_addr = inet_addr(ip); + addressSize = sizeof(sa); } -inline void prebindOptions( int sock ){
- cout << "doing prebind option" << endl;
- int x = 1;
- if ( setsockopt( sock , SOL_SOCKET, SO_REUSEADDR, &x, sizeof(x)) < 0 )
- cout << "Failed to set socket opt, SO_REUSEADDR" << endl;
-}
-
-
-#endif
-
-struct SockAddr {
- SockAddr() { addressSize = sizeof(sockaddr_in); memset(&sa, 0, sizeof(sa)); }
- SockAddr(int sourcePort); /* listener side */
- SockAddr(const char *ip, int port); /* EndPoint (remote) side, or if you want to specify which interface locally */
-
- struct sockaddr_in sa;
- socklen_t addressSize;
-
- bool isLocalHost() const {
-#if defined(_WIN32)
- return sa.sin_addr.S_un.S_addr == 0x100007f;
-#else
- return sa.sin_addr.s_addr == 0x100007f;
-#endif
- }
-
- string toString() {
- stringstream out;
- out << inet_ntoa(sa.sin_addr) << ':'
- << sa.sin_port;
- return out.str();
- }
-
- unsigned getPort() { return sa.sin_port; }
-
- bool operator==(const SockAddr& r) const {
- return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr &&
- sa.sin_port == r.sa.sin_port;
- }
- bool operator!=(const SockAddr& r) const { return !(*this == r); }
- bool operator<(const SockAddr& r) const {
- if( sa.sin_port >= r.sa.sin_port )
- return false;
- return sa.sin_addr.s_addr < r.sa.sin_addr.s_addr;
- }
-};
-
-const int MaxMTU = 16384;
-
-class UDPConnection {
-public:
- UDPConnection() { sock = 0; }
- ~UDPConnection() { if( sock ) { closesocket(sock); sock = 0; } }
- bool init(const SockAddr& myAddr);
- int recvfrom(char *buf, int len, SockAddr& sender);
- int sendto(char *buf, int len, const SockAddr& EndPoint);
- int mtu(const SockAddr& sa) {
- return sa.isLocalHost() ? 16384 : 1480;
- }
-
- SOCKET sock;
-};
-
-inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) {
- return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize);
-}
-
-inline int UDPConnection::sendto(char *buf, int len, const SockAddr& EndPoint) {
- if( 0 && rand() < (RAND_MAX>>4) ) {
- cout << " NOTSENT ";
- // cout << curTimeMillis() << " .TEST: NOT SENDING PACKET" << endl;
- return 0;
- }
- return ::sendto(sock, buf, len, 0, (sockaddr *) &EndPoint.sa, EndPoint.addressSize);
-}
-
-inline bool UDPConnection::init(const SockAddr& myAddr) {
- sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
- if( sock == INVALID_SOCKET ) {
- cout << "invalid socket? " << errno << endl;
- return false;
- }
- //cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;
- if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) {
- cout << "udp init failed" << endl;
- closesocket(sock);
- sock = 0;
- return false;
- }
- socklen_t optLen;
- int rcvbuf;
- if (getsockopt(sock,
- SOL_SOCKET,
- SO_RCVBUF,
- (char*)&rcvbuf,
- &optLen) != -1)
- cout << "SO_RCVBUF:" << rcvbuf << endl;
- return true;
-}
-
-inline SockAddr::SockAddr(int sourcePort) {
- memset(sa.sin_zero, 0, sizeof(sa.sin_zero));
- sa.sin_family = AF_INET;
- sa.sin_port = htons(sourcePort);
- sa.sin_addr.s_addr = htonl(INADDR_ANY);
- addressSize = sizeof(sa);
-}
-
-inline SockAddr::SockAddr(const char *ip, int port) {
- memset(sa.sin_zero, 0, sizeof(sa.sin_zero));
- sa.sin_family = AF_INET;
- sa.sin_port = htons(port);
- sa.sin_addr.s_addr = inet_addr(ip);
- addressSize = sizeof(sa);
-}
diff --git a/util/util.cpp b/util/util.cpp index 612a27f589c..92e0edb7423 100644 --- a/util/util.cpp +++ b/util/util.cpp @@ -1,48 +1,48 @@ -#include "stdafx.h"
-#include "goodies.h"
-
-unsigned occasion = 0;
-bool goingAway = false;
-
-bool isPrime(int n) {
- int z = 2;
- while( 1 ) {
- if( z*z > n )
- break;
- if( n % z == 0 )
- return false;
- z++;
- }
- return true;
-}
-
-int nextPrime(int n) {
- n |= 1; // 2 goes to 3...don't care...
- while( !isPrime(n) )
- n += 2;
- return n;
-}
-
-struct UtilTest {
- UtilTest() {
- assert( WrappingInt(0) <= WrappingInt(0) );
- assert( WrappingInt(0) <= WrappingInt(1) );
- assert( !(WrappingInt(1) <= WrappingInt(0)) );
- assert( (WrappingInt(0xf0000000) <= WrappingInt(0)) );
- assert( (WrappingInt(0xf0000000) <= WrappingInt(9000)) );
- assert( !(WrappingInt(300) <= WrappingInt(0xe0000000)) );
-
- assert( tdiff(3, 4) == 1 );
- assert( tdiff(4, 3) == -1 );
- assert( tdiff(0xffffffff, 0) == 1 );
-
- assert( isPrime(3) );
- assert( isPrime(2) );
- assert( isPrime(13) );
- assert( isPrime(17) );
- assert( !isPrime(9) );
- assert( !isPrime(6) );
- assert( nextPrime(4) == 5 );
- assert( nextPrime(8) == 11 );
- }
-} utilTest;
+#include "stdafx.h" +#include "goodies.h" + +unsigned occasion = 0; +bool goingAway = false; + +bool isPrime(int n) { + int z = 2; + while( 1 ) { + if( z*z > n ) + break; + if( n % z == 0 ) + return false; + z++; + } + return true; +} + +int nextPrime(int n) { + n |= 1; // 2 goes to 3...don't care... + while( !isPrime(n) ) + n += 2; + return n; +} + +struct UtilTest { + UtilTest() { + assert( WrappingInt(0) <= WrappingInt(0) ); + assert( WrappingInt(0) <= WrappingInt(1) ); + assert( !(WrappingInt(1) <= WrappingInt(0)) ); + assert( (WrappingInt(0xf0000000) <= WrappingInt(0)) ); + assert( (WrappingInt(0xf0000000) <= WrappingInt(9000)) ); + assert( !(WrappingInt(300) <= WrappingInt(0xe0000000)) ); + + assert( tdiff(3, 4) == 1 ); + assert( tdiff(4, 3) == -1 ); + assert( tdiff(0xffffffff, 0) == 1 ); + + assert( isPrime(3) ); + assert( isPrime(2) ); + assert( isPrime(13) ); + assert( isPrime(17) ); + assert( !isPrime(9) ); + assert( !isPrime(6) ); + assert( nextPrime(4) == 5 ); + assert( nextPrime(8) == 11 ); + } +} utilTest; |