Merge branch 'master' of ssh://git.10gen.com/data/gitroot/p

12 files changed, 167 insertions, 40 deletions

diff --git a/db/cloner.cpp b/db/cloner.cpp
index cf2699a6295..cc1cd582dd0 100644
--- a/db/cloner.cpp
+++ b/db/cloner.cpp
@@ -23,6 +23,7 @@
 #include "jsobj.h"
 #include "query.h"
 #include "commands.h"
+#include "db.h"
 
 extern int port;
 bool userCreateNS(const char *ns, JSObj& j, string& err);
@@ -36,9 +37,20 @@ public:
 };
 
 void Cloner::copy(const char *from_collection, const char *to_collection) {
-	auto_ptr<DBClientCursor> c( conn.query(from_collection, emptyObj) );
+//    cout << "COPY: " << from_collection << " -> " << to_collection << endl;
+//    cout << "      client: " << client->name << endl;
+    auto_ptr<DBClientCursor> c;
+    {
+        dbtemprelease r;
+        c = auto_ptr<DBClientCursor>( conn.query(from_collection, emptyObj) );
+    }
 	assert( c.get() );
-	while( c->more() ) { 
+    while( 1 ) {
+        {
+            dbtemprelease r;
+            if( !c->more() )
+                break;
+        }
 		JSObj js = c->next();
 		theDataFileMgr.insert(to_collection, (void*) js.objdata(), js.objsize());
 	}
@@ -54,32 +66,46 @@ bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb) {
             return false;
         }
 	}
-	if( !conn.connect(masterHost, errmsg) )
-		return false;
-
+    /* todo: we can put thesee releases inside dbclient or a dbclient specialization.
+       or just wait until we get rid of global lcok anyway. 
+       */
 	string ns = fromdb + ".system.namespaces";
-
-	auto_ptr<DBClientCursor> c( conn.query(ns.c_str(), emptyObj) );
+	auto_ptr<DBClientCursor> c;
+    {
+        dbtemprelease r;
+        if( !conn.connect(masterHost, errmsg) )
+            return false;
+        c = auto_ptr<DBClientCursor>( conn.query(ns.c_str(), emptyObj) );
+    }
 	if( c.get() == 0 ) {
 		errmsg = "query failed " + ns;
 		return false;
 	}
 
-	while( c->more() ) { 
+    while( 1 ) {
+        {
+            dbtemprelease r;
+            if( !c->more() )
+                break;
+        }
 		JSObj collection = c->next();
 		Element e = collection.findElement("name");
 		assert( !e.eoo() );
 		assert( e.type() == String );
 		const char *from_name = e.valuestr();
-		if( strstr(from_name, ".system.") || strchr(from_name, '$') )
+        if( strstr(from_name, ".system.") || strchr(from_name, '$') ) {
+//            cout << "TEMP: skip " << from_name << endl;
 			continue;
+        }
 		JSObj options = collection.getObjectField("options");
         /* change name "<fromdb>.collection" -> <todb>.collection */
         const char *p = strchr(from_name, '.');
         assert(p);
         string to_name = todb + p;
-		if( !options.isEmpty() ) {
+		//if( !options.isEmpty() ) {
+        {
 			string err;
+//            cout << "TEMP: usercreatens " << to_name << " client:" << client->name << endl;
 			userCreateNS(to_name.c_str(), options, err);
 		}
 		copy(from_name, to_name.c_str());
@@ -130,8 +156,7 @@ public:
             errmsg = "parms missing - {copydb: 1, fromhost: <hostname>, fromdb: <db>, todb: <db>}";
             return false;
         }
-        string temp = todb + ".";
-        setClient(temp.c_str());
+        setClient(todb.c_str());
         bool res = cloneFrom(fromhost.c_str(), errmsg, fromdb);
         client = 0;
         return res;
diff --git a/db/commands.cpp b/db/commands.cpp
index 95ee8117763..ce343e98924 100644
--- a/db/commands.cpp
+++ b/db/commands.cpp
@@ -51,13 +51,14 @@ Command::Command(const char *_name) : name(_name) {
 
 bool dbEval(JSObj& cmd, JSObjBuilder& result) { 
 	Element e = cmd.firstElement();
-	assert( e.type() == Code );
-	const char *code = e.valuestr();
+	assert( e.type() == Code || e.type() == CodeWScope );
+	const char *code = e.type() == Code ? e.valuestr() : e.codeWScopeCode();
+
 	if ( ! JavaJS ) {
 		result.append("errmsg", "db side execution is disabled");
 		return false;
 	}
-
+	
 	jlong f = JavaJS->functionCreate(code);
 	if( f == 0 ) { 
 		result.append("errmsg", "compile failed");
@@ -65,6 +66,8 @@ bool dbEval(JSObj& cmd, JSObjBuilder& result) {
 	}
 
 	Scope s;
+	if ( e.type() == CodeWScope )
+	  s.init( e.codeWScopeScopeData() );
 	s.setString("$client", client->name.c_str());
 	Element args = cmd.findElement("args");
 	if( args.type() == Array ) {
@@ -321,7 +324,7 @@ bool _runCommands(const char *ns, JSObj& jsobj, stringstream& ss, BufBuilder &b,
         if( !ok ) 
             anObjBuilder.append("errmsg", errmsg);
     }
-	else if( e.type() == Code ) { 
+    else if( e.type() == Code || e.type() == CodeWScope ) { 
 		valid = true;
 		ok = dbEval(jsobj, anObjBuilder);
 	}
diff --git a/db/db.cpp b/db/db.cpp
index 1ef144abf73..c8b9a3d44d7 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -858,6 +858,11 @@ int main(int argc, char* argv[], char *envp[] )
 			quicktest();
 			return 0;
 		}
+		if( strcmp(argv[1], "javatest") == 0 ) {
+                        JavaJS = new JavaJSImpl();
+			javajstest();
+			return 0;
+		}
 		if( strcmp(argv[1], "test2") == 0 ) {
 			return test2();
 		}
@@ -970,14 +975,16 @@ int main(int argc, char* argv[], char *envp[] )
 	return 0;
 }
 
-//#if !defined(_WIN32)
-//int main( int argc, char *argv[], char *envp[] ) {
-//	return _tmain(argc, 0);
-//}
-//#endif
+void foo() { 
+  boost::mutex z;
+  boost::detail::thread::lock_ops<boost::mutex>::lock(z);
+  cout << "inside lock" << endl;
+  boost::detail::thread::lock_ops<boost::mutex>::unlock(z);
+}
 
 #undef exit
 void dbexit(int rc, const char *why) { 
+  foo();
 	log() << "  dbexit: " << why << "; flushing op log and files" << endl;
 	flushOpLog();
 
@@ -988,3 +995,4 @@ void dbexit(int rc, const char *why) {
 	log() << "  dbexit: really exiting now" << endl;
 	exit(rc);
 }
+
diff --git a/db/db.h b/db/db.h
index 023b789fde0..896ec85d4db 100644
--- a/db/db.h
+++ b/db/db.h
@@ -34,3 +34,36 @@ struct dblock {
     }
 };
 
+/* a scoped release of a mutex temporarily -- like a scopedlock but reversed. 
+*/
+struct temprelease {
+    boost::mutex& m;
+    temprelease(boost::mutex& _m) : m(_m) { 
+        boost::detail::thread::lock_ops<boost::mutex>::unlock(m);
+    }
+    ~temprelease() { 
+        boost::detail::thread::lock_ops<boost::mutex>::lock(m);
+    }
+};
+
+#include "pdfile.h"
+
+struct dbtemprelease {
+    string clientname;
+    dbtemprelease() {
+        if( client ) 
+            clientname = client->name;
+        dbLocked--;
+        assert( dbLocked == 0 );
+        boost::detail::thread::lock_ops<boost::mutex>::unlock(dbMutex);
+    }
+    ~dbtemprelease() { 
+        boost::detail::thread::lock_ops<boost::mutex>::lock(dbMutex);
+        dbLocked++;
+        assert( dbLocked == 1 );
+        if( clientname.empty() )
+            client = 0;
+        else
+            setClient(clientname.c_str());
+    }
+};
diff --git a/db/javajs.cpp b/db/javajs.cpp
index 6cb7dd2e5e3..9d4d558731b 100644
--- a/db/javajs.cpp
+++ b/db/javajs.cpp
@@ -180,6 +180,8 @@ JavaJSImpl::JavaJSImpl(const char *appserverPath){
   jassert( _dbjni );
 
   _scopeCreate = _mainEnv->GetStaticMethodID( _dbhook , "scopeCreate" , "()J" );
+  _scopeInit = _mainEnv->GetStaticMethodID( _dbhook , "scopeInit" , "(JLjava/nio/ByteBuffer;)Z" );
+  _scopeSetThis = _mainEnv->GetStaticMethodID( _dbhook , "scopeSetThis" , "(JLjava/nio/ByteBuffer;)Z" );
   _scopeReset = _mainEnv->GetStaticMethodID( _dbhook , "scopeReset" , "(J)Z" );
   _scopeFree = _mainEnv->GetStaticMethodID( _dbhook , "scopeFree" , "(J)V" );
 
@@ -199,6 +201,8 @@ JavaJSImpl::JavaJSImpl(const char *appserverPath){
   _invoke = _mainEnv->GetStaticMethodID( _dbhook , "invoke" , "(JJ)I" );
 
   jassert( _scopeCreate );  
+  jassert( _scopeInit );
+  jassert( _scopeSetThis );
   jassert( _scopeReset );
   jassert( _scopeFree );
 
@@ -278,6 +282,26 @@ int JavaJSImpl::scopeSetObject( jlong id , const char * field , JSObj * obj ){
   return _getEnv()->CallStaticBooleanMethod( _dbhook , _scopeSetObject , id , _getEnv()->NewStringUTF( field ) , bb );
 }
 
+int JavaJSImpl::scopeInit( jlong id , JSObj * obj ){
+  if ( ! obj )
+    return 0;
+
+  jobject bb = _getEnv()->NewDirectByteBuffer( (void*)(obj->objdata()) , (jlong)(obj->objsize()) );
+  jassert( bb );
+
+  return _getEnv()->CallStaticBooleanMethod( _dbhook , _scopeInit , id , bb );
+}
+
+int JavaJSImpl::scopeSetThis( jlong id , JSObj * obj ){
+  if ( ! obj )
+    return 0;
+
+  jobject bb = _getEnv()->NewDirectByteBuffer( (void*)(obj->objdata()) , (jlong)(obj->objsize()) );
+  jassert( bb );
+
+  return _getEnv()->CallStaticBooleanMethod( _dbhook , _scopeSetThis , id , bb );
+}
+
 // scope getters
 
 char JavaJSImpl::scopeGetType( jlong id , const char * field ){
@@ -540,7 +564,7 @@ int javajstest() {
   
   if ( debug ) cout << "func5 start" << endl;
   jassert( JavaJS.scopeSetObject( scope , "c" , &obj ) );
-  jlong func5 = JavaJS.functionCreate( "assert( 517 == c.foo );" );
+  jlong func5 = JavaJS.functionCreate( "assert.eq( 517 , c.foo );" );
   jassert( func5 );
   jassert( ! JavaJS.invoke( scope , func5 ) );
   if ( debug ) cout << "func5 done" << endl;
diff --git a/db/javajs.h b/db/javajs.h
index af904fa7c90..64693286be4 100644
--- a/db/javajs.h
+++ b/db/javajs.h
@@ -52,6 +52,8 @@ class JavaJSImpl {
   ~JavaJSImpl();
   
   jlong scopeCreate();
+  int scopeInit( jlong id , JSObj * obj );
+  int scopeSetThis( jlong id , JSObj * obj );
   jboolean scopeReset( jlong id );
   void scopeFree( jlong id );
 
@@ -117,6 +119,8 @@ class JavaJSImpl {
   jclass _dbjni;
   
   jmethodID _scopeCreate;
+  jmethodID _scopeInit;
+  jmethodID _scopeSetThis;
   jmethodID _scopeReset;
   jmethodID _scopeFree;
 
@@ -146,6 +150,11 @@ class Scope {
   Scope() { s = JavaJS->scopeCreate(); }
   ~Scope() { JavaJS->scopeFree(s); s = 0; }
   void reset() { JavaJS->scopeReset(s); }
+
+  void init( const char * data ){
+    JSObj o( data , 0 );
+    JavaJS->scopeInit( s , & o );
+  }
   
   double getNumber(const char *field) { return JavaJS->scopeGetNumber(s,field); }
   string getString(const char *field) { return JavaJS->scopeGetString(s,field); }
diff --git a/db/jsobj.cpp b/db/jsobj.cpp
index dc981941167..8aa517aebf1 100644
--- a/db/jsobj.cpp
+++ b/db/jsobj.cpp
@@ -106,6 +106,10 @@ int Element::size() const {
 		case String:
 			x = valuestrsize() + 4 + 1;
 			break;
+	        case CodeWScope:
+			x = objsize() + 1;
+			break;
+			
         case DBRef:
           x = valuestrsize() + 4 + 12 + 1;
           break;
diff --git a/db/jsobj.h b/db/jsobj.h
index 4b630b0de27..a2b5e416aa5 100644
--- a/db/jsobj.h
+++ b/db/jsobj.h
@@ -36,7 +36,7 @@ class JSObjBuilder;
 */
 enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5, 
               Undefined=6, jstOID=7, Bool=8, Date=9 , jstNULL=10, RegEx=11 ,
-              DBRef=12, Code=13, Symbol=14, JSTypeMax=14, MaxKey=127 };
+              DBRef=12, Code=13, Symbol=14, CodeWScope=15 , JSTypeMax=15, MaxKey=127 };
 
 /* subtypes of BinData.
    bdtCustom and above are ones that the JS compiler understands, but are
@@ -75,6 +75,7 @@ struct OID {
      BinData:   <int len> <byte subtype> <byte[len] data>
      Code:      a function (not a closure): same format as String.
      Symbol:    a language symbol (say a python symbol).  same format as String.
+     Code With Scope: <total size><String><Object>
 */
 
 #pragma pack(pop)
@@ -126,6 +127,12 @@ public:
 	// for strings.  also gives you start of the real data for an embedded object
 	const char * valuestr() const { return value() + 4; }
 
+	const char * codeWScopeCode() const { return value() + 8; }
+	const char * codeWScopeScopeData() const { 
+	  // TODO fix
+	  return codeWScopeCode() + strlen( codeWScopeCode() ) + 1;
+	}
+	
 	JSObj embeddedObject();
 
 	const char *regex() { assert(type() == RegEx); return value(); }
diff --git a/db/matcher.cpp b/db/matcher.cpp
index 24c5fdf8ab8..b5835389fb4 100644
--- a/db/matcher.cpp
+++ b/db/matcher.cpp
@@ -66,10 +66,12 @@ MiniLex minilex;
 
 class Where { 
 public:
-	Where() { codeCopy = 0; }
+        Where() { codeCopy = 0; jsScope = 0; }
 	~Where() {
 		JavaJS->scopeFree(scope);
 		delete codeCopy;
+		if ( jsScope )
+		  delete jsScope;
 		scope = 0; func = 0; codeCopy = 0;
 	}
 	jlong scope, func;
@@ -78,7 +80,8 @@ public:
 	bool fullObject;
 	int nFields;
 	char *codeCopy;
-
+        JSObj *jsScope;
+  
 	void setFunc(const char *code) {
 		codeCopy = new char[strlen(code)+1];
 		strcpy(codeCopy,code);
@@ -124,23 +127,31 @@ JSMatcher::JSMatcher(JSObj &_jsobj) :
    in(0), where(0), jsobj(_jsobj), nRegex(0)
 {
 	nBuilders = 0;
-
+	
 	JSElemIter i(jsobj);
 	n = 0;
 	while( i.more() ) {
 		Element e = i.next();
 		if( e.eoo() )
 			break;
-
-		if( e.type() == Code && strcmp(e.fieldName(), "$where")==0 ) { 
+		
+		if( ( e.type() == CodeWScope || e.type() == Code ) && strcmp(e.fieldName(), "$where")==0 ) { 
 			// $where: function()...
 			assert( where == 0 );
 			where = new Where();
-			const char *code = e.valuestr();
 			massert( "$where query, but jni is disabled", JavaJS );
 			where->scope = JavaJS->scopeCreate();
 			JavaJS->scopeSetString(where->scope, "$client", client->name.c_str());
-			where->setFunc(code);
+			
+			if ( e.type() == CodeWScope ){
+			  where->setFunc( e.codeWScopeCode() );
+			  where->jsScope = new JSObj( e.codeWScopeScopeData() , 0 );
+			}
+			else {
+			  const char *code = e.valuestr();
+			  where->setFunc(code);
+			}
+			
 			continue;
 		}
 
@@ -401,9 +412,15 @@ bool JSMatcher::matches(JSObj& jsobj, bool *deep) {
 	if( where ) { 
 		if( where->func == 0 )
 			return false; // didn't compile
-		if( jsobj.objsize() < 200 || where->fullObject ) {
-			JavaJS->scopeSetObject(where->scope, "obj", &jsobj);
-		} else {
+		
+		if( 1 || jsobj.objsize() < 200 || where->fullObject ) {
+		  if ( where->jsScope ){
+		    JavaJS->scopeInit( where->scope , where->jsScope );
+		  }
+		  JavaJS->scopeSetThis(where->scope, &jsobj);		  
+		  JavaJS->scopeSetObject(where->scope, "obj", &jsobj);		  
+		} 
+		else {
 			JSObjBuilder b;
 			where->buildSubset(jsobj, b);
 			JSObj temp = b.done();
diff --git a/db/namespace.cpp b/db/namespace.cpp
index eea9f1e66fe..327ebbee8a7 100644
--- a/db/namespace.cpp
+++ b/db/namespace.cpp
@@ -326,7 +326,7 @@ void NamespaceDetailsTransient::computeIndexKeys() {
    options: { capped : ..., size : ... }
 */
 void addNewNamespaceToCatalog(const char *ns, JSObj *options = 0) {
-	log() << "New namespace: " << ns << endl;
+	OCCASIONALLY log() << "New namespace: " << ns << '\n';
 	if( strstr(ns, "system.namespaces") ) { 
 		// system.namespaces holds all the others, so it is not explicitly listed in the catalog.
 		// TODO: fix above should not be strstr!
diff --git a/db/namespace.h b/db/namespace.h
index fc9de774c73..96cbf3c9aa8 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -305,11 +305,8 @@ inline void nsToClient(const char *ns, char *client) {
 	const char *p = ns;
 	char *q = client;
 	while( *p != '.' ) { 
-		if( *p == 0 ) { 
-			assert(false);
-			*client = 0;
-			return;
-		}
+		if( *p == 0 ) 
+            break;
 		*q++ = *p++;
 	}
 	*q = 0;
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
index 9180e6fabff..9b71b73baa0 100644
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -106,7 +106,7 @@ bool userCreateNS(const char *ns, JSObj& j, string& err) {
 	/* todo: do this only when we have allocated space successfully? or we could insert with a { ok: 0 } field
              and then go back and set to ok : 1 after we are done.
 	*/
-	addNewNamespaceToCatalog(ns, &j);
+    addNewNamespaceToCatalog(ns, j.isEmpty() ? 0 : &j);
 
 	int ies = initialExtentSize(128);
 	Element e = j.findElement("size");