first commit

30 files changed, 2163 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000000..d15ee82bf49
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
+*~
+*.o
+*.aps
+*.tar.gz
+*.suo
+*.ncb
+*.idb 
+*.obj 
+*.opt 
+*.pch 
+db/Debug
diff --git a/db/db b/db/db
new file mode 100644
index 00000000000..dbace406a37
--- /dev/null
+++ b/db/db
diff --git a/db/db.cpp b/db/db.cpp
new file mode 100644
index 00000000000..5f8ae6d265e
--- /dev/null
+++ b/db/db.cpp
@@ -0,0 +1,269 @@
+// db.cpp : Defines the entry point for the console application.

+//

+

+#include "stdafx.h"

+#include "db.h"

+#include "../grid/message.h"

+#include "../util/mmap.h"

+#include "../util/hashtab.h"

+#include "pdfile.h"

+#include "jsobj.h"

+#include "query.h"

+

+struct MyStartupTests {

+	MyStartupTests() {

+		assert( sizeof(OID) == 12 );

+	}

+} mystartupdbcpp;

+

+/* example for

+	var zz = { x: 3, y: "foo", v: { z:5, arr: [1, 2] } }

+	zz.v.arr.prop = "zoo";

+*/

+

+void quicktest() { 

+	cout << "quicktest\n";

+

+	MemoryMappedFile mmf;

+	char *m = (char *) mmf.map("/tmp/abc", 16384);

+	//	cout << "mmf reads: " << m << endl;

+	strcpy_s(m, 1000, "hello worldz");

+}

+

+MessagingPort dbMsgPort;

+void pdfileInit();

+

+class DbMessage {

+public:

+	DbMessage(Message& _m) : m(_m) {

+		int *r = (int *) _m.data;

+		reserved = *r;

+		r++;

+		data = (const char *) r;

+		nextjsobj = data;

+	}

+

+	const char * getns() { return data; }

+	void getns(Namespace& ns) {

+		ns = data;

+	}

+

+	OID* getOID() {

+		return (OID *) (data + strlen(data) + 1); // skip namespace

+	}

+

+	void getQueryStuff(const char *&query, int& ntoreturn) {

+		int *i = (int *) (data + strlen(data) + 1);

+		ntoreturn = *i;

+		i++;

+		query = (const char *) i;

+	}

+

+	/* for insert and update msgs */

+	bool moreJSObjs() { return nextjsobj != 0; }

+	JSObj nextJsObj() {

+		if( nextjsobj == data )

+			nextjsobj += strlen(data) + 1; // skip namespace

+		JSObj js(nextjsobj);

+		if( js.size <= 4 )

+			nextjsobj = null;

+		else

+			nextjsobj += js.size;

+		return js;

+	}

+

+private:

+	Message& m;

+	int reserved;

+	const char *data;

+	const char *nextjsobj;

+};

+

+Record* findByOID(const char *ns, OID *oid) {

+	// temp implementation

+	Cursor c = theDataFileMgr.findAll(ns);

+	while( c.ok() ) {

+		Record *r = c.current();

+		JSObj js(r);

+		OID *i = js.getOID();

+		if( i && *oid == *i )

+			return r;

+		c.advance();

+	}

+	return 0;

+}

+

+void updateByOID(const char *ns, char *objdata, int objsize, OID *oid) {

+	Record *r = findByOID(ns, oid);

+	if( r == 0 ) { 

+		cout << "updateByOID: no such record " << ns << endl;

+		return;

+	}

+	if( objsize > r->netLength() ) {

+		cout << "ERROR: updateByOID: growing records not implemented yet." << endl;

+		return;

+	}

+	/* note: need to be smarter if it gets a lot smaller??? */

+	/* this really dumb for now as it gets smaller but doesn't allow regrowth 

+	to the original size! */

+	memcpy(r->data, objdata, objsize);

+	r->setNewLength(objsize);

+}

+

+#pragma pack(push)

+#pragma pack(1)

+struct EmptyObject {

+	EmptyObject() { len = 5; jstype = EOO; }

+	int len;

+	char jstype;

+} emptyObject;

+#pragma pack(pop)

+

+void query(Message& m) {

+	DbMessage d(m);

+	const char *query;

+	int ntoreturn;

+	d.getQueryStuff(query, ntoreturn);

+

+	QueryResult* msgdata = runQuery(d.getns(), query, ntoreturn);

+	Message resp;

+	resp.setData(msgdata, true);

+	dbMsgPort.reply(m, resp);

+}

+

+void getbyoid(Message& m) {

+	DbMessage d(m);

+	Record *r = findByOID(d.getns(), d.getOID());

+	Message resp;

+	if( r == 0 )

+		resp.setData(opReply, (char *) &emptyObject, emptyObject.len);

+	else

+		resp.setData(opReply, r->data, r->netLength());

+	dbMsgPort.reply(m, resp);

+}

+

+void dbinsert(Message& m) {

+	DbMessage d(m);

+	while( d.moreJSObjs() ) {

+		JSObj js = d.nextJsObj();

+		if( m.data->operation == dbInsert ) {

+			theDataFileMgr.insert(d.getns(), (void*) js.objdata(), js.objsize());

+		} else {

+			// update

+			OID *oid = js.getOID();

+			if( oid == null )

+				cout << "error: no oid on update -- that isn't coded yet" << endl;

+			else

+				updateByOID(d.getns(), (char *) js.objdata(), js.objsize(), oid);

+		}

+	}

+}

+

+void run() { 

+	dbMsgPort.init(MessagingPort::DBPort);

+

+	pdfileInit();

+

+	theDataFileMgr.insert("sys.unittest.pdfile", "hello world", 12);

+	cout << "findAll:\n";

+	Cursor c = theDataFileMgr.findAll("sys.unittest.pdfile");

+	while( c.ok() ) {

+		Record* r = c.current();

+		cout << "  gotrec " << r->netLength() << ' ' << 

+			r->data << '\n';

+		c.advance();

+	}

+	cout << endl;

+

+	Message m;

+	while( 1 ) { 

+		cout << "waiting for msg..." << endl;

+		m.reset();

+		if( !dbMsgPort.recv(m) ) {

+			cout << "recv() returned false" << endl;

+			break;

+		}

+		cout << "  got msg" << endl;

+		cout << "  op:" << m.data->operation << " len:" << m.data->len << endl;

+

+		if( m.data->operation == dbMsg ) { 

+			bool end = strcmp("end", m.data->_data) == 0;

+			Message resp;

+			resp.setData(opReply, "i am fine");

+			dbMsgPort.reply(m, resp);

+			if( end ) {

+				cout << "    end msg" << endl;

+				break;

+			}

+		}

+		else if( m.data->operation == dbUpdate || dbInsert ) {

+			dbinsert(m);

+		}

+		else if( m.data->operation == dbGetByOID ) {

+			getbyoid(m);

+		}

+		else if( m.data->operation == dbQuery ) { 

+			query(m);

+		}

+		else if( m.data->operation == dbGetMore ) {

+			cout << "dbGetMore: not implemented!" << endl;

+		}

+		else {

+			cout << "    operation isn't supported (???)" << endl;

+		}

+	}

+}

+

+void msg(const char *m) { 

+	MessagingPort p;

+	p.init(29999);

+

+	SockAddr db("127.0.0.1", MessagingPort::DBPort);

+

+	Message send;

+	Message response;

+

+	send.setData(1000, m);

+

+	cout << "contacting DB..." << endl;

+	bool ok = p.call(db, send, response);

+	cout << "ok: " << ok << endl;

+	cout << "  " << response.data->id << endl;

+	cout << "  " << response.data->len << endl;

+	cout << "  " << response.data->operation << endl;

+	cout << "  " << response.data->reserved << endl;

+	cout << "  " << response.data->responseTo << endl;

+	cout << "  " << response.data->_data << endl;

+

+}

+

+int main(int argc, char* argv[], char *envp[] )

+{

+	quicktest();

+

+	if( argc >= 2 ) {

+		if( strcmp(argv[1], "quicktest") == 0 )

+			return 0;

+		if( strcmp(argv[1], "msg") == 0 ) {

+			msg(argc >= 3 ? argv[2] : "ping");

+			return 0;

+		}

+		if( strcmp(argv[1], "run") == 0 ) {

+			run();

+			return 0;

+		}

+	}

+

+	cout << "usage:\n";

+	cout << "  quicktest    just check basic assertions and exit" << endl;

+	cout << "  msg [msg]    send a request to the db server" << endl;

+	cout << "  msg end      shut down" << endl;

+	cout << "  run          run db" << endl;

+	return 0;

+}

+

+//#if !defined(_WIN32)

+//int main( int argc, char *argv[], char *envp[] ) {

+//	return _tmain(argc, 0);

+//}

+//#endif

diff --git a/db/db.h b/db/db.h
new file mode 100644
index 00000000000..648f027acf7
--- /dev/null
+++ b/db/db.h
@@ -0,0 +1,2 @@
+#include "../stdafx.h"

+

diff --git a/db/db.rc b/db/db.rc
new file mode 100644
index 00000000000..11c5d52f7ae
--- /dev/null
+++ b/db/db.rc
@@ -0,0 +1,61 @@
+// Microsoft Visual C++ generated resource script.

+//

+#include "resource.h"

+

+#define APSTUDIO_READONLY_SYMBOLS

+/////////////////////////////////////////////////////////////////////////////

+//

+// Generated from the TEXTINCLUDE 2 resource.

+//

+#include "afxres.h"

+

+/////////////////////////////////////////////////////////////////////////////

+#undef APSTUDIO_READONLY_SYMBOLS

+

+/////////////////////////////////////////////////////////////////////////////

+// English (U.S.) resources

+

+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)

+LANGUAGE 9, 1

+#pragma code_page(1252)

+

+#ifdef APSTUDIO_INVOKED

+/////////////////////////////////////////////////////////////////////////////

+//

+// TEXTINCLUDE

+//

+

+1 TEXTINCLUDE  

+BEGIN

+    "resource.h\0"

+END

+

+2 TEXTINCLUDE  

+BEGIN

+    "#include ""afxres.h""\r\n"

+    "\0"

+END

+

+3 TEXTINCLUDE  

+BEGIN

+    "\r\n"

+    "\0"

+END

+

+#endif    // APSTUDIO_INVOKED

+

+#endif    // English (U.S.) resources

+/////////////////////////////////////////////////////////////////////////////

+

+

+

+#ifndef APSTUDIO_INVOKED

+/////////////////////////////////////////////////////////////////////////////

+//

+// Generated from the TEXTINCLUDE 3 resource.

+//

+

+

+/////////////////////////////////////////////////////////////////////////////

+#endif    // not APSTUDIO_INVOKED

+

diff --git a/db/db.sln b/db/db.sln
new file mode 100644
index 00000000000..9b9a7a9b6bd
--- /dev/null
+++ b/db/db.sln
@@ -0,0 +1,20 @@
+

+Microsoft Visual Studio Solution File, Format Version 10.00

+# Visual C++ Express 2008

+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db", "db.vcproj", "{215B2D68-0A70-4D10-8E75-B31010C62A91}"

+EndProject

+Global

+	GlobalSection(SolutionConfigurationPlatforms) = preSolution

+		Debug|Win32 = Debug|Win32

+		Release|Win32 = Release|Win32

+	EndGlobalSection

+	GlobalSection(ProjectConfigurationPlatforms) = postSolution

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.Build.0 = Debug|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.ActiveCfg = Release|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.Build.0 = Release|Win32

+	EndGlobalSection

+	GlobalSection(SolutionProperties) = preSolution

+		HideSolutionNode = FALSE

+	EndGlobalSection

+EndGlobal

diff --git a/db/db.vcproj b/db/db.vcproj
new file mode 100644
index 00000000000..a92406cf103
--- /dev/null
+++ b/db/db.vcproj
@@ -0,0 +1,290 @@
+<?xml version="1.0" encoding="Windows-1252"?>

+<VisualStudioProject

+	ProjectType="Visual C++"

+	Version="9.00"

+	Name="db"

+	ProjectGUID="{215B2D68-0A70-4D10-8E75-B31010C62A91}"

+	RootNamespace="db"

+	Keyword="Win32Proj"

+	TargetFrameworkVersion="196613"

+	>

+	<Platforms>

+		<Platform

+			Name="Win32"

+		/>

+	</Platforms>

+	<ToolFiles>

+	</ToolFiles>

+	<Configurations>

+		<Configuration

+			Name="Debug|Win32"

+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

+			IntermediateDirectory="$(ConfigurationName)"

+			ConfigurationType="1"

+			CharacterSet="1"

+			>

+			<Tool

+				Name="VCPreBuildEventTool"

+			/>

+			<Tool

+				Name="VCCustomBuildTool"

+			/>

+			<Tool

+				Name="VCXMLDataGeneratorTool"

+			/>

+			<Tool

+				Name="VCWebServiceProxyGeneratorTool"

+			/>

+			<Tool

+				Name="VCMIDLTool"

+			/>

+			<Tool

+				Name="VCCLCompilerTool"

+				Optimization="0"

+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"

+				MinimalRebuild="true"

+				BasicRuntimeChecks="3"

+				RuntimeLibrary="3"

+				UsePrecompiledHeader="2"

+				WarningLevel="3"

+				DebugInformationFormat="4"

+			/>

+			<Tool

+				Name="VCManagedResourceCompilerTool"

+			/>

+			<Tool

+				Name="VCResourceCompilerTool"

+			/>

+			<Tool

+				Name="VCPreLinkEventTool"

+			/>

+			<Tool

+				Name="VCLinkerTool"

+				AdditionalDependencies="ws2_32.lib"

+				LinkIncremental="2"

+				GenerateDebugInformation="true"

+				SubSystem="1"

+				TargetMachine="1"

+			/>

+			<Tool

+				Name="VCALinkTool"

+			/>

+			<Tool

+				Name="VCManifestTool"

+			/>

+			<Tool

+				Name="VCXDCMakeTool"

+			/>

+			<Tool

+				Name="VCBscMakeTool"

+			/>

+			<Tool

+				Name="VCFxCopTool"

+			/>

+			<Tool

+				Name="VCAppVerifierTool"

+			/>

+			<Tool

+				Name="VCPostBuildEventTool"

+			/>

+		</Configuration>

+		<Configuration

+			Name="Release|Win32"

+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

+			IntermediateDirectory="$(ConfigurationName)"

+			ConfigurationType="1"

+			CharacterSet="1"

+			WholeProgramOptimization="1"

+			>

+			<Tool

+				Name="VCPreBuildEventTool"

+			/>

+			<Tool

+				Name="VCCustomBuildTool"

+			/>

+			<Tool

+				Name="VCXMLDataGeneratorTool"

+			/>

+			<Tool

+				Name="VCWebServiceProxyGeneratorTool"

+			/>

+			<Tool

+				Name="VCMIDLTool"

+			/>

+			<Tool

+				Name="VCCLCompilerTool"

+				Optimization="2"

+				EnableIntrinsicFunctions="true"

+				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"

+				RuntimeLibrary="2"

+				EnableFunctionLevelLinking="true"

+				UsePrecompiledHeader="2"

+				WarningLevel="3"

+				DebugInformationFormat="3"

+			/>

+			<Tool

+				Name="VCManagedResourceCompilerTool"

+			/>

+			<Tool

+				Name="VCResourceCompilerTool"

+			/>

+			<Tool

+				Name="VCPreLinkEventTool"

+			/>

+			<Tool

+				Name="VCLinkerTool"

+				LinkIncremental="1"

+				GenerateDebugInformation="true"

+				SubSystem="1"

+				OptimizeReferences="2"

+				EnableCOMDATFolding="2"

+				TargetMachine="1"

+			/>

+			<Tool

+				Name="VCALinkTool"

+			/>

+			<Tool

+				Name="VCManifestTool"

+			/>

+			<Tool

+				Name="VCXDCMakeTool"

+			/>

+			<Tool

+				Name="VCBscMakeTool"

+			/>

+			<Tool

+				Name="VCFxCopTool"

+			/>

+			<Tool

+				Name="VCAppVerifierTool"

+			/>

+			<Tool

+				Name="VCPostBuildEventTool"

+			/>

+		</Configuration>

+	</Configurations>

+	<References>

+	</References>

+	<Files>

+		<Filter

+			Name="Source Files"

+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"

+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"

+			>

+			<File

+				RelativePath=".\db.cpp"

+				>

+			</File>

+			<File

+				RelativePath="..\grid\message.cpp"

+				>

+			</File>

+			<File

+				RelativePath="..\util\mmap.cpp"

+				>

+			</File>

+			<File

+				RelativePath=".\pdfile.cpp"

+				>

+			</File>

+			<File

+				RelativePath=".\query.cpp"

+				>

+			</File>

+			<File

+				RelativePath="..\util\sock.cpp"

+				>

+			</File>

+			<File

+				RelativePath="..\stdafx.cpp"

+				>

+				<FileConfiguration

+					Name="Debug|Win32"

+					>

+					<Tool

+						Name="VCCLCompilerTool"

+						UsePrecompiledHeader="1"

+					/>

+				</FileConfiguration>

+			</File>

+		</Filter>

+		<Filter

+			Name="Header Files"

+			Filter="h;hpp;hxx;hm;inl;inc;xsd"

+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"

+			>

+			<File

+				RelativePath="..\util\builder.h"

+				>

+			</File>

+			<File

+				RelativePath=".\db.h"

+				>

+			</File>

+			<File

+				RelativePath="..\util\hashtab.h"

+				>

+			</File>

+			<File

+				RelativePath=".\jsobj.h"

+				>

+			</File>

+			<File

+				RelativePath="..\grid\message.h"

+				>

+			</File>

+			<File

+				RelativePath="..\util\mmap.h"

+				>

+			</File>

+			<File

+				RelativePath=".\pdfile.h"

+				>

+			</File>

+			<File

+				RelativePath=".\query.h"

+				>

+			</File>

+			<File

+				RelativePath=".\resource.h"

+				>

+			</File>

+			<File

+				RelativePath="..\util\sock.h"

+				>

+			</File>

+			<File

+				RelativePath="..\stdafx.h"

+				>

+			</File>

+			<File

+				RelativePath=".\storage.h"

+				>

+			</File>

+		</Filter>

+		<Filter

+			Name="Resource Files"

+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"

+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"

+			>

+			<File

+				RelativePath=".\db.rc"

+				>

+			</File>

+			<File

+				RelativePath="..\targetver.h"

+				>

+			</File>

+		</Filter>

+		<Filter

+			Name="misc"

+			>

+			<File

+				RelativePath=".\makefile"

+				>

+			</File>

+		</Filter>

+	</Files>

+	<Globals>

+	</Globals>

+</VisualStudioProject>

diff --git a/db/db.vcproj.Dwight-PC.Dwight Merriman.user b/db/db.vcproj.Dwight-PC.Dwight Merriman.user
new file mode 100644
index 00000000000..08418941e1f
--- /dev/null
+++ b/db/db.vcproj.Dwight-PC.Dwight Merriman.user
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="Windows-1252"?>

+<VisualStudioUserFile

+	ProjectType="Visual C++"

+	Version="9.00"

+	ShowAllFiles="false"

+	>

+	<Configurations>

+		<Configuration

+			Name="Debug|Win32"

+			>

+			<DebugSettings

+				Command="$(TargetPath)"

+				WorkingDirectory=""

+				CommandArguments="run"

+				Attach="false"

+				DebuggerType="3"

+				Remote="1"

+				RemoteMachine="DWIGHT-PC"

+				RemoteCommand=""

+				HttpUrl=""

+				PDBPath=""

+				SQLDebugging=""

+				Environment=""

+				EnvironmentMerge="true"

+				DebuggerFlavor="0"

+				MPIRunCommand=""

+				MPIRunArguments=""

+				MPIRunWorkingDirectory=""

+				ApplicationCommand=""

+				ApplicationArguments=""

+				ShimCommand=""

+				MPIAcceptMode=""

+				MPIAcceptFilter=""

+			/>

+		</Configuration>

+		<Configuration

+			Name="Release|Win32"

+			>

+			<DebugSettings

+				Command="$(TargetPath)"

+				WorkingDirectory=""

+				CommandArguments=""

+				Attach="false"

+				DebuggerType="3"

+				Remote="1"

+				RemoteMachine="DWIGHT-PC"

+				RemoteCommand=""

+				HttpUrl=""

+				PDBPath=""

+				SQLDebugging=""

+				Environment=""

+				EnvironmentMerge="true"

+				DebuggerFlavor=""

+				MPIRunCommand=""

+				MPIRunArguments=""

+				MPIRunWorkingDirectory=""

+				ApplicationCommand=""

+				ApplicationArguments=""

+				ShimCommand=""

+				MPIAcceptMode=""

+				MPIAcceptFilter=""

+			/>

+		</Configuration>

+	</Configurations>

+</VisualStudioUserFile>

diff --git a/db/jsobj.h b/db/jsobj.h
new file mode 100644
index 00000000000..e599a0765db
--- /dev/null
+++ b/db/jsobj.h
@@ -0,0 +1,118 @@
+// jsobj.h

+

+#include "../stdafx.h"

+#include "pdfile.h"

+

+#pragma pack(push)

+#pragma pack(1)

+

+/* BinData = binary data types. 

+   EOO = end of object

+*/

+enum JSType { EOO = 0, Number=1, String=2, Object=3, Array=4, BinData=5, Undefined=6, jstOID=7, Bool=8, Date=9 };

+

+/* subtypes of BinData. 

+   bdtCustom and above are ones that the JS compiler understands, but are

+   opaque to the database.

+*/

+enum BinDataType { Function=1, ByteArray=2, bdtCustom=128 };

+

+/*	Object id's are optional for JSObjects.  

+	When present they should be the first object member added.

+*/

+struct OID { 

+	long long a;

+	unsigned b;

+	bool operator==(const OID& r) { return a==r.a&&b==r.b; }

+};

+

+/* marshalled js object format:

+

+   <unsigned totalSize> {<byte JSType><string FieldName><Data>}* EOO

+      totalSize includes itself.

+

+   Data:

+     EOO: nothing follows

+     Undefined: nothing follows

+     OID: an OID object

+     Number: <double>

+     String: <unsigned strlen><string>       

+     Object: a nested object, which terminates with EOO.

+     Array:

+       <unsigned length>

+       {Object}[length]  

+       a nested object, which is the object properties of the array  

+     BinData:

+       <byte subtype>

+       <unsigned len>

+       <byte[len] data>

+*/

+

+/* db operation message format 

+

+   unsigned opid;         // arbitary; will be echoed back

+   byte operation;

+   

+   Update:

+      int reserved;

+      string collection;  // name of the collection (namespace)

+      a series of JSObjects terminated with a null object (i.e., just EOO)

+   Insert:

+      int reserved;

+      string collection;

+      a series of JSObjects terminated with a null object (i.e., just EOO)

+   GetByOID:

+      int reserved;

+      string collection;

+      OID oid;

+   Query:

+      int reserved;

+      string collection;

+	  unsigned nToReturn; // how many you want back as the beginning of the cursor data

+      string query;

+   GetMore:

+      int reserved;

+      unsigned cursorID;

+      unsigned nToReturn;

+

+   byte EOM

+*/

+

+/* db response format

+

+   GetByOID operation:

+      marshalled JSObject returned.  always specified, even if an error.

+

+   Query or GetMore: see query.h

+      int reserved;

+      unsigned cursorID;

+      unsigned startOfs;

+      unsigned nReturned;

+      list of marshalled JSObjects;

+*/

+

+#pragma pack(pop)

+

+class JSObj {

+public:

+	JSObj(const char *_data) : data(_data) {

+		size = *((int*) data);

+	}

+	JSObj(Record *r) { 

+		size = r->netLength();

+		data = r->data;

+	}

+

+	const char *objdata() { return data + 4; } // skip the length field.

+	int objsize() { return size - 4; }

+

+	OID* getOID() {

+		const char *p = objdata();

+		if( *p != jstOID )

+			return 0;

+		return (OID *) ++p;

+	}

+

+	int size;

+	const char *data;

+};

diff --git a/db/makefile b/db/makefile
new file mode 100644
index 00000000000..aa99e2aa880
--- /dev/null
+++ b/db/makefile
@@ -0,0 +1,20 @@
+# makefile for our db project
+
+FLAGS=-I ..
+
+OBJS=../stdafx.o ../util/sock.o ../grid/message.o ../util/mmap.o pdfile.o
+
+.cpp.o:
+	g++ -c $(FLAGS) $< -o $@
+
+# Our convention is that passing 'quicktest' on the command line means run 
+# fast regressions and then immediately exit.  That way you know immediately if you
+# broke something horribly.
+
+db: $(OBJS) db.o
+	g++ $(FLAGS) -o $@ $(OBJS) db.o
+	./db quicktest
+
+clean:
+	-rm -f $(OBJS) db.o
+	-rm -f db
diff --git a/db/makefile.txt b/db/makefile.txt
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/db/makefile.txt
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
new file mode 100644
index 00000000000..b0557f04e05
--- /dev/null
+++ b/db/pdfile.cpp
@@ -0,0 +1,166 @@
+// pdfile.cpp

+

+#include "stdafx.h"

+#include "pdfile.h"

+#include "db.h"

+#include "../util/mmap.h"

+#include "../util/hashtab.h"

+

+#include <map>

+#include <string>

+

+DataFileMgr theDataFileMgr;

+

+/* just temporary */

+const int ExtentSize = 1024 * 1024;

+

+/*---------------------------------------------------------------------*/ 

+

+class NamespaceIndex {

+public:

+	NamespaceIndex() { }

+

+	void init() { 

+		const int LEN = 16 * 1024 * 1024;

+		void *p = f.map("/data/namespace.idx", LEN);

+		ht = new HashTable<Namespace,DiskLoc>(p, LEN, "namespace index");

+	}

+

+	void add(const char *ns, DiskLoc& loc) { 

+		Namespace n(ns);

+		ht->put(n, loc);

+	}

+

+	bool find(const char *ns, DiskLoc& loc) { 

+		Namespace n(ns);

+		DiskLoc *l = ht->get(n);

+		if( l ) {

+			loc = *l;

+			return true;

+		}

+		return false;

+	}

+

+private:

+	MemoryMappedFile f;

+	HashTable<Namespace,DiskLoc> *ht;

+} namespaceIndex;

+

+/*---------------------------------------------------------------------*/ 

+

+void PhysicalDataFile::open(const char *filename, int length) {

+	header = (PDFHeader *) mmf.map(filename, length);

+	assert(header);

+	header->init(length);

+}

+

+Extent* PhysicalDataFile::newExtent(const char *ns, DiskLoc& loc) {

+	int left = header->unusedLength - ExtentSize;

+	if( left < 0 ) {

+		cout << "ERROR: newExtent: no more room for extents. write more code" << endl;

+		assert(false);

+		exit(2);

+	}

+	int offset = header->unused.getOfs();

+	header->unused.setOfs( offset + ExtentSize );

+	header->unusedLength -= ExtentSize;

+	loc.setOfs(offset);

+	Extent *e = _getExtent(loc);

+	e->init(ns, ExtentSize, offset);

+	return e;

+}

+

+/*---------------------------------------------------------------------*/ 

+

+/* assumes already zeroed -- insufficient for block 'reuse' perhaps */

+void Extent::init(const char *nsname, int _length, int _offset) { 

+	magic = 0x41424344;

+	myLoc.setOfs(_offset);

+	ns = nsname;

+	length = _length;

+	firstRecord.Null(); lastRecord.Null();

+

+	firstEmptyRegion = myLoc;

+	firstEmptyRegion.inc( (extentData-(char*)this) );

+

+	Record *empty1 = (Record *) extentData;

+	Record *empty = getRecord(firstEmptyRegion);

+	assert( empty == empty1 );

+	empty->lengthWithHeaders = _length - (extentData - (char *) this);

+	empty->next.Null();

+}

+

+Record* Extent::newRecord(int len) {

+	if( firstEmptyRegion.isNull() )

+		return 0;

+

+	assert(len > 0);

+	int newRecSize = len + Record::HeaderSize;

+	DiskLoc newRecordLoc = firstEmptyRegion;

+	Record *r = getRecord(newRecordLoc);

+	int left = r->netLength() - len;

+	if( left < 0 ) {

+		/* this might be wasteful if huge variance in record sizes in a namespace */

+		firstEmptyRegion.Null();

+		return 0;

+	}

+

+	DiskLoc nextEmpty = r->next;

+	r->lengthWithHeaders = newRecSize;

+	r->next.Null();

+	if( !lastRecord.isNull() ) {

+		assert(getRecord(lastRecord)->next.isNull());

+		getRecord(lastRecord)->next = newRecordLoc;

+	}

+	lastRecord = newRecordLoc;

+

+	if( firstRecord.isNull() )

+		firstRecord = newRecordLoc;

+

+	if( left < Record::HeaderSize + 32 ) { 

+		firstEmptyRegion.Null();

+	}

+	else {

+		firstEmptyRegion.inc(newRecSize);

+		Record *empty = getRecord(firstEmptyRegion);

+		empty->next = nextEmpty;

+		empty->lengthWithHeaders = left;

+	}

+

+	return r;

+}

+

+/*---------------------------------------------------------------------*/ 

+

+Cursor DataFileMgr::findAll(const char *ns) {

+	DiskLoc loc;

+	bool found = namespaceIndex.find(ns, loc);

+	if( !found ) {

+		cout << "info: findAll() namespace does not exist: " << ns << endl;

+		return Cursor(DiskLoc());

+	}

+	Extent *e = temp.getExtent(loc);

+	return Cursor( e->firstRecord );

+}

+

+void DataFileMgr::insert(const char *ns, void *buf, int len) {

+	DiskLoc loc;

+	bool found = namespaceIndex.find(ns, loc);

+	if( !found ) {

+		cout << "New namespace: " << ns << endl;

+		temp.newExtent(ns, loc);

+		namespaceIndex.add(ns, loc);

+	}

+	Extent *e = temp.getExtent(loc);

+	Record *r = e->newRecord(len); /*todo: if zero returned, need new extent */

+	memcpy(r->data, buf, len);

+}

+

+void DataFileMgr::init() {

+	temp.open("/data/temp.dat", 64 * 1024 * 1024);

+}

+

+void pdfileInit() {

+	namespaceIndex.init();

+	theDataFileMgr.init();

+}

diff --git a/db/pdfile.h b/db/pdfile.h
new file mode 100644
index 00000000000..5f80d4134fd
--- /dev/null
+++ b/db/pdfile.h
@@ -0,0 +1,197 @@
+// pdfile.h

+

+#pragma once

+

+#include "../stdafx.h"

+#include "../util/mmap.h"

+#include "storage.h"

+

+struct PDFHeader;

+struct Extent;

+struct Record;

+

+/*---------------------------------------------------------------------*/ 

+

+struct Namespace {

+	Namespace(const char *ns) { 

+		*this = ns;

+	}

+	Namespace& operator=(const char *ns) { 

+		memset(buf, 0, 128); /* this is just to keep stuff clean in the files for easy dumping and reading */

+		strcpy_s(buf, 128, ns); return *this; 

+	}

+

+	bool operator==(const Namespace& r) { return strcmp(buf, r.buf) == 0; }

+	int hash() const {

+		unsigned x = 0;

+		const char *p = buf;

+		while( *p ) { 

+			x = x * 131 + *p;

+			p++;

+		}

+		return (x & 0x7fffffff) | 0x8000000; // must be > 0

+	}

+

+	char buf[128];

+};

+

+/*---------------------------------------------------------------------*/ 

+

+#pragma pack(push)

+#pragma pack(1)

+

+struct Record {

+	enum { HeaderSize = 12 };

+	DiskLoc next;

+	int lengthWithHeaders;

+	char data[4];

+	bool haveNext() { return !next.isNull(); }

+	int netLength() { return lengthWithHeaders - HeaderSize; }

+	void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }

+};

+

+/* extents are regions where all the records within the region 

+   belong to the same namespace.

+*/

+struct Extent {

+	unsigned magic;

+	DiskLoc myLoc;

+	Namespace ns; /* which namespace this extent is for.  this is just for troubleshooting really */

+	int length;   /* size of the extent, including these fields */

+	DiskLoc firstEmptyRegion;

+	DiskLoc firstRecord, lastRecord;

+	char extentData[4];

+

+	/* assumes already zeroed -- insufficient for block 'reuse' perhaps */

+	void init(const char *nsname, int _length, int _offset);

+

+	void assertOk() { assert(magic == 0x41424344); }

+

+	Record* newRecord(int len);

+

+	Record* getRecord(DiskLoc dl) {

+		assert( !dl.isNull() );

+		assert( dl.sameFile(myLoc) );

+		int x = dl.getOfs() - myLoc.getOfs();

+		assert( x > 0 );

+		return (Record *) (((char *) this) + x);

+	}

+};

+

+/*

+      ----------------------

+      Header

+      ----------------------

+      Extent (for a particular namespace)

+        Record

+        ...

+        Record (some chained for unused space)

+      ----------------------

+      more Extents...

+      ----------------------

+*/

+

+/* data file header */

+struct PDFHeader {

+	int version;

+	int versionMinor;

+	int fileLength;

+	DiskLoc unused; /* unused is the portion of the file that doesn't belong to any allocated extents. -1 = no more */

+	int unusedLength;

+	int reserved[8192 - 4*4];

+

+	char data[4];

+

+	static int headerSize() { return sizeof(PDFHeader) - 4; }

+

+	bool uninitialized() { if( version == 0 ) return true; assert(version == 3); return false; }

+

+	Record* getRecord(DiskLoc dl) {

+		int ofs = dl.getOfs();

+		assert( ofs >= headerSize() );

+		return (Record*) (((char *) this) + ofs);

+	}

+

+	void init(int filelength) {

+		if( uninitialized() ) {

+			assert(filelength > 32768 );

+			fileLength = filelength;

+			version = 3;

+			versionMinor = 0;

+			unused.setOfs( headerSize() );

+			assert( (data-(char*)this) == headerSize() );

+			unusedLength = fileLength - headerSize() - 16;

+			memcpy(data+unusedLength, "      \nthe end\n", 16); 

+		}

+	}

+};

+

+#pragma pack(pop)

+

+class PhysicalDataFile {

+	friend class DataFileMgr;

+	friend class Cursor;

+public:

+	void open(const char *filename, int length = 64 * 1024 * 1024);

+

+private:

+	Extent* newExtent(const char *ns, DiskLoc& loc);

+	Extent* getExtent(DiskLoc loc);

+	Extent* _getExtent(DiskLoc loc);

+	Record* recordAt(DiskLoc dl) { return header->getRecord(dl); }

+

+	MemoryMappedFile mmf;

+	PDFHeader *header;

+	int length;

+};

+

+inline Extent* PhysicalDataFile::_getExtent(DiskLoc loc) {

+	loc.assertOk();

+	Extent *e = (Extent *) (((char *)header) + loc.getOfs());

+	return e;

+}

+

+inline Extent* PhysicalDataFile::getExtent(DiskLoc loc) {

+	Extent *e = _getExtent(loc);

+	return e;

+}

+

+class Cursor;

+

+class DataFileMgr {

+	friend class Cursor;

+public:

+	void init();

+

+	void insert(const char *ns, void *buf, int len);

+	Cursor findAll(const char *ns);

+

+private:

+	PhysicalDataFile temp;

+};

+

+extern DataFileMgr theDataFileMgr;

+

+class Cursor {

+public:

+	bool ok() { return !curr.isNull(); }

+	bool eof() { return !ok(); }

+	Record* current() {

+		assert( ok() );

+		return theDataFileMgr.temp.recordAt(curr); 

+	}

+	bool advance() { 

+		if( eof() )

+			return false;

+		Record *r = current();

+		curr = r->next;

+		return ok();

+	}

+

+	Cursor(DiskLoc dl) : curr(dl) { }

+	Cursor() { }

+

+private:

+	DiskLoc curr;

+};

+

diff --git a/db/query.cpp b/db/query.cpp
new file mode 100644
index 00000000000..0c70d770f15
--- /dev/null
+++ b/db/query.cpp
@@ -0,0 +1,48 @@
+// query.cpp

+

+#include "stdafx.h"

+#include "query.h"

+#include "pdfile.h"

+#include "jsobj.h"

+#include "../util/builder.h"

+

+int nextCursorId = 1;

+

+QueryResult* runQuery(const char *ns, const char *query, int ntoreturn) {

+

+	/* temp implementation -- just returns everything! */

+

+	BufBuilder b;

+

+	QueryResult *qr = 0;

+	b.skip(qr->data - ((char *)qr));

+

+	int n = 0;

+	Cursor c = theDataFileMgr.findAll(ns);

+	while( c.ok() ) {

+		Record *r = c.current();

+

+		JSObj js(r);

+		// check criteria here.

+

+		b.append(r->netLength()+4);

+		b.append(r->data, r->netLength());

+		n++;

+

+		if( n >= ntoreturn )

+			break;

+

+		c.advance();

+	}

+

+	qr = (QueryResult *) b.buf();

+	qr->len = b.len();

+	qr->reserved = 0;

+	qr->operation = opReply;

+	qr->cursorId = nextCursorId++;

+	qr->startOfs = 0;

+	qr->nReturned = n;

+	b.decouple();

+

+	return qr;

+}

diff --git a/db/query.h b/db/query.h
new file mode 100644
index 00000000000..7b101cd7816
--- /dev/null
+++ b/db/query.h
@@ -0,0 +1,25 @@
+// query.h

+

+#pragma once

+

+#include "../stdafx.h"

+#include "../grid/message.h"

+

+/*

+   Query or GetMore:

+      int reserved;

+      unsigned cursorID;

+      unsigned startOfs;

+      unsigned nReturned;

+      list of marshalled JSObjects;

+*/ 

+

+struct QueryResult : public MsgData {

+	int cursorId;

+	int startOfs;

+	int nReturned;

+	char data[4];

+};

+

+QueryResult* runQuery(const char *ns, const char *query, int ntoreturn);

+

diff --git a/db/resource.h b/db/resource.h
new file mode 100644
index 00000000000..6fafd161abb
--- /dev/null
+++ b/db/resource.h
@@ -0,0 +1,14 @@
+//{{NO_DEPENDENCIES}}

+// Microsoft Visual C++ generated include file.

+// Used by db.rc

+

+// Next default values for new objects

+// 

+#ifdef APSTUDIO_INVOKED

+#ifndef APSTUDIO_READONLY_SYMBOLS

+#define _APS_NEXT_RESOURCE_VALUE        101

+#define _APS_NEXT_COMMAND_VALUE         40001

+#define _APS_NEXT_CONTROL_VALUE         1001

+#define _APS_NEXT_SYMED_VALUE           101

+#endif

+#endif

diff --git a/db/send b/db/send
new file mode 100644
index 00000000000..fe6a0899528
--- /dev/null
+++ b/db/send
diff --git a/db/storage.h b/db/storage.h
new file mode 100644
index 00000000000..43bf5bfc4a4
--- /dev/null
+++ b/db/storage.h
@@ -0,0 +1,37 @@
+/* storage.h

+

+   Storage subsystem management.

+   Lays out our datafiles on disk, manages disk space.

+*/

+

+#pragma once

+

+#pragma pack(push)

+#pragma pack(1)

+

+class DiskLoc {

+	int reserved; /* this will be volume, file #, etc. */

+	int ofs;

+public:

+	DiskLoc() { reserved = -1; ofs = -1; }

+

+	bool isNull() { return ofs == -1; }

+	void Null() { reserved = -1; ofs = -1; }

+	void assertOk() { assert(!isNull()); }

+

+	int getOfs() { return ofs; }

+	void setOfs(int _ofs) { 

+		reserved = -2;

+		ofs = _ofs;

+	}

+

+	void inc(int amt) {

+		assert( !isNull() );

+		ofs += amt;

+	}

+

+	bool sameFile(DiskLoc b) { return reserved == b.reserved; /* not really done...*/ }

+

+};

+

+#pragma pack(pop)

diff --git a/grid/message.cpp b/grid/message.cpp
new file mode 100644
index 00000000000..48b91d90bc8
--- /dev/null
+++ b/grid/message.cpp
@@ -0,0 +1,153 @@
+/* message

+

+   todo: authenticate; encrypt?

+*/

+

+#include "stdafx.h"

+#include "message.h"

+

+const int FragMax = 1480;

+const int MSS = FragMax - 8;

+

+#pragma pack(push)

+#pragma pack(1)

+

+struct Fragment {

+	enum { MinFragmentLen = 8 + 1 };

+	int msgId;

+	short fragmentLen;

+	short fragmentNo;

+	char data[1];

+	int fragmentDataLen() { return fragmentLen - 8; }

+

+	bool ok(int nRead) { 

+		if( nRead < MinFragmentLen || fragmentLen > nRead || fragmentLen < MinFragmentLen ) {

+			cout << "recv: fragment bad. fragmentLen:" << fragmentLen << " nRead:" << nRead << endl;

+			return false;

+		}

+		if( fragmentNo == 0 && fragmentLen < MinFragmentLen + MsgDataHeaderSize ) { 

+			cout << "recv: bad first fragment. fragmentLen:" << fragmentLen << endl;

+			return false;

+		}

+		return true;

+	} 

+

+	MsgData* startOfMsgData() { assert(fragmentNo == 0); return (MsgData *) data; }

+};

+#pragma pack(pop)

+

+int NextMsgId = -1000;

+struct MsgStart {

+	MsgStart() {

+		srand(3);

+		NextMsgId = rand();

+		assert(MsgDataHeaderSize == 20);

+		assert(sizeof(Fragment) == 9);

+	}

+} msgstart;

+

+MessagingPort::MessagingPort() {

+}

+

+MessagingPort::~MessagingPort() { 

+}

+

+void MessagingPort::init(int myUdpPort) {

+	SockAddr me(myUdpPort);

+	if( !conn.init(me) ) {

+		cout << "conn init failure in MessagingPort::init " << myUdpPort << endl;

+		exit(2);

+	}

+}

+

+/* this is a temp implementation.  it will only work if there is a single entity messaging the receiver! */

+bool MessagingPort::recv(Message& m) {

+	int n = conn.recvfrom(buf, BufSize, m.from);

+	Fragment *ff = (Fragment *) buf;

+	if( !ff->ok(n) )

+		return false;

+

+	MsgData *somd = ff->startOfMsgData();

+	int totalLen = somd->len;

+	if( ff->fragmentDataLen() >= totalLen ) {

+		// it's a short message, we got it all in one packet

+		m.setData(somd, false);

+		return true;

+	}

+

+	/* we'll need to read more */

+	char *msgData = (char *) malloc(totalLen);

+	char *p = msgData;

+	memcpy(p, somd, ff->fragmentDataLen());

+	int sofar = ff->fragmentDataLen();

+	int wanted = totalLen;

+	p += ff->fragmentDataLen();

+	wanted -= ff->fragmentDataLen();

+

+	/* note out of order, retransmits not done.  just get us going on localhost */

+	int msgid = ff->msgId;

+	int expectedFragmentNo = 1;

+	SockAddr from;

+	while( 1 ) {

+		char b[FragMax];

+		int n = conn.recvfrom(b, sizeof(b), from);

+		Fragment *f = (Fragment *) b; 

+		if( !f->ok(n) )

+			return false;

+		if( f->msgId != msgid || f->fragmentNo != expectedFragmentNo ) {

+			cout << "bad fragment" << endl;

+			return false;

+		}

+		if( from != m.from ) {

+			cout << "wrong sender? impl not done for multiple 'clients'" << endl;

+			assert(false);

+			return false;

+		}

+

+		memcpy(p, f->startOfMsgData(), f->fragmentDataLen());

+		p += f->fragmentDataLen();

+		wanted -= f->fragmentDataLen();

+		expectedFragmentNo++;

+

+		if( wanted <= 0 ) {

+			assert( wanted == 0 );

+			break;

+		}

+	}

+

+	return true;

+}

+

+void MessagingPort::reply(Message& received, Message& response) {

+	say(received.from, response, received.data->id);

+}

+

+bool MessagingPort::call(SockAddr& to, Message& toSend, Message& response) {

+	say(to, toSend);

+	bool ok = recv(response);

+	if( !ok )

+		return false;

+	assert( response.data->responseTo == toSend.data->id);

+	return true;

+}

+

+void MessagingPort::say(SockAddr& to, Message& toSend, int responseTo) {

+	toSend.data->reserved = 0;

+	toSend.data->id = NextMsgId++;

+	toSend.data->responseTo = responseTo;

+

+	int left = toSend.data->len;

+	assert( left > 0 && left <= 16 * 1024 * 1024 );

+	Fragment *f = (Fragment *) buf;

+	f->msgId = toSend.data->id;

+	f->fragmentNo = 0;

+	char *p = (char *) toSend.data;

+	while( left > 0 ) { 

+		int l = left > MSS ? MSS : left;

+		f->fragmentLen = l + 8;

+		memcpy(f->data, p, l);

+		p += l;

+		left -= l;

+		conn.sendto(buf, l+8, to);

+	}

+}

diff --git a/grid/message.h b/grid/message.h
new file mode 100644
index 00000000000..b72497a8d4a
--- /dev/null
+++ b/grid/message.h
@@ -0,0 +1,98 @@
+// message.h

+

+#pragma once

+

+#include "../util/sock.h"

+

+class Message;

+

+class MessagingPort {

+public:

+	enum { DBPort = 27017 };

+

+	MessagingPort();

+	~MessagingPort();

+

+	void init(int myUdpPort);

+

+	/* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's.

+	   also, the Message data will go out of scope on the subsequent recv call. 

+	*/

+	bool recv(Message& m);

+	void reply(Message& received, Message& response);

+	bool call(SockAddr& to, Message& toSend, Message& response);

+	void say(SockAddr& to, Message& toSend, int responseTo = -1);

+

+private:

+	UDPConnection conn;

+	enum { BufSize = 64 * 1024 };

+	char buf[BufSize];

+};

+

+#pragma pack(push)

+#pragma pack(1)

+

+enum Operations { 

+	opReply = 1,     /* reply. responseTo is set. */

+

+	dbMsg = 1000,    /* generic msg command followed by a string */

+

+	dbUpdate = 2001, /* update object */

+	dbInsert = 2002,

+	dbGetByOID = 2003,

+	dbQuery = 2004,

+	dbGetMore = 2005

+};

+

+struct MsgData {

+	int len; /* len of the msg, including this field */

+	int reserved;

+	int id; /* request/reply id's match... */

+	int responseTo; /* id of the message we are responding to */

+	int operation;

+	char _data[4];

+

+	int dataLen();

+};

+const int MsgDataHeaderSize = sizeof(MsgData) - 4;

+inline int MsgData::dataLen() { return len - MsgDataHeaderSize; }

+

+#pragma pack(pop)

+

+class Message {

+public:

+	Message() { data = 0; }

+	~Message() { reset(); }

+

+	SockAddr from;

+	MsgData *data;

+

+	void reset() {

+		if( freeIt && data )

+			free(data);

+		data = 0;

+	}

+

+	void setData(MsgData *d, bool _freeIt) {

+		assert( data == 0 ); 

+		freeIt = _freeIt;

+		data = d;

+	}

+	void setData(int operation, const char *msgtxt) {

+		setData(operation, msgtxt, strlen(msgtxt)+1);

+	}

+	void setData(int operation, const char *msgdata, int len) {

+		assert(data == 0);

+		int dataLen = len + sizeof(MsgData) - 4;

+		MsgData *d = (MsgData *) malloc(dataLen);

+		memcpy(d->_data, msgdata, len);

+		d->len = dataLen;

+		d->operation = operation;

+		freeIt= true;

+		data = d;

+	}

+

+private:

+	bool freeIt;

+};

+

diff --git a/stdafx.cpp b/stdafx.cpp
new file mode 100644
index 00000000000..c26ad91edc8
--- /dev/null
+++ b/stdafx.cpp
@@ -0,0 +1,14 @@
+// stdafx.cpp : source file that includes just the standard includes

+// db.pch will be the pre-compiled header

+// stdafx.obj will contain the pre-compiled type information

+

+#include "stdafx.h"

+

+// TODO: reference any additional headers you need in STDAFX.H

+// and not in this file

+

+struct MyAsserts {

+	MyAsserts() {

+

+	}

+} myassertsstdafx;

diff --git a/stdafx.h b/stdafx.h
new file mode 100644
index 00000000000..55b7250f2c6
--- /dev/null
+++ b/stdafx.h
@@ -0,0 +1,43 @@
+// stdafx.h : include file for standard system include files,

+// or project specific include files that are used frequently, but

+// are changed infrequently

+//

+

+#pragma once

+

+#include "targetver.h"

+

+#include "assert.h"

+#include <stdio.h>

+#include <sstream>

+

+//#if defined(_WIN32)

+//#include <tchar.h>

+//#else

+typedef char _TCHAR;

+//#endif

+

+#include <iostream>

+#include <fstream>

+using namespace std;

+

+#if !defined(_WIN32)

+typedef int HANDLE;

+inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); }

+#else

+typedef void *HANDLE;

+#endif

+

+//#if defined(CHAR)

+//#error CHAR already defined?

+//#endif

+

+//#if defined(_WIN32_WINNT)

+//typedef wchar_t CHAR;

+//#else

+// more to be done...linux unicode is 32 bit.

+//typedef unsigned short CHAR; // 16 bit unicode

+//#endif

+

+#define null (0)

+

diff --git a/stdafx.h.gch b/stdafx.h.gch
new file mode 100644
index 00000000000..d616f62ad94
--- /dev/null
+++ b/stdafx.h.gch
diff --git a/targetver.h b/targetver.h
new file mode 100644
index 00000000000..3965af59bcd
--- /dev/null
+++ b/targetver.h
@@ -0,0 +1,5 @@
+#pragma once

+#ifndef _WIN32_WINNT            // Allow use of features specific to Windows Vista or later.

+#define _WIN32_WINNT 0x0600     // Change this to the appropriate value to target other versions of Windows.

+#endif

+

diff --git a/util/builder.h b/util/builder.h
new file mode 100644
index 00000000000..886ea230c7a
--- /dev/null
+++ b/util/builder.h
@@ -0,0 +1,15 @@
+/* builder.h

+

+*/

+

+#include "../stdafx.h"

+

+class BufBuilder {

+public:

+	void skip(int n) { }

+	char* buf() { return 0; }

+	void decouple() { }

+	void append(int) { }

+	void append(void *, int len) { }

+	int len() { return 0; }

+};

diff --git a/util/hashtab.h b/util/hashtab.h
new file mode 100644
index 00000000000..9c45425d06b
--- /dev/null
+++ b/util/hashtab.h
@@ -0,0 +1,96 @@
+/* hashtab.h

+

+   Simple, fixed size hash table.  Darn simple.

+

+   Uses a contiguous block of memory, so you can put it in a memory mapped file very easily.

+*/

+

+#include "../stdafx.h"

+#include <map>

+

+#pragma pack(push)

+#pragma pack(1)

+

+/* you should define:

+

+   int Key::hash() return > 0 always.

+*/

+

+template <

+	class Key,

+	class Type

+>

+class HashTable {

+private:

+	const char *name;

+	struct Node {

+		int hash;

+		Key k;

+		Type value;

+	} *nodes;

+	int n;

+

+	int _find(const Key& k, bool& found) {

+		found = false;

+		int h = k.hash();

+		int i = h % n;

+		int start = i;

+		int chain = 0;

+		while( 1 ) {

+			if( nodes[i].hash == 0 ) {

+				return i;

+			}

+			if( nodes[i].hash == h && nodes[i].k == k ) {

+				found = true;

+				return i;

+			}

+			chain++;

+			i = (i+1) % n;

+			if( i == start ) { 

+				cout << "warning: hashtable is full " << name << endl;

+				return -1;

+			}

+			if( chain == 200 )

+				cout << "warning: hashtable long chain " << name << endl;

+		}

+	}

+

+public:

+	/* buf must be all zeroes on initialization. */

+	HashTable(void *buf, int buflen, const char *_name) : name(_name) { 

+		int m = sizeof(Node);

+		n = buflen / m;

+		if( (n & 1) == 0 )

+			n--;

+		nodes = (Node *) buf;

+		assert(nodes[n-1].hash == 0);

+		assert(nodes[0].hash == 0);

+	}

+

+	Type* get(const Key& k) { 

+		bool found;

+		int i = _find(k, found);

+		if( found )

+			return &nodes[i].value;

+		return 0;

+	}

+

+	void put(const Key& k, const Type& value) {

+		bool found;

+		int i = _find(k, found);

+		if( i < 0 )

+			return;

+		if( !found ) {

+			nodes[i].k = k;

+			nodes[i].hash = k.hash();

+		}

+		else {

+			assert( nodes[i].hash == k.hash() );

+		}

+		nodes[i].value = value;

+	}

+

+};

+

+#pragma pack(pop)

+

diff --git a/util/mmap.cpp b/util/mmap.cpp
new file mode 100644
index 00000000000..5151ea2ad00
--- /dev/null
+++ b/util/mmap.cpp
@@ -0,0 +1,128 @@
+// mmap.cpp

+

+#include "stdafx.h"

+#include "mmap.h"

+

+#if defined(_WIN32) 

+

+#include "windows.h"

+

+MemoryMappedFile::MemoryMappedFile() {

+	fd = 0; maphandle = 0; view = 0;

+}

+

+MemoryMappedFile::~MemoryMappedFile() {

+	if( view )

+		UnmapViewOfFile(view);

+	view = 0;

+	if( maphandle )

+		CloseHandle(maphandle);

+	maphandle = 0;

+	if( fd )

+		CloseHandle(fd);

+	fd = 0;

+}

+

+std::wstring toWideString(const char *s) {

+ //const std::basic_string<TCHAR> s) {

+  std::basic_ostringstream<TCHAR> buf;

+  buf << s;

+  return buf.str();

+}

+

+void* MemoryMappedFile::map(const char *filename, int length) {

+	std::wstring filenamew = toWideString(filename);

+

+	fd = CreateFile(

+		filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ,

+		NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);

+	if( fd == INVALID_HANDLE_VALUE ) {

+		cout << "CreateFile failed " << filename << endl;

+		return 0;

+	}

+

+	maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL);

+	if( maphandle == NULL ) {

+		cout << "CreateFileMapping failed " << filename << endl;

+		return 0;

+	}

+

+	view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0);

+	if( view == 0 )

+		cout << "MapViewOfFile failed " << filename << endl;

+

+	return view;

+}

+

+void MemoryMappedFile::flush(bool) {

+}

+

+#else

+

+#include <errno.h>

+#include <sys/mman.h>

+#include <sys/types.h>

+#include <sys/stat.h>

+#include <fcntl.h>

+

+MemoryMappedFile::MemoryMappedFile() {

+	fd = 0; maphandle = 0; view = 0; len = 0;

+}

+

+MemoryMappedFile::~MemoryMappedFile() {

+	if( view )

+		munmap(view, len);

+	view = 0;

+

+	if( fd )

+		close(fd);

+	fd = 0;

+}

+

+void* MemoryMappedFile::map(const char *filename, int length) {

+	len = length;

+

+	fd = open(filename, O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);

+	if( !fd ) {

+		cout << "couldn't open " << filename << ' ' << errno << endl;

+		return 0;

+	}

+

+	/* make sure the file is the full desired length */

+	off_t filelen = lseek(fd, 0, SEEK_END);

+	if( filelen < length ) { 

+		cout << "map: file length=" << filelen << " want:" << length << endl;

+		if( filelen != 0 ) { 

+			cout << "  failing mapping" << endl;

+			return 0;

+		}

+		cout << "  writing file to full length with zeroes..." << endl;

+		int z = 8192;

+		char buf[z];

+		memset(buf, 0, z);

+		int left = length;

+		while( 1 ) { 

+			if( left <= z ) {

+				write(fd, buf, left);

+				break;

+			}

+			write(fd, buf, z);

+			left -= z;

+		}

+		cout << "  done" << endl;

+	}

+

+	lseek(fd, length, SEEK_SET);

+	write(fd, "", 1);

+

+	view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);

+	return view;

+}

+

+void MemoryMappedFile::flush(bool sync) {

+	if( msync(view, len, sync ? MS_SYNC : MS_ASYNC) )

+		cout << "msync error " << errno << endl;

+}

+

+#endif

+

diff --git a/util/mmap.h b/util/mmap.h
new file mode 100644
index 00000000000..aae4eed0f32
--- /dev/null
+++ b/util/mmap.h
@@ -0,0 +1,22 @@
+// mmap.h

+

+#pragma once

+

+class MemoryMappedFile {

+public:

+	MemoryMappedFile();

+	~MemoryMappedFile();

+

+	/* only smart enough right now to deal with files of a fixed length. 

+	   creates if DNE

+	*/

+	void* map(const char *filename, int length);

+

+	void flush(bool sync);

+

+private:

+	HANDLE fd;

+	HANDLE maphandle;

+	void *view;

+	int len;

+};

diff --git a/util/sock.cpp b/util/sock.cpp
new file mode 100644
index 00000000000..d5fb7af1742
--- /dev/null
+++ b/util/sock.cpp
@@ -0,0 +1,158 @@
+// sock.cpp

+

+#include "stdafx.h"

+#include "sock.h"

+

+void sendtest() { 

+	cout << "sendtest\n"; 

+	SockAddr me(27016);

+	SockAddr dest("127.0.0.1", 27015);

+	UDPConnection c;

+	if( c.init(me) ) {

+		char buf[256];

+		cout << "sendto: ";

+		cout << c.sendto(buf, sizeof(buf), dest) << " errno:" << h_errno << endl;

+	}

+	cout << "end\n";

+}

+

+void listentest() { 

+	cout << "listentest\n"; 

+	SockAddr me(27015);

+	SockAddr sender;

+	UDPConnection c;

+	if( c.init(me) ) {

+		char buf[256];

+		cout << "recvfrom: ";

+		cout << c.recvfrom(buf, sizeof(buf), sender) << " errno:" << h_errno << endl;

+	}

+	cout << "end listentest\n";

+}

+

+void xmain();

+struct SockStartupTests {

+	SockStartupTests() {

+#if defined(_WIN32)

+        WSADATA d;

+		if( WSAStartup(MAKEWORD(2,2), &d) != 0 ) {

+			cout << "ERROR: wsastartup failed " << errno << endl;

+			exit(1);

+		}

+#endif

+		//cout << "ntohl:" << ntohl(256) << endl;

+		//sendtest();

+		//listentest();

+	}

+} sstests;

+

+#if 0

+void smain() {

+

+  WSADATA wsaData;

+  SOCKET RecvSocket;

+  sockaddr_in RecvAddr;

+  int Port = 27015;

+  char RecvBuf[1024];

+  int  BufLen = 1024;

+  sockaddr_in SenderAddr;

+  int SenderAddrSize = sizeof(SenderAddr);

+

+  //-----------------------------------------------

+  // Initialize Winsock

+  WSAStartup(MAKEWORD(2,2), &wsaData);

+

+  //-----------------------------------------------

+  // Create a receiver socket to receive datagrams

+  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

+

+  //-----------------------------------------------

+  // Bind the socket to any address and the specified port.

+  RecvAddr.sin_family = AF_INET;

+  RecvAddr.sin_port = htons(Port);

+  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);

+

+  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));

+

+  //-----------------------------------------------

+  // Call the recvfrom function to receive datagrams

+  // on the bound socket.

+  printf("Receiving datagrams...\n");

+  recvfrom(RecvSocket, 

+    RecvBuf, 

+    BufLen, 

+    0, 

+    (SOCKADDR *)&SenderAddr, 

+    &SenderAddrSize);

+

+  //-----------------------------------------------

+  // Close the socket when finished receiving datagrams

+  printf("Finished receiving. Closing socket.\n");

+  closesocket(RecvSocket);

+

+  //-----------------------------------------------

+  // Clean up and exit.

+  printf("Exiting.\n");

+  WSACleanup();

+  return;

+}

+

+

+

+

+void xmain() {

+

+  WSADATA wsaData;

+  SOCKET RecvSocket;

+  sockaddr_in RecvAddr;

+  int Port = 27015;

+  char RecvBuf[1024];

+  int  BufLen = 1024;

+  sockaddr_in SenderAddr;

+  int SenderAddrSize = sizeof(SenderAddr);

+

+  //-----------------------------------------------

+  // Initialize Winsock

+  WSAStartup(MAKEWORD(2,2), &wsaData);

+

+  //-----------------------------------------------

+  // Create a receiver socket to receive datagrams

+

+  RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

+

+  //-----------------------------------------------

+  // Bind the socket to any address and the specified port.

+  RecvAddr.sin_family = AF_INET;

+  RecvAddr.sin_port = htons(Port);

+  RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY);

+

+  SockAddr a(Port);

+  bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize);

+//  bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr));

+

+  SockAddr b;

+

+  //-----------------------------------------------

+  // Call the recvfrom function to receive datagrams

+  // on the bound socket.

+  printf("Receiving datagrams...\n");

+  recvfrom(RecvSocket, 

+    RecvBuf, 

+    BufLen, 

+    0,

+	(SOCKADDR *) &b.sa, &b.addressSize);

+//    (SOCKADDR *)&SenderAddr, 

+//    &SenderAddrSize);

+

+  //-----------------------------------------------

+  // Close the socket when finished receiving datagrams

+  printf("Finished receiving. Closing socket.\n");

+  closesocket(RecvSocket);

+

+  //-----------------------------------------------

+  // Clean up and exit.

+  printf("Exiting.\n");

+  WSACleanup();

+  return;

+}

+

+#endif

diff --git a/util/sock.h b/util/sock.h
new file mode 100644
index 00000000000..3a6c1eddfb4
--- /dev/null
+++ b/util/sock.h
@@ -0,0 +1,88 @@
+// sock.h

+

+#pragma once 

+

+//#include "socket.h"

+

+#if defined(_WIN32)

+#include <winsock2.h>

+#include <ws2tcpip.h>

+typedef int socklen_t;

+#else

+#include <sys/socket.h>

+#include <sys/types.h>

+#include <sys/socket.h>

+#include <netinet/in.h>

+#include <arpa/inet.h>

+#include <errno.h>

+inline void closesocket(int s) { close(s); }

+const int INVALID_SOCKET = -1;

+typedef int SOCKET;

+#define h_errno errno

+#endif

+

+struct SockAddr {

+	SockAddr() { addressSize = sizeof(sockaddr_in); }

+	SockAddr(int sourcePort); /* source side */ 

+	SockAddr(const char *ip, int port); /* dest (remote) side, or if you want to specify which interface locally */

+

+	struct sockaddr_in sa;

+	socklen_t addressSize;

+

+	bool operator==(const SockAddr& r) const { 

+		return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr &&

+			sa.sin_port == r.sa.sin_port;

+	}

+	bool operator!=(const SockAddr& r) const { return !(*this == r); }

+};

+

+class UDPConnection {

+public:

+	UDPConnection() { sock = 0; }

+	~UDPConnection() { if( sock ) { closesocket(sock); sock = 0; } }

+	bool init(const SockAddr& myAddr);

+	int recvfrom(char *buf, int len, SockAddr& sender);

+	int sendto(char *buf, int len, const SockAddr& dest);

+

+	SOCKET sock;

+};

+

+inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) {

+	return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize);

+}

+

+inline int UDPConnection::sendto(char *buf, int len, const SockAddr& dest) {

+	return ::sendto(sock, buf, len, 0, (sockaddr *) &dest.sa, dest.addressSize);

+}

+

+inline bool UDPConnection::init(const SockAddr& myAddr) {

+	sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

+	if( sock == INVALID_SOCKET ) {

+		cout << "invalid socket? " << errno << endl;

+		return false;

+	}

+	cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;

+	if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) { 

+		cout << "udp init failed" << endl;

+		closesocket(sock);

+		sock = 0;

+		return false;

+	}

+	return true;

+}

+

+inline SockAddr::SockAddr(int sourcePort) {

+	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));

+	sa.sin_family = AF_INET;

+	sa.sin_port = htons(sourcePort);

+	sa.sin_addr.s_addr = htonl(INADDR_ANY);

+	addressSize = sizeof(sa);

+}

+

+inline SockAddr::SockAddr(const char *ip, int port) {

+	memset(sa.sin_zero, 0, sizeof(sa.sin_zero));

+	sa.sin_family = AF_INET;

+	sa.sin_port = htons(port);

+	sa.sin_addr.s_addr = inet_addr(ip);

+	addressSize = sizeof(sa);

+}