compression lib

3 files changed, 112 insertions, 4 deletions

diff --git a/dbtests/perftests.cpp b/dbtests/perftests.cpp
index b90c039a662..a34700b74c9 100644
--- a/dbtests/perftests.cpp
+++ b/dbtests/perftests.cpp
@@ -36,6 +36,7 @@
 #include "../util/checksum.h"
 #include "../util/version.h"
 #include "../db/key.h"
+#include "../util/compress.h"
 
 using namespace bson;
 
@@ -193,6 +194,8 @@ namespace PerfTests {
             catch(...) { }
         }
 
+        virtual unsigned batchSize() { return 50; }
+
         void say(unsigned long long n, int ms, string s) {
             unsigned long long rps = n*1000/ms;
             cout << "stats " << setw(33) << left << s << ' ' << right << setw(9) << rps << ' ' << right << setw(5) << ms << "ms ";
@@ -281,9 +284,9 @@ namespace PerfTests {
 
             dur::stats._intervalMicros = 0; // no auto rotate
             dur::stats.curr->reset();
-            Timer t;
+            mongo::Timer t;
             unsigned long long n = 0;
-            const unsigned Batch = 50;
+            const unsigned Batch = batchSize();
 
             if( hlm == 0 ) { 
                 // means just do once
@@ -321,7 +324,7 @@ namespace PerfTests {
                 const char *test2name = timed2();
                 if( test2name ) {
                     dur::stats.curr->reset();
-                    Timer t;
+                    mongo::Timer t;
                     unsigned long long n = 0;
                     while( 1 ) {
                         unsigned i;
@@ -603,6 +606,47 @@ namespace PerfTests {
         virtual bool showDurStats() { return false; }
     };
 
+    class Compress : public B {
+    public:
+        const unsigned sz;
+        void *p;
+        Compress() : sz(1024*1024*100+3) { }
+        virtual unsigned batchSize() { return 1; }
+        string name() { return "compress"; }
+        virtual bool showDurStats() { return false; }
+        virtual int howLongMillis() { return 4000; } 
+        unsigned long long expectation() { return 1000000; }
+        void prep() { 
+            p = malloc(sz);
+            // this isn't a fair test as it is mostly rands but we just want a rough perf check
+            static int last;
+            for (unsigned i = 0; i<sz; i++) {
+                int r = rand();
+                if( (r & 0x300) == 0x300 )
+                    r = last;
+                ((char*)p)[i] = r;
+                last = r;
+            }
+        }
+        size_t last;
+        string res;
+        void timed() {
+            mongo::Timer t;
+            string out;
+            size_t len = compress((const char *) p, sz, &out);
+            bool ok = uncompress(out.c_str(), out.size(), &res);
+            ASSERT(ok);
+            static unsigned once;
+            if( once++ == 0 )
+                cout << "compress round trip " << sz/(1024.0*1024) / (t.millis()/1000.0) << "MB/sec\n";
+            //cout << len / (1024.0/1024) << " compressed" << endl;
+        }
+        void post() {
+            ASSERT( memcmp(res.c_str(), p, sz) == 0 );
+            free(p);
+        }
+    };
+
     // test speed of checksum method
     class ChecksumTest : public B {
     public:
@@ -612,6 +656,7 @@ namespace PerfTests {
         virtual int howLongMillis() { return 2000; } 
         int expectationTimeMillis() { return 5000; }
         virtual bool showDurStats() { return false; }
+        virtual unsigned batchSize() { return 1; }
 
         void *p;
 
@@ -827,6 +872,8 @@ namespace PerfTests {
             }
             else {
                 add< Dummy >();
+                add< ChecksumTest >();
+                add< Compress >();
                 add< TLS >();
                 add< Malloc >();
                 add< Timer >();
@@ -843,7 +890,6 @@ namespace PerfTests {
                 add< BSONIter >();
                 add< BSONGetFields1 >();
                 add< BSONGetFields2 >();
-                add< ChecksumTest >();
                 add< TaskQueueTest >();
                 add< InsertDup >();
                 add< Insert1 >();
diff --git a/dbtests/test.vcxproj b/dbtests/test.vcxproj
index b37ed57dd7d..b9d80de953e 100644
--- a/dbtests/test.vcxproj
+++ b/dbtests/test.vcxproj
@@ -258,8 +258,16 @@
     <ClInclude Include="..\db\resource.h" />

     <ClInclude Include="..\db\scanandorder.h" />

     <ClInclude Include="..\db\security.h" />

+    <ClInclude Include="..\third_party\snappy\config.h" />

+    <ClInclude Include="..\third_party\snappy\snappy-c.h" />

+    <ClInclude Include="..\third_party\snappy\snappy-internal.h" />

+    <ClInclude Include="..\third_party\snappy\snappy-sinksource.h" />

+    <ClInclude Include="..\third_party\snappy\snappy-stubs-internal.h" />

+    <ClInclude Include="..\third_party\snappy\snappy-stubs-public.h" />

+    <ClInclude Include="..\third_party\snappy\snappy.h" />

     <ClInclude Include="..\util\builder.h" />

     <ClInclude Include="..\util\checksum.h" />

+    <ClInclude Include="..\util\compress.h" />

     <ClInclude Include="..\util\concurrency\list.h" />

     <ClInclude Include="..\util\concurrency\task.h" />

     <ClInclude Include="..\util\concurrency\value.h" />

@@ -608,9 +616,27 @@
     <ClCompile Include="..\s\shard.cpp" />

     <ClCompile Include="..\s\shardconnection.cpp" />

     <ClCompile Include="..\s\shardkey.cpp" />

+    <ClCompile Include="..\third_party\snappy\snappy-sinksource.cc">

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader>

+    </ClCompile>

+    <ClCompile Include="..\third_party\snappy\snappy.cc">

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader>

+    </ClCompile>

     <ClCompile Include="..\util\alignedbuilder.cpp">

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

     </ClCompile>

+    <ClCompile Include="..\util\compress.cpp">

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader>

+    </ClCompile>

     <ClCompile Include="..\util\concurrency\spin_lock.cpp">

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>

     </ClCompile>

diff --git a/dbtests/test.vcxproj.filters b/dbtests/test.vcxproj.filters
index 1c832cd17ba..0691238e6c0 100755
--- a/dbtests/test.vcxproj.filters
+++ b/dbtests/test.vcxproj.filters
@@ -56,6 +56,9 @@
     <Filter Include="bson">

       <UniqueIdentifier>{e6652333-c77f-420c-af8e-72d55bc095fe}</UniqueIdentifier>

     </Filter>

+    <Filter Include="misc and third party\snappy">

+      <UniqueIdentifier>{fbc4416f-ca67-4e63-a1ea-49027de7e080}</UniqueIdentifier>

+    </Filter>

   </ItemGroup>

   <ItemGroup>

     <ClInclude Include="..\..\boostw\boost_1_34_1\boost\config\auto_link.hpp">

@@ -304,6 +307,30 @@
     <ClInclude Include="..\server.h">

       <Filter>db\h</Filter>

     </ClInclude>

+    <ClInclude Include="..\third_party\snappy\config.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy-c.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy-internal.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy-sinksource.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy-stubs-internal.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\third_party\snappy\snappy-stubs-public.h">

+      <Filter>misc and third party\snappy</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\compress.h">

+      <Filter>misc and third party</Filter>

+    </ClInclude>

   </ItemGroup>

   <ItemGroup>

     <Library Include="..\..\js\js64r.lib">

@@ -857,6 +884,15 @@
     <ClCompile Include="..\util\concurrency\spin_lock.cpp">

       <Filter>util\concurrency</Filter>

     </ClCompile>

+    <ClCompile Include="..\third_party\snappy\snappy.cc">

+      <Filter>misc and third party\snappy</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\compress.cpp">

+      <Filter>misc and third party</Filter>

+    </ClCompile>

+    <ClCompile Include="..\third_party\snappy\snappy-sinksource.cc">

+      <Filter>misc and third party\snappy</Filter>

+    </ClCompile>

   </ItemGroup>

   <ItemGroup>

     <None Include="..\SConstruct">