SCons updates to support variant directories.

This patch is a reorganization of our build files, which brings them slightly
closer in line with standard SCons organization.

In particular, the SConstruct file sets up the various "build environment"
objects, by examining the local system and command line parameters.  Then, it
delegates to some SConscript files, which describe build rules, like how to
compile "mongod" from source.

Typically, you would create several SConscript files for a project this large,
after breaking the project into logical sub projects, such as "platform
abstraction", "data manager", "query optimizer", etc.  That will be future work.
For now, we only separate out the special rules for executing smoke tests into
SConscript.smoke.  Pretty much all other build rules are in src/mongo/SConscript.

"tools" are placed in site_scons/site_tools.

This patch also includes better support for building and tracking dependencies
among static libraries ("libdeps" and "MergeLibrary"), and some incumbent, minor
restructuring.

This patch introduces a "warning" message from SCons about framework.o having
two rules that generate it.  It is harmless, for now, and will be removed in
future work.

Future work also includes eliminating use of the SCons "Glob" utility, and
restructuring the source code into sensible components.

6 files changed, 454 insertions, 0 deletions

diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py
new file mode 100644
index 00000000000..24b857c16e8
--- /dev/null
+++ b/site_scons/libdeps.py
@@ -0,0 +1,189 @@
+"""Extension to SCons providing advanced static library dependency tracking.
+
+These modifications to a build environment, which can be attached to
+StaticLibrary and Program builders via a call to setup_environment(env),
+cause the build system to track library dependencies through static libraries,
+and to add them to the link command executed when building programs.
+
+For example, consider a program 'try' that depends on a lib 'tc', which in
+turn uses a symbol from a lib 'tb' which in turn uses a library from 'ta'.
+Without this package, the Program declaration for "try" looks like this:
+
+Program('try', ['try.c', 'path/to/${LIBPREFIX}tc${LIBSUFFIX}',
+                'path/to/${LIBPREFIX}tc${LIBSUFFIX}',
+                'path/to/${LIBPREFIX}tc${LIBSUFFIX}',])
+
+With this library, we can instead write the following
+
+Program('try', ['try.c'], LIBDEPS=['path/to/tc'])
+StaticLibrary('tc', ['c.c'], LIBDEPS=['path/to/tb'])
+StaticLibrary('tb', ['b.c'], LIBDEPS=['path/to/ta'])
+StaticLibrary('ta', ['a.c'])
+
+And the build system will figure out that it needs to link libta.a and libtb.a
+when building 'try'.
+"""
+
+# Copyright (c) 2010, Corensic Inc., All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import os
+
+import SCons.Errors
+import SCons.Scanner
+import SCons.Util
+
+class DependencyCycleError(SCons.Errors.UserError):
+    """Exception representing a cycle discovered in library dependencies."""
+
+    def __init__(self, first_node ):
+        super(DependencyCycleError, self).__init__()
+        self.cycle_nodes = [first_node]
+
+    def __str__(self):
+        return " => ".join(str(n) for n in self.cycle_nodes)
+
+def __get_libdeps(node, env_var):
+    """Given a SCons Node, return its library dependencies.
+
+    Computes the dependencies if they're not already cached.
+    """
+
+    cached_var_name = env_var + '_cached'
+
+    if not hasattr(node.attributes, cached_var_name):
+        setattr(node.attributes, cached_var_name, __compute_libdeps(node, env_var))
+    return getattr(node.attributes, cached_var_name)
+
+def __compute_libdeps(node, env_var):
+    """Recursively identify all library dependencies for a node."""
+
+    if getattr(node.attributes, 'libdeps_exploring', False):
+        raise DependencyCycleError(node)
+
+    env = node.get_env()
+    deps = set()
+    node.attributes.libdeps_exploring = True
+    try:
+        try:
+            for child in env.Flatten(env.get(env_var, [])):
+                if not child:
+                    continue
+                deps.add(child)
+                deps.update(__get_libdeps(child, env_var))
+
+        except DependencyCycleError, e:
+            if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]:
+                e.cycle_nodes.append(node)
+                raise
+    finally:
+        node.attributes.libdeps_exploring = False
+
+    return deps
+
+def update_scanner(builder):
+    """Update the scanner for "builder" to also scan library dependencies."""
+
+    old_scanner = builder.target_scanner
+
+    if old_scanner:
+        path_function = old_scanner.path_function
+        def new_scanner(node, env, path=()):
+            result = set(old_scanner.function(node, env, path))
+            result.update(__get_libdeps(node, 'LIBDEPS'))
+            result.update(__get_libdeps(node, 'SYSLIBDEPS'))
+            return sorted(result)
+    else:
+        path_function = None
+        def new_scanner(node, env, path=()):
+            result = set(__get_libdeps(node, 'LIBDEPS'))
+            result.update(__get_libdeps(node, 'SYSLIBDEPS'))
+            return sorted(result)
+
+    builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner,
+                                                    path_function=path_function)
+
+def get_libdeps(source, target, env, for_signature):
+    """Implementation of the special _LIBDEPS environment variable.
+
+    Expands to the library dependencies for a target.
+    """
+
+    if for_signature:
+        return []
+    return list(__get_libdeps(target[0], 'LIBDEPS'))
+
+def get_syslibdeps(source, target, env, for_signature):
+    if for_signature:
+        return[]
+    deps = list(__get_libdeps(target[0], 'SYSLIBDEPS'))
+    return deps
+
+def libdeps_emitter(target, source, env):
+    """SCons emitter that takes values from the LIBDEPS environment variable and
+    converts them to File node objects, binding correct path information into
+    those File objects.
+
+    Emitters run on a particular "target" node during the initial execution of
+    the SConscript file, rather than during the later build phase.  When they
+    run, the "env" environment's working directory information is what you
+    expect it to be -- that is, the working directory is considered to be the
+    one that contains the SConscript file.  This allows specification of
+    relative paths to LIBDEPS elements.
+
+    This emitter also adds LIBSUFFIX and LIBPREFIX appropriately.
+    """
+
+    libdep_files = []
+    lib_suffix = env.subst('$LIBSUFFIX', target=target, source=source)
+    lib_prefix = env.subst('$LIBPREFIX', target=target, source=source)
+    for dep in env.Flatten([env.get('LIBDEPS', [])]):
+        full_path = env.subst(str(dep), target=target, source=source)
+        dir_name = os.path.dirname(full_path)
+        file_name = os.path.basename(full_path)
+        if not file_name.startswith(lib_prefix):
+            file_name = '${LIBPREFIX}' + file_name
+        if not file_name.endswith(lib_suffix):
+            file_name += '${LIBSUFFIX}'
+        libdep_files.append(env.File(os.path.join(dir_name, file_name)))
+
+    env['LIBDEPS'] = libdep_files
+
+    return target, source
+
+def setup_environment(env):
+    """Set up the given build environment to do LIBDEPS tracking."""
+
+    env['_LIBDEPS'] = get_libdeps
+    env['_SYSLIBDEPS'] = ' ${_stripixes(LIBLINKPREFIX, SYSLIBDEPS, LIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__)} '
+    env['_SHLIBDEPS'] = '$SHLIBDEP_GROUP_START ${_concat(SHLIBDEPPREFIX, __env__.subst(_LIBDEPS, target=TARGET, source=SOURCE), SHLIBDEPSUFFIX, __env__, target=TARGET, source=SOURCE)} $SHLIBDEP_GROUP_END'
+
+    env['LIBDEPS'] = SCons.Util.CLVar()
+    env['SYSLIBDEPS'] = SCons.Util.CLVar()
+    env.Append(LIBEMITTER=libdeps_emitter,
+               PROGEMITTER=libdeps_emitter,
+               SHLIBEMITTER=libdeps_emitter)
+    env.Prepend(_LIBFLAGS=' $LINK_LIBGROUP_START $_LIBDEPS $LINK_LIBGROUP_END $_SYSLIBDEPS ')
+    for builder_name in ('Program', 'SharedLibrary', 'LoadableModule'):
+        try:
+            update_scanner(env['BUILDERS'][builder_name])
+        except KeyError:
+            pass
diff --git a/site_scons/site_tools/gch.py b/site_scons/site_tools/gch.py
new file mode 100644
index 00000000000..8db5403ffcc
--- /dev/null
+++ b/site_scons/site_tools/gch.py
@@ -0,0 +1,112 @@
+# $Id$
+#
+# SCons builder for gcc's precompiled headers
+# Copyright (C) 2006  Tim Blechmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING.  If not, write to
+# the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# $Revision$
+# $LastChangedRevision$
+# $LastChangedDate$
+# $LastChangedBy$
+
+import SCons.Action
+import SCons.Builder
+import SCons.Scanner.C
+import SCons.Util
+import SCons.Script
+
+SCons.Script.EnsureSConsVersion(0,96,92)
+
+GchAction = SCons.Action.Action('$GCHCOM', '$GCHCOMSTR')
+GchShAction = SCons.Action.Action('$GCHSHCOM', '$GCHSHCOMSTR')
+
+def gen_suffix(env, sources):
+    return sources[0].get_suffix() + env['GCHSUFFIX']
+
+def header_path(env, node):
+    assert len(node.sources) == 1
+    path = node.sources[0].path
+    return path
+
+GchShBuilder = SCons.Builder.Builder(action = GchShAction,
+                                     source_scanner = SCons.Scanner.C.CScanner(),
+                                     suffix = gen_suffix)
+
+GchBuilder = SCons.Builder.Builder(action = GchAction,
+                                   source_scanner = SCons.Scanner.C.CScanner(),
+                                   suffix = gen_suffix)
+
+def static_pch_emitter(target,source,env):
+    SCons.Defaults.StaticObjectEmitter( target, source, env )
+
+    scanner = SCons.Scanner.C.CScanner()
+    path = scanner.path(env)
+    deps = scanner(source[0], env, path)
+
+    if env.has_key('Gch') and env['Gch']:
+        if header_path(env, env['Gch']) in [x.path for x in deps]:
+            env.Depends(target, env['Gch'])
+
+    return (target, source)
+
+def shared_pch_emitter(target,source,env):
+    SCons.Defaults.SharedObjectEmitter( target, source, env )
+
+    scanner = SCons.Scanner.C.CScanner()
+    path = scanner.path(env)
+    deps = scanner(source[0], env, path)
+
+    if env.has_key('GchSh') and env['GchSh']:
+        if header_path(env, env['GchSh']) in [x.path for x in deps]:
+            env.Depends(target, env['GchSh'])
+    return (target, source)
+
+def generate(env):
+    """
+    Add builders and construction variables for the Gch builder.
+    """
+    env.Append(BUILDERS = {
+        'gch': env.Builder(
+        action = GchAction,
+        target_factory = env.fs.File,
+        ),
+        'gchsh': env.Builder(
+        action = GchShAction,
+        target_factory = env.fs.File,
+        ),
+        })
+
+    try:
+        bld = env['BUILDERS']['Gch']
+        bldsh = env['BUILDERS']['GchSh']
+    except KeyError:
+        bld = GchBuilder
+        bldsh = GchShBuilder
+        env['BUILDERS']['Gch'] = bld
+        env['BUILDERS']['GchSh'] = bldsh
+
+    env['GCHCOM']     = '$CXX -o $TARGET -x c++-header -c $CXXFLAGS $_CCCOMCOM $SOURCE'
+    env['GCHSHCOM']   = '$CXX -o $TARGET -x c++-header -c $SHCXXFLAGS $_CCCOMCOM $SOURCE'
+    env['GCHSUFFIX']  = '.gch'
+
+    for suffix in SCons.Util.Split('.c .C .cc .cxx .cpp .c++'):
+        env['BUILDERS']['StaticObject'].add_emitter( suffix, static_pch_emitter )
+        env['BUILDERS']['SharedObject'].add_emitter( suffix, shared_pch_emitter )
+
+
+def exists(env):
+    return env.Detect('g++')
diff --git a/site_scons/site_tools/jsheader.py b/site_scons/site_tools/jsheader.py
new file mode 100644
index 00000000000..c271948599b
--- /dev/null
+++ b/site_scons/site_tools/jsheader.py
@@ -0,0 +1,53 @@
+import os
+
+from SCons.Builder import Builder
+
+def jsToH(target, source, env):
+
+    outFile = str( target[0] )
+
+    h =  ['#include "bson/stringdata.h"'
+        ,'namespace mongo {'
+        ,'struct JSFile{ const char* name; const StringData& source; };'
+        ,'namespace JSFiles{'
+         ]
+
+    def cppEscape(s):
+        s = s.strip()
+        s = s.replace( '\\', '\\\\' )
+        s = s.replace( '"', r'\"' )
+        return s
+
+    for s in source:
+        filename = str(s)
+        objname = os.path.split(filename)[1].split('.')[0]
+        stringname = '_jscode_raw_' + objname
+
+        h.append('const StringData ' + stringname + " = ")
+
+        for l in open( filename, 'r' ):
+            h.append( '"' + cppEscape(l) + r'\n" ' )
+
+        h.append(";")
+        h.append('extern const JSFile %s;'%objname) #symbols aren't exported w/o this
+        h.append('const JSFile %s = { "%s", %s };'%(objname, filename.replace('\\', '/'), stringname))
+
+    h.append("} // namespace JSFiles")
+    h.append("} // namespace mongo")
+    h.append("")
+
+    text = '\n'.join(h);
+
+    out = open( outFile, 'wb' )
+    try:
+        out.write( text )
+    finally:
+        out.close()
+
+jshBuilder = Builder( action=jsToH )
+
+def generate(env, **kw):
+    env.Append( BUILDERS=dict( JSHeader=jshBuilder ) )
+
+def exists(env):
+    return True
diff --git a/site_scons/site_tools/mergelib.py b/site_scons/site_tools/mergelib.py
new file mode 100644
index 00000000000..8c63d3aacce
--- /dev/null
+++ b/site_scons/site_tools/mergelib.py
@@ -0,0 +1,24 @@
+"""Builder for static libraries composed of the contents of other static libraries.
+
+The following rule creates a library "mylib" whose contents are the contents of
+"firstlib", "secondlib", and all LIBDEPS dependencies of "firstlib" and
+"secondlib".  This creates self-contained static and shared libraries that can
+be distributed to customers.
+
+MergeLibrary('mylib', ['firstlib', 'secondlib'])
+MergeSharedLibrary('mylib', ['firstlib', 'secondlib'])
+
+This file provides the platform-independent tool, which just selects and imports
+the platform-specific tool providing MergeLibrary and MergeSharedLibrary.
+"""
+
+import sys
+
+def exists( env ):
+    return True
+
+def generate( env ):
+    if sys.platform == 'win32':
+        env.Tool( 'mergelibwin' )
+    else:
+        env.Tool( 'mergelibposix' )
diff --git a/site_scons/site_tools/mergelibposix.py b/site_scons/site_tools/mergelibposix.py
new file mode 100644
index 00000000000..82807ed6b4f
--- /dev/null
+++ b/site_scons/site_tools/mergelibposix.py
@@ -0,0 +1,42 @@
+"""Builder for static libraries composed of the contents of other static libraries.
+
+The following rule creates a library "mylib" whose contents are the contents of
+"firstlib", "secondlib", and all LIBDEPS dependencies of "firstlib" and
+"secondlib".  This creates self-contained static and shared libraries that can
+be distributed to customers.
+
+MergeLibrary('mylib', ['firstlib', 'secondlib'])
+MergeSharedLibrary('mylib', ['firstlib', 'secondlib'])
+
+This implementation is for posix systems whose linkers can generate "relocatable
+objects" (usually with the -r option).
+"""
+
+import libdeps
+from SCons.Action import Action
+from SCons.Builder import Builder
+
+def merge_library_method( env, target, source, LIBDEPS=None, **kwargs ):
+    robj_name = env.subst( '${TARGET}-mergelib', target=target, source=source )
+    robj = env.RelocatableObject( robj_name, [], LIBDEPS=source, **kwargs )
+    return env.Library( target, robj, LIBDEPS=LIBDEPS or [], **kwargs )
+
+def merge_shared_library_method( env, target, source, LIBDEPS=None, **kwargs ):
+    robj_name = env.subst( '${TARGET}-mergeshlib', target=target, source=source )
+    robj = env.RelocatableObject( robj_name, [], LIBDEPS=source, **kwargs )
+    return env.SharedLibrary( target, robj, LIBDEPS=LIBDEPS or [], **kwargs )
+
+def exists( env ):
+    return True
+
+def generate( env ):
+    env['_RELOBJDEPSFLAGS'] = '$RELOBJ_LIBDEPS_START ${_concat("$RELOBJ_LIBDEPS_ITEM ", __env__.subst(_LIBDEPS, target=TARGET, source=SOURCE), "", __env__, target=TARGET, source=SOURCE)} $RELOBJ_LIBDEPS_END'
+    env['RELOBJCOM'] = 'ld -o $TARGET -r $SOURCES $_RELOBJDEPSFLAGS'
+    relobj_builder = Builder( action='$RELOBJCOM',
+                              prefix="$OBJPREFIX",
+                              suffix="$OBJSUFFIX",
+                              emitter=libdeps.libdeps_emitter )
+    libdeps.update_scanner( relobj_builder )
+    env['BUILDERS']['RelocatableObject'] = relobj_builder
+    env.AddMethod( merge_library_method, 'MergeLibrary' )
+    env.AddMethod( merge_shared_library_method, 'MergeSharedLibrary' )
diff --git a/site_scons/site_tools/mergelibwin.py b/site_scons/site_tools/mergelibwin.py
new file mode 100644
index 00000000000..680ccdd5b9a
--- /dev/null
+++ b/site_scons/site_tools/mergelibwin.py
@@ -0,0 +1,34 @@
+"""Builder for static libraries composed of the contents of other static libraries.
+
+The following rule creates a library "mylib" whose contents are the contents of
+"firstlib", "secondlib", and all LIBDEPS dependencies of "firstlib" and
+"secondlib".  This creates self-contained static and shared libraries that can
+be distributed to customers.
+
+MergeLibrary('mylib', ['firstlib', 'secondlib'])
+MergeSharedLibrary('mylib', ['firstlib', 'secondlib'])
+
+This implementation is for win32 systems using msvc.
+"""
+
+import libdeps
+from SCons.Action import Action
+from SCons.Builder import Builder
+
+def merge_library_method( env, target, source, LIBDEPS=None, **kwargs ):
+    return env._MergeLibrary( target, [], LIBDEPS=source, **kwargs )
+
+def exists( env ):
+    return True
+
+def generate( env ):
+    merge_library = Builder(
+        action='${TEMPFILE("$AR $ARFLAGS /OUT:$TARGET $_LIBDEPS")}',
+        src_prefix='$LIBPREFIX',
+        src_suffix='$LIBSUFFIX',
+        prefix='$LIBPREFIX',
+        suffix='$LIBSUFFIX',
+        emitter=libdeps.libdeps_emitter )
+    libdeps.update_scanner( merge_library )
+    env['BUILDERS']['_MergeLibrary'] = merge_library
+    env.AddMethod( merge_library_method, 'MergeLibrary' )