SERVER-7118 Change "mongo modules" to use SConscript files for modules.

Also allows you to have modules in mongos and the shell, as well as mongod.
Requires changes to the modules, to have SConscript files, and define libraries.
Allows modules to have unit tests, interesting linking rules, dependencies into
mongo, etc.

Still may need to do some work on includes.  The mongo-enterprise module has very
simple include requirements, today.

1 files changed, 107 insertions, 95 deletions

diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py
index 42fc52094ca..501ec182560 100644
--- a/buildscripts/moduleconfig.py
+++ b/buildscripts/moduleconfig.py
@@ -1,131 +1,143 @@
-"""Utility functions for SCons to discover and configure
-MongoDB modules (sub-trees of db/modules/). This file exports
-two functions:
-
-    discover_modules, which returns a dictionary of module name
-        to the imported python module object for the module's
-        build.py file
-
-    configure_modules, which runs per-module configuration, and
-        is given the SCons environment, its own path, etc
-
-Each module must have a "build.py" script, which is expected to
-have a "configure" function, and optionally a "test" function
-if the module exposes per-module tests.
+"""Utility functions for SCons to discover and configure MongoDB modules.
+
+A MongoDB module is an organized collection of source code and build rules that can be provided at
+compile-time to alter or extend the behavior of MongoDB.  The files comprising a single MongoDB
+module are arranged in a directory hierarchy, rooted in a directory whose name is by convention the
+module name, and containing in that root directory at least two files: a build.py file and a
+SConscript file.
+
+MongoDB modules are discovered by a call to the discover_modules() function, whose sole parameter is
+the directory which is the immediate parent of all module directories.  The exact directory is
+chosen by the SConstruct file, which is the direct consumer of this python module.  The only rule is
+that it must be a subdirectory of the src/ directory, to correctly work with the SCons variant
+directory system that separates build products for source.
+
+Once discovered, modules are configured by the configure_modules() function, and the build system
+integrates their SConscript files into the rest of the build.
+
+MongoDB module build.py files implement a single function, configure(conf, env), which they may use
+to configure the supplied "env" object.  The configure functions may add extra LIBDEPS to mongod,
+mongos and the mongo shell (TODO: other mongo tools and the C++ client), and through those libraries
+alter those programs' behavior.
+
+MongoDB module SConscript files can describe libraries, programs and unit tests, just as other
+MongoDB SConscript files do.
 """
 
 __all__ = ('discover_modules', 'configure_modules', 'register_module_test')
 
 import imp
-from os import listdir, makedirs
-from os.path import abspath, dirname, join, isdir, isfile
-
-def discover_modules(mongo_root):
-    """Scan <mongo_root>/db/modules/ for directories that
-    look like MongoDB modules (i.e. they contain a "build.py"
-    file), and return a dictionary of module name (the directory
-    name) to build.py python modules.
+import inspect
+import os
+
+def discover_modules(module_root):
+    """Scans module_root for subdirectories that look like MongoDB modules.
+
+    Returns a list of imported build.py module objects.
     """
-    found_modules = {}
+    found_modules = []
 
-    module_root = abspath(join(mongo_root, 'db', 'modules'))
-    if not isdir(module_root):
+    if not os.path.isdir(module_root):
         return found_modules
 
-    for name in listdir(module_root):
-        root = join(module_root, name)
-        if '.' in name or not isdir(root):
+    for name in os.listdir(module_root):
+        root = os.path.join(module_root, name)
+        if name.startswith('.') or not os.path.isdir(root):
             continue
 
-        build_py = join(root, 'build.py')
+        build_py = os.path.join(root, 'build.py')
         module = None
 
-        if isfile(build_py):
+        if os.path.isfile(build_py):
             print "adding module: %s" % name
             fp = open(build_py, "r")
-            module = imp.load_module("module_" + name, fp, build_py, (".py", "r", imp.PY_SOURCE))
-            found_modules[name] = module
-            fp.close()
+            try:
+                module = imp.load_module("module_" + name, fp, build_py,
+                                         (".py", "r", imp.PY_SOURCE))
+                if getattr(module, "name", None) is None:
+                    module.name = name
+                found_modules.append(module)
+            finally:
+                fp.close()
 
     return found_modules
 
 def configure_modules(modules, conf, env):
-    """
-    Run the configure() function in the build.py python modules
-    for each module listed in the modules dictionary (as created
-    by discover_modules). The configure() function should use the
-    prepare the Mongo build system for building the module.
-
-    build.py files may specify a "customIncludes" flag, which, if
-    True, causes configure() to be called with three arguments:
-    the SCons Configure() object, the SCons environment, and an
-    empty list which should be modified in-place by the configure()
-    function; if false, configure() is called with only the first
-    two arguments, and the source files are discovered with a
-    glob against the <module_root>/src/*.cpp.
-
-    Returns a dictionary mapping module name to a list of source
-    files to be compiled for the module.
-    """
-    source_map = {}
+    """ Run the configure() function in the build.py python modules for each module in "modules"
+    (as created by discover_modules).
 
-    for name, module in modules.items():
+    The configure() function should prepare the Mongo build system for building the module.
+    """
+    for module in modules:
+        name = module.name
         print "configuring module: %s" % name
 
-        root = dirname(module.__file__)
-        module_sources = []
+        root = os.path.dirname(module.__file__)
+        module.configure(conf, env)
 
-        if getattr(module, "customIncludes", False):
-            # then the module configures itself and its
-            # configure() takes 3 args
-            module.configure(conf, env, module_sources)
-        else:
-            # else we glob the files in the module's src/
-            # subdirectory, and its configure() takes 2 args
-            module.configure(conf, env)
-            module_sources.extend(env.Glob(join(root, "src/*.cpp")))
+def get_module_sconscripts(modules):
+    sconscripts = []
+    for m in modules:
+        module_dir_path = __get_src_relative_path(os.path.join(os.path.dirname(m.__file__)))
+        sconscripts.append(os.path.join(module_dir_path, 'SConscript'))
+    return sconscripts
 
-        if not module_sources:
-            print "WARNING: no source files for module %s, module will not be built." % name
-        else:
-            source_map[name] = module_sources
+def __get_src_relative_path(path):
+    """Return a path relative to ./src.
 
-    _setup_module_tests_file(str(env.File(env['MODULETEST_LIST'])))
+    The src directory is important because of its relationship to BUILD_DIR,
+    established in the SConstruct file.  For variant directories to work properly
+    in SCons, paths relative to the src or BUILD_DIR must often be generated.
+    """
+    src_dir = os.path.abspath('src')
+    path = os.path.abspath(os.path.normpath(path))
+    if not path.startswith(src_dir):
+        raise ValueError('Path "%s" is not relative to the src directory "%s"' % (path, src_dir))
+    result = path[len(src_dir) + 1:]
+    return result
+
+def __get_module_path(module_frame_depth):
+    """Return the path to the MongoDB module whose build.py is executing "module_frame_depth" frames
+    above this function, relative to the "src" directory.
+    """
+    module_filename = inspect.stack()[module_frame_depth + 1][1]
+    return os.path.dirname(__get_src_relative_path(module_filename))
 
-    return source_map
+def __get_module_src_path(module_frame_depth):
+    """Return the path relative to the SConstruct file of the MongoDB module's source tree.
 
-module_tests = []
-def register_module_test(*command):
-    """Modules can register tests as part of their configure(), which
-    are commands whose exit status indicates the success or failure of
-    the test.
+    module_frame_depth is the number of frames above the current one in which one can find a
+    function from the MongoDB module's build.py function.
+    """
+    return os.path.join('src', __get_module_path(module_frame_depth + 1))
 
-    Use this function from configure() like:
+def __get_module_build_path(module_frame_depth):
+    """Return the path relative to the SConstruct file of the MongoDB module's build tree.
 
-        register_module_test('/usr/bin/python', '/path/to/module/tests/foo.py')
-        register_module_test('/bin/bash', '/path/to/module/tests/bar.sh')
+    module_frame_depth is the number of frames above the current one in which one can find a
+    function from the MongoDB module's build.py function.
+    """
+    return os.path.join('$BUILD_DIR', __get_module_path(module_frame_depth + 1))
 
-    The registered test commands can be run with "scons smokeModuleTests"
+def get_current_module_src_path():
+    """Return the path relative to the SConstruct file of the current MongoDB module's source tree.
+
+    May only meaningfully be called from within build.py
     """
-    command = ' '.join(command)
-    module_tests.append(command)
+    return __get_module_src_path(1)
+
+def get_current_module_build_path():
+    """Return the path relative to the SConstruct file of the current MongoDB module's build tree.
 
-def _setup_module_tests_file(test_file):
-    """Modules' configure() functions may have called register_module_test,
-    in which case, we need to record the registered tests' commands into
-    a text file which smoke.py and SCons know how to work with.
+    May only meaningfully be called from within build.py
     """
-    if not module_tests:
-        return
 
-    folder = dirname(test_file)
-    if not isdir(folder):
-        makedirs(folder)
+    return __get_module_build_path(1)
 
-    fp = file(test_file, 'w')
-    for test in module_tests:
-        fp.write(test)
-        fp.write('\n')
-    fp.close()
-    print "Generated %s" % test_file
+def get_current_module_libdep_name(libdep_rel_path):
+    """Return a $BUILD_DIR relative path to a "libdep_rel_path", where "libdep_rel_path"
+    is specified relative to the MongoDB module's build.py file.
 
+    May only meaningfully be called from within build.py
+    """
+    return os.path.join(__get_module_build_path(1), libdep_rel_path)