diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | multivenv_experiment/README.md | 99 | ||||
| -rw-r--r-- | multivenv_experiment/notbstalphaelement/bstplugin.py | 10 | ||||
| -rw-r--r-- | multivenv_experiment/notbstalphaelement/setup.py | 8 | ||||
| -rw-r--r-- | multivenv_experiment/notbstbetaelement/bstplugin.py | 10 | ||||
| -rw-r--r-- | multivenv_experiment/notbstbetaelement/setup.py | 8 | ||||
| -rw-r--r-- | multivenv_experiment/notbstgammaelement/bstplugin.py | 10 | ||||
| -rw-r--r-- | multivenv_experiment/notbstgammaelement/setup.py | 8 | ||||
| -rw-r--r-- | multivenv_experiment/notbuildstream/notbuildstream.py | 74 | ||||
| -rw-r--r-- | multivenv_experiment/notbuildstream/setup.py | 12 |
10 files changed, 240 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore index 5c258fad4..b02de5fa2 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ doc/source/buildstream.rst doc/source/buildstream.*.rst doc/build/ versioneer.pyc +*.egg-info diff --git a/multivenv_experiment/README.md b/multivenv_experiment/README.md new file mode 100644 index 000000000..1ca97f1de --- /dev/null +++ b/multivenv_experiment/README.md @@ -0,0 +1,99 @@ +Multi-venv experiment +===================== + +This is an experiment to test the feasability of supporting a venv per plugin, +all running under the same interpreter. + +It shows: + + * Installing two separate venvs with different dependency versions + * Running an interpretor which loads a plugin in both separate + venvs (it could even be the same plugin, and need not be a + "BuildStream" plugin, but some python module loaded on demand) + * Prove that we infact have separation (perhaps by having the plugin + just print the versions of it's dependencies). + +The approach taken in this experiment is to push all modules required by the +plugin into a PluginBase space. + +Major problems discovered +------------------------- + +- We don't override `sys.modules`, so this in jinja2 v2.10 won't work: `del + sys.modules['jinja2._identifier']`. A first attempt to override `sys.modules` + resulted in mysterious failures not detailed here. + +- Relative-imports work in a way that doesn't seem to be obvious, which is + incompatible with the hack to rewrite top-level imports. See the + `import_override` function for more details. + +- Global imports don't seem to work with pure PluginBase, e.g. jinja2 + itself will do `from jinja2.environment import Environment, Template` in its + `__init__.py`, which fails. This is overcome with the `import_override` + hackery in the experiment. + +- the `jinja2.__version__` reported by the plugin is actually the version of + `pluginbase` instead. Interestingly the `jinja2.__version__` reported in the + main app is different. + +- the `jinja2.evalcontextfilter` accessed by the plugin is different from the + one accessed in the main app. That's probably for the same reason as the + above point. + +Setup +----- + +Create a Python venv for each subdirectory, e.g. + + python3 -m venv ~/PyVenv/notbuildstream + python3 -m venv ~/PyVenv/notbstalphaelement + python3 -m venv ~/PyVenv/notbstbetaelement + python3 -m venv ~/PyVenv/notbstgammaelement + +Then, for each directory: + +- Activate the appropriate venv. e.g. `. ~/PyVenv/notbuildstream/bin/activate`. +- For each subdirectory, `pip install SUBDIRECTORY`. Don't use `-e`, as it + seems that `.egg-link`s are a separate case that need special consideration. + +Make sure that the reference to `lib/python3.7/site-packages` in +`notbuildstream.py` is fixed up as appropriate for your venvs. + +Running +------- + +Enter the venv for `notbuildstream`, and invoke it with the paths to the other +venvs, e.g. + + notbst ~/PyVenv/ ~/PyVenv/notbst{alpha,beta,gamma}element + +You should see somthing like: + +``` +venv: /Users/jevripiotis/PyVenv/notbstalphaelement +Alpha +jinja2.__version__: 1.0.0 +jinja2.evalcontextfilter: None +main: jinja2: <module 'pluginbase._internalspace._sp5d362f8c6220a8c7f6ec3263825004f6.jinja2' from '/Users/jevripiotis/PyVenv/notbstalphaelement/lib/python3.7/site-packages/jinja2/__init__.py'> +main: jinja2.__version__: 2.8 +main: jinja2.__file__: /Users/jevripiotis/PyVenv/notbstalphaelement/lib/python3.7/site-packages/jinja2/__init__.py +main: Has evalcontextfilter: <function evalcontextfilter at 0x10b66c9d8> + +venv: /Users/jevripiotis/PyVenv/notbstbetaelement +Beta +jinja2.__version__: 1.0.0 +jinja2.evalcontextfilter: None +main: jinja2: <module 'pluginbase._internalspace._sp97ff7e741eeaeaf9c282906561d1b456.jinja2' from '/Users/jevripiotis/PyVenv/notbstbetaelement/lib/python3.7/site-packages/jinja2/__init__.py'> +main: jinja2.__version__: unknown +main: jinja2.__file__: /Users/jevripiotis/PyVenv/notbstbetaelement/lib/python3.7/site-packages/jinja2/__init__.py +main: Has evalcontextfilter: None + +venv: /Users/jevripiotis/PyVenv/notbstgammaelement +Traceback (most recent call last): + File "/Users/jevripiotis/PyVenv/notbuildstream/bin/notbst", line 11, in <module> + load_entry_point('notbuildstream', 'console_scripts', 'notbst')() +--- 8< --- snip long stacktrace --- 8< --- + File "/Users/jevripiotis/PyVenv/notbstgammaelement/lib/python3.7/site-packages/jinja2/lexer.py", line 50, in <module> + del sys.modules['jinja2._identifier'] +KeyError: 'jinja2._identifier' +``` diff --git a/multivenv_experiment/notbstalphaelement/bstplugin.py b/multivenv_experiment/notbstalphaelement/bstplugin.py new file mode 100644 index 000000000..e30671f6c --- /dev/null +++ b/multivenv_experiment/notbstalphaelement/bstplugin.py @@ -0,0 +1,10 @@ +import sys + +import jinja2 + + +class Element: + def __init__(self, bst_context): + print("Alpha") + print(f"jinja2.__version__: {jinja2.__version__}") + print("jinja2.evalcontextfilter:", getattr(jinja2, "evalcontextfilter", None)) diff --git a/multivenv_experiment/notbstalphaelement/setup.py b/multivenv_experiment/notbstalphaelement/setup.py new file mode 100644 index 000000000..cf758fb26 --- /dev/null +++ b/multivenv_experiment/notbstalphaelement/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name="notbstalphaelement", + version="0.1", + py_modules=["bstplugin"], + install_requires=["Jinja2==2.8"], +) diff --git a/multivenv_experiment/notbstbetaelement/bstplugin.py b/multivenv_experiment/notbstbetaelement/bstplugin.py new file mode 100644 index 000000000..5e41f75cf --- /dev/null +++ b/multivenv_experiment/notbstbetaelement/bstplugin.py @@ -0,0 +1,10 @@ +import sys + +import jinja2 + + +class Element: + def __init__(self, bst_context): + print("Beta") + print(f"jinja2.__version__: {jinja2.__version__}") + print("jinja2.evalcontextfilter:", getattr(jinja2, "evalcontextfilter", None)) diff --git a/multivenv_experiment/notbstbetaelement/setup.py b/multivenv_experiment/notbstbetaelement/setup.py new file mode 100644 index 000000000..da7c2d86f --- /dev/null +++ b/multivenv_experiment/notbstbetaelement/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name="notbstbetaelement", + version="0.1", + py_modules=["bstplugin"], + install_requires=["Jinja2==2.3"], +) diff --git a/multivenv_experiment/notbstgammaelement/bstplugin.py b/multivenv_experiment/notbstgammaelement/bstplugin.py new file mode 100644 index 000000000..cfd41f260 --- /dev/null +++ b/multivenv_experiment/notbstgammaelement/bstplugin.py @@ -0,0 +1,10 @@ +import sys + +import jinja2 + + +class Element: + def __init__(self, bst_context): + print("Gamma") + print(f"jinja2.__version__: {jinja2.__version__}") + print("jinja2.evalcontextfilter:", getattr(jinja2, "evalcontextfilter", None)) diff --git a/multivenv_experiment/notbstgammaelement/setup.py b/multivenv_experiment/notbstgammaelement/setup.py new file mode 100644 index 000000000..3bf3a2e81 --- /dev/null +++ b/multivenv_experiment/notbstgammaelement/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name="notbstgammaelement", + version="0.1", + py_modules=["bstplugin"], + install_requires=["Jinja2==2.10"], +) diff --git a/multivenv_experiment/notbuildstream/notbuildstream.py b/multivenv_experiment/notbuildstream/notbuildstream.py new file mode 100644 index 000000000..cce44d6dd --- /dev/null +++ b/multivenv_experiment/notbuildstream/notbuildstream.py @@ -0,0 +1,74 @@ +import builtins +import sys +import contextlib + +import click + +import pluginbase + + +# TODO: Make this work instead by overriding everything except BuildStream and +# the standard library. +@contextlib.contextmanager +def import_override(*override_modules): + def myimport(name, globals_=None, locals_=None, fromlist=None, level=None): + # XXX: In the case of 'from . import A, B' we run into trouble if + # we remap the name. We should fully understand why and what + # guarantees there are before considering using any of this. + # + # When this is the case, it seems that 'fromlist' will be an empty + # list. Again, we need to know the guarantees here. + # + if fromlist != []: + for m in override_modules: + if name.startswith(m): + name = "notbuildstream.plugins." + name + return builtins_import(name, globals_, locals_, fromlist, level) + + builtins_import = builtins.__import__ + try: + builtins.__import__ = myimport + yield + finally: + builtins.__import__ = builtins_import + + +@click.command("notbuildstream") +@click.argument( + "plugin_venvs", + nargs=-1, + metavar="PATH", + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +def cli(plugin_venvs): + + pbase = pluginbase.PluginBase(package="notbuildstream.plugins") + psource_list = [] + + for venv in plugin_venvs: + print(f"venv: {venv}") + + # XXX: We should determine this path using some standard mechanism. + search_path = [venv + "/lib/python3.7/site-packages", venv] + + psource = pbase.make_plugin_source(searchpath=search_path, identifier=venv) + psource_list.append(psource) + + with import_override("jinja2", "markupsafe"): + with psource: + # TODO: use entrypoints and lookup plugins with pkgconfig. + plugin = psource.load_plugin("bstplugin") + element = plugin.Element("a") + jinja2 = psource.load_plugin("jinja2") + print(f"main: jinja2: {jinja2}") + print(f"main: jinja2.__version__: {jinja2.__version__}") + print(f"main: jinja2.__file__: {jinja2.__file__}") + print( + "main: Has evalcontextfilter:", + getattr(jinja2, "evalcontextfilter", None), + ) + print() + + +if __name__ == "__main__": + sys.exit(cli()) diff --git a/multivenv_experiment/notbuildstream/setup.py b/multivenv_experiment/notbuildstream/setup.py new file mode 100644 index 000000000..f0a79823d --- /dev/null +++ b/multivenv_experiment/notbuildstream/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + +setup( + name="notbuildstream", + version="0.1", + py_modules=["notbuildstream"], + install_requires=["Click", "pluginbase"], + entry_points=""" + [console_scripts] + notbst=notbuildstream:cli + """, +) |