Add easy python profiling support

When the PEAS_PYTHON_PROFILE enviroment variable
is set profiling information will be output when all python
plugins are unloaded.

6 files changed, 253 insertions, 14 deletions

diff --git a/loaders/python/Makefile.am b/loaders/python/Makefile.am
index 565e6fb..6d869c7 100644
--- a/loaders/python/Makefile.am
+++ b/loaders/python/Makefile.am
@@ -16,8 +16,9 @@ AM_CPPFLAGS = \
 loader_LTLIBRARIES = libpythonloader.la
 
 libpythonloader_la_SOURCES = \
-	peas-plugin-loader-python.c 	\
-	peas-plugin-loader-python.h
+	peas-plugin-loader-python.c 		\
+	peas-plugin-loader-python.h		\
+	peas-plugin-loader-python-resources.c
 
 libpythonloader_la_LDFLAGS = \
 	$(LOADER_LIBTOOL_FLAGS)		\
@@ -30,5 +31,12 @@ libpythonloader_la_LIBADD = \
 	$(PYGOBJECT_LIBS)			\
 	$(PYTHON2_LIBS)
 
+loader_resources_deps = $(shell $(GLIB_COMPILE_RESOURCES) --sourcedir=$(srcdir) --generate-dependencies $(srcdir)/peas-plugin-loader-python.gresource.xml)
+peas-plugin-loader-python-resources.c: $(srcdir)/peas-plugin-loader-python.gresource.xml $(loader_resources_deps)
+	$(AM_V_GEN) $(GLIB_COMPILE_RESOURCES) --internal --target=$@ --sourcedir=$(srcdir) --generate-source $(srcdir)/peas-plugin-loader-python.gresource.xml
+
+EXTRA_DIST = $(loader_resources_deps)
+CLEANFILES = peas-plugin-loader-python-resources.c
+
 gcov_sources = $(libpythonloader_la_SOURCES)
 include $(top_srcdir)/Makefile.gcov
diff --git a/loaders/python/peas-plugin-loader-python-internal.py b/loaders/python/peas-plugin-loader-python-internal.py
new file mode 100644
index 0000000..df07cb9
--- /dev/null
+++ b/loaders/python/peas-plugin-loader-python-internal.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+#  Copyright (C) 2014 - Garrett Regier
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU Library General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+import cProfile
+import os
+import pstats
+import sys
+import threading
+import weakref
+
+
+class Hooks(object):
+    def __init__(self):
+        self.profiling_enabled = os.getenv('PEAS_PYTHON_PROFILE') is not None
+        if not self.profiling_enabled:
+            return
+
+        sort = os.getenv('PEAS_PYTHON_PROFILE')
+        self.stat_sort = ('time',) if sort == '' else sort.split(';')
+
+        self.stats = None
+        self.stats_lock = threading.Lock()
+
+        self.thread_refs = []
+        self.thread_local = threading.local()
+
+        threading.setprofile(self.init_thread)
+
+        self.profile = cProfile.Profile()
+        self.profile.enable()
+
+    def add_stats(self, profile):
+        with self.stats_lock:
+            if self.stats is None:
+                self.stats = pstats.Stats(profile)
+
+            else:
+                self.stats.add(profile)
+
+    def init_thread(self, *unused):
+        # Only call once per thread
+        sys.setprofile(None)
+
+        thread_profile = cProfile.Profile()
+
+        def thread_finished(thread_ref):
+            thread_profile.disable()
+            self.add_stats(thread_profile)
+
+            self.thread_refs.remove(thread_ref)
+
+        # Need something to weakref, the
+        # current thread does not support it
+        thread_ref = set()
+        self.thread_local.ref = thread_ref
+
+        self.thread_refs.append(weakref.ref(thread_ref, thread_finished))
+
+        # Only enable the profile at the end
+        thread_profile.enable()
+
+    def all_plugins_unloaded(self):
+        if not self.profiling_enabled:
+            return
+
+        self.profile.disable()
+        self.add_stats(self.profile)
+
+        with self.stats_lock:
+            self.stats.strip_dirs().sort_stats(*self.stat_sort).print_stats()
+
+        # Need to create a new profile to avoid adding the stats twice
+        self.profile = cProfile.Profile()
+        self.profile.enable()
+
+    def exit(self):
+        if not self.profiling_enabled:
+            return
+
+        self.profile.disable()
+
+
+hooks = Hooks()
+
+# ex:ts=4:et:
diff --git a/loaders/python/peas-plugin-loader-python.c b/loaders/python/peas-plugin-loader-python.c
index 0aa1d3d..e0f587e 100644
--- a/loaders/python/peas-plugin-loader-python.c
+++ b/loaders/python/peas-plugin-loader-python.c
@@ -46,6 +46,7 @@ struct _PeasPluginLoaderPythonPrivate {
   guint init_failed : 1;
   guint must_finalize_python : 1;
   PyThreadState *py_thread_state;
+  PyObject *hooks;
 };
 
 typedef struct {
@@ -65,6 +66,23 @@ peas_register_types (PeasObjectModule *module)
                                               PEAS_TYPE_PLUGIN_LOADER_PYTHON);
 }
 
+/* NOTE: This must not be called with the GIL held */
+static void
+internal_python_hook (PeasPluginLoaderPython *pyloader,
+                      const gchar            *name)
+{
+  PyGILState_STATE state = PyGILState_Ensure ();
+  PyObject *result;
+
+  result = PyObject_CallMethod (pyloader->priv->hooks, (gchar *) name, NULL);
+  Py_XDECREF (result);
+
+  if (PyErr_Occurred ())
+    PyErr_Print ();
+
+  PyGILState_Release (state);
+}
+
 /* NOTE: This must be called with the GIL held */
 static PyTypeObject *
 find_python_extension_type (PeasPluginInfo *info,
@@ -279,6 +297,12 @@ peas_plugin_loader_python_unload (PeasPluginLoader *loader,
   PeasPluginLoaderPython *pyloader = PEAS_PLUGIN_LOADER_PYTHON (loader);
 
   g_hash_table_remove (pyloader->priv->loaded_plugins, info);
+
+  /* We have to use this as a hook as the
+   * loader will not be finalized by applications
+   */
+  if (g_hash_table_size (pyloader->priv->loaded_plugins) == 0)
+    internal_python_hook (pyloader, "all_plugins_unloaded");
 }
 
 static void
@@ -412,7 +436,8 @@ peas_plugin_loader_python_initialize (PeasPluginLoader *loader)
   PeasPluginLoaderPython *pyloader = PEAS_PLUGIN_LOADER_PYTHON (loader);
   PyGILState_STATE state = 0;
   long hexversion;
-  PyObject *gettext, *result;
+  GBytes *internal_python;
+  PyObject *gettext, *result, *builtins_module, *code, *globals;
   const gchar *prgname;
 #if PY_VERSION_HEX < 0x03000000
   const char *argv[] = { NULL, NULL };
@@ -494,17 +519,11 @@ peas_plugin_loader_python_initialize (PeasPluginLoader *loader)
   g_free (argv[0]);
 #endif
 
-  /* Note that we don't call this with the GIL held,
-   * since we haven't initialised pygobject yet
-   */
   if (!peas_plugin_loader_python_add_module_path (pyloader, PEAS_PYEXECDIR))
     {
       g_warning ("Error initializing Python Plugin Loader: "
                  "failed to add the module path");
 
-      if (PyErr_Occurred ())
-        PyErr_Print ();
-
       goto python_init_error;
     }
 
@@ -517,7 +536,6 @@ peas_plugin_loader_python_initialize (PeasPluginLoader *loader)
     {
       g_warning ("Error initializing Python Plugin Loader: "
                  "PyGObject initialization failed");
-      PyErr_Print ();
 
       goto python_init_error;
     }
@@ -552,6 +570,90 @@ peas_plugin_loader_python_initialize (PeasPluginLoader *loader)
       goto python_init_error;
     }
 
+#if PY_VERSION_HEX < 0x03000000
+  builtins_module = PyImport_ImportModule ("__builtin__");
+#else
+  builtins_module = PyImport_ImportModule ("builtins");
+#endif
+
+  if (builtins_module == NULL)
+    goto python_init_error;
+
+  internal_python = g_resources_lookup_data ("/org/gnome/libpeas/loaders/"
+#if PY_VERSION_HEX < 0x03000000
+                                             "python/"
+#else
+                                             "python3/"
+#endif
+                                             "internal.py",
+                                             G_RESOURCE_LOOKUP_FLAGS_NONE,
+                                             NULL);
+
+  if (internal_python == NULL)
+    {
+      g_warning ("Error initializing Python Plugin Loader: "
+                 "failed to locate internal python code");
+
+      goto python_init_error;
+    }
+
+  /* Compile it manually so the filename is available */
+  code = Py_CompileString (g_bytes_get_data (internal_python, NULL),
+                           "peas-plugin-loader-python-internal.py",
+                           Py_file_input);
+
+  g_bytes_unref (internal_python);
+
+  if (code == NULL)
+    {
+      g_warning ("Error initializing Python Plugin Loader: "
+                 "failed to compile internal python code");
+
+      goto python_init_error;
+    }
+
+  globals = PyDict_New ();
+  if (globals == NULL)
+    {
+      Py_DECREF (code);
+      goto python_init_error;
+    }
+
+  if (PyDict_SetItemString (globals, "__builtins__",
+                            PyModule_GetDict (builtins_module)) != 0)
+    {
+      Py_DECREF (globals);
+      Py_DECREF (code);
+      goto python_init_error;
+    }
+
+  result = PyEval_EvalCode ((gpointer) code, globals, globals);
+  Py_XDECREF (result);
+
+  Py_DECREF (code);
+
+  if (PyErr_Occurred ())
+    {
+      g_warning ("Error initializing Python Plugin Loader: "
+                 "failed to run internal python code");
+
+      Py_DECREF (globals);
+      goto python_init_error;
+    }
+
+  pyloader->priv->hooks = PyDict_GetItemString (globals, "hooks");
+  Py_XINCREF (pyloader->priv->hooks);
+
+  Py_DECREF (globals);
+
+  if (pyloader->priv->hooks == NULL)
+    {
+      g_warning ("Error initializing Python Plugin Loader: "
+                 "failed to find internal python hooks");
+
+      goto python_init_error;
+    }
+
   /* Python has been successfully initialized */
   pyloader->priv->init_failed = FALSE;
 
@@ -564,12 +666,12 @@ peas_plugin_loader_python_initialize (PeasPluginLoader *loader)
 
 python_init_error:
 
+  if (PyErr_Occurred ())
+    PyErr_Print ();
+
   g_warning ("Please check the installation of all the Python "
              "related packages required by libpeas and try again");
 
-  if (PyErr_Occurred ())
-    PyErr_Clear ();
-
   if (!pyloader->priv->must_finalize_python)
     PyGILState_Release (state);
 
@@ -611,6 +713,14 @@ peas_plugin_loader_python_finalize (GObject *object)
 
   if (Py_IsInitialized ())
     {
+      if (pyloader->priv->hooks != NULL)
+        {
+          internal_python_hook (pyloader, "exit");
+
+          /* Borrowed Reference */
+          pyloader->priv->hooks = NULL;
+        }
+
       if (pyloader->priv->py_thread_state)
         {
           PyEval_RestoreThread (pyloader->priv->py_thread_state);
diff --git a/loaders/python/peas-plugin-loader-python.gresource.xml b/loaders/python/peas-plugin-loader-python.gresource.xml
new file mode 100644
index 0000000..ebc8595
--- /dev/null
+++ b/loaders/python/peas-plugin-loader-python.gresource.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<gresources>
+  <gresource prefix="/org/gnome/libpeas/loaders/python">
+    <file alias="internal.py">peas-plugin-loader-python-internal.py</file>
+  </gresource>
+</gresources>
diff --git a/loaders/python3/Makefile.am b/loaders/python3/Makefile.am
index 5ce2354..cc0be42 100644
--- a/loaders/python3/Makefile.am
+++ b/loaders/python3/Makefile.am
@@ -17,7 +17,8 @@ loader_LTLIBRARIES = libpython3loader.la
 
 libpython3loader_la_SOURCES = \
 	$(top_srcdir)/loaders/python/peas-plugin-loader-python.c	\
-	$(top_srcdir)/loaders/python/peas-plugin-loader-python.h
+	$(top_srcdir)/loaders/python/peas-plugin-loader-python.h	\
+	peas-plugin-loader-python3-resources.c
 
 libpython3loader_la_LDFLAGS = \
 	$(LOADER_LIBTOOL_FLAGS)		\
@@ -30,5 +31,12 @@ libpython3loader_la_LIBADD = \
 	$(PYGOBJECT_LIBS)			\
 	$(PYTHON3_LIBS)
 
+loader_resources_deps = $(shell $(GLIB_COMPILE_RESOURCES) --sourcedir=$(srcdir) --generate-dependencies $(srcdir)/peas-plugin-loader-python3.gresource.xml)
+peas-plugin-loader-python3-resources.c: $(srcdir)/peas-plugin-loader-python3.gresource.xml $(loader_resources_deps)
+	$(AM_V_GEN) $(GLIB_COMPILE_RESOURCES) --internal --target=$@ --sourcedir=$(srcdir) --generate-source $(srcdir)/peas-plugin-loader-python3.gresource.xml
+
+EXTRA_DIST = $(loader_resources_deps)
+CLEANFILES = peas-plugin-loader-python3-resources.c
+
 gcov_sources = $(libpython3loader_la_SOURCES)
 include $(top_srcdir)/Makefile.gcov
diff --git a/loaders/python3/peas-plugin-loader-python3.gresource.xml b/loaders/python3/peas-plugin-loader-python3.gresource.xml
new file mode 100644
index 0000000..314ac6a
--- /dev/null
+++ b/loaders/python3/peas-plugin-loader-python3.gresource.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<gresources>
+  <gresource prefix="/org/gnome/libpeas/loaders/python3">
+    <file alias="internal.py">../python/peas-plugin-loader-python-internal.py</file>
+  </gresource>
+</gresources>