From 16a13276f26e2b4b0cad35c66a527bb8d128d587 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Jul 2011 10:28:17 +0200 Subject: Added basic frame for pygit2 - it just needs some basic methods to be implemented now - depending on the performance, it might actually receive some more work --- git/db/interface.py | 4 +- git/db/py/base.py | 4 +- git/db/pygit2/__init__.py | 11 +++ git/db/pygit2/complex.py | 90 ++++++++++++++++++++++++ git/test/db/dulwich/test_base.py | 3 +- git/test/db/pygit2/__init__.py | 4 ++ git/test/db/pygit2/lib.py | 23 ++++++ git/test/db/pygit2/test_base.py | 32 +++++++++ git/test/performance/db/test_looseodb_dulwich.py | 9 ++- git/test/performance/db/test_looseodb_pygit2.py | 13 ++++ git/test/performance/db/test_odb_dulwich.py | 9 ++- git/test/performance/db/test_odb_pygit2.py | 13 ++++ 12 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 git/db/pygit2/__init__.py create mode 100644 git/db/pygit2/complex.py create mode 100644 git/test/db/pygit2/__init__.py create mode 100644 git/test/db/pygit2/lib.py create mode 100644 git/test/db/pygit2/test_base.py create mode 100644 git/test/performance/db/test_looseodb_pygit2.py create mode 100644 git/test/performance/db/test_odb_pygit2.py (limited to 'git') diff --git a/git/db/interface.py b/git/db/interface.py index 9ad74cc1..2ff44f26 100644 --- a/git/db/interface.py +++ b/git/db/interface.py @@ -164,7 +164,9 @@ class RootPathDB(object): def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing - to pontentially access datafiles""" + to pontentially access datafiles + :param rela_path: if not None or '', the relative path will be appended + to the database root path. Otherwise you will obtain the database root path itself""" raise NotImplementedError() #} END interface diff --git a/git/db/py/base.py b/git/db/py/base.py index 49f28a8d..d34e8b12 100644 --- a/git/db/py/base.py +++ b/git/db/py/base.py @@ -108,7 +108,9 @@ class PureRootPathDB(RootPathDB): def root_path(self): return self._root_path - def db_path(self, rela_path): + def db_path(self, rela_path=None): + if not rela_path: + return self._root_path return join(self._root_path, rela_path) #} END interface diff --git a/git/db/pygit2/__init__.py b/git/db/pygit2/__init__.py new file mode 100644 index 00000000..af005662 --- /dev/null +++ b/git/db/pygit2/__init__.py @@ -0,0 +1,11 @@ +"""Pygit2 module initialization""" + +def init_pygit2(): + """:raise ImportError: if pygit2 is not present""" + try: + import pygit2 + except ImportError: + raise ImportError("Could not find 'pygit2' in the PYTHONPATH - pygit2 functionality is not available") + #END handle pygit2 import + +init_pygit2() diff --git a/git/db/pygit2/complex.py b/git/db/pygit2/complex.py new file mode 100644 index 00000000..67635a1e --- /dev/null +++ b/git/db/pygit2/complex.py @@ -0,0 +1,90 @@ + +__all__ = ['Pygit2GitODB', 'Pygit2GitDB', 'Pygit2CompatibilityGitDB'] + +from git.db.py.complex import PureGitODB +from git.db.py.base import ( + PureRepositoryPathsMixin, + PureConfigurationMixin, + PureIndexDB, + ) +from git.db.py.resolve import PureReferencesMixin +from git.db.py.transport import PureTransportDB +from git.db.py.submodule import PureSubmoduleDB + +from git.db.cmd.complex import CmdHighLevelRepository, GitCommandMixin +from git.db.compat import RepoCompatibilityInterface + +from pygit2 import Repository as Pygit2Repo + +from git.base import OInfo, OStream +from git.fun import type_id_to_type_map, type_to_type_id_map + +from cStringIO import StringIO +import os + + +class Pygit2GitODB(PureGitODB): + """A full fledged database to read and write object files from all kinds of sources.""" + + def __init__(self, objects_root): + """Initalize this instance""" + PureGitODB.__init__(self, objects_root) + if hasattr(self, 'git_dir'): + wd = self.git_dir + else: + wd = os.path.dirname(objects_root) + #END try to figure out good entry for pygit2 - it needs the .gitdir + print objects_root + print wd + self._py2_repo = Pygit2Repo(wd) + + def __getattr__(self, attr): + try: + # supply LazyMixin with this call first + return super(Pygit2GitODB, self).__getattr__(attr) + except AttributeError: + # now assume its on the pygit2 repository ... for now + return getattr(self._py2_repo, attr) + #END handle attr + + #{ Object DBR + + # def info(self, binsha): + # type_id, uncomp_data = self._py2_repo.object_store.get_raw(binsha) + # return OInfo(binsha, type_id_to_type_map[type_id], len(uncomp_data)) + # + # def stream(self, binsha): + # type_id, uncomp_data = self._py2_repo.object_store.get_raw(binsha) + # return OStream(binsha, type_id_to_type_map[type_id], len(uncomp_data), StringIO(uncomp_data)) + # + # #}END object dbr + # + # #{ Object DBW + # + # def store(self, istream): + # obj = ShaFile.from_raw_string(type_to_type_id_map[istream.type], istream.read()) + # self._py2_repo.object_store.add_object(obj) + # istream.binsha = obj.sha().digest() + # return istream + + #}END object dbw + +class Pygit2GitDB( PureRepositoryPathsMixin, PureConfigurationMixin, + PureReferencesMixin, PureSubmoduleDB, + PureIndexDB, + PureTransportDB, # not fully implemented + GitCommandMixin, + CmdHighLevelRepository, + Pygit2GitODB): # must come last, as it doesn't pass on __init__ with super + + + def __init__(self, root_path): + """Initialize ourselves on the .git directory, or the .git/objects directory.""" + PureRepositoryPathsMixin._initialize(self, root_path) + super(Pygit2GitDB, self).__init__(self.objects_dir) + + +class Pygit2CompatibilityGitDB(RepoCompatibilityInterface, Pygit2GitDB): + """Basic pygit2 compatibility database""" + pass + diff --git a/git/test/db/dulwich/test_base.py b/git/test/db/dulwich/test_base.py index 50e64131..78416518 100644 --- a/git/test/db/dulwich/test_base.py +++ b/git/test/db/dulwich/test_base.py @@ -20,7 +20,7 @@ else: #END handle imports -class TestPyDBBase(RepoBase): +class TestDulwichDBBase(RepoBase): __metaclass__ = DulwichRequiredMetaMixin RepoCls = DulwichDB @@ -28,6 +28,5 @@ class TestPyDBBase(RepoBase): @with_rw_repo('HEAD', bare=False) def test_basics(self, rw_repo): db = DulwichDB(rw_repo.working_tree_dir) - print db.git_dir diff --git a/git/test/db/pygit2/__init__.py b/git/test/db/pygit2/__init__.py new file mode 100644 index 00000000..8a681e42 --- /dev/null +++ b/git/test/db/pygit2/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php diff --git a/git/test/db/pygit2/lib.py b/git/test/db/pygit2/lib.py new file mode 100644 index 00000000..356df9dc --- /dev/null +++ b/git/test/db/pygit2/lib.py @@ -0,0 +1,23 @@ +"""pygit2 specific utilities, as well as all the default ones""" + +from git.test.lib import ( + InheritedTestMethodsOverrideWrapperMetaClsAutoMixin, + needs_module_or_skip + ) + +__all__ = ['needs_pygit2_or_skip', 'Pygit2RequiredMetaMixin'] + +#{ Decoorators + +def needs_pygit2_or_skip(func): + """Skip this test if we have no pygit2 - print warning""" + return needs_module_or_skip('pygit2')(func) + +#}END decorators + +#{ MetaClasses + +class Pygit2RequiredMetaMixin(InheritedTestMethodsOverrideWrapperMetaClsAutoMixin): + decorator = [needs_pygit2_or_skip] + +#} END metaclasses diff --git a/git/test/db/pygit2/test_base.py b/git/test/db/pygit2/test_base.py new file mode 100644 index 00000000..246a1643 --- /dev/null +++ b/git/test/db/pygit2/test_base.py @@ -0,0 +1,32 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +from lib import * +from git.test.lib import TestBase, with_rw_repo +from git.test.db.base import RepoBase + + + +try: + import pygit2 +except ImportError: + # om this case, all other pygit2 tests will be skipped + # Need to properly initialize the class though, otherwise it would fail + from git.db.complex import PureCompatibilityGitDB as Pygit2DB +else: + # now we know pygit2 is available, to do futher imports + from git.db.pygit2.complex import Pygit2CompatibilityGitDB as Pygit2DB + +#END handle imports + +class TestPyGit2DBBase(RepoBase): + __metaclass__ = Pygit2RequiredMetaMixin + RepoCls = Pygit2DB + + @needs_pygit2_or_skip + @with_rw_repo('HEAD', bare=False) + def test_basics(self, rw_repo): + db = Pygit2DB(rw_repo.working_tree_dir) + + diff --git a/git/test/performance/db/test_looseodb_dulwich.py b/git/test/performance/db/test_looseodb_dulwich.py index cf27a528..e123ebf1 100644 --- a/git/test/performance/db/test_looseodb_dulwich.py +++ b/git/test/performance/db/test_looseodb_dulwich.py @@ -1,6 +1,13 @@ -from git.db.dulwich.complex import DulwichGitODB +try: + from git.db.dulwich.complex import DulwichGitODB +except ImportError: + from git.db.py.complex import PureGitODB as DulwichGitODB +#END handle import + +from git.test.db.dulwich.lib import DulwichRequiredMetaMixin from looseodb_impl import TestLooseDBWPerformanceBase class TestPureLooseDB(TestLooseDBWPerformanceBase): + __metaclass__ = DulwichRequiredMetaMixin LooseODBCls = DulwichGitODB diff --git a/git/test/performance/db/test_looseodb_pygit2.py b/git/test/performance/db/test_looseodb_pygit2.py new file mode 100644 index 00000000..326af9fb --- /dev/null +++ b/git/test/performance/db/test_looseodb_pygit2.py @@ -0,0 +1,13 @@ +try: + from git.db.pygit2.complex import Pygit2GitODB +except ImportError: + from git.db.py.complex import PureGitODB as Pygit2GitODB +#END handle import + +from git.test.db.pygit2.lib import Pygit2RequiredMetaMixin +from looseodb_impl import TestLooseDBWPerformanceBase + +class TestPureLooseDB(TestLooseDBWPerformanceBase): + __metaclass__ = Pygit2RequiredMetaMixin + LooseODBCls = Pygit2GitODB + diff --git a/git/test/performance/db/test_odb_dulwich.py b/git/test/performance/db/test_odb_dulwich.py index 069c5b43..aa886e08 100644 --- a/git/test/performance/db/test_odb_dulwich.py +++ b/git/test/performance/db/test_odb_dulwich.py @@ -1,6 +1,13 @@ -from git.db.dulwich.complex import DulwichCompatibilityGitDB +try: + from git.db.dulwich.complex import DulwichCompatibilityGitDB +except ImportError: + from git.db.complex import PureCompatibilityGitDB as DulwichCompatibilityGitDB +#END handle dulwich compatibility + +from git.test.db.dulwich.lib import DulwichRequiredMetaMixin from odb_impl import TestObjDBPerformanceBase class TestPureDB(TestObjDBPerformanceBase): + __metaclass__ = DulwichRequiredMetaMixin RepoCls = DulwichCompatibilityGitDB diff --git a/git/test/performance/db/test_odb_pygit2.py b/git/test/performance/db/test_odb_pygit2.py new file mode 100644 index 00000000..aa886e08 --- /dev/null +++ b/git/test/performance/db/test_odb_pygit2.py @@ -0,0 +1,13 @@ +try: + from git.db.dulwich.complex import DulwichCompatibilityGitDB +except ImportError: + from git.db.complex import PureCompatibilityGitDB as DulwichCompatibilityGitDB +#END handle dulwich compatibility + +from git.test.db.dulwich.lib import DulwichRequiredMetaMixin +from odb_impl import TestObjDBPerformanceBase + +class TestPureDB(TestObjDBPerformanceBase): + __metaclass__ = DulwichRequiredMetaMixin + RepoCls = DulwichCompatibilityGitDB + -- cgit v1.2.1 From a5a0fa2d3befd21534de91e4e2013fbe885995c6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Jul 2011 17:34:19 +0200 Subject: Fixed up tests to actually use pygit2. Its worth noting that the performance tests only work reliably in a patched up version, or the next point release. --- git/db/pygit2/complex.py | 33 ++++++++++++++++------------- git/test/performance/db/test_odb_dulwich.py | 2 +- git/test/performance/db/test_odb_pygit2.py | 14 ++++++------ 3 files changed, 26 insertions(+), 23 deletions(-) (limited to 'git') diff --git a/git/db/pygit2/complex.py b/git/db/pygit2/complex.py index 67635a1e..c1563bca 100644 --- a/git/db/pygit2/complex.py +++ b/git/db/pygit2/complex.py @@ -17,7 +17,8 @@ from git.db.compat import RepoCompatibilityInterface from pygit2 import Repository as Pygit2Repo from git.base import OInfo, OStream -from git.fun import type_id_to_type_map, type_to_type_id_map +from git.fun import type_id_to_type_map, type_to_type_id_map +from git.util import hex_to_bin from cStringIO import StringIO import os @@ -49,23 +50,25 @@ class Pygit2GitODB(PureGitODB): #{ Object DBR - # def info(self, binsha): - # type_id, uncomp_data = self._py2_repo.object_store.get_raw(binsha) - # return OInfo(binsha, type_id_to_type_map[type_id], len(uncomp_data)) - # - # def stream(self, binsha): - # type_id, uncomp_data = self._py2_repo.object_store.get_raw(binsha) - # return OStream(binsha, type_id_to_type_map[type_id], len(uncomp_data), StringIO(uncomp_data)) - # + def info(self, binsha): + type_id, uncomp_data = self._py2_repo.read(binsha) + return OInfo(binsha, type_id_to_type_map[type_id], len(uncomp_data)) + + def stream(self, binsha): + type_id, uncomp_data = self._py2_repo.read(binsha) + return OStream(binsha, type_id_to_type_map[type_id], len(uncomp_data), StringIO(uncomp_data)) + # #}END object dbr # # #{ Object DBW - # - # def store(self, istream): - # obj = ShaFile.from_raw_string(type_to_type_id_map[istream.type], istream.read()) - # self._py2_repo.object_store.add_object(obj) - # istream.binsha = obj.sha().digest() - # return istream + def store(self, istream): + # TODO: remove this check once the required functionality was merged in pygit2 + if hasattr(self._py2_repo, 'write'): + istream.binsha = hex_to_bin(self._py2_repo.write(type_to_type_id_map[istream.type], istream.read())) + return istream + else: + return super(Pygit2GitODB, self).store(istream) + #END handle write support #}END object dbw diff --git a/git/test/performance/db/test_odb_dulwich.py b/git/test/performance/db/test_odb_dulwich.py index aa886e08..6802483c 100644 --- a/git/test/performance/db/test_odb_dulwich.py +++ b/git/test/performance/db/test_odb_dulwich.py @@ -7,7 +7,7 @@ except ImportError: from git.test.db.dulwich.lib import DulwichRequiredMetaMixin from odb_impl import TestObjDBPerformanceBase -class TestPureDB(TestObjDBPerformanceBase): +class TestDulwichDB(TestObjDBPerformanceBase): __metaclass__ = DulwichRequiredMetaMixin RepoCls = DulwichCompatibilityGitDB diff --git a/git/test/performance/db/test_odb_pygit2.py b/git/test/performance/db/test_odb_pygit2.py index aa886e08..bb7ed8a9 100644 --- a/git/test/performance/db/test_odb_pygit2.py +++ b/git/test/performance/db/test_odb_pygit2.py @@ -1,13 +1,13 @@ try: - from git.db.dulwich.complex import DulwichCompatibilityGitDB + from git.db.pygit2.complex import Pygit2CompatibilityGitDB except ImportError: - from git.db.complex import PureCompatibilityGitDB as DulwichCompatibilityGitDB -#END handle dulwich compatibility + from git.db.complex import PureCompatibilityGitDB as Pygit2CompatibilityGitDB +#END handle pygit2 compatibility -from git.test.db.dulwich.lib import DulwichRequiredMetaMixin +from git.test.db.pygit2.lib import Pygit2RequiredMetaMixin from odb_impl import TestObjDBPerformanceBase -class TestPureDB(TestObjDBPerformanceBase): - __metaclass__ = DulwichRequiredMetaMixin - RepoCls = DulwichCompatibilityGitDB +class TestPygit2DB(TestObjDBPerformanceBase): + __metaclass__ = Pygit2RequiredMetaMixin + RepoCls = Pygit2CompatibilityGitDB -- cgit v1.2.1