diff options
| -rw-r--r-- | NEWS | 15 | ||||
| -rw-r--r-- | README | 8 | ||||
| -rw-r--r-- | TODO | 24 | ||||
| -rw-r--r-- | lib/testresources/__init__.py | 111 | ||||
| -rw-r--r-- | lib/testresources/tests/test_optimising_test_suite.py | 114 | ||||
| -rw-r--r-- | lib/testresources/tests/test_resourced_test_case.py | 6 | ||||
| -rw-r--r-- | lib/testresources/tests/test_test_resource.py | 126 | ||||
| -rwxr-xr-x | test_all.py | 2 |
8 files changed, 258 insertions, 148 deletions
@@ -19,6 +19,11 @@ IN DEVELOPMENT * Expanded TODO. (Jonathan Lange) + * Resources can now be reset by overriding TestResource.reset, which for + some resources is significantly cheaper. If checking for dirtiness is + expensive, isDirty can also be overridden. + (James Henstridge, Robert Collins) + * Started keeping a NEWS file! (Jonathan Lange) BUG FIXES: @@ -29,8 +34,18 @@ IN DEVELOPMENT * All resources are dropped when a test with no declared resources is run. (James Henstridge) + * A dirty or changed dependency of a resource makes the resource dirty too. + (Robert Collins, #324202) + API CHANGES: + * adsorbSuite is now deprecated in favour of addTest. addTest now flattens + standard library TestSuites and distributes custom TestSuite across + their member tests. (Jonathan Lange) + + * ResourcedTestCase.setUpResources and tearDownResources are now instance + methods, not static methods. (Jonathan Lange) + * All methods on TestResource are now instance methods, and thus tests should use instances of TestResource subclasses, not the classes themselves. (Jonathan Lange) @@ -73,9 +73,11 @@ minimise the number of setup and tear downs required. It attempts to achieve this by callling getResource() and finishedWith() around the sequence of tests that use a specific resource. -OptimisingTestSuite has a new method over normal TestSuites: -adsorbSuite(test_case_or_suite), which scans another test suite and -incorporates all of its tests directly into the OptimisingTestSuite. +Tests are added to an OptimisingTestSuite as normal. Any standard library +TestSuite objects will be flattened, while any custom TestSuite subclasses +will be distributed across their member tests. This means that any custom +logic in test suites should be preserved, at the price of some level of +optimisation. Because the test suite does the optimisation, you can control the amount of optimising that takes place by adding more or fewer tests to a single @@ -50,30 +50,6 @@ Ideas * There are now many simple test helpers. These can probably be consolidated. -* We want to support adding other "special" test suites to - OptimisingTestSuite. In particular, if we add a test suite that provides - services to its tests to an OptimisingTestSuite, adsorbSuite should not - totally flatten the suite, but instead keep the suite, even if it changes - the structure of the tests. - - e.g. addTest maintains the structure: - >>> OptimisingTestSuite().addTest(SpecialSuite([a, b]))._tests - [SpecialSuite([a, b])] - - Currently, adsorbSuite destroys all suite structure: - >>> OptimisingTestSuite().adsorbSuite(SpecialSuite([a, b]))._tests - [a, b] - - Instead, it should preserve the suite while changing the structure: - >>> OptimisingTestSuite().adsorbSuite(SpecialSuite([a, b]))._tests - [SpecialSuite(a), SpecialSuite(b)] - - All of the tests in each of the resulting new SpecialSuites should have - identical resource requirements so we can still optimise. - - Once it does this, we should deprecate adsorbSuite and move this - functionality to addTest / addTests. - * 'TestResource' isn't a very good name. Since the switch to instance-based resources, it's even worse, since the objects are more like resource factories or resource managers. Other possible names involve 'asset', diff --git a/lib/testresources/__init__.py b/lib/testresources/__init__.py index 2fec2c0..5ef36ca 100644 --- a/lib/testresources/__init__.py +++ b/lib/testresources/__init__.py @@ -25,18 +25,6 @@ def test_suite(): return testresources.tests.test_suite() -def iterate_tests(test_suite_or_case): - """Iterate through all of the test cases in `test_suite_or_case`.""" - try: - suite = iter(test_suite_or_case) - except TypeError: - yield test_suite_or_case - else: - for test in suite: - for subtest in iterate_tests(test): - yield subtest - - def split_by_resources(tests): """Split a list of tests by the resources that the tests use. @@ -60,14 +48,29 @@ class OptimisingTestSuite(unittest.TestSuite): """A resource creation optimising TestSuite.""" def adsorbSuite(self, test_case_or_suite): - """Add `test_case_or_suite`, unwrapping any suites we find. + """Deprecated. Use addTest instead.""" + self.addTest(test_case_or_suite) - This means that any containing TestSuites will be removed. These - suites might have their own unittest extensions, so be careful with - this. + def addTest(self, test_case_or_suite): + """Add `test_case_or_suite`, unwrapping standard TestSuites. + + This means that any containing unittest.TestSuites will be removed, + while any custom test suites will be 'distributed' across their + members. Thus addTest(CustomSuite([a, b])) will result in + CustomSuite([a]) and CustomSuite([b]) being added to this suite. """ - for test in iterate_tests(test_case_or_suite): - self.addTest(test) + try: + tests = iter(test_case_or_suite) + except TypeError: + unittest.TestSuite.addTest(self, test_case_or_suite) + return + if unittest.TestSuite == test_case_or_suite.__class__: + for test in tests: + self.adsorbSuite(test) + else: + for test in tests: + unittest.TestSuite.addTest( + self, test_case_or_suite.__class__([test])) def cost_of_switching(self, old_resource_set, new_resource_set): """Cost of switching from 'old_resource_set' to 'new_resource_set'. @@ -196,12 +199,11 @@ class TestResource(object): def __init__(self): self._dirty = False - self._needsReset = False self._uses = 0 self._currentResource = None self.resources = list(getattr(self.__class__, "resources", [])) - def clean_all(self, resource): + def _clean_all(self, resource): """Clean the dependencies from resource, and then resource itself.""" self.clean(resource) for name, manager in self.resources: @@ -231,7 +233,7 @@ class TestResource(object): """ self._uses -= 1 if self._uses == 0: - self.clean_all(resource) + self._clean_all(resource) self._setResource(None) def getResource(self): @@ -243,23 +245,31 @@ class TestResource(object): that it is no longer needed. """ if self._uses == 0: - self._setResource(self.make_all()) - elif self._needsReset: + self._setResource(self._make_all()) + elif self.isDirty(): self._setResource(self.reset(self._currentResource)) self._uses += 1 return self._currentResource - def markUsed(self, resource): - """Mark the resource as having been used. - - A used resource must be reset before it is used again. + def isDirty(self): + """Return True if this managers cached resource is dirty. + + Calling when the resource is not currently held has undefined + behaviour. """ - if not self._needsReset: - self._needsReset = True - for name, manager in self.resources: - manager.markUsed(getattr(resource, name)) - - def make_all(self): + if self._dirty: + return True + for name, mgr in self.resources: + if mgr.isDirty(): + return True + res = mgr.getResource() + try: + if res is not getattr(self._currentResource, name): + return True + finally: + mgr.finishedWith(res) + + def _make_all(self): """Make the dependencies of this resource and this resource.""" dependency_resources = {} for name, resource in self.resources: @@ -278,20 +288,6 @@ class TestResource(object): raise NotImplementedError( "Override make to construct resources.") - def reset(self, resource): - """Override this to reset resources. - - By default, the resource will be cleaned then remade if it had - previously been `dirtied`. - - :param resource: The existing resource. - :return: The new resource. - """ - if self._dirty: - self.clean_all(resource) - resource = self.make_all() - return resource - def neededResources(self): """Return the resources needed for this resource, including self. @@ -309,11 +305,28 @@ class TestResource(object): result.append(self) return result + def reset(self, old_resource): + """Override this to reset resources. + + By default, the resource will be cleaned then remade if it had + previously been `dirtied`. + + This function needs to take the dependent resource stack into + consideration as _make_all and _clean_all do. + + :return: The new resource. + """ + if self._dirty: + self._clean_all(old_resource) + resource = self._make_all() + else: + resource = old_resource + return resource + def _setResource(self, new_resource): """Set the current resource to a new value.""" self._currentResource = new_resource self._dirty = False - self._needsReset = False class ResourcedTestCase(unittest.TestCase): @@ -334,8 +347,6 @@ class ResourcedTestCase(unittest.TestCase): """Set up any resources that this test needs.""" for resource in self.resources: setattr(self, resource[0], resource[1].getResource()) - for resource in self.resources: - resource[1].markUsed(getattr(self, resource[0])) def tearDown(self): self.tearDownResources() diff --git a/lib/testresources/tests/test_optimising_test_suite.py b/lib/testresources/tests/test_optimising_test_suite.py index 4bf06d3..6be88e4 100644 --- a/lib/testresources/tests/test_optimising_test_suite.py +++ b/lib/testresources/tests/test_optimising_test_suite.py @@ -32,6 +32,16 @@ def test_suite(): return result +class CustomSuite(unittest.TestSuite): + """Custom TestSuite that's comparable using == and !=.""" + + def __eq__(self, other): + return (self.__class__ == other.__class__ + and self._tests == other._tests) + def __ne__(self, other): + return not self.__eq__(other) + + class MakeCounter(testresources.TestResource): """Test resource that counts makes and cleans.""" @@ -39,13 +49,17 @@ class MakeCounter(testresources.TestResource): testresources.TestResource.__init__(self) self.cleans = 0 self.makes = 0 + self.calls = [] def clean(self, resource): self.cleans += 1 + self.calls.append(('clean', resource)) def make(self, dependency_resources): self.makes += 1 - return "boo" + resource = "boo %d" % self.makes + self.calls.append(('make', resource)) + return resource class TestOptimisingTestSuite(testtools.TestCase): @@ -55,14 +69,14 @@ class TestOptimisingTestSuite(testtools.TestCase): class TestCaseForTesting(unittest.TestCase): def runTest(self): if test_running_hook: - test_running_hook() + test_running_hook(self) return TestCaseForTesting('runTest') def makeResourcedTestCase(self, resource_manager, test_running_hook): """Make a ResourcedTestCase.""" class ResourcedTestCaseForTesting(testresources.ResourcedTestCase): def runTest(self): - test_running_hook() + test_running_hook(self) test_case = ResourcedTestCaseForTesting('runTest') test_case.resources = [('_default', resource_manager)] return test_case @@ -71,36 +85,61 @@ class TestOptimisingTestSuite(testtools.TestCase): testtools.TestCase.setUp(self) self.optimising_suite = testresources.OptimisingTestSuite() - def testAdsorbTest(self): - # Adsorbing a single test case is the same as adding one using - # addTest. + def testAddTest(self): + # Adding a single test case is the same as adding one using the + # standard addTest. case = self.makeTestCase() - self.optimising_suite.adsorbSuite(case) + self.optimising_suite.addTest(case) self.assertEqual([case], self.optimising_suite._tests) - def testAdsorbTestSuite(self): - # Adsorbing a test suite will is the same as adding all the tests in + def testAddTestSuite(self): + # Adding a standard test suite is the same as adding all the tests in # that suite. case = self.makeTestCase() suite = unittest.TestSuite([case]) - self.optimising_suite.adsorbSuite(suite) + self.optimising_suite.addTest(suite) self.assertEqual([case], self.optimising_suite._tests) - def testAdsorbFlattensAllSuiteStructure(self): - # adsorbSuite will get rid of all suite structure when adding a test, - # no matter how much nesting is going on. + def testAddFlattensStandardSuiteStructure(self): + # addTest will get rid of all unittest.TestSuite structure when adding + # a test, no matter how much nesting is going on. case1 = self.makeTestCase() case2 = self.makeTestCase() case3 = self.makeTestCase() suite = unittest.TestSuite( [unittest.TestSuite([case1, unittest.TestSuite([case2])]), case3]) - self.optimising_suite.adsorbSuite(suite) + self.optimising_suite.addTest(suite) self.assertEqual([case1, case2, case3], self.optimising_suite._tests) + def testAddDistributesNonStandardSuiteStructure(self): + # addTest distributes all non-standard TestSuites across their + # members. + case1 = self.makeTestCase() + case2 = self.makeTestCase() + inner_suite = unittest.TestSuite([case2]) + suite = CustomSuite([case1, inner_suite]) + self.optimising_suite.addTest(suite) + self.assertEqual( + [CustomSuite([case1]), CustomSuite([inner_suite])], + self.optimising_suite._tests) + + def testAddPullsNonStandardSuitesUp(self): + # addTest flattens standard TestSuites, even those that contain custom + # suites. When it reaches the custom suites, it distributes them + # across their members. + case1 = self.makeTestCase() + case2 = self.makeTestCase() + inner_suite = CustomSuite([case1, case2]) + self.optimising_suite.addTest( + unittest.TestSuite([unittest.TestSuite([inner_suite])])) + self.assertEqual( + [CustomSuite([case1]), CustomSuite([case2])], + self.optimising_suite._tests) + def testSingleCaseResourceAcquisition(self): sample_resource = MakeCounter() - def getResourceCount(): + def getResourceCount(test): self.assertEqual(sample_resource._uses, 2) case = self.makeResourcedTestCase(sample_resource, getResourceCount) self.optimising_suite.addTest(case) @@ -112,7 +151,7 @@ class TestOptimisingTestSuite(testtools.TestCase): def testResourceReuse(self): make_counter = MakeCounter() - def getResourceCount(): + def getResourceCount(test): self.assertEqual(make_counter._uses, 2) case = self.makeResourcedTestCase(make_counter, getResourceCount) case2 = self.makeResourcedTestCase(make_counter, getResourceCount) @@ -147,21 +186,58 @@ class TestOptimisingTestSuite(testtools.TestCase): def testResourcesDroppedForNonResourcedTestCase(self): sample_resource = MakeCounter() - def resourced_case_hook(): + def resourced_case_hook(test): self.assertTrue(sample_resource._uses > 0) self.optimising_suite.addTest(self.makeResourcedTestCase( sample_resource, resourced_case_hook)) - - def normal_case_hook(): + def normal_case_hook(test): # The resource should not be acquired when the normal test # runs. self.assertEqual(sample_resource._uses, 0) self.optimising_suite.addTest(self.makeTestCase(normal_case_hook)) + result = unittest.TestResult() + self.optimising_suite.run(result) + self.assertEqual(result.testsRun, 2) + self.assertEqual([], result.failures) + self.assertEqual([], result.errors) + self.assertEqual(result.wasSuccessful(), True) + + def testDirtiedResourceNotRecreated(self): + make_counter = MakeCounter() + def dirtyResource(test): + make_counter.dirtied(test._default) + case = self.makeResourcedTestCase(make_counter, dirtyResource) + self.optimising_suite.addTest(case) + result = unittest.TestResult() + self.optimising_suite.run(result) + self.assertEqual(result.testsRun, 1) + self.assertEqual(result.wasSuccessful(), True) + # The resource should only have been made once. + self.assertEqual(make_counter.makes, 1) + def testDirtiedResourceCleanedUp(self): + make_counter = MakeCounter() + def testOne(test): + make_counter.calls.append('test one') + make_counter.dirtied(test._default) + def testTwo(test): + make_counter.calls.append('test two') + case1 = self.makeResourcedTestCase(make_counter, testOne) + case2 = self.makeResourcedTestCase(make_counter, testTwo) + self.optimising_suite.addTest(case1) + self.optimising_suite.addTest(case2) result = unittest.TestResult() self.optimising_suite.run(result) self.assertEqual(result.testsRun, 2) self.assertEqual(result.wasSuccessful(), True) + # Two resources should have been created and cleaned up + self.assertEqual(make_counter.calls, + [('make', 'boo 1'), + 'test one', + ('clean', 'boo 1'), + ('make', 'boo 2'), + 'test two', + ('clean', 'boo 2')]) class TestSplitByResources(testtools.TestCase): diff --git a/lib/testresources/tests/test_resourced_test_case.py b/lib/testresources/tests/test_resourced_test_case.py index 461cacb..73fffac 100644 --- a/lib/testresources/tests/test_resourced_test_case.py +++ b/lib/testresources/tests/test_resourced_test_case.py @@ -86,12 +86,6 @@ class TestResourcedTestCase(testtools.TestCase): self.resourced_case.setUpResources() self.assertEqual(self.resource_manager._uses, 1) - def testSetUpResourcesMarksUsed(self): - # resources used by a test need resetting afterwards. - self.resourced_case.resources = [("foo", self.resource_manager)] - self.resourced_case.setUpResources() - self.assertEqual(self.resource_manager._needsReset, True) - def testTearDownResourcesDeletesResourceAttributes(self): self.resourced_case.resources = [("foo", self.resource_manager)] self.resourced_case.setUpResources() diff --git a/lib/testresources/tests/test_test_resource.py b/lib/testresources/tests/test_test_resource.py index 7c1a146..4d89469 100644 --- a/lib/testresources/tests/test_test_resource.py +++ b/lib/testresources/tests/test_test_resource.py @@ -29,6 +29,18 @@ def test_suite(): return result +class MockResourceInstance(object): + + def __init__(self, name): + self._name = name + + def __cmp__(self, other): + return cmp(self.__dict__, other.__dict__) + + def __repr__(self): + return self._name + + class MockResource(testresources.TestResource): """Mock resource that logs the number of make and clean calls.""" @@ -42,7 +54,7 @@ class MockResource(testresources.TestResource): def make(self, dependency_resources): self.makes += 1 - return "Boo!" + return MockResourceInstance("Boo!") class MockResettableResource(MockResource): @@ -54,7 +66,8 @@ class MockResettableResource(MockResource): def reset(self, resource): self.resets += 1 - return resource + "!" + resource._name += "!" + return resource class TestTestResource(testtools.TestCase): @@ -69,10 +82,6 @@ class TestTestResource(testtools.TestCase): resource_manager = testresources.TestResource() self.assertEqual(False, resource_manager._dirty) - def testInitiallyNotNeedsReset(self): - resource_manager = testresources.TestResource() - self.assertEqual(False, resource_manager._needsReset) - def testInitiallyUnused(self): resource_manager = testresources.TestResource() self.assertEqual(0, resource_manager._uses) @@ -146,6 +155,41 @@ class TestTestResource(testtools.TestCase): resource_manager.getResource() self.assertEqual(1, resource_manager.makes) + def testIsDirty(self): + resource_manager = MockResource() + r = resource_manager.getResource() + resource_manager.dirtied(r) + self.assertTrue(resource_manager.isDirty()) + resource_manager.finishedWith(r) + + def testIsDirtyIsTrueIfDependenciesChanged(self): + resource_manager = MockResource() + dep1 = MockResource() + dep2 = MockResource() + dep3 = MockResource() + resource_manager.resources.append(("dep1", dep1)) + resource_manager.resources.append(("dep2", dep2)) + resource_manager.resources.append(("dep3", dep3)) + r = resource_manager.getResource() + dep2.dirtied(r.dep2) + r2 =dep2.getResource() + self.assertTrue(resource_manager.isDirty()) + resource_manager.finishedWith(r) + dep2.finishedWith(r2) + + def testIsDirtyIsTrueIfDependenciesAreDirty(self): + resource_manager = MockResource() + dep1 = MockResource() + dep2 = MockResource() + dep3 = MockResource() + resource_manager.resources.append(("dep1", dep1)) + resource_manager.resources.append(("dep2", dep2)) + resource_manager.resources.append(("dep3", dep3)) + r = resource_manager.getResource() + dep2.dirtied(r.dep2) + self.assertTrue(resource_manager.isDirty()) + resource_manager.finishedWith(r) + def testRepeatedGetResourceCallsMakeResourceOnceOnly(self): resource_manager = MockResource() resource_manager.getResource() @@ -157,10 +201,27 @@ class TestTestResource(testtools.TestCase): resource_manager.getResource() resource = resource_manager.getResource() self.assertEqual(1, resource_manager.makes) - resource_manager.markUsed(resource) + resource_manager.dirtied(resource) resource_manager.getResource() self.assertEqual(1, resource_manager.makes) self.assertEqual(1, resource_manager.resets) + resource_manager.finishedWith(resource) + + def testUsedResourceResetBetweenUses(self): + resource_manager = MockResettableResource() + # take two refs; like happens with OptimisingTestSuite. + resource_manager.getResource() + resource = resource_manager.getResource() + resource_manager.dirtied(resource) + resource_manager.finishedWith(resource) + # Get again, but its been dirtied. + resource = resource_manager.getResource() + resource_manager.finishedWith(resource) + resource_manager.finishedWith(resource) + # The resource is made once, reset once and cleaned once. + self.assertEqual(1, resource_manager.makes) + self.assertEqual(1, resource_manager.resets) + self.assertEqual(1, resource_manager.cleans) def testFinishedWithDecrementsUses(self): resource_manager = MockResource() @@ -209,13 +270,6 @@ class TestTestResource(testtools.TestCase): resource_manager.finishedWith(resource) self.assertEqual(False, resource_manager._dirty) - def testFinishedWithMarksNeedsReset(self): - resource_manager = MockResource() - resource = resource_manager.getResource() - resource_manager.markUsed(resource) - resource_manager.finishedWith(resource) - self.assertEqual(False, resource_manager._needsReset) - def testResourceAvailableBetweenFinishedWithCalls(self): resource_manager = MockResource() resource = resource_manager.getResource() @@ -223,28 +277,6 @@ class TestTestResource(testtools.TestCase): resource_manager.finishedWith(resource) self.assertIs(resource, resource_manager._currentResource) - def testMarkUsedSetsNeedsReset(self): - resource_manager = MockResettableResource() - resource = resource_manager.getResource() - self.assertEqual(False, resource_manager._needsReset) - resource_manager.markUsed(resource) - self.assertEqual(True, resource_manager._needsReset) - - def testUsedResourceResetBetweenUses(self): - resource_manager = MockResettableResource() - resource_manager.getResource() - resource = resource_manager.getResource() - resource_manager.markUsed(resource) - resource_manager.finishedWith(resource) - resource = resource_manager.getResource() - resource_manager.markUsed(resource) - resource_manager.finishedWith(resource) - resource_manager.finishedWith(resource) - # The resource is made once, reset once and cleaned once. - self.assertEqual(1, resource_manager.makes) - self.assertEqual(1, resource_manager.resets) - self.assertEqual(1, resource_manager.cleans) - # The default implementation of reset() performs a make/clean if # the dirty flag is set. def testDirtiedSetsDirty(self): @@ -254,6 +286,19 @@ class TestTestResource(testtools.TestCase): resource_manager.dirtied(resource) self.assertEqual(True, resource_manager._dirty) + def testDirtyingResourceTriggersCleanOnGet(self): + resource_manager = MockResource() + resource1 = resource_manager.getResource() + resource2 = resource_manager.getResource() + resource_manager.dirtied(resource2) + resource_manager.finishedWith(resource2) + self.assertEqual(0, resource_manager.cleans) + resource3 = resource_manager.getResource() + self.assertEqual(1, resource_manager.cleans) + resource_manager.finishedWith(resource3) + resource_manager.finishedWith(resource1) + self.assertEqual(2, resource_manager.cleans) + def testDefaultResetMethodPreservesCleanResource(self): resource_manager = MockResource() resource = resource_manager.getResource() @@ -280,12 +325,3 @@ class TestTestResource(testtools.TestCase): self.assertEqual(1, resource_manager.makes) resource = resource_manager.getResource() self.assertEqual(2, resource_manager.makes) - - def testMarkUsedWhenUnused(self): - resource_manager = MockResource() - resource = resource_manager.getResource() - resource_manager.finishedWith(resource) - resource_manager.markUsed(resource) - self.assertEqual(1, resource_manager.makes) - resource = resource_manager.getResource() - self.assertEqual(2, resource_manager.makes) diff --git a/test_all.py b/test_all.py index c0288bd..e265c90 100755 --- a/test_all.py +++ b/test_all.py @@ -54,7 +54,7 @@ class EarlyStoppingTextTestResult(unittest._TextTestResult): self.stop() def addFailure(self, test, err): - unittest._TextTestResult.addError(self, test, err) + unittest._TextTestResult.addFailure(self, test, err) if self.stopOnFailure(): self.stop() |