diff options
Diffstat (limited to 'test/suite/test_join01.py')
| -rw-r--r-- | test/suite/test_join01.py | 409 |
1 files changed, 152 insertions, 257 deletions
diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 4aa2bc6e269..f8d96a2718a 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -35,10 +35,44 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios class test_join01(wttest.WiredTigerTestCase): nentries = 100 - scenarios = [ + type_scen = [ ('table', dict(ref='table')), ('index', dict(ref='index')) ] + bloom0_scen = [ + ('bloom0=0', dict(joincfg0='')), + ('bloom0=1000', dict(joincfg0=',strategy=bloom,count=1000')), + ('bloom0=10000', dict(joincfg0=',strategy=bloom,count=10000')), + ] + bloom1_scen = [ + ('bloom1=0', dict(joincfg1='')), + ('bloom1=1000', dict(joincfg1=',strategy=bloom,count=1000')), + ('bloom1=10000', dict(joincfg1=',strategy=bloom,count=10000')), + ] + projection_scen = [ + ('no-projection', dict(do_proj=False)), + ('projection', dict(do_proj=True)) + ] + nested_scen = [ + ('simple', dict(do_nested=False)), + ('nested', dict(do_nested=True)) + ] + stats_scen = [ + ('no-stats', dict(do_stats=False)), + ('stats', dict(do_stats=True)) + ] + order_scen = [ + ('order=0', dict(join_order=0)), + ('order=1', dict(join_order=1)), + ('order=2', dict(join_order=2)), + ('order=3', dict(join_order=3)), + ] + scenarios = number_scenarios(multiply_scenarios('.', type_scen, + bloom0_scen, bloom1_scen, + projection_scen, + nested_scen, stats_scen, + order_scen)) + # We need statistics for these tests. conn_config = 'statistics=(all)' @@ -52,9 +86,29 @@ class test_join01(wttest.WiredTigerTestCase): return [s, rs, sort3] # Common function for testing iteration of join cursors - def iter_common(self, jc, do_proj): + def iter_common(self, jc, do_proj, do_nested, join_order): # See comments in join_common() - expect = [73, 82, 62, 83, 92] + # The order that the results are seen depends on + # the ordering of the joins. Specifically, the first + # join drives the order that results are seen. + if do_nested: + if join_order == 0: + expect = [73, 82, 83, 92] + elif join_order == 1: + expect = [73, 82, 83, 92] + elif join_order == 2: + expect = [82, 92, 73, 83] + elif join_order == 3: + expect = [92, 73, 82, 83] + else: + if join_order == 0: + expect = [73, 82, 62, 83, 92] + elif join_order == 1: + expect = [62, 73, 82, 83, 92] + elif join_order == 2: + expect = [62, 82, 92, 73, 83] + elif join_order == 3: + expect = [73, 82, 62, 83, 92] while jc.next() == 0: [k] = jc.get_keys() i = k - 1 @@ -64,7 +118,9 @@ class test_join01(wttest.WiredTigerTestCase): [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) if len(expect) == 0 or i != expect[0]: - self.tty(' result ' + str(i) + ' is not in: ' + str(expect)) + self.tty('ERROR: ' + str(i) + ' is not next in: ' + + str(expect)) + self.tty('JOIN ORDER=' + str(join_order) + ', NESTED=' + str(do_nested)) self.assertTrue(i == expect[0]) expect.remove(i) self.assertEquals(0, len(expect)) @@ -81,6 +137,8 @@ class test_join01(wttest.WiredTigerTestCase): 'join: index:join01:index2: ' + statdesc ] if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) + elif self.do_proj: + expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -118,11 +176,46 @@ class test_join01(wttest.WiredTigerTestCase): self.assertTrue(len(expectstats) == 0, 'missing expected values in stats: ' + str(expectstats)) + def session_record_join(self, jc, refc, config, order, joins): + joins.append([order, [jc, refc, config]]) + + def session_play_one_join(self, firsturi, jc, refc, config): + if refc.uri == firsturi and config != None: + config = config.replace('strategy=bloom','') + #self.tty('->join(jc, uri="' + refc.uri + + # '", config="' + str(config) + '"') + self.session.join(jc, refc, config) + + def session_play_joins(self, joins, join_order): + #self.tty('->') + firsturi = None + for [i, joinargs] in joins: + if i >= join_order: + if firsturi == None: + firsturi = joinargs[1].uri + self.session_play_one_join(firsturi, *joinargs) + for [i, joinargs] in joins: + if i < join_order: + if firsturi == None: + firsturi = joinargs[1].uri + self.session_play_one_join(firsturi, *joinargs) + # Common function for testing the most basic functionality # of joins - def join_common(self, joincfg0, joincfg1, do_proj, do_stats): + def test_join(self): + joincfg0 = self.joincfg0 + joincfg1 = self.joincfg1 + do_proj = self.do_proj + do_nested = self.do_nested + do_stats = self.do_stats + join_order = self.join_order #self.tty('join_common(' + joincfg0 + ',' + joincfg1 + ',' + - # str(do_proj) + ')') + # str(do_proj) + ',' + str(do_nested) + ',' + + # str(do_stats) + ',' + str(join_order) + ')') + + closeme = [] + joins = [] # cursors to be joined + self.session.create('table:join01', 'key_format=r' + ',value_format=SSi,columns=(k,v0,v1,v2)') self.session.create('index:join01:index0','columns=(v0)') @@ -143,7 +236,7 @@ class test_join01(wttest.WiredTigerTestCase): # We join on index2 first, not using bloom indices. # This defines the order that items are returned. - # index2 is sorts multiples of 3 first (see gen_values()) + # index2 sorts multiples of 3 first (see gen_values()) # and by using 'gt' and key 99, we'll skip multiples of 3, # and examine primary keys 2,5,8,...,95,98,1,4,7,...,94,97. jc = self.session.open_cursor('join:table:join01' + proj_suffix, @@ -152,7 +245,7 @@ class test_join01(wttest.WiredTigerTestCase): c2 = self.session.open_cursor('index:join01:index2(v1)', None, None) c2.set_key(99) # skips all entries w/ primary key divisible by three self.assertEquals(0, c2.search()) - self.session.join(jc, c2, 'compare=gt') + self.session_record_join(jc, c2, 'compare=gt', 0, joins) # Then select all the numbers 0-99 whose string representation # sort >= '60'. @@ -163,285 +256,87 @@ class test_join01(wttest.WiredTigerTestCase): c0 = self.session.open_cursor('table:join01', None, None) c0.set_key(60) self.assertEquals(0, c0.search()) - self.session.join(jc, c0, 'compare=ge' + joincfg0) + self.session_record_join(jc, c0, 'compare=ge' + joincfg0, 1, joins) # Then select all numbers whose reverse string representation # is in '20' < x < '40'. c1a = self.session.open_cursor('index:join01:index1(v1)', None, None) c1a.set_key('21') self.assertEquals(0, c1a.search()) - self.session.join(jc, c1a, 'compare=gt' + joincfg1) + self.session_record_join(jc, c1a, 'compare=gt' + joincfg1, 2, joins) c1b = self.session.open_cursor('index:join01:index1(v1)', None, None) c1b.set_key('41') self.assertEquals(0, c1b.search()) - self.session.join(jc, c1b, 'compare=lt' + joincfg1) + self.session_record_join(jc, c1b, 'compare=lt' + joincfg1, 2, joins) # Numbers that satisfy these 3 conditions (with ordering implied by c2): # [73, 82, 62, 83, 92]. # # After iterating, we should be able to reset and iterate again. + if do_nested: + # To test nesting, we create two new levels of conditions: + # + # x == 72 or x == 73 or x == 82 or x == 83 or + # (x >= 90 and x <= 99) + # + # that will get AND-ed into our existing join. The expected + # result is [73, 82, 83, 92]. + # + # We don't specify the projection here, it should be picked up + # from the 'enclosing' join. + nest1 = self.session.open_cursor('join:table:join01', None, None) + nest2 = self.session.open_cursor('join:table:join01', None, None) + + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key('90') + self.assertEquals(0, nc.search()) + self.session.join(nest2, nc, 'compare=ge') # joincfg left out + closeme.append(nc) + + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key('99') + self.assertEquals(0, nc.search()) + self.session.join(nest2, nc, 'compare=le') + closeme.append(nc) + + self.session.join(nest1, nest2, "operation=or") + + for val in [ '72', '73', '82', '83' ]: + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key(val) + self.assertEquals(0, nc.search()) + self.session.join(nest1, nc, 'compare=eq,operation=or' + + joincfg0) + closeme.append(nc) + self.session_record_join(jc, nest1, None, 3, joins) + + self.session_play_joins(joins, join_order) + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 0) - self.iter_common(jc, do_proj) + jc.reset() + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 1) jc.reset() - self.iter_common(jc, do_proj) + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 2) jc.reset() - self.iter_common(jc, do_proj) + self.iter_common(jc, do_proj, do_nested, join_order) jc.close() c2.close() c1a.close() c1b.close() c0.close() + if do_nested: + nest1.close() + nest2.close() + for c in closeme: + c.close() self.session.drop('table:join01') - # Test joins with basic functionality - def test_join(self): - bloomcfg1000 = ',strategy=bloom,count=1000' - bloomcfg10000 = ',strategy=bloom,count=10000' - for cfga in [ '', bloomcfg1000, bloomcfg10000 ]: - for cfgb in [ '', bloomcfg1000, bloomcfg10000 ]: - for do_proj in [ False, True ]: - #self.tty('cfga=' + cfga + - # ', cfgb=' + cfgb + - # ', doproj=' + str(do_proj)) - self.join_common(cfga, cfgb, do_proj, False) - - def test_join_errors(self): - self.session.create('table:join01', 'key_format=r,value_format=SS' - ',columns=(k,v0,v1)') - self.session.create('table:join01B', 'key_format=r,value_format=SS' - ',columns=(k,v0,v1)') - self.session.create('index:join01:index0','columns=(v0)') - self.session.create('index:join01:index1','columns=(v1)') - self.session.create('index:join01B:index0','columns=(v0)') - jc = self.session.open_cursor('join:table:join01', None, None) - tc = self.session.open_cursor('table:join01', None, None) - fc = self.session.open_cursor('file:join01.wt', None, None) - ic0 = self.session.open_cursor('index:join01:index0', None, None) - ic0again = self.session.open_cursor('index:join01:index0', None, None) - ic1 = self.session.open_cursor('index:join01:index1', None, None) - icB = self.session.open_cursor('index:join01B:index0', None, None) - tcB = self.session.open_cursor('table:join01B', None, None) - - tc.set_key(1) - tc.set_value('val1', 'val1') - tc.insert() - tcB.set_key(1) - tcB.set_value('val1', 'val1') - tcB.insert() - fc.next() - - # Joining using a non join-cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(tc, ic0, 'compare=ge'), - '/not a join cursor/') - # Joining a table cursor, not index - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, fc, 'compare=ge'), - '/not an index or table cursor/') - # Joining a non positioned cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - ic0.set_key('val1') - # Joining a non positioned cursor (no search or next has been done) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - ic0.set_key('valXX') - self.assertEqual(ic0.search(), wiredtiger.WT_NOTFOUND) - # Joining a non positioned cursor after failed search - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - - # position the cursors now - ic0.set_key('val1') - ic0.search() - ic0again.next() - icB.next() - - # Joining non matching index - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, icB, 'compare=ge'), - '/table for join cursor does not match/') - - # The cursor must be positioned - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic1, 'compare=ge'), - '/requires reference cursor be positioned/') - ic1.next() - - # The first cursor joined cannot be bloom - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic1, - 'compare=ge,strategy=bloom,count=1000'), - '/first joined cursor cannot specify strategy=bloom/') - - # This succeeds. - self.session.join(jc, ic1, 'compare=ge'), - - # With bloom filters, a count is required - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge,strategy=bloom'), - '/count must be nonzero/') - - # This succeeds. - self.session.join(jc, ic0, 'compare=ge,strategy=bloom,count=1000'), - - bloom_config = ',strategy=bloom,count=1000' - # Cannot use the same index cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, - 'compare=le' + bloom_config), - '/index cursor already used in a join/') - - # When joining with the same index, need compatible compares - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=ge' + bloom_config), - '/join has overlapping ranges/') - - # Another incompatible compare - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=gt' + bloom_config), - '/join has overlapping ranges/') - - # Compare is compatible, but bloom args need to match - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=le'), - '/join has incompatible strategy/') - - # Counts need to match for bloom filters - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=le,strategy=bloom,' - 'count=100'), '/count.* does not match previous count/') - - # This succeeds - self.session.join(jc, ic0again, 'compare=le,strategy=bloom,count=1000') - - # Need to do initial next() before getting key/values - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.get_keys(), - '/join cursor must be advanced with next/') - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.get_values(), - '/join cursor must be advanced with next/') - - # Operations on the joined cursor are frozen until the join is closed. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.next(), - '/index cursor is being used in a join/') - - # Operations on the joined cursor are frozen until the join is closed. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.prev(), - '/index cursor is being used in a join/') - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.reset(), - '/index cursor is being used in a join/') - - # Only a small number of operations allowed on a join cursor - msg = "/Unsupported cursor/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.search(), msg) - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.prev(), msg) - - self.assertEquals(jc.next(), 0) - self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) - - # Only after the join cursor is closed can we use the index cursor - # normally - jc.close() - self.assertEquals(ic0.next(), wiredtiger.WT_NOTFOUND) - self.assertEquals(ic0.prev(), 0) - - # common code for making sure that cursors can be - # implicitly closed, no matter the order they are created - def cursor_close_common(self, joinfirst): - self.session.create('table:join01', 'key_format=r' + - ',value_format=SS,columns=(k,v0,v1)') - self.session.create('index:join01:index0','columns=(v0)') - self.session.create('index:join01:index1','columns=(v1)') - c = self.session.open_cursor('table:join01', None, None) - for i in range(0, self.nentries): - c.set_key(*self.gen_key(i)) - c.set_value(*self.gen_values(i)) - c.insert() - c.close() - - if joinfirst: - jc = self.session.open_cursor('join:table:join01', None, None) - c0 = self.session.open_cursor('index:join01:index0', None, None) - c1 = self.session.open_cursor('index:join01:index1', None, None) - c0.next() # index cursors must be positioned - c1.next() - if not joinfirst: - jc = self.session.open_cursor('join:table:join01', None, None) - self.session.join(jc, c0, 'compare=ge') - self.session.join(jc, c1, 'compare=ge') - self.session.close() - self.session = None - - def test_cursor_close1(self): - self.cursor_close_common(True) - - def test_cursor_close2(self): - self.cursor_close_common(False) - - # test statistics using the framework set up for this test - def test_stats(self): - bloomcfg1000 = ',strategy=bloom,count=1000' - bloomcfg10 = ',strategy=bloom,count=10' - self.join_common(bloomcfg1000, bloomcfg1000, False, True) - - # Intentially run with an underconfigured Bloom filter, - # statistics should pick up some false positives. - self.join_common(bloomcfg10, bloomcfg10, False, True) - - # test statistics with a simple one index join cursor - def test_simple_stats(self): - self.session.create("table:join01b", - "key_format=i,value_format=i,columns=(k,v)") - self.session.create("index:join01b:index", "columns=(v)") - - cursor = self.session.open_cursor("table:join01b", None, None) - cursor[1] = 11 - cursor[2] = 12 - cursor[3] = 13 - cursor.close() - - cursor = self.session.open_cursor("index:join01b:index", None, None) - cursor.set_key(11) - cursor.search() - - jcursor = self.session.open_cursor("join:table:join01b", None, None) - self.session.join(jcursor, cursor, "compare=gt") - - while jcursor.next() == 0: - [k] = jcursor.get_keys() - [v] = jcursor.get_values() - - statcur = self.session.open_cursor("statistics:join", jcursor, None) - found = False - while statcur.next() == 0: - [desc, pvalue, value] = statcur.get_values() - #self.tty(str(desc) + "=" + str(pvalue)) - found = True - self.assertEquals(found, True) - - jcursor.close() - cursor.close() - - if __name__ == '__main__': wttest.run() |