1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
#!/usr/bin/env python
#
# Public Domain 2014-2016 MongoDB, Inc.
# Public Domain 2008-2014 WiredTiger, Inc.
#
# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# test_compact02.py
# Test that compact reduces the file size.
#
import wiredtiger, wttest
from wiredtiger import stat
from wtscenario import make_scenarios
# Test basic compression
class test_compact02(wttest.WiredTigerTestCase):
types = [
('table', dict(uri='table:test_compact02')),
]
cacheSize = [
('default', dict(cacheSize='')),
('1mb', dict(cacheSize='cache_size=1MB')),
('10gb', dict(cacheSize='cache_size=10GB')),
]
# There's a balance between the pages we create and the size of the records
# being stored: compaction doesn't work on tables with many overflow items
# because we don't rewrite them. Experimentally, 8KB is as small as the test
# can go. Additionally, we can't set the maximum page size too large because
# there won't be enough pages to rewrite. Experimentally, 128KB works.
fileConfig = [
('default', dict(fileConfig='')),
('8KB', dict(fileConfig='leaf_page_max=8kb')),
('64KB', dict(fileConfig='leaf_page_max=64KB')),
('128KB', dict(fileConfig='leaf_page_max=128KB')),
]
scenarios = make_scenarios(types, cacheSize, fileConfig)
# We want about 22K records that total about 130Mb. That is an average
# of 6196 bytes per record. Half the records should be smaller, about
# 2700 bytes (about 30Mb) and the other half should be larger, 9666 bytes
# per record (about 100Mb).
#
# Test flow is as follows.
#
# 1. Create a table with the data, alternating record size.
# 2. Checkpoint and get stats on the table to confirm the size.
# 3. Delete the half of the records with the larger record size.
# 4. Checkpoint so compact finds something to work with.
# 5. Call compact.
# 6. Get stats on compacted table.
#
nrecords = 22000
bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666
smallvalue = "ihgfedcba" * 303 # 9*303 == 2727
fullsize = nrecords / 2 * len(bigvalue) + nrecords / 2 * len(smallvalue)
# Return the size of the file
def getSize(self):
# To allow this to work on systems without ftruncate,
# get the portion of the file allocated, via 'statistics=(all)',
# not the physical file size, via 'statistics=(size)'.
cstat = self.session.open_cursor(
'statistics:' + self.uri, None, 'statistics=(all)')
sz = cstat[stat.dsrc.block_size][2]
cstat.close()
return sz
# This test varies the cache size and so needs to set up its own connection.
# Override the standard methods.
def setUpConnectionOpen(self, dir):
return None
def setUpSessionOpen(self, conn):
return None
def ConnectionOpen(self, cacheSize):
self.home = '.'
conn_params = 'create,' + \
cacheSize + ',error_prefix="%s: ",' % self.shortid() + \
'statistics=(all),' + \
'eviction_dirty_target=99,eviction_dirty_trigger=99'
try:
self.conn = wiredtiger.wiredtiger_open(self.home, conn_params)
except wiredtiger.WiredTigerError as e:
print "Failed conn at '%s' with config '%s'" % (dir, conn_params)
self.session = self.conn.open_session(None)
# Create a table, add keys with both big and small values.
def test_compact02(self):
self.ConnectionOpen(self.cacheSize)
mb = 1024 * 1024
params = 'key_format=i,value_format=S,' + self.fileConfig
# 1. Create a table with the data, alternating record size.
self.session.create(self.uri, params)
c = self.session.open_cursor(self.uri, None)
for i in range(self.nrecords):
if i % 2 == 0:
c[i] = str(i) + self.bigvalue
else:
c[i] = str(i) + self.smallvalue
c.close()
# 2. Checkpoint and get stats on the table to confirm the size.
self.session.checkpoint()
sz = self.getSize()
self.pr('After populate ' + str(sz / mb) + 'MB')
self.assertGreater(sz, self.fullsize)
# 3. Delete the half of the records with the larger record size.
c = self.session.open_cursor(self.uri, None)
count = 0
for i in range(self.nrecords):
if i % 2 == 0:
count += 1
c.set_key(i)
c.remove()
c.close()
self.pr('Removed total ' + str((count * 9666) / mb) + 'MB')
# 4. Checkpoint
self.session.checkpoint()
# 5. Call compact.
self.session.compact(self.uri, None)
# 6. Get stats on compacted table.
sz = self.getSize()
self.pr('After compact ' + str(sz / mb) + 'MB')
# After compact, the file size should be less than half the full size.
self.assertLess(sz, self.fullsize / 2)
if __name__ == '__main__':
wttest.run()
|
