1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
# Copyright (C) 2009 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""A manager of caches."""
from bzrlib import lru_cache, trace
from bzrlib.plugins.fastimport import helpers
class CacheManager(object):
def __init__(self, info=None, verbose=False, inventory_cache_size=10):
"""Create a manager of caches.
:param info: a ConfigObj holding the output from
the --info processor, or None if no hints are available
"""
self.verbose = verbose
# dataref -> data. datref is either :mark or the sha-1.
# Sticky blobs aren't removed after being referenced.
self._blobs = {}
self._sticky_blobs = {}
# revision-id -> Inventory cache
# these are large and we probably don't need too many as
# most parents are recent in history
self.inventories = lru_cache.LRUCache(inventory_cache_size)
# import commmit-ids -> revision-id lookup table
# we need to keep all of these but they are small
self.revision_ids = {}
# (path, branch_ref) -> file-ids - as generated.
# (Use store_file_id/fetch_fileid methods rather than direct access.)
self._file_ids = {}
# Head tracking: last ref, last id per ref & map of commit ids to ref*s*
self.last_ref = None
self.last_ids = {}
self.heads = {}
# Work out the blobs to make sticky - None means all
self._blobs_to_keep = None
if info is not None:
try:
self._blobs_to_keep = info['Blob usage tracking']['multi']
except KeyError:
# info not in file - possible when no blobs used
pass
def dump_stats(self, note=trace.note):
"""Dump some statistics about what we cached."""
# TODO: add in inventory stastistics
note("Cache statistics:")
self._show_stats_for(self._sticky_blobs, "sticky blobs", note=note)
self._show_stats_for(self.revision_ids, "revision-ids", note=note)
self._show_stats_for(self._file_ids, "file-ids", note=note,
tuple_key=True)
# These aren't interesting so omit from the output, at least for now
#self._show_stats_for(self._blobs, "other blobs", note=note)
#self._show_stats_for(self.last_ids, "last-ids", note=note)
#self._show_stats_for(self.heads, "heads", note=note)
def _show_stats_for(self, dict, label, note=trace.note, tuple_key=False):
"""Dump statistics about a given dictionary.
By the key and value need to support len().
"""
count = len(dict)
if tuple_key:
size = sum(map(len, (''.join(k) for k in dict.keys())))
else:
size = sum(map(len, dict.keys()))
size += sum(map(len, dict.values()))
kbytes = size * 1.0 / 1024
note(" %-12s: %8.1f kB (%d %s)" % (label, kbytes, count,
helpers.single_plural(count, "item", "items")))
def clear_all(self):
"""Free up any memory used by the caches."""
self._blobs.clear()
self._sticky_blobs.clear()
self.revision_ids.clear()
self._file_ids.clear()
self.last_ids.clear()
self.heads.clear()
self.inventories.clear()
def store_blob(self, id, data):
"""Store a blob of data."""
if (self._blobs_to_keep is None or data == '' or
id in self._blobs_to_keep):
self._sticky_blobs[id] = data
else:
self._blobs[id] = data
def fetch_blob(self, id):
"""Fetch a blob of data."""
try:
return self._sticky_blobs[id]
except KeyError:
return self._blobs.pop(id)
def store_file_id(self, branch_ref, path, id):
"""Store the path to file-id mapping for a branch."""
key = self._fileid_key(path, branch_ref)
self._file_ids[key] = id
def fetch_file_id(self, branch_ref, path):
"""Lookup the file-id for a path in a branch.
Raises KeyError if unsuccessful.
"""
key = self._fileid_key(path, branch_ref)
return self._file_ids[key]
def _fileid_key(self, path, branch_ref):
return (path, branch_ref)
def delete_path(self, branch_ref, path):
"""Remove a path from caches."""
# We actually want to remember what file-id we gave a path,
# even when that file is deleted, so doing nothing is correct.
# It's quite possible for a path to be deleted twice where
# the first time is in a merge branch (but the same branch_ref)
# and the second time is when that branch is merged to mainline.
pass
def rename_path(self, branch_ref, old_path, new_path):
"""Rename a path in the caches."""
# In this case, we need to forget the file-id we gave a path,
# otherwise, we'll get duplicate file-ids in the repository
# if a new file is created at the old path.
old_key = self._fileid_key(old_path, branch_ref)
new_key = self._fileid_key(new_path, branch_ref)
try:
old_file_id = self._file_ids[old_key]
except KeyError:
# The old_key has already been removed, most likely
# in a merge branch.
pass
else:
self._file_ids[new_key] = old_file_id
del self._file_ids[old_key]
def track_heads(self, cmd):
"""Track the repository heads given a CommitCommand.
:param cmd: the CommitCommand
:return: the list of parents in terms of commit-ids
"""
# Get the true set of parents
if cmd.from_ is not None:
parents = [cmd.from_]
else:
last_id = self.last_ids.get(cmd.ref)
if last_id is not None:
parents = [last_id]
else:
parents = []
parents.extend(cmd.merges)
# Track the heads
self.track_heads_for_ref(cmd.ref, cmd.id, parents)
return parents
def track_heads_for_ref(self, cmd_ref, cmd_id, parents=None):
if parents is not None:
for parent in parents:
if parent in self.heads:
del self.heads[parent]
self.heads.setdefault(cmd_id, set()).add(cmd_ref)
self.last_ids[cmd_ref] = cmd_id
self.last_ref = cmd_ref
|
