From b01a5f8c2aac5815c14cfe8315888e7ba78c1fb6 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Thu, 28 Feb 2013 11:39:57 +0200 Subject: [Search] Refactor: implement asynchronous search To prevent UI from blocking and to prevent excessive searches, the custom model needs to be refactored to not do synchronous searches in on_get_value(). This is achieved by * adding SearchModel.set_request_range() so the view can inform the model of the required rows * Adding a dict-based cache for the search results * Adding row-inserted/changed/deleted signals to the model Fixes #2 Signed-off-by: Jussi Kukkonen --- src/msd/msd_main_window.py | 28 ++++- src/msd/msd_search.py | 282 ++++++++++++++++++++++++++++++++------------- src/msd/msd_upnp.py | 6 +- 3 files changed, 233 insertions(+), 83 deletions(-) diff --git a/src/msd/msd_main_window.py b/src/msd/msd_main_window.py index e72c344..5891a5a 100644 --- a/src/msd/msd_main_window.py +++ b/src/msd/msd_main_window.py @@ -122,17 +122,39 @@ class MainWindow(object): self.__sort_order.set_sort_by(sort_by) tv = column.get_tree_view() model = tv.get_model() - model.flush() + + # remove the model first to avoid handling lots of useless signals tv.set_model(None) + model.flush() tv.set_model(model) - def __create_column(self, treeview, name, col, width, sort_by): + def __cell_data_func(self, column, cell, model, tree_iter): + # Only search model supports set_request_range at the moment + if self.__search_view.get_model() == model: + path = model.get_path (tree_iter) + + # This could be a lot smarter: should fetch data so that + # there's always at least 1 visible_range preloaded: + # that way e.g. pressing PgDn would not show "Loading" + requested_range = model.get_request_range() + if (path[0] >= requested_range[0] and + path[0] <= requested_range[1]): + return + + visible_range = self.__search_view.get_visible_range() + if (visible_range): + visible_count = visible_range[1][0] - visible_range[0][0] + model.set_request_range(max(0, visible_range[0][0] - visible_count // 2), + min(len(model) - 1, visible_range[1][0] + visible_count // 2)) + + def __create_column(self, treeview, name, col, width, sort_by, cell_data_func=None): renderer = gtk.CellRendererText() column = gtk.TreeViewColumn(name, renderer) column.set_sizing(gtk.TREE_VIEW_COLUMN_FIXED) column.set_fixed_width(width) column.add_attribute(renderer, 'text', col) column.connect("clicked", self.__column_clicked, sort_by) + column.set_cell_data_func(renderer, cell_data_func); treeview.append_column(column) def __close_overlay(self): @@ -192,7 +214,7 @@ class MainWindow(object): treeview.set_headers_visible(True) treeview.set_fixed_height_mode(True) - self.__create_column(treeview, "Title", 0, 300, "DisplayName") + self.__create_column(treeview, "Title", 0, 300, "DisplayName", self.__cell_data_func) self.__create_column(treeview, "Date", 2, 100, "Date") self.__create_column(treeview, "Type", 3, 75, "Type") self.__create_column(treeview, "Author", 1, 100, "Artist") diff --git a/src/msd/msd_search.py b/src/msd/msd_search.py index caa0d3c..8b7516b 100644 --- a/src/msd/msd_search.py +++ b/src/msd/msd_search.py @@ -16,6 +16,7 @@ # 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. # # Mark Ryan +# Jussi Kukkonen # import pygtk @@ -23,22 +24,86 @@ pygtk.require('2.0') import gtk import cStringIO import dateutil.parser -import datetime from msd_sort_order import * from msd_upnp import * + +# Use a dictionary to store results so we only end up using memory for +# cached items. Class will call on_inserted/on_changed/on_deleted +# as needed. +# Note that this 'sparse dict' implementation is not 100% complete: e.g. +# "for x in results_dict" does not return the 'empty' rows. +class SearchResultArray(dict): + def __init__(self, empty_value, length=0, + on_inserted=None, on_changed=None, on_deleted=None): + dict.__init__(self) + self.__empty_value = empty_value + self.__length = length + self.__on_inserted = on_inserted + self.__on_changed = on_changed + self.__on_deleted = on_deleted + + def __len__(self): + return self.__length + + def __contains__(self, key): + return key < self.__length + + def __setitem__(self, key, value): + new = False + if (key >= self.__length): + self.__length = key + 1 + new = True + dict.__setitem__ (self, key, value) + if (new): + self.__on_inserted(key) + else: + self.__on_changed(key) + + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + if (key >= 0 and key < self.__length): + return self.__empty_value + raise + + def set_length(self, length): + if length > self.__length: + for key in xrange(self.__length, length): + self.__length = key + 1 + self.__on_inserted(key) + elif length < self.__length: + for key in xrange(self.__length - 1, length - 1, -1): + try: + del self[key] + except: + pass + self.__length = key + self.__on_deleted(key) + + def get_cached_item_count(self): + return dict.__len__(self) + + class SearchModel(gtk.GenericTreeModel): columns = (("DisplayName", str), ("Artist", str), ("Date", str), - ("Type",str), ("Path", str), ("URLs", str)) + ("Type",str), ("Path", str), ("URL", str), + ("Loaded", bool)) filter = ["Artist", "DisplayName", "URLs", "Date", "Path", "Type"] - buffer_size = 50 + # maximum number of items to fetch at a time + max_items_per_search = 50 + + # Minimum number of items to query if server does not + # tell us how many results there are + min_items_default = 100 @staticmethod def __create_query_string(query, images, videos, music): - search_string = None + search_string = '' if images or videos or music: q_buffer = cStringIO.StringIO() @@ -66,57 +131,143 @@ class SearchModel(gtk.GenericTreeModel): return search_string - def __get_search_items(self, start, count): - if self.__items: - end = start - while (end < start + SearchModel.buffer_size and - end < self.__max_items and not self.__items[end]): - end = end + 1 - else: - end = count - if start < end: - count = end - start - try: - sort_descriptor = self.__sort_order.get_upnp_sort_order() - items, max_items = self.__root.search(self.__search_string, - start, count, - SearchModel.filter, - sort_descriptor) + @staticmethod + def __create_row(item): + try: + date = dateutil.parser.parse(item['Date']).strftime("%x") + except: + date = None + media_type = item.get('Type', '').capitalize().split('.', 1)[0] + return [item.get('DisplayName', None), + item.get('Artist', None), + date, + media_type, + item.get('Path', None), + item.get('URLs', [None])[0], + True] + + def __on_search_reply(self, items, max_items): + # if server does not tell how many results there are, set + # a sensible minimum + if (max_items == 0): + self.__request_count = max(self.__request_count, + self.min_items_default) + + # 'add' empty rows before actual results + index = self.__request_start + self.__result_count + if index > len(self.__items): + self.__items.set_length(index) + + # add actual fetched results + for item in items: + self.__items[index] = self.__create_row (item) + index = index + 1 + + # 'add' (or remove) empty rows after actual results + if max_items != 0: + self.__items.set_length(max_items) + + self.__result_count = self.__result_count + len(items) + print ("%d rows fetched (%d/%d rows cached)" + % (len(items), self.__items.get_cached_item_count(), len(self.__items))) + + # Was a new search request made while this one was executed? + if (self.__restart_count > 0): + self.__start_search (self.__restart_start, self.__restart_count) + return - max_items = max(max_items, len(items)) + # should we stop searching now? + if len(items) == 0 or self.__result_count >= self.__request_count: + self.__search_in_progress = False + if max_items == 0 and self.__result_count >= self.__request_count: + print "max_items not known, may have more rows available..." + self.__may_have_more_results = True + return - # TODO: I need to inform list view if max item has changed? + self.__get_search_items() - if max_items != self.__max_items: - self.__max_items = max_items - self.__items = [None] * self.__max_items - for item in items: - self.__items[start] = item - start = start + 1 - except Exception: - pass + + def __on_search_error(self, error): + self.__search_in_progress = False + print "Search failed: %s" % error + + def __get_search_items(self): + start = self.__request_start + self.__result_count + count = min(self.__request_count - self.__result_count, + SearchModel.max_items_per_search) + + self.__root.search(self.__search_string, + start, count, + SearchModel.filter, + self.__sort_order.get_upnp_sort_order(), + self.__on_search_reply, + self.__on_search_error) + + def __start_search(self, start, count): + self.__search_in_progress = True + self.__may_have_more_results = False + self.__result_count = 0 + self.__restart_start = 0 + self.__restart_count = 0 + self.__request_start = start + self.__request_count = count + self.__get_search_items() + + def get_request_range (self): + return self.__request_range + + def set_request_range (self, start, end): + self.__request_range = [start, end] + # skip any rows in beginning or end that are already loaded + try: + while self.__items[start][6] and start <= end: + start = start + 1 + while self.__items[end][6] and start <= end: + end = end - 1 + except: + pass + + if start > end: + return + + if (self.__search_in_progress): + self.__restart_count = end - start + 1 + self.__restart_start = start + else: + self.__start_search (start, end - start + 1) + + def __on_inserted (self, row): + path = (row,) + self.row_inserted (path, self.get_iter (path)) + + def __on_changed (self, row): + path = (row,) + self.row_changed (path, self.get_iter (path)) + + def __on_deleted (self, row): + self.row_deleted ((row,)) def __init__(self, root, query, images, videos, music, sort_order): gtk.GenericTreeModel.__init__(self) - - self.__items = None - self.__max_items = 0 + empty_array = ["Loading...",None, None, None, None, False] + self.__items = SearchResultArray(empty_array, + on_inserted = self.__on_inserted, + on_changed = self.__on_changed, + on_deleted = self.__on_deleted) self.__root = root self.__sort_order = sort_order + self.__search_in_progress = False self.__search_string = SearchModel.__create_query_string(query, images, videos, music) - if self.__search_string: - self.__get_search_items(0, SearchModel.buffer_size) + self.set_request_range (0, SearchModel.max_items_per_search - 1) def flush(self): - i = 0 - while i < self.__max_items: - self.__items[i] = None - i = i + 1 + self.__items.set_length (0) + self.set_request_range (0, SearchModel.max_items_per_search - 1) def on_get_flags(self): - return gtk.TREE_MODEL_LIST_ONLY | gtk.TREE_MODEL_ITERS_PERSIST + return gtk.TREE_MODEL_LIST_ONLY def on_get_n_columns(self): return len(SearchModel.columns) @@ -125,50 +276,25 @@ class SearchModel(gtk.GenericTreeModel): return SearchModel.columns[n][1] def on_get_iter(self, path): - if path[0] >= self.__max_items: - raise ValueError("Invalid Path") - return path[0] + # return internal row reference (key) for use in on_* methods + retval = None + if len(self.__items) > 0 and path[0] < len(self.__items): + retval = path[0] + return retval def on_get_path(self, rowref): return (rowref, ) def on_get_value(self, rowref, col): - retval = None - self.__get_search_items(rowref, SearchModel.buffer_size) - if rowref < self.__max_items and self.__items and self.__items[rowref]: - key = SearchModel.columns[col][0] - if key in self.__items[rowref]: - data = self.__items[rowref][key] - if col == 2: - date = dateutil.parser.parse(data) - retval = date.strftime("%x") - elif col == 3: - data = data[0].upper() + data[1:] - period = data.find('.') - if period >=0: - retval = data[:period] - else: - retval = data - elif col == 5: - retval = data[0] - else: - retval = data - elif col == 1: - retval = "Unknown" - elif col == 2: - retval = datetime.date.today().strftime("%x") - else: - retval = "" - else: - retval = "" - - return retval + try: + return self.__items[rowref][col] + except KeyError: + return None def on_iter_next(self, rowref): retval = None - rowref = rowref + 1 - if rowref < self.__max_items: - retval = rowref + if rowref + 1 < len(self.__items): + retval = rowref + 1 return retval def on_iter_children(self, rowref): @@ -183,12 +309,12 @@ class SearchModel(gtk.GenericTreeModel): def on_iter_n_children(self, rowref): retval = 0 if not rowref: - retval = self.__max_items + retval = len(self.__items) return retval def on_iter_nth_child(self, rowref, child): retval = None - if not rowref and child < self.__max_items: + if not rowref and child < len(self.__items): retval = child return retval diff --git a/src/msd/msd_upnp.py b/src/msd/msd_upnp.py index f2bba50..bb81c67 100644 --- a/src/msd/msd_upnp.py +++ b/src/msd/msd_upnp.py @@ -41,9 +41,11 @@ class Container(MediaObject): self.__containerIF = dbus.Interface(obj, 'org.gnome.UPnP.MediaContainer2') - def search(self, query, offset, count, fltr, sort=""): + def search(self, query, offset, count, fltr, sort="", on_reply=None, on_error=None): return self.__containerIF.SearchObjectsEx(query, offset, count, fltr, - sort) + sort, + reply_handler=on_reply, + error_handler=on_error) def list_children(self, offset, count, fltr, sort=""): return self.__containerIF.ListChildrenEx(offset, count, fltr, sort) -- cgit v1.2.1