[Search] Refactor: implement asynchronous search

To prevent UI from blocking and to prevent excessive searches, the
custom model needs to be refactored to not do synchronous searches in
on_get_value().

This is achieved by
 * adding  SearchModel.set_request_range() so the view can inform the
   model of the required rows
 * Adding a dict-based cache for the search results
 * Adding row-inserted/changed/deleted signals to the model

Fixes #2

Signed-off-by: Jussi Kukkonen <jussi.kukkonen@intel.com>

3 files changed, 233 insertions, 83 deletions

diff --git a/src/msd/msd_main_window.py b/src/msd/msd_main_window.py
index e72c344..5891a5a 100644
--- a/src/msd/msd_main_window.py
+++ b/src/msd/msd_main_window.py
@@ -122,17 +122,39 @@ class MainWindow(object):
         self.__sort_order.set_sort_by(sort_by)
         tv = column.get_tree_view()
         model = tv.get_model()
-        model.flush()
+
+        # remove the model first to avoid handling lots of useless signals
         tv.set_model(None)
+        model.flush()
         tv.set_model(model)
 
-    def __create_column(self, treeview, name, col, width, sort_by):
+    def __cell_data_func(self, column, cell, model, tree_iter):
+        # Only search model supports set_request_range at the moment
+        if self.__search_view.get_model() == model:
+            path = model.get_path (tree_iter)
+
+            # This could be a lot smarter: should fetch data so that
+            # there's always at least 1 visible_range preloaded:
+            # that way e.g. pressing PgDn would not show "Loading"
+            requested_range = model.get_request_range()
+            if (path[0] >= requested_range[0] and
+                path[0] <= requested_range[1]):
+                return
+
+            visible_range = self.__search_view.get_visible_range()
+            if (visible_range):
+                visible_count = visible_range[1][0] - visible_range[0][0]
+                model.set_request_range(max(0, visible_range[0][0] - visible_count // 2),
+                                        min(len(model) - 1, visible_range[1][0] + visible_count // 2))
+
+    def __create_column(self, treeview, name, col, width, sort_by, cell_data_func=None):
         renderer = gtk.CellRendererText()
         column = gtk.TreeViewColumn(name, renderer)
         column.set_sizing(gtk.TREE_VIEW_COLUMN_FIXED)
         column.set_fixed_width(width)
         column.add_attribute(renderer, 'text', col)
         column.connect("clicked", self.__column_clicked, sort_by)
+        column.set_cell_data_func(renderer, cell_data_func);
         treeview.append_column(column)
 
     def __close_overlay(self):
@@ -192,7 +214,7 @@ class MainWindow(object):
         treeview.set_headers_visible(True)
         treeview.set_fixed_height_mode(True)
 
-        self.__create_column(treeview, "Title", 0, 300, "DisplayName")
+        self.__create_column(treeview, "Title", 0, 300, "DisplayName", self.__cell_data_func)
         self.__create_column(treeview, "Date", 2, 100, "Date")
         self.__create_column(treeview, "Type", 3, 75, "Type")
         self.__create_column(treeview, "Author", 1, 100, "Artist")
diff --git a/src/msd/msd_search.py b/src/msd/msd_search.py
index caa0d3c..8b7516b 100644
--- a/src/msd/msd_search.py
+++ b/src/msd/msd_search.py
@@ -16,6 +16,7 @@
 # 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Mark Ryan <mark.d.ryan@intel.com>
+# Jussi Kukkonen <jussi.kukkonen@intel.com>
 #
 
 import pygtk
@@ -23,22 +24,86 @@ pygtk.require('2.0')
 import gtk
 import cStringIO
 import dateutil.parser
-import datetime
 
 from msd_sort_order import *
 from msd_upnp import *
 
+
+# Use a dictionary to store results so we only end up using memory for 
+# cached items. Class will call on_inserted/on_changed/on_deleted
+# as needed.
+# Note that this 'sparse dict' implementation is not 100% complete: e.g.
+# "for x in results_dict" does not return the 'empty' rows.
+class SearchResultArray(dict):
+    def __init__(self, empty_value, length=0,
+                 on_inserted=None, on_changed=None, on_deleted=None):
+        dict.__init__(self)
+        self.__empty_value = empty_value
+        self.__length = length
+        self.__on_inserted = on_inserted
+        self.__on_changed = on_changed
+        self.__on_deleted = on_deleted
+
+    def __len__(self):
+        return self.__length
+
+    def __contains__(self, key):
+        return key < self.__length
+
+    def __setitem__(self, key, value):
+        new = False
+        if (key >= self.__length):
+            self.__length = key + 1
+            new = True
+        dict.__setitem__ (self, key, value)
+        if (new):
+            self.__on_inserted(key)
+        else:
+            self.__on_changed(key)
+
+    def __getitem__(self, key):
+        try:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            if (key >= 0 and key < self.__length):
+                return self.__empty_value
+            raise
+
+    def set_length(self, length):
+        if length > self.__length:
+            for key in xrange(self.__length, length):
+                self.__length = key + 1
+                self.__on_inserted(key)
+        elif length < self.__length:
+            for key in xrange(self.__length - 1, length - 1, -1):
+                try:
+                    del self[key]
+                except:
+                    pass
+                self.__length = key
+                self.__on_deleted(key)
+
+    def get_cached_item_count(self):
+        return dict.__len__(self)
+
+
 class SearchModel(gtk.GenericTreeModel):
     columns = (("DisplayName", str), ("Artist", str), ("Date", str),
-               ("Type",str), ("Path", str), ("URLs", str))
+               ("Type",str), ("Path", str), ("URL", str),
+               ("Loaded", bool))
     filter = ["Artist", "DisplayName", "URLs", "Date", "Path",
               "Type"]
 
-    buffer_size = 50
+    # maximum number of items to fetch at a time
+    max_items_per_search = 50
+
+    # Minimum number of items to query if server does not
+    # tell us how many results there are
+    min_items_default = 100
 
     @staticmethod
     def __create_query_string(query, images, videos, music):
-        search_string = None
+        search_string = ''
 
         if images or videos or music:
             q_buffer = cStringIO.StringIO()
@@ -66,57 +131,143 @@ class SearchModel(gtk.GenericTreeModel):
 
         return search_string
 
-    def __get_search_items(self, start, count):
-        if self.__items:
-            end = start
-            while (end < start + SearchModel.buffer_size and
-                   end < self.__max_items and not self.__items[end]):
-                end = end + 1
-        else:
-            end = count
 
-        if start < end:
-            count = end - start
-            try:
-                sort_descriptor = self.__sort_order.get_upnp_sort_order()
-                items, max_items = self.__root.search(self.__search_string,
-                                                      start, count,
-                                                      SearchModel.filter,
-                                                      sort_descriptor)
+    @staticmethod
+    def __create_row(item):
+        try:
+            date = dateutil.parser.parse(item['Date']).strftime("%x")
+        except:
+            date = None
+        media_type = item.get('Type', '').capitalize().split('.', 1)[0]
+        return [item.get('DisplayName', None),
+                item.get('Artist', None),
+                date,
+                media_type,
+                item.get('Path', None),
+                item.get('URLs', [None])[0],
+                True]
+
+    def __on_search_reply(self, items, max_items):
+        # if server does not tell how many results there are, set
+        # a sensible minimum
+        if (max_items == 0):
+            self.__request_count = max(self.__request_count,
+                                       self.min_items_default)
+
+        # 'add' empty rows before actual results
+        index = self.__request_start + self.__result_count
+        if index > len(self.__items):
+            self.__items.set_length(index)
+
+        # add actual fetched results
+        for item in items:
+            self.__items[index] = self.__create_row (item)
+            index = index + 1
+
+        # 'add' (or remove) empty rows after actual results
+        if max_items != 0:
+            self.__items.set_length(max_items)
+
+        self.__result_count = self.__result_count + len(items)
+        print ("%d rows fetched (%d/%d rows cached)"
+               % (len(items), self.__items.get_cached_item_count(), len(self.__items)))
+
+        # Was a new search request made while this one was executed?
+        if (self.__restart_count > 0):
+            self.__start_search (self.__restart_start, self.__restart_count)
+            return
 
-                max_items = max(max_items, len(items))
+        # should we stop searching now?
+        if len(items) == 0 or self.__result_count >= self.__request_count:
+            self.__search_in_progress = False
+            if max_items == 0 and self.__result_count >= self.__request_count:
+                print "max_items not known, may have more rows available..."
+                self.__may_have_more_results = True
+            return
 
-                # TODO: I need to inform list view if max item has changed?
+        self.__get_search_items()
 
-                if max_items != self.__max_items:
-                    self.__max_items = max_items
-                    self.__items = [None] * self.__max_items
-                for item in items:
-                    self.__items[start] = item
-                    start = start + 1
-            except Exception:
-                pass
+
+    def __on_search_error(self, error):
+        self.__search_in_progress = False
+        print "Search failed: %s" % error
+
+    def __get_search_items(self):
+        start = self.__request_start + self.__result_count
+        count = min(self.__request_count - self.__result_count,
+                    SearchModel.max_items_per_search)
+
+        self.__root.search(self.__search_string,
+                           start, count,
+                           SearchModel.filter,
+                           self.__sort_order.get_upnp_sort_order(),
+                           self.__on_search_reply,
+                           self.__on_search_error)
+
+    def __start_search(self, start, count):
+        self.__search_in_progress = True
+        self.__may_have_more_results = False
+        self.__result_count = 0
+        self.__restart_start = 0
+        self.__restart_count = 0
+        self.__request_start = start
+        self.__request_count = count
+        self.__get_search_items()
+
+    def get_request_range (self):
+        return self.__request_range
+
+    def set_request_range (self, start, end):
+        self.__request_range = [start, end]
+        # skip any rows in beginning or end that are already loaded
+        try:
+            while self.__items[start][6] and start <= end:
+                start = start + 1
+            while self.__items[end][6] and start <= end:
+                end = end - 1
+        except:
+            pass
+
+        if start > end:
+            return
+
+        if (self.__search_in_progress):
+            self.__restart_count = end - start + 1
+            self.__restart_start = start
+        else:
+            self.__start_search (start, end - start + 1)
+
+    def __on_inserted (self, row):
+        path = (row,)
+        self.row_inserted (path, self.get_iter (path))
+
+    def __on_changed (self, row):
+        path = (row,)
+        self.row_changed (path, self.get_iter (path))
+
+    def __on_deleted (self, row):
+        self.row_deleted ((row,))
 
     def __init__(self, root, query, images, videos, music, sort_order):
         gtk.GenericTreeModel.__init__(self)
-
-        self.__items = None
-        self.__max_items = 0
+        empty_array = ["Loading...",None, None, None, None, False]
+        self.__items = SearchResultArray(empty_array,
+                                         on_inserted = self.__on_inserted,
+                                         on_changed = self.__on_changed,
+                                         on_deleted = self.__on_deleted)
         self.__root = root
         self.__sort_order = sort_order
+        self.__search_in_progress = False
         self.__search_string = SearchModel.__create_query_string(query, images,
                                                                  videos, music)
-        if self.__search_string:
-            self.__get_search_items(0, SearchModel.buffer_size)
+        self.set_request_range (0, SearchModel.max_items_per_search - 1)
 
     def flush(self):
-        i = 0
-        while i < self.__max_items:
-            self.__items[i] = None
-            i = i + 1
+        self.__items.set_length (0)
+        self.set_request_range (0, SearchModel.max_items_per_search - 1)
 
     def on_get_flags(self):
-        return gtk.TREE_MODEL_LIST_ONLY | gtk.TREE_MODEL_ITERS_PERSIST
+        return gtk.TREE_MODEL_LIST_ONLY
 
     def on_get_n_columns(self):
         return len(SearchModel.columns)
@@ -125,50 +276,25 @@ class SearchModel(gtk.GenericTreeModel):
         return SearchModel.columns[n][1]
 
     def on_get_iter(self, path):
-        if path[0] >= self.__max_items:
-            raise ValueError("Invalid Path")
-        return path[0]
+        # return internal row reference (key) for use in on_* methods
+        retval = None
+        if len(self.__items) > 0 and path[0] < len(self.__items):
+            retval = path[0]
+        return retval
 
     def on_get_path(self, rowref):
         return (rowref, )
 
     def on_get_value(self, rowref, col):
-        retval = None
-        self.__get_search_items(rowref, SearchModel.buffer_size)
-        if rowref < self.__max_items and self.__items and self.__items[rowref]:
-            key = SearchModel.columns[col][0]
-            if key in self.__items[rowref]:
-                data = self.__items[rowref][key]
-                if col == 2:
-                    date = dateutil.parser.parse(data)
-                    retval = date.strftime("%x")
-                elif col == 3:
-                    data = data[0].upper() + data[1:]
-                    period = data.find('.')
-                    if period >=0:
-                        retval = data[:period]
-                    else:
-                        retval = data
-                elif col == 5:
-                    retval = data[0]
-                else:
-                    retval = data
-            elif col == 1:
-                retval = "Unknown"
-            elif col == 2:
-                retval = datetime.date.today().strftime("%x")
-            else:
-                retval = ""
-        else:
-            retval = ""
-
-        return retval
+        try:
+            return self.__items[rowref][col]
+        except KeyError:
+            return None
 
     def on_iter_next(self, rowref):
         retval = None
-        rowref = rowref + 1
-        if rowref < self.__max_items:
-            retval = rowref
+        if rowref + 1 < len(self.__items):
+            retval = rowref + 1
         return retval
 
     def on_iter_children(self, rowref):
@@ -183,12 +309,12 @@ class SearchModel(gtk.GenericTreeModel):
     def on_iter_n_children(self, rowref):
         retval = 0
         if not rowref:
-            retval = self.__max_items
+            retval = len(self.__items)
         return retval
 
     def on_iter_nth_child(self, rowref, child):
         retval = None
-        if not rowref and child < self.__max_items:
+        if not rowref and child < len(self.__items):
             retval = child
         return retval
 
diff --git a/src/msd/msd_upnp.py b/src/msd/msd_upnp.py
index f2bba50..bb81c67 100644
--- a/src/msd/msd_upnp.py
+++ b/src/msd/msd_upnp.py
@@ -41,9 +41,11 @@ class Container(MediaObject):
         self.__containerIF = dbus.Interface(obj,
                                             'org.gnome.UPnP.MediaContainer2')
 
-    def search(self, query, offset, count, fltr, sort=""):
+    def search(self, query, offset, count, fltr, sort="", on_reply=None, on_error=None):
         return self.__containerIF.SearchObjectsEx(query, offset, count, fltr,
-                                                  sort)
+                                                  sort,
+                                                  reply_handler=on_reply,
+                                                  error_handler=on_error)
 
     def list_children(self, offset, count, fltr, sort=""):
         return self.__containerIF.ListChildrenEx(offset, count, fltr, sort)