extstore: more page mover algorithm fixes

wasn't checking for "dirty" flag on a class before deciding it was too free,
so it would assign/remove/assign/remove/assign/remove, and tend to
over-remove slightly. Over time the global page pool would simply grow.

Also adjusts timing so draining the global page pool works better. multiple
classes can be drained into at once by nature of zeroing out the free limits
one at a time.

2 files changed, 35 insertions, 19 deletions

diff --git a/scripts/memcached-automove-extstore b/scripts/memcached-automove-extstore
index 3419620..fa7a11c 100755
--- a/scripts/memcached-automove-extstore
+++ b/scripts/memcached-automove-extstore
@@ -74,6 +74,7 @@ def determine_move(history, stats, diffs, memfree):
     w = history['w'][-1]
     oldest = (-1, 0)
     youngest = (-1, sys.maxsize)
+    too_free = False
     # Most bytes free
     decision = (-1, -1)
     if int(stats['slab_global_page_pool']) < memfree[0] / 2:
@@ -107,11 +108,17 @@ def determine_move(history, stats, diffs, memfree):
         age = window_check(history, sid, 'age') / len(history['w'])
 
         # if > 2.5 pages free, and not dirty, reassign to global page pool
-        if slab['free_chunks'] > slab['chunks_per_page'] * MIN_PAGES_FOR_RECLAIM:
-            if window_check(history, sid, 'dirty') == 0 and small_slab == True:
+        if slab['free_chunks'] > slab['chunks_per_page'] * MIN_PAGES_FOR_RECLAIM and too_free == False:
+            dirt = window_check(history, sid, 'dirty')
+            excess = window_check(history, sid, 'excess_free')
+            if small_slab == True and dirt == 0:
                 # If we're a small slab, don't hang on to free memory forever.
                 decision = (sid, 0)
-                break
+                too_free = True
+            elif small_slab == False and dirt == 0 \
+                    and excess >= len(history['w']):
+                decision = (sid, 0)
+                too_free = True
 
         # are we the oldest slab class? (and a valid target)
         if small_slab == False:
@@ -122,18 +129,18 @@ def determine_move(history, stats, diffs, memfree):
                 youngest = (sid, age)
 
 
-    if pool_high and youngest[0] != -1:
+    if w.get('slab_pool_high') and youngest[0] != -1:
         # if global pool is too high, feed youngest large class.
         # decision = (0, youngest[0])
         # No current way to assign directly from global to a class.
         # minimize the free chunks limiter and re-check.
         decision = (youngest[0], -2)
-    elif pool_low and oldest[0] != -1:
+    elif too_free == False and pool_low and oldest[0] != -1:
         # if pool is too low, take from oldest large class.
         if args.verbose:
             print("oldest:  [class: {}] [age: {}]".format(int(oldest[0]), oldest[1]))
         decision = (oldest[0], 0)
-    elif youngest[0] != -1 and oldest[0] != -1 and youngest[0] != oldest[0]:
+    elif too_free == False and youngest[0] != -1 and oldest[0] != -1 and youngest[0] != oldest[0]:
         # youngest is outside of the tolerance ratio, move a page around.
         if args.verbose:
             print("old:   [class: {}] [age: {:.2f}]\nyoung: [class: {}] [age: {:.2f}]".format(
@@ -288,16 +295,20 @@ while True:
                 decision = (-1, -1)
                 if int(stats['evictions']) > 0:
                     decision = determine_move(history, stats, diffs, memfree)
-                    if int(decision[0]) > 0 and decision[1] >= 0:
+                    if int(decision[0]) > 0 and int(decision[1]) >= 0:
                         print("moving page from, to:", decision)
                         if args.automove:
                             run_move(s, decision)
                     elif int(decision[0]) > 0 and decision[1] == -2:
-                        # we've been told to zero out the memory limit and
-                        # sleep for a bit.
-                        last_memfree_check = 0
+                        # we've been told to zero out the memory limit.
+                        # let the memfree check run "soon".
+                        # this (purposefully) can slowly remove limits on
+                        # multiple classes if global pool stays too high.
+                        last_memfree_check = time() - 58
                         s.write("extstore free_memchunks {} {}\r\n".format(decision[0], 0))
                         s.readline()
+                        if args.verbose:
+                            print("relaxing free:", decision[0])
 
 
                 # Minimize sleeping if we just moved a page to global pool.
diff --git a/slab_automove_extstore.c b/slab_automove_extstore.c
index ae69879..5663da8 100644
--- a/slab_automove_extstore.c
+++ b/slab_automove_extstore.c
@@ -144,6 +144,7 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) {
     uint64_t oldest_age = 0;
     int youngest = -1;
     uint64_t youngest_age = ~0;
+    bool too_free = false;
     *src = -1;
     *dst = -1;
 
@@ -191,17 +192,18 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) {
         uint64_t age = w_sum.age / a->window_size;
 
         // if > N free chunks and not dirty, make decision.
-        if (a->sam_after[n].free_chunks > a->sam_after[n].chunks_per_page * MIN_PAGES_FOR_RECLAIM) {
-            if (small_slab && w_sum.dirty == 0) {
+        if (a->sam_after[n].free_chunks > a->sam_after[n].chunks_per_page * MIN_PAGES_FOR_RECLAIM
+                && w_sum.dirty == 0) {
+            if (small_slab) {
                 *src = n;
                 *dst = 0;
-                break;
-            } else if (!small_slab && w_sum.excess_free >= a->window_size / 2
-                    && a->sam_after[n].total_pages > MIN_PAGES_FOR_SOURCE) {
+                too_free = true;
+            } else if (!small_slab && w_sum.excess_free >= a->window_size) {
                 // If large slab and free chunks haven't decreased for a full
                 // window, reclaim pages.
                 *src = n;
                 *dst = 0;
+                too_free = true;
             }
         }
 
@@ -229,7 +231,7 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) {
     if (a->window_cur < a->window_size)
         return;
 
-    if (wg_sum.pool_high && youngest != -1) {
+    if (wg_sum.pool_high >= a->window_size && !wg_sum.pool_low && youngest != -1) {
         /**src = 0;
         *dst = youngest;*/
         /* TODO: No current way to directly assign page from 0 to elsewhere.
@@ -238,12 +240,15 @@ void slab_automove_extstore_run(void *arg, int *src, int *dst) {
          * If set rates are very high and the pool is too low, this can bottom
          * out...
          */
-        a->last_memcheck_run = 0;
+        // schedule a memcheck run for "soon" to keep the limit zeroed out
+        // while the pool stays too high. This will also allow multiple
+        // classes to zero out over time.
+        a->last_memcheck_run = current_time - (MEMCHECK_PERIOD - 2);
         a->settings->ext_free_memchunks[youngest] = 0;
-    } else if (wg_sum.pool_low && oldest != -1) {
+    } else if (!too_free && wg_sum.pool_low && oldest != -1) {
         *src = oldest;
         *dst = 0;
-    } else if (youngest != -1 && oldest != -1 && youngest != oldest) {
+    } else if (!too_free && youngest != -1 && oldest != -1 && youngest != oldest) {
         // if we have a youngest and oldest, and oldest is outside the ratio.
         if (a->sam_after[youngest].free_chunks <= a->free_mem[youngest]
                 && youngest_age < ((double)oldest_age * a->max_age_ratio)) {