extstore: make defaults more aggressive

extstore has a background thread which examines slab classes for items
to flush to disk. The thresholds for flushing to disk are managed by a
specialized "slab automove" algorithm. This algorithm was written in
2017 and not tuned since.

Most serious users set "ext_item_age=0" and force flush all items. This
is partially because the defaults do not flush aggressively enough,
which causes memory to run out and evictions to happen.

This change simplifies the slab automove portion. Instead of balancing
free chunks of memory per slab class, it sets a target of a certain
number of free global pages.

The extstore flusher thread also uses the page pool and some low chunk
limits to decide when to start flushing. Its sleep routines have also
been adjusted as it could oversleep too easily.

A few other small changes were required to avoid over-moving slab pages
around.

1 files changed, 10 insertions, 10 deletions

diff --git a/storage.c b/storage.c
index 890437e..f82d96b 100644
--- a/storage.c
+++ b/storage.c
@@ -572,6 +572,7 @@ static int storage_write(void *storage, const int clsid, const int item_age) {
 static pthread_t storage_write_tid;
 static pthread_mutex_t storage_write_plock;
 #define WRITE_SLEEP_MIN 500
+#define MIN_PAGES_FREE 3
 
 static void *storage_write_thread(void *arg) {
     void *storage = arg;
@@ -591,6 +592,7 @@ static void *storage_write_thread(void *arg) {
     while (1) {
         // cache per-loop to avoid calls to the slabs_clsid() search loop
         int min_class = slabs_clsid(settings.ext_item_size);
+        unsigned int global_pages = global_page_pool_size(NULL);
         bool do_sleep = true;
         counter++;
         if (to_sleep > settings.ext_max_sleep)
@@ -601,7 +603,6 @@ static void *storage_write_thread(void *arg) {
             bool mem_limit_reached = false;
             unsigned int chunks_free;
             int item_age;
-            int target = settings.ext_free_memchunks[x];
             if (min_class > x || (backoff[x] && (counter % backoff[x] != 0))) {
                 // Long sleeps means we should retry classes sooner.
                 if (to_sleep > WRITE_SLEEP_MIN * 10)
@@ -610,13 +611,15 @@ static void *storage_write_thread(void *arg) {
             }
 
             // Avoid extra slab lock calls during heavy writing.
+            unsigned int chunks_perpage = 0;
             chunks_free = slabs_available_chunks(x, &mem_limit_reached,
-                    NULL);
+                    &chunks_perpage);
+            unsigned int target = chunks_perpage * MIN_PAGES_FREE;
 
             // storage_write() will fail and cut loop after filling write buffer.
             while (1) {
                 // if we are low on chunks and no spare, push out early.
-                if (chunks_free < target && mem_limit_reached) {
+                if (chunks_free < target && global_pages <= settings.ext_global_pool_min) {
                     item_age = 0;
                 } else {
                     item_age = settings.ext_item_age;
@@ -624,7 +627,6 @@ static void *storage_write_thread(void *arg) {
                 if (storage_write(storage, x, item_age)) {
                     chunks_free++; // Allow stopping if we've done enough this loop
                     did_move = true;
-                    do_sleep = false;
                     if (to_sleep > WRITE_SLEEP_MIN)
                         to_sleep /= 2;
                 } else {
@@ -635,7 +637,7 @@ static void *storage_write_thread(void *arg) {
             if (!did_move) {
                 backoff[x]++;
             } else if (backoff[x]) {
-                backoff[x] /= 2;
+                backoff[x] = 1;
             }
         }
 
@@ -643,7 +645,7 @@ static void *storage_write_thread(void *arg) {
         pthread_mutex_unlock(&storage_write_plock);
         if (do_sleep) {
             usleep(to_sleep);
-            to_sleep *= 2;
+            to_sleep++;
         }
         pthread_mutex_lock(&storage_write_plock);
     }
@@ -1379,10 +1381,8 @@ void *storage_init(void *conf) {
         settings.ext_drop_under = cf->storage_file->page_count / 4;
     }
     crc32c_init();
-    /* Init free chunks to zero. */
-    for (int x = 0; x < MAX_NUMBER_OF_SLAB_CLASSES; x++) {
-        settings.ext_free_memchunks[x] = 0;
-    }
+
+    settings.ext_global_pool_min = 0;
     storage = extstore_init(cf->storage_file, ext_cf, &eres);
     if (storage == NULL) {
         fprintf(stderr, "Failed to initialize external storage: %s\n",