Make LRU crawler work from maint thread.

Wasn't sending the condition signal after a refactor :(

Also adds some stats to inspect how much work the LRU crawler is doing, and
removes some printf noise for the LRU maintainer.

6 files changed, 22 insertions, 10 deletions

diff --git a/doc/protocol.txt b/doc/protocol.txt
index f0476f0..d63c144 100644
--- a/doc/protocol.txt
+++ b/doc/protocol.txt
@@ -582,6 +582,7 @@ integers separated by a colon (treat this as a floating point number).
 | slab_reassign_running | bool    | If a slab page is being moved             |
 | slabs_moved           | 64u     | Total slab pages moved                    |
 | crawler_reclaimed     | 64u     | Total items freed by LRU Crawler          |
+| crawler_items-checked | 64u     | Total items examined by LRU Crawler       |
 | lrutail_reflocked     | 64u     | Times LRU tail was found with active ref. |
 |                       |         | Items can be evicted to avoid OOM errors. |
 | moves_to_cold         | 64u     | Items moved from HOT/WARM to COLD LRU's   |
@@ -589,6 +590,7 @@ integers separated by a colon (treat this as a floating point number).
 | moves_within_lru      | 64u     | Items reshuffled within HOT or WARM LRU's |
 | direct_reclaims       | 64u     | Times worker threads had to directly      |
 |                       |         | reclaim or evict items.                   |
+| lru_crawler_starts    | 64u     | Times an LRU crawler was started          |
 | lru_maintainer_juggles                                                      |
 |                       | 64u     | Number of times the LRU bg thread woke up |
 |-----------------------+---------+-------------------------------------------|
diff --git a/items.c b/items.c
index d91b41a..dba615c 100644
--- a/items.c
+++ b/items.c
@@ -36,6 +36,7 @@ typedef struct {
     uint64_t expired_unfetched;
     uint64_t evicted_unfetched;
     uint64_t crawler_reclaimed;
+    uint64_t crawler_items_checked;
     uint64_t lrutail_reflocked;
     uint64_t moves_to_cold;
     uint64_t moves_to_warm;
@@ -508,6 +509,7 @@ void item_stats_totals(ADD_STAT add_stats, void *c) {
             totals.evicted += itemstats[i].evicted;
             totals.reclaimed += itemstats[i].reclaimed;
             totals.crawler_reclaimed += itemstats[i].crawler_reclaimed;
+            totals.crawler_items_checked += itemstats[i].crawler_items_checked;
             totals.lrutail_reflocked += itemstats[i].lrutail_reflocked;
             totals.moves_to_cold += itemstats[i].moves_to_cold;
             totals.moves_to_warm += itemstats[i].moves_to_warm;
@@ -526,6 +528,8 @@ void item_stats_totals(ADD_STAT add_stats, void *c) {
                 (unsigned long long)totals.reclaimed);
     APPEND_STAT("crawler_reclaimed", "%llu",
                 (unsigned long long)totals.crawler_reclaimed);
+    APPEND_STAT("crawler_items_checked", "%llu",
+                (unsigned long long)totals.crawler_items_checked);
     APPEND_STAT("lrutail_reflocked", "%llu",
                 (unsigned long long)totals.lrutail_reflocked);
     if (settings.lru_maintainer_thread) {
@@ -565,6 +569,7 @@ void item_stats(ADD_STAT add_stats, void *c) {
             totals.expired_unfetched += itemstats[i].expired_unfetched;
             totals.evicted_unfetched += itemstats[i].evicted_unfetched;
             totals.crawler_reclaimed += itemstats[i].crawler_reclaimed;
+            totals.crawler_items_checked += itemstats[i].crawler_items_checked;
             totals.lrutail_reflocked += itemstats[i].lrutail_reflocked;
             totals.moves_to_cold += itemstats[i].moves_to_cold;
             totals.moves_to_warm += itemstats[i].moves_to_warm;
@@ -605,6 +610,8 @@ void item_stats(ADD_STAT add_stats, void *c) {
                             "%llu", (unsigned long long)totals.evicted_unfetched);
         APPEND_NUM_FMT_STAT(fmt, n, "crawler_reclaimed",
                             "%llu", (unsigned long long)totals.crawler_reclaimed);
+        APPEND_NUM_FMT_STAT(fmt, n, "crawler_items_checked",
+                            "%llu", (unsigned long long)totals.crawler_items_checked);
         APPEND_NUM_FMT_STAT(fmt, n, "lrutail_reflocked",
                             "%llu", (unsigned long long)totals.lrutail_reflocked);
         if (settings.lru_maintainer_thread) {
@@ -1004,8 +1011,6 @@ static void *lru_maintainer_thread(void *arg) {
         STATS_LOCK();
         stats.lru_maintainer_juggles++;
         STATS_UNLOCK();
-        if (settings.verbose > 2)
-            fprintf(stderr, "LRU maintainer thread running\n");
         /* We were asked to immediately wake up and poke a particular slab
          * class due to a low watermark being hit */
         if (lru_maintainer_check_clsid != 0) {
@@ -1029,9 +1034,6 @@ static void *lru_maintainer_thread(void *arg) {
             lru_maintainer_crawler_check();
             last_crawler_check = current_time;
         }
-
-        if (settings.verbose > 2)
-            fprintf(stderr, "LRU maintainer thread sleeping\n");
     }
     pthread_mutex_unlock(&lru_maintainer_lock);
     if (settings.verbose > 2)
@@ -1197,6 +1199,7 @@ static item *crawler_crawl_q(item *it) {
 static void item_crawler_evaluate(item *search, uint32_t hv, int i) {
     int slab_id = CLEAR_LRU(i);
     crawlerstats_t *s = &crawlerstats[slab_id];
+    itemstats[i].crawler_items_checked++;
     if ((search->exptime != 0 && search->exptime < current_time)
         || is_flushed(search)) {
         itemstats[i].crawler_reclaimed++;
@@ -1383,6 +1386,10 @@ static int do_lru_crawler_start(uint32_t id, uint32_t remaining) {
         pthread_mutex_unlock(&lru_locks[sid]);
     }
     if (starts) {
+        STATS_LOCK();
+        stats.lru_crawler_running = true;
+        stats.lru_crawler_starts++;
+        STATS_UNLOCK();
         pthread_mutex_lock(&lru_crawler_stats_lock);
         memset(&crawlerstats[id], 0, sizeof(crawlerstats_t));
         crawlerstats[id].start_time = current_time;
@@ -1397,6 +1404,9 @@ static int lru_crawler_start(uint32_t id, uint32_t remaining) {
         return 0;
     }
     starts = do_lru_crawler_start(id, remaining);
+    if (starts) {
+        pthread_cond_signal(&lru_crawler_cond);
+    }
     pthread_mutex_unlock(&lru_crawler_lock);
     return starts;
 }
@@ -1441,9 +1451,6 @@ enum crawler_result_type lru_crawler_crawl(char *slabs) {
     }
     if (starts) {
         pthread_cond_signal(&lru_crawler_cond);
-        STATS_LOCK();
-        stats.lru_crawler_running = true;
-        STATS_UNLOCK();
         pthread_mutex_unlock(&lru_crawler_lock);
         return CRAWLER_OK;
     } else {
diff --git a/memcached.c b/memcached.c
index c8aada5..c9e4b11 100644
--- a/memcached.c
+++ b/memcached.c
@@ -184,6 +184,7 @@ static void stats_init(void) {
     stats.accepting_conns = true; /* assuming we start in this state. */
     stats.slab_reassign_running = false;
     stats.lru_crawler_running = false;
+    stats.lru_crawler_starts = 0;
 
     /* make the time we started always be 2 seconds before we really
        did, so time(0) - time.started is never zero.  if so, things
@@ -2633,6 +2634,7 @@ static void server_stats(ADD_STAT add_stats, conn *c) {
     }
     if (settings.lru_crawler) {
         APPEND_STAT("lru_crawler_running", "%u", stats.lru_crawler_running);
+        APPEND_STAT("lru_crawler_starts", "%u", stats.lru_crawler_starts);
     }
     if (settings.lru_maintainer_thread) {
         APPEND_STAT("lru_maintainer_juggles", "%llu", (unsigned long long)stats.lru_maintainer_juggles);
diff --git a/memcached.h b/memcached.h
index 58071ba..9d86767 100644
--- a/memcached.h
+++ b/memcached.h
@@ -285,6 +285,7 @@ struct stats {
     uint64_t      evicted_unfetched; /* items evicted but never touched */
     bool          slab_reassign_running; /* slab reassign in progress */
     uint64_t      slabs_moved;       /* times slabs were moved around */
+    uint64_t      lru_crawler_starts; /* Number of item crawlers kicked off */
     bool          lru_crawler_running; /* crawl in progress */
     uint64_t      lru_maintainer_juggles; /* number of LRU bg pokes */
 };
diff --git a/t/binary.t b/t/binary.t
index fe74009..3ee0823 100755
--- a/t/binary.t
+++ b/t/binary.t
@@ -2,7 +2,7 @@
 
 use strict;
 use warnings;
-use Test::More tests => 3627;
+use Test::More tests => 3639;
 use FindBin qw($Bin);
 use lib "$Bin/lib";
 use MemcachedTest;
diff --git a/t/stats.t b/t/stats.t
index 7f805f5..92f96f0 100755
--- a/t/stats.t
+++ b/t/stats.t
@@ -70,7 +70,7 @@ my $sock = $server->sock;
 my $stats = mem_stats($sock);
 
 # Test number of keys
-is(scalar(keys(%$stats)), 51, "51 stats values");
+is(scalar(keys(%$stats)), 52, "52 stats values");
 
 # Test initial state
 foreach my $key (qw(curr_items total_items bytes cmd_get cmd_set get_hits evictions get_misses