external storage base commit

been squashing reorganizing, and pulling code off to go upstream ahead
of merging the whole branch.

1 files changed, 61 insertions, 4 deletions

diff --git a/memcached.h b/memcached.h
index 69c3b50..97c78f6 100644
--- a/memcached.h
+++ b/memcached.h
@@ -5,6 +5,8 @@
  * structures and function prototypes.
  */
 
+#define EXTSTORE 1
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -24,6 +26,11 @@
 #include "cache.h"
 #include "logger.h"
 
+#ifdef EXTSTORE
+#include "extstore.h"
+#include "crc32c.h"
+#endif
+
 #include "sasl_defs.h"
 
 /** Maximum length of a key. */
@@ -266,6 +273,14 @@ struct slab_stats {
     X(auth_errors) \
     X(idle_kicks) /* idle connections killed */
 
+#ifdef EXTSTORE
+#define EXTSTORE_THREAD_STATS_FIELDS \
+    X(get_extstore) \
+    X(recache_from_extstore) \
+    X(miss_from_extstore) \
+    X(badcrc_from_extstore)
+#endif
+
 /**
  * Stats stored per-thread.
  */
@@ -273,6 +288,9 @@ struct thread_stats {
     pthread_mutex_t   mutex;
 #define X(name) uint64_t    name;
     THREAD_STATS_FIELDS
+#ifdef EXTSTORE
+    EXTSTORE_THREAD_STATS_FIELDS
+#endif
 #undef X
     struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];
     uint64_t lru_hits[POWER_LARGEST];
@@ -384,6 +402,13 @@ struct settings {
     unsigned int logger_buf_size; /* size of per-thread logger buffer */
     bool drop_privileges;   /* Whether or not to drop unnecessary process privileges */
     bool relaxed_privileges;   /* Relax process restrictions when running testapp */
+#ifdef EXTSTORE
+    unsigned int ext_item_size; /* minimum size of items to store externally */
+    unsigned int ext_item_age; /* max age of tail item before storing ext. */
+    unsigned int ext_recache_rate; /* counter++ % recache_rate == 0 > recache */
+    unsigned int ext_wbuf_size; /* read only note for the engine */
+    double ext_max_frag; /* ideal maximum page fragmentation */
+#endif
 };
 
 extern struct stats stats;
@@ -404,6 +429,10 @@ extern struct settings settings;
 /* If an item's storage are chained chunks. */
 #define ITEM_CHUNKED 32
 #define ITEM_CHUNK 64
+#ifdef EXTSTORE
+/* ITEM_data bulk is external to item */
+#define ITEM_HDR 128
+#endif
 
 /**
  * Structure for storing items within memcached.
@@ -471,7 +500,13 @@ typedef struct _strchunk {
     uint8_t          slabs_clsid; /* Same as above. */
     char data[];
 } item_chunk;
-
+#ifdef EXTSTORE
+typedef struct {
+    unsigned int page_version; /* from IO header */
+    unsigned int offset; /* from IO header */
+    unsigned short page_id; /* from IO header */
+} item_hdr;
+#endif
 typedef struct {
     pthread_t thread_id;        /* unique ID of this thread */
     struct event_base *base;    /* libevent handle this thread uses */
@@ -481,14 +516,31 @@ typedef struct {
     struct thread_stats stats;  /* Stats generated by this thread */
     struct conn_queue *new_conn_queue; /* queue of new connections to handle */
     cache_t *suffix_cache;      /* suffix cache */
+#ifdef EXTSTORE
+    cache_t *io_cache;          /* IO objects */
+    void *storage;              /* data object for storage system */
+#endif
     logger *l;                  /* logger buffer */
     void *lru_bump_buf;         /* async LRU bump buffer */
 } LIBEVENT_THREAD;
-
+typedef struct conn conn;
+#ifdef EXTSTORE
+typedef struct _io_wrap {
+    obj_io io;
+    struct _io_wrap *next;
+    conn *c;
+    item *hdr_it;             /* original header item. */
+    unsigned int iovec_start; /* start of the iovecs for this IO */
+    unsigned int iovec_count; /* total number of iovecs */
+    unsigned int iovec_data;  /* specific index of data iovec */
+    bool miss;                /* signal a miss to unlink hdr_it */
+    bool badcrc;              /* signal a crc failure */
+    bool active; // FIXME: canary for test. remove
+} io_wrap;
+#endif
 /**
  * The structure representing a connection into memcached.
  */
-typedef struct conn conn;
 struct conn {
     int    sfd;
     sasl_conn_t *sasl_conn;
@@ -549,7 +601,12 @@ struct conn {
     int    suffixsize;
     char   **suffixcurr;
     int    suffixleft;
-
+#ifdef EXTSTORE
+    int io_wrapleft;
+    unsigned int recache_counter;
+    io_wrap *io_wraplist; /* linked list of io_wraps */
+    bool io_queued; /* FIXME: debugging flag */
+#endif
     enum protocol protocol;   /* which protocol this connection speaks */
     enum network_transport transport; /* what transport is used by this connection */