test: test_workload_gen: Destroy collections.

2 files changed, 110 insertions, 56 deletions

diff --git a/src/test/test_workload_gen/workload_generator.cc b/src/test/test_workload_gen/workload_generator.cc
index b77e6519cab..8b7ca2f34c7 100644
--- a/src/test/test_workload_gen/workload_generator.cc
+++ b/src/test/test_workload_gen/workload_generator.cc
@@ -29,11 +29,10 @@ const coll_t WorkloadGenerator::META_COLL("meta");
 const coll_t WorkloadGenerator::TEMP_COLL("temp");
 
 WorkloadGenerator::WorkloadGenerator(vector<const char*> args) :
-    m_allow_coll_destruction(false),
-    m_prob_destroy_coll(def_prob_destroy_coll),
-    m_prob_create_coll(def_prob_create_coll),
+    m_destroy_coll_every_nr_runs(def_destroy_coll_every_nr_runs),
     m_num_colls(def_num_colls), m_num_obj_per_coll(def_num_obj_per_coll),
-    m_store(0), m_in_flight(0), m_lock("State Lock") {
+    m_store(0), m_nr_runs(0),
+    m_in_flight(0), m_lock("State Lock"), m_next_coll_nr(0) {
 
   init_args(args);
   dout(0) << "data         = " << g_conf->osd_data << dendl;
@@ -46,15 +45,12 @@ WorkloadGenerator::WorkloadGenerator(vector<const char*> args) :
   m_store->mkfs();
   m_store->mount();
 
-  m_osr.resize(m_num_colls);
-
   init();
 }
 
 void WorkloadGenerator::init_args(vector<const char*> args) {
   for (std::vector<const char*>::iterator i = args.begin(); i != args.end();) {
     string val;
-    int allow_coll_dest;
 
     if (ceph_argparse_double_dash(args, i)) {
       break;
@@ -64,10 +60,11 @@ void WorkloadGenerator::init_args(vector<const char*> args) {
     } else if (ceph_argparse_witharg(args, i, &val, "-O", "--num-objects",
         (char*) NULL)) {
       m_num_obj_per_coll = strtoll(val.c_str(), NULL, 10);
-    } else if (ceph_argparse_binary_flag(args, i, &allow_coll_dest, NULL,
-        "--allow-coll-destruction", (char*) NULL)) {
-      m_allow_coll_destruction = (allow_coll_dest ? true : false);
     }
+//    else if (ceph_argparse_binary_flag(args, i, &allow_coll_dest, NULL,
+//        "--allow-coll-destruction", (char*) NULL)) {
+//      m_allow_coll_destruction = (allow_coll_dest ? true : false);
+//    }
   }
 }
 
@@ -84,31 +81,53 @@ void WorkloadGenerator::init() {
   wait_for_ready();
 
   char buf[100];
+  char meta_buf[100];
   for (int i = 0; i < m_num_colls; i++) {
     memset(buf, 0, 100);
+    memset(meta_buf, 0, 100);
     snprintf(buf, 100, "0.%d_head", i);
-    coll_t coll(buf);
+    snprintf(meta_buf, 100, "pglog_0.%d_head", i);
+    coll_entry_t *entry = new coll_entry_t(i, buf, meta_buf);
 
-    dout(0) << "Creating collection " << coll.to_str() << dendl;
+    dout(0) << "Creating collection " << entry->coll.to_str() << dendl;
 
     t = new ObjectStore::Transaction;
 
-    t->create_collection(coll);
-
-    memset(buf, 0, 100);
-    snprintf(buf, 100, "pglog_0.%d_head", i);
-    hobject_t coll_meta_obj(sobject_t(object_t(buf), CEPH_NOSNAP));
-    t->touch(META_COLL, coll_meta_obj);
+    t->create_collection(entry->coll);
+    t->touch(META_COLL, entry->meta_obj);
 
-    m_store->queue_transaction(&m_osr[i], t,
+    m_store->queue_transaction(&(entry->osr), t,
         new C_WorkloadGeneratorOnReadable(this, t));
     m_in_flight++;
+
+    m_collections.insert(entry);
+    m_next_coll_nr++;
   }
 
+  dout(0) << "Currently in-flight collections: " << m_in_flight << dendl;
+
   wait_for_done();
+  m_nr_runs = 0;
   dout(0) << "Done initializing!" << dendl;
 }
 
+int WorkloadGenerator::get_uniform_random_value(int min, int max) {
+  boost::uniform_int<> value(min, max);
+  return value(m_rng);
+}
+
+WorkloadGenerator::coll_entry_t
+*WorkloadGenerator::get_rnd_coll_entry(bool erase = false) {
+  set<WorkloadGenerator::coll_entry_t*>::iterator i = m_collections.begin();
+  int index = get_uniform_random_value(0, m_collections.size()-1);
+  for ( ; index > 0; --index, ++i) ;
+
+  WorkloadGenerator::coll_entry_t *entry = *i;
+  if (erase)
+    m_collections.erase(i);
+  return entry;
+}
+
 int WorkloadGenerator::get_random_collection_nr() {
   return (rand() % m_num_colls);
 }
@@ -207,46 +226,58 @@ void WorkloadGenerator::do_append_log(ObjectStore::Transaction *t,
   t->write(META_COLL, log_obj, st.st_size, bl.length(), bl);
 }
 
-void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t) {
+void WorkloadGenerator::do_destroy_collection(ObjectStore::Transaction *t,
+    WorkloadGenerator::coll_entry_t *entry) {
 
-}
+  m_nr_runs = 0;
+  entry->osr.flush();
+  vector<hobject_t> ls;
+  m_store->collection_list(entry->coll, ls);
+  dout(0) << "Destroying collection '" << entry->coll.to_str()
+      << "' (" << ls.size() << " objects)" << dendl;
 
-void WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t) {
+  vector<hobject_t>::iterator it;
+  for (it = ls.begin(); it < ls.end(); it++) {
+    t->remove(entry->coll, *it);
+  }
 
+  t->remove_collection(entry->coll);
+  t->remove(META_COLL, entry->meta_obj);
 }
 
-bool WorkloadGenerator::allow_collection_destruction() {
-  return (this->m_allow_coll_destruction);
+void WorkloadGenerator::do_create_collection(ObjectStore::Transaction *t) {
+
 }
 
 void WorkloadGenerator::run() {
 
   do {
+    if (!m_collections.size()) {
+      dout(0) << "We ran out of collections!" << dendl;
+      break;
+    }
+
     m_lock.Lock();
     wait_for_ready();
 
-    int coll_nr = get_random_collection_nr();
-    int obj_nr = get_random_object_nr(coll_nr);
-
-    bool do_destroy = false;
-    if (allow_collection_destruction()) {
-      int rnd_probability = (1 + (rand() % 100));
-
-      if (rnd_probability <= m_prob_destroy_coll)
-        do_destroy = true;
-    }
+    ObjectStore::Transaction *t = new ObjectStore::Transaction;
 
-    coll_t coll = get_collection_by_nr(coll_nr);
-    hobject_t obj = get_object_by_nr(obj_nr);
+    bool destroy_collection = should_destroy_collection();
+    coll_entry_t *entry = get_rnd_coll_entry(destroy_collection);
 
-    ObjectStore::Transaction *t = new ObjectStore::Transaction;
+    if (destroy_collection) {
+      do_destroy_collection(t, entry);
+    } else {
+      int obj_nr = get_random_object_nr(entry->id);
+      hobject_t obj = get_object_by_nr(obj_nr);
 
-    do_write_object(t, coll, obj);
-    do_setattr_object(t, coll, obj);
-    do_setattr_collection(t, coll);
-    do_append_log(t, coll);
+      do_write_object(t, entry->coll, obj);
+      do_setattr_object(t, entry->coll, obj);
+      do_setattr_collection(t, entry->coll);
+      do_append_log(t, entry->coll);
+    }
 
-    m_store->queue_transaction(&m_osr[coll_nr], t,
+    m_store->queue_transaction(&(entry->osr), t,
         new C_WorkloadGeneratorOnReadable(this, t));
 
     m_in_flight++;
@@ -277,6 +308,5 @@ int main(int argc, const char *argv[]) {
   wrkldgen.reset(wrkldgen_ptr);
   wrkldgen->run();
   wrkldgen->print_results();
-
   return 0;
 }
diff --git a/src/test/test_workload_gen/workload_generator.h b/src/test/test_workload_gen/workload_generator.h
index 7de4d524b8a..2110e3b49a2 100644
--- a/src/test/test_workload_gen/workload_generator.h
+++ b/src/test/test_workload_gen/workload_generator.h
@@ -15,21 +15,37 @@
 
 #include "os/FileStore.h"
 #include <boost/scoped_ptr.hpp>
-#include <queue>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_int.hpp>
 #include <set>
 
+typedef boost::mt11213b rngen_t;
+
 class WorkloadGenerator {
+public:
+  struct coll_entry_t {
+    int id;
+    coll_t coll;
+    hobject_t meta_obj;
+    ObjectStore::Sequencer osr;
+
+    coll_entry_t(int i, char *coll_buf, char *meta_obj_buf)
+    : id(i), coll(coll_buf),
+      meta_obj(sobject_t(object_t(meta_obj_buf), CEPH_NOSNAP)) {
+    }
+  };
+
 private:
+
   /* kept in upper case for consistency with coll_t's */
   static const coll_t META_COLL;
   static const coll_t TEMP_COLL;
 
   static const int max_in_flight = 50;
 
+  static const int def_destroy_coll_every_nr_runs = 100;
   static const int def_num_obj_per_coll = 6000;
   static const int def_num_colls = 30;
-  static const int def_prob_destroy_coll = 30;
-  static const int def_prob_create_coll = 10;
 
   static const size_t min_write_bytes = 1;
   static const size_t max_write_mb = 5;
@@ -42,25 +58,23 @@ private:
 
   static const size_t log_append_bytes = 1024;
 
-  /* probabilities for creating or destroying a collection */
-  bool m_allow_coll_destruction;
-  int m_prob_destroy_coll;
-  int m_prob_create_coll;
-
+  int m_destroy_coll_every_nr_runs;
   int m_num_colls;
   int m_num_obj_per_coll;
 
   boost::scoped_ptr<ObjectStore> m_store;
 
+  int m_nr_runs;
   int m_in_flight;
-  vector<ObjectStore::Sequencer> m_osr;
+//  vector<ObjectStore::Sequencer> m_osr;
 
   Mutex m_lock;
   Cond m_cond;
 
-  set<coll_t> m_available_collections;
-  set<coll_t> m_removed_collections;
+  set<coll_entry_t*> m_collections;
+  int m_next_coll_nr;
 
+  rngen_t m_rng;
 
   void wait_for_ready() {
     while (m_in_flight >= max_in_flight)
@@ -76,6 +90,8 @@ private:
   void init_args(vector<const char*> args);
   void init();
 
+  int get_uniform_random_value(int min, int max);
+  coll_entry_t *get_rnd_coll_entry(bool erase);
   int get_random_collection_nr();
   int get_random_object_nr(int coll_nr);
 
@@ -93,8 +109,10 @@ private:
   void do_setattr_collection(ObjectStore::Transaction *t, coll_t coll);
   void do_append_log(ObjectStore::Transaction *t, coll_t coll);
 
-  bool allow_collection_destruction();
-  void do_destroy_collection(ObjectStore::Transaction *t);
+  bool should_destroy_collection() {
+    return (m_nr_runs >= m_destroy_coll_every_nr_runs);
+  }
+  void do_destroy_collection(ObjectStore::Transaction *t, coll_entry_t *entry);
   void do_create_collection(ObjectStore::Transaction *t);
 
 public:
@@ -117,6 +135,7 @@ public:
       dout(0) << "Got one back!" << dendl;
       Mutex::Locker locker(m_state->m_lock);
       m_state->m_in_flight--;
+      m_state->m_nr_runs++;
       m_state->m_cond.Signal();
     }
   };
@@ -125,4 +144,9 @@ public:
   void print_results(void);
 };
 
+bool operator<(const WorkloadGenerator::coll_entry_t& l,
+    const WorkloadGenerator::coll_entry_t& r) {
+      return (l.id < r.id);
+}
+
 #endif /* WORKLOAD_GENERATOR_H_ */