Merge branch 'next'

19 files changed, 237 insertions, 34 deletions

diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst
index f53173e2145..213fa4eb028 100644
--- a/doc/radosgw/config-ref.rst
+++ b/doc/radosgw/config-ref.rst
@@ -202,3 +202,25 @@ set automatically.
 :Description: Whether rgw should use dns cname record of the request hostname field (if hostname is not equal to ``rgw dns name``)
 :Type: Boolean
 :Default: ``false``
+
+
+
+``rgw enable ops log``
+:Description: Whether rgw will log each successful operation
+:Type: Boolean
+:Default: ``true``
+
+``rgw ops log rados``
+:Description: Whether rgw operations logging should be written into the RADOS backend
+:Type: Boolean
+:Default: ``true``
+
+``rgw ops log socket path``
+:Description: The path of a unix domain socket to which operations logging data will be written.
+:Type: String
+:Default: N/A
+
+``rgw ops log data backlog``
+:Description: Total backlog data size for unix domain socket operations logging
+:Type: Integer
+:Default: ``5ul << 20``
diff --git a/doc/radosgw/config.rst b/doc/radosgw/config.rst
index c9605d06cef..32ce66d0426 100644
--- a/doc/radosgw/config.rst
+++ b/doc/radosgw/config.rst
@@ -224,7 +224,31 @@ For details on RADOS Gateway administration, see `radosgw-admin`_.
    in quotes, or simply regenerating the key and ensuring that it 
    does not have an escape character.
 
+Configuring the Operations Logging
+==================================
 
+By default, the RADOS Gateway will log every successful operation in the RADOS backend.
+This means that every request, whether it is a read request or a write request will
+generate a RADOS operation that writes data. This does not come without cost, and may
+affect overall performance. Turning off logging completely can be done by adding the
+following config option to ceph.conf::
+
+        rgw enable ops log = false
+
+Another way to reduce the logging load is to send operations logging data to a unix domain
+socket, instead of writing it to the RADOS backend::
+
+        rgw ops log rados = false
+        rgw enable ops log = true
+        rgw ops log socket path = <path to socket>
+
+When specifying a unix domain socket, it is also possible to specify the maximum amount
+of memory that will be used to keep the data backlog::
+
+        rgw ops log data backlog = <size in bytes>
+
+Any backlogged data in excess to the specified size will be lost, so socket needs to be
+constantly read.
 
 Enabling Swift Access
 =====================
diff --git a/src/check_version b/src/check_version
index e31ca498dab..8600c556f4c 100755
--- a/src/check_version
+++ b/src/check_version
@@ -1,12 +1,13 @@
 #!/bin/sh
 
-if [ ! -d ../.git ]; then
-    echo "not updating .git_version (no ../.git)"
+dname=`dirname $0`
+
+if [ ! -d $dname/../.git ]; then
+    echo "not updating .git_version (no $dname/../.git)"
     exit 0
 fi
 
-dname=`dirname $0`
-cur=`git rev-parse HEAD 2>/dev/null; git describe 2>/dev/null`
+cur=`cd $dname && git rev-parse HEAD 2>/dev/null; git describe 2>/dev/null`
 [ -e $1 ] && old=`cat $1`
 
 if [ "$cur" != "$old" ]; then
diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8d4a5ac63d4..641aba170d7 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -5100,7 +5100,8 @@ int Client::getdir(const char *relpath, list<string>& contents)
 
 
 /****** file i/o **********/
-int Client::open(const char *relpath, int flags, mode_t mode) 
+int Client::open(const char *relpath, int flags, mode_t mode, int stripe_unit,
+    int stripe_count, int object_size, const char *data_pool)
 {
   ldout(cct, 3) << "open enter(" << relpath << ", " << flags << "," << mode << ") = " << dendl;
   Mutex::Locker lock(client_lock);
@@ -5124,7 +5125,8 @@ int Client::open(const char *relpath, int flags, mode_t mode)
     r = path_walk(dirpath, &dir);
     if (r < 0)
       return r;
-    r = _create(dir, dname.c_str(), flags, mode, &in, &fh);
+    r = _create(dir, dname.c_str(), flags, mode, &in, &fh, stripe_unit,
+                stripe_count, object_size, data_pool);
     created = true;
   }
   if (r < 0)
@@ -5156,6 +5158,13 @@ int Client::open(const char *relpath, int flags, mode_t mode)
   return r;
 }
 
+int Client::open(const char *relpath, int flags, mode_t mode)
+{
+  /* Use default file striping parameters */
+  return open(relpath, flags, mode, file_stripe_unit, file_stripe_count,
+      object_size, NULL);
+}
+
 int Client::lookup_hash(inodeno_t ino, inodeno_t dirino, const char *name)
 {
   Mutex::Locker lock(client_lock);
@@ -6598,7 +6607,8 @@ int Client::ll_mknod(vinodeno_t parent, const char *name, mode_t mode, dev_t rde
   return r;
 }
 
-int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp, int uid, int gid) 
+int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp,
+    int stripe_unit, int stripe_count, int object_size, const char *data_pool, int uid, int gid)
 { 
   ldout(cct, 3) << "_create(" << dir->ino << " " << name << ", 0" << oct << mode << dec << ")" << dendl;
   
@@ -6622,8 +6632,8 @@ int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode
   req->head.args.open.flags = flags | O_CREAT;
   req->head.args.open.mode = mode;
   
-  req->head.args.open.stripe_unit = file_stripe_unit;
-  req->head.args.open.stripe_count = file_stripe_count;
+  req->head.args.open.stripe_unit = stripe_unit;
+  req->head.args.open.stripe_count = stripe_count;
   req->head.args.open.object_size = object_size;
   req->head.args.open.file_replication = file_replication;
   req->dentry_drop = CEPH_CAP_FILE_SHARED;
diff --git a/src/client/Client.h b/src/client/Client.h
index f6f288a5bba..ae75d2df9fe 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -529,7 +529,9 @@ private:
   int _setxattr(Inode *in, const char *name, const void *value, size_t len, int flags, int uid=-1, int gid=-1);
   int _removexattr(Inode *in, const char *nm, int uid=-1, int gid=-1);
   int _open(Inode *in, int flags, mode_t mode, Fh **fhp, int uid=-1, int gid=-1);
-  int _create(Inode *in, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp, int uid=-1, int gid=-1);
+  int _create(Inode *in, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp,
+      int stripe_unit, int stripe_count, int object_size, const char *data_pool,
+	      int uid=-1, int gid=-1);
   loff_t _lseek(Fh *fh, loff_t offset, int whence);
   int _read(Fh *fh, int64_t offset, uint64_t size, bufferlist *bl);
   int _write(Fh *fh, int64_t offset, uint64_t size, const char *buf);
@@ -620,6 +622,7 @@ public:
   // file ops
   int mknod(const char *path, mode_t mode, dev_t rdev=0);
   int open(const char *path, int flags, mode_t mode=0);
+  int open(const char *path, int flags, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool);
   int lookup_hash(inodeno_t ino, inodeno_t dirino, const char *name);
   int lookup_ino(inodeno_t ino);
   int close(int fd);
diff --git a/src/include/cephfs/libcephfs.h b/src/include/cephfs/libcephfs.h
index 3ebbac13226..71bf25f6aed 100644
--- a/src/include/cephfs/libcephfs.h
+++ b/src/include/cephfs/libcephfs.h
@@ -602,6 +602,25 @@ int ceph_mknod(struct ceph_mount_info *cmount, const char *path, mode_t mode, de
 int ceph_open(struct ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
 
 /**
+ * Create and/or open a file with a specific file layout.
+ *
+ * @param cmount the ceph mount handle to use for performing the open.
+ * @param path the path of the file to open.  If the flags parameter includes O_CREAT,
+ *        the file will first be created before opening.
+ * @param flags a set of option masks that control how the file is created/opened.
+ * @param mode the permissions to place on the file if the file does not exist and O_CREAT
+ *        is specified in the flags.
+ * @param stripe_unit the stripe unit size (option, -1 for default)
+ * @param stripe_count the stripe count (optional, -1 for default)
+ * @param object_size the object size (optional, -1 for default)
+ * @param data_pool name of target data pool name (optional, NULL or empty string for default)
+ * @returns a non-negative file descriptor number on success or a negative error code on failure.
+ */
+int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
+ 		     mode_t mode, int stripe_unit, int stripe_count, int object_size,
+ 		     const char *data_pool);
+
+/**
  * Close the open file.
  *
  * @param cmount the ceph mount handle to use for performing the close.
diff --git a/src/java/java/com/ceph/fs/CephMount.java b/src/java/java/com/ceph/fs/CephMount.java
index d814804b92a..41b38f5aa09 100644
--- a/src/java/java/com/ceph/fs/CephMount.java
+++ b/src/java/java/com/ceph/fs/CephMount.java
@@ -396,6 +396,27 @@ public class CephMount {
   private static synchronized native int native_ceph_open(long mountp, String path, int flags, int mode);
 
   /**
+   * Open a file with a specific file layout.
+   *
+   * @param path Path of file to open or create.
+   * @param flags Open flags.
+   * @param mode Permission mode.
+   * @param stripe_unit File layout stripe unit size.
+   * @param stripe_count File layout stripe count.
+   * @param object_size Size of each object.
+   * @param data_pool The target data pool.
+   * @return File descriptor.
+   */
+  public int open(String path, int flags, int mode, int stripe_unit, int stripe_count,
+      int object_size, String data_pool) throws FileNotFoundException {
+    return native_ceph_open_layout(instance_ptr, path, flags, mode, stripe_unit,
+        stripe_count, object_size, data_pool);
+  }
+
+  private static synchronized native int native_ceph_open_layout(long mountp, String path,
+      int flags, int mode, int stripe_unit, int stripe_count, int object_size, String data_pool);
+
+  /**
    * Close an open file.
    *
    * @param fd The file descriptor.
diff --git a/src/java/native/libcephfs_jni.cc b/src/java/native/libcephfs_jni.cc
index 13132900d16..c94e136a980 100644
--- a/src/java/native/libcephfs_jni.cc
+++ b/src/java/native/libcephfs_jni.cc
@@ -1413,6 +1413,58 @@ JNIEXPORT jint JNICALL Java_com_ceph_fs_CephMount_native_1ceph_1open
 
 /*
  * Class:     com_ceph_fs_CephMount
+ * Method:    native_ceph_open_layout
+ * Signature: (JLjava/lang/String;IIIIILjava/lang/String;)I
+ */
+JNIEXPORT jint JNICALL Java_com_ceph_fs_CephMount_native_1ceph_1open_1layout
+  (JNIEnv *env, jclass clz, jlong j_mntp, jstring j_path, jint j_flags, jint j_mode,
+   jint stripe_unit, jint stripe_count, jint object_size, jstring j_data_pool)
+{
+	struct ceph_mount_info *cmount = get_ceph_mount(j_mntp);
+	CephContext *cct = ceph_get_mount_context(cmount);
+	const char *c_path, *c_data_pool = NULL;
+	int ret, flags = fixup_open_flags(j_flags);
+
+	CHECK_ARG_NULL(j_path, "@path is null", -1);
+	CHECK_MOUNTED(cmount, -1);
+
+	c_path = env->GetStringUTFChars(j_path, NULL);
+	if (!c_path) {
+		cephThrowInternal(env, "Failed to pin memory");
+		return -1;
+	}
+
+	if (j_data_pool) {
+		c_data_pool = env->GetStringUTFChars(j_data_pool, NULL);
+		if (!c_data_pool) {
+			env->ReleaseStringUTFChars(j_path, c_path);
+			cephThrowInternal(env, "Failed to pin memory");
+			return -1;
+		}
+	}
+
+	ldout(cct, 10) << "jni: open_layout: path " << c_path << " flags " << flags
+		<< " mode " << (int)j_mode << " stripe_unit " << stripe_unit
+		<< " stripe_count " << stripe_count << " object_size " << object_size
+		<< " data_pool " << (c_data_pool ? c_data_pool : "<NULL>") << dendl;
+
+	ret = ceph_open_layout(cmount, c_path, flags, (int)j_mode,
+			(int)stripe_unit, (int)stripe_count, (int)object_size, c_data_pool);
+
+	ldout(cct, 10) << "jni: open_layout: exit ret " << ret << dendl;
+
+	env->ReleaseStringUTFChars(j_path, c_path);
+	if (j_data_pool)
+		env->ReleaseStringUTFChars(j_data_pool, c_data_pool);
+
+  if (ret < 0)
+    handle_error(env, ret);
+
+	return ret;
+}
+
+/*
+ * Class:     com_ceph_fs_CephMount
  * Method:    native_ceph_close
  * Signature: (JI)I
  */
diff --git a/src/java/test/com/ceph/fs/CephMountTest.java b/src/java/test/com/ceph/fs/CephMountTest.java
index 026d2a9ed0d..c48b06ff419 100644
--- a/src/java/test/com/ceph/fs/CephMountTest.java
+++ b/src/java/test/com/ceph/fs/CephMountTest.java
@@ -623,6 +623,15 @@ public class CephMountTest {
     mount.unlink(path);
   }
 
+  @Test
+  public void test_open_layout() throws Exception {
+    String path = makePath();
+    int fd = mount.open(path, CephMount.O_WRONLY|CephMount.O_CREAT, 0,
+        (1<<20), 1, (1<<20), null);
+    mount.close(fd);
+    mount.unlink(path);
+  }
+
   /*
    * open/close
    */
diff --git a/src/java/test/com/ceph/fs/CephUnmountedTest.java b/src/java/test/com/ceph/fs/CephUnmountedTest.java
index 79f9d7ac18c..7074168ee10 100644
--- a/src/java/test/com/ceph/fs/CephUnmountedTest.java
+++ b/src/java/test/com/ceph/fs/CephUnmountedTest.java
@@ -95,6 +95,11 @@ public class CephUnmountedTest {
   }
 
   @Test(expected=CephNotMountedException.class)
+  public void test_open_layout() throws Exception {
+    mount.open("/a/path", 0, 0, 0, 0, 0, null);
+  }
+
+  @Test(expected=CephNotMountedException.class)
   public void test_close() throws Exception {
     mount.close(0);
   }
diff --git a/src/libcephfs.cc b/src/libcephfs.cc
index f23b959c9b1..02cf0864972 100644
--- a/src/libcephfs.cc
+++ b/src/libcephfs.cc
@@ -610,6 +610,15 @@ extern "C" int ceph_open(struct ceph_mount_info *cmount, const char *path,
   return cmount->get_client()->open(path, flags, mode);
 }
 
+extern "C" int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
+    mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool)
+{
+  if (!cmount->is_mounted())
+    return -ENOTCONN;
+  return cmount->get_client()->open(path, flags, mode, stripe_unit,
+      stripe_count, object_size, data_pool);
+}
+
 extern "C" int ceph_close(struct ceph_mount_info *cmount, int fd)
 {
   if (!cmount->is_mounted())
diff --git a/src/logrotate.conf b/src/logrotate.conf
index aef3572918b..97d4fcdc545 100644
--- a/src/logrotate.conf
+++ b/src/logrotate.conf
@@ -4,11 +4,11 @@
     compress
     sharedscripts
     postrotate
-        if [ -x `which invoke-rc.d` ]; then
+        if which invoke-rc.d && [ -x `which invoke-rc.d` ]; then
             invoke-rc.d ceph reload >/dev/null
-        elif [ -x `which service` ]; then
+        elif which service && [ -x `which service` ]; then
             service ceph reload >/dev/null
-        elif [ -x `which initctl` ]; then
+        elif which initctl && [ -x `which initctl` ]; then
             # upstart reload isn't very helpful here:
             #   https://bugs.launchpad.net/upstart/+bug/1012938
             for type in mon osd mds; do
diff --git a/src/osdc/ObjectCacher.cc b/src/osdc/ObjectCacher.cc
index c933bad164a..807a35e4fb6 100644
--- a/src/osdc/ObjectCacher.cc
+++ b/src/osdc/ObjectCacher.cc
@@ -734,7 +734,9 @@ void ObjectCacher::bh_write(BufferHead *bh)
 {
   assert(lock.is_locked());
   ldout(cct, 7) << "bh_write " << *bh << dendl;
-  
+
+  bh->ob->get();
+
   // finishers
   C_WriteCommit *oncommit = new C_WriteCommit(this, bh->ob->oloc.pool,
                                               bh->ob->get_soid(), bh->start(), bh->length());
@@ -842,6 +844,8 @@ void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start,
 
     // is the entire object set now clean and fully committed?
     ObjectSet *oset = ob->oset;
+    ob->put();
+
     if (flush_set_callback &&
 	was_dirty_or_tx > 0 &&
 	oset->dirty_or_tx == 0) {        // nothing dirty/tx
diff --git a/src/osdc/ObjectCacher.h b/src/osdc/ObjectCacher.h
index b431463cdee..1e7293bb189 100644
--- a/src/osdc/ObjectCacher.h
+++ b/src/osdc/ObjectCacher.h
@@ -182,8 +182,6 @@ class ObjectCacher {
     int dirty_or_tx;
 
     map< tid_t, list<Context*> > waitfor_commit;
-    list<Context*> waitfor_rd;
-    list<Context*> waitfor_wr;
 
   public:
     Object(const Object& other);
@@ -215,11 +213,9 @@ class ObjectCacher {
     void set_object_locator(object_locator_t& l) { oloc = l; }
 
     bool can_close() {
-      if (data.empty() &&
-	  waitfor_commit.empty() &&
-	  waitfor_rd.empty() && waitfor_wr.empty() &&
-	  dirty_or_tx == 0) {
-	assert(lru_is_expireable());
+      if (lru_is_expireable()) {
+	assert(data.empty());
+	assert(waitfor_commit.empty());
 	return true;
       }
       return false;
diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index c2ea5253a54..99eb090e9dc 100644
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -582,15 +582,6 @@ int RGWPutACLs_ObjStore::get_params()
   return ret;
 }
 
-int RGWInitMultipart_ObjStore::get_params()
-{
-  if (!s->args.exists("uploads")) {
-    ret = -ENOTSUP;
-  }
-
-  return ret;
-}
-
 static int read_all_chunked_input(req_state *s, char **pdata, int *plen)
 {
 #define READ_CHUNK 4096
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 4ac5a5383e3..35c34a6cdba 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -152,8 +152,6 @@ class RGWInitMultipart_ObjStore : public RGWInitMultipart {
 public:
   RGWInitMultipart_ObjStore() {}
   ~RGWInitMultipart_ObjStore() {}
-
-  virtual int get_params();
 };
 
 class RGWCompleteMultipart_ObjStore : public RGWCompleteMultipart {
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 4ae3258f67d..f200db847fa 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -1237,7 +1237,7 @@ int RGWInitMultipart_ObjStore_S3::get_params()
 
   policy = s3policy;
 
-  return RGWInitMultipart_ObjStore::get_params();
+  return 0;
 }
 
 void RGWInitMultipart_ObjStore_S3::send_response()
@@ -1578,8 +1578,11 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_post()
 {
   if (s->args.exists("uploadId"))
     return new RGWCompleteMultipart_ObjStore_S3;
-  else
+
+  if (s->args.exists("uploads"))
     return new RGWInitMultipart_ObjStore_S3;
+
+  return NULL;
 }
 
 int RGWHandler_ObjStore_S3::init_from_header(struct req_state *s, int default_formatter, bool configurable_format)
diff --git a/src/rgw/rgw_rest_usage.cc b/src/rgw/rgw_rest_usage.cc
index 428744538e5..e087bcc512c 100644
--- a/src/rgw/rgw_rest_usage.cc
+++ b/src/rgw/rgw_rest_usage.cc
@@ -2,6 +2,8 @@
 #include "rgw_usage.h"
 #include "rgw_rest_usage.h"
 
+#include "include/str_list.h"
+
 #define dout_subsys ceph_subsys_rgw
 
 class RGWOp_Usage_Get : public RGWRESTOp {
@@ -31,6 +33,18 @@ void RGWOp_Usage_Get::execute() {
   RESTArgs::get_bool(s, "show-entries", true, &show_entries);
   RESTArgs::get_bool(s, "show-summary", true, &show_summary);
 
+  string cat_str;
+  RESTArgs::get_string(s, "categories", cat_str, &cat_str);
+
+  if (!cat_str.empty()) {
+    list<string> cat_list;
+    list<string>::iterator iter;
+    get_str_list(cat_str, cat_list);
+    for (iter = cat_list.begin(); iter != cat_list.end(); ++iter) {
+      categories[*iter] = true;
+    }
+  }
+
   http_ret = RGWUsage::show(store, uid, start, end, show_entries, show_summary, &categories, flusher);
 }
 
diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc
index f398f4d4d2d..42fb411362d 100644
--- a/src/test/libcephfs/test.cc
+++ b/src/test/libcephfs/test.cc
@@ -149,6 +149,28 @@ TEST(LibCephFS, Mount) {
   ceph_shutdown(cmount);
 }
 
+TEST(LibCephFS, Open_layout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  /* valid layout */
+  char test_layout_file[256];
+  sprintf(test_layout_file, "test_layout_%d_b", getpid());
+  int fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), NULL);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  /* invalid layout */
+  sprintf(test_layout_file, "test_layout_%d_c", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 1, 19, NULL);
+  ASSERT_EQ(fd, -EINVAL);
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
 TEST(LibCephFS, Dir_ls) {
 
   pid_t mypid = getpid();