kclient: do async prealloc

5 files changed, 85 insertions, 50 deletions

diff --git a/src/kernel/inode.c b/src/kernel/inode.c
index bbd3730efb1..7987b36deca 100644
--- a/src/kernel/inode.c
+++ b/src/kernel/inode.c
@@ -301,10 +301,6 @@ struct inode *ceph_alloc_inode(struct super_block *sb)
 	INIT_LIST_HEAD(&ci->i_listener_list);
 	spin_lock_init(&ci->i_listener_lock);
 
-	ci->vfs_inode.i_mapping->a_ops = &ceph_aops;
-	ci->vfs_inode.i_mapping->backing_dev_info =
-		&ceph_client(sb)->backing_dev_info;
-
 	return &ci->vfs_inode;
 }
 
@@ -432,6 +428,10 @@ static void init_inode_ops(struct inode *inode)
 		derr(0, "%p BAD mode 0%o S_IFMT 0%o\n", inode, inode->i_mode,
 		     inode->i_mode & S_IFMT);
 	}
+
+	inode->i_mapping->a_ops = &ceph_aops;
+	inode->i_mapping->backing_dev_info =
+		&ceph_client(inode->i_sb)->backing_dev_info;
 }
 
 int ceph_async_create(struct inode *dir, struct dentry *dentry,
@@ -450,6 +450,8 @@ int ceph_async_create(struct inode *dir, struct dentry *dentry,
 	if ((ceph_inode(dir)->i_ceph_flags & CEPH_I_COMPLETE) == 0)
 		return -EPERM;
 
+	return -1; // NO
+
 	vino.ino = ceph_mdsc_prealloc_dequeue(mdsc);
 	if (!vino.ino)
 		return -EAGAIN;
diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c
index 4d428c57b83..5d7e80cab43 100644
--- a/src/kernel/mds_client.c
+++ b/src/kernel/mds_client.c
@@ -637,29 +637,7 @@ out:
 /*
  * ino preallocation
  */
-u64 ceph_mdsc_prealloc_dequeue(struct ceph_mds_client *mdsc)
-{
-	u64 r = 0;
-
-	mutex_lock(&mdsc->inoq.mutex);	
-	if (mdsc->inoq.numi) {
-		r = mdsc->inoq.inos[mdsc->inoq.head].start;
-		mdsc->inoq.inos[mdsc->inoq.head].start++;
-		mdsc->inoq.inos[mdsc->inoq.head].len--;
-		mdsc->inoq.numi--;
-		if (mdsc->inoq.inos[mdsc->inoq.head].len == 0) {
-			mdsc->inoq.num--;
-			mdsc->inoq.head++;
-			if (mdsc->inoq.head == mdsc->inoq.max)
-				mdsc->inoq.head = 0;
-		}
-	}
-	mutex_unlock(&mdsc->inoq.mutex);
-	dout(20, "prealloc_dequeue %llx\n", r);
-	return r;
-}
-
-int prealloc_enqueue(struct ceph_mds_client *mdsc, u64 first, int num)
+static int prealloc_enqueue(struct ceph_mds_client *mdsc, u64 first, int num)
 {
 	int room;
 	int ret;
@@ -711,38 +689,88 @@ out:
 	return ret;
 }
 
-int request_prealloc(struct ceph_mds_client *mdsc)
+/*
+ * called under mdsc->mutex only if error.
+ */
+static void prealloc_completion(struct ceph_mds_client *mdsc,
+				struct ceph_mds_request *req)
+{
+	if (!IS_ERR(req->r_reply)) {
+		struct ceph_mds_reply_head *h = req->r_reply->front.iov_base;
+		int err = le32_to_cpu(h->result);
+
+		dout(10, "prealloc_completion got %llx~%d\n",
+		     le64_to_cpu(h->ino), err);
+		prealloc_enqueue(mdsc, le64_to_cpu(h->ino), err);
+	} else {
+		int err = PTR_ERR(req->r_reply);
+		dout(10, "prealloc_completion err %d\n", err);
+	}
+
+	ceph_mdsc_put_request(req);
+}
+
+static int request_prealloc(struct ceph_mds_client *mdsc)
 {
 	struct ceph_mds_request *req;
-	struct ceph_mds_request_head *rhead;
-	int target = 1024;
-	int num, err;
+	int min = mdsc->client->mount_args.prealloc_min;
+	int max = mdsc->client->mount_args.prealloc_max;
+	int willhave;
+	int want;
 
 	mutex_lock(&mdsc->inoq.mutex);
-	num = target - mdsc->inoq.numi - mdsc->inoq.requesting;
-	dout(10, "request_prealloc have %d want %d requesting %d .. %d\n",
-	     mdsc->inoq.num, target, mdsc->inoq.requesting, num);
-	if (target < mdsc->inoq.numi + mdsc->inoq.requesting ||
-	    num < mdsc->inoq.requesting) {
+	willhave = mdsc->inoq.numi + mdsc->inoq.requesting;
+	want = max - mdsc->inoq.numi - mdsc->inoq.requesting;
+	dout(10, "request_prealloc have %d+%d=%d, target %d-%d, want %d\n",
+	     mdsc->inoq.num, mdsc->inoq.requesting, willhave, min, max, want);
+	if (willhave > min) {
 		mutex_unlock(&mdsc->inoq.mutex);
 		return 0;
 	}
-	mdsc->inoq.requesting += num;
+	mdsc->inoq.requesting += want;
 	mutex_unlock(&mdsc->inoq.mutex);
 
-	req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_PREALLOC,
-				       0, NULL, 0, NULL, NULL, USE_TABLE_MDS);
+	req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_PREALLOC, NULL, NULL,
+				       NULL, NULL, USE_TABLE_MDS);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
-	rhead = req->r_request->front.iov_base;
-	rhead->args.prealloc.num = cpu_to_le32(num);
-	err = ceph_mdsc_do_request(mdsc, NULL, req);
-	if (err > 0) {
-		struct ceph_mds_reply_head *h = req->r_reply->front.iov_base;
-		prealloc_enqueue(mdsc, le64_to_cpu(h->ino), err);
-	}	
-	ceph_mdsc_put_request(req);
-	return err;	
+	req->r_args.prealloc.num = cpu_to_le32(want);
+	req->r_callback = prealloc_completion;
+	ceph_mdsc_submit_request(mdsc, req);
+	return 0;
+}
+
+void ceph_mdsc_request_prealloc(struct ceph_mds_client *mdsc)
+{
+	request_prealloc(mdsc);
+}	
+
+u64 ceph_mdsc_prealloc_dequeue(struct ceph_mds_client *mdsc)
+{
+	u64 r = 0;
+	int want_more = 0;
+
+	mutex_lock(&mdsc->inoq.mutex);	
+	if (mdsc->inoq.numi) {
+		r = mdsc->inoq.inos[mdsc->inoq.head].start;
+		mdsc->inoq.inos[mdsc->inoq.head].start++;
+		mdsc->inoq.inos[mdsc->inoq.head].len--;
+		mdsc->inoq.numi--;
+		if (mdsc->inoq.inos[mdsc->inoq.head].len == 0) {
+			mdsc->inoq.num--;
+			mdsc->inoq.head++;
+			if (mdsc->inoq.head == mdsc->inoq.max)
+				mdsc->inoq.head = 0;
+		}
+	}
+	if (mdsc->inoq.num < mdsc->client->mount_args.prealloc_min)
+		want_more = 1;
+	mutex_unlock(&mdsc->inoq.mutex);
+	dout(20, "prealloc_dequeue %llx\n", r);
+
+	if (want_more)
+		request_prealloc(mdsc);  /* a little sloppy... */
+	return r;
 }
 
 /*
@@ -2204,8 +2232,6 @@ static void delayed_work(struct work_struct *work)
 	if (want_map)
 		ceph_monc_request_mdsmap(&mdsc->client->monc, want_map);
 
-	request_prealloc(mdsc);
-
 	schedule_delayed(mdsc);
 }
 
diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h
index b19227f7836..2bc424bb111 100644
--- a/src/kernel/mds_client.h
+++ b/src/kernel/mds_client.h
@@ -313,5 +313,6 @@ extern struct ceph_mds_request *ceph_mdsc_get_listener_req(struct inode *inode,
 						    u64 tid);
 
 extern u64 ceph_mdsc_prealloc_dequeue(struct ceph_mds_client *mdsc);
+extern void ceph_mdsc_request_prealloc(struct ceph_mds_client *mdsc);
 
 #endif
diff --git a/src/kernel/super.c b/src/kernel/super.c
index e5205218356..1d96ddcdb56 100644
--- a/src/kernel/super.c
+++ b/src/kernel/super.c
@@ -474,6 +474,8 @@ static int parse_mount_args(int flags, char *options, const char *dev_name,
 	args->flags = CEPH_MOUNT_DEFAULT;
 	args->osd_timeout = 5;    /* seconds */
 	args->mount_timeout = 30; /* seconds */
+	args->prealloc_min = 512;
+	args->prealloc_max = 1024;
 	args->snapdir_name = ".snap";
 
 	/* ip1[:port1][,ip2[:port2]...]:/subdir/in/fs */
@@ -818,6 +820,9 @@ static int ceph_mount(struct ceph_client *client, struct vfsmount *mnt,
 	mnt->mnt_root = root;
 	mnt->mnt_sb = client->sb;
 	client->mount_state = CEPH_MOUNT_MOUNTED;
+
+	ceph_mdsc_request_prealloc(&client->mdsc);
+
 	dout(10, "mount success\n");
 	err = 0;
 
diff --git a/src/kernel/super.h b/src/kernel/super.h
index 82a3b909f9c..7d135af442f 100644
--- a/src/kernel/super.h
+++ b/src/kernel/super.h
@@ -68,6 +68,7 @@ struct ceph_mount_args {
 	int wsize;
 	int rsize;            /* max readahead */
 	int osd_timeout;
+	int prealloc_min, prealloc_max;
 	char *snapdir_name;   /* default ".snap" */
 };