block: move stats from disk to part0

Move stats related fields - stamp, in_flight, dkstats - from disk to
part0 and unify stat handling such that...

* part_stat_*() now updates part0 together if the specified partition
  is not part0.  ie. part_stat_*() are now essentially all_stat_*().

* {disk|all}_stat_*() are gone.

* part_round_stats() is updated similary.  It handles part0 stats
  automatically and disk_round_stats() is killed.

* part_{inc|dec}_in_fligh() is implemented which automatically updates
  part0 stats for parts other than part0.

* disk_map_sector_rcu() is updated to return part0 if no part matches.
  Combined with the above changes, this makes NULL special case
  handling in callers unnecessary.

* Separate stats show code paths for disk are collapsed into part
  stats show code paths.

* Rename disk_stat_lock/unlock() to part_stat_lock/unlock()

While at it, reposition stat handling macros a bit and add missing
parentheses around macro parameters.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

1 files changed, 45 insertions, 114 deletions

diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 3d15b42dc352..c90e1b4fbe5a 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -145,13 +145,6 @@ struct gendisk {
 	struct timer_rand_state *random;
 
 	atomic_t sync_io;		/* RAID */
-	unsigned long stamp;
-	int in_flight;
-#ifdef	CONFIG_SMP
-	struct disk_stats *dkstats;
-#else
-	struct disk_stats dkstats;
-#endif
 	struct work_struct async_notify;
 #ifdef  CONFIG_BLK_DEV_INTEGRITY
 	struct blk_integrity *integrity;
@@ -232,46 +225,18 @@ extern struct hd_struct *disk_map_sector_rcu(struct gendisk *disk,
  * internal use only.
  */
 #ifdef	CONFIG_SMP
-#define disk_stat_lock()	({ rcu_read_lock(); get_cpu(); })
-#define disk_stat_unlock()	do { put_cpu(); rcu_read_unlock(); } while (0)
-
-#define disk_stat_add(cpu, gendiskp, field, addnd)			\
-	(per_cpu_ptr(gendiskp->dkstats, cpu)->field += addnd)
-
-#define disk_stat_read(gendiskp, field)					\
-({									\
-	typeof(gendiskp->dkstats->field) res = 0;			\
-	int i;								\
-	for_each_possible_cpu(i)					\
-		res += per_cpu_ptr(gendiskp->dkstats, i)->field;	\
-	res;								\
-})
-
-static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
-{
-	int i;
-
-	for_each_possible_cpu(i)
-		memset(per_cpu_ptr(gendiskp->dkstats, i), value,
-				sizeof(struct disk_stats));
-}		
+#define part_stat_lock()	({ rcu_read_lock(); get_cpu(); })
+#define part_stat_unlock()	do { put_cpu(); rcu_read_unlock(); } while (0)
 
-#define part_stat_add(cpu, part, field, addnd)				\
-	(per_cpu_ptr(part->dkstats, cpu)->field += addnd)
-
-#define all_stat_add(cpu, gendiskp, part, field, addnd, sector)		\
-({									\
-	if (part)							\
-		part_stat_add(cpu, part, field, addnd);			\
-	disk_stat_add(cpu, gendiskp, field, addnd);			\
-})
+#define __part_stat_add(cpu, part, field, addnd)			\
+	(per_cpu_ptr((part)->dkstats, (cpu))->field += (addnd))
 
 #define part_stat_read(part, field)					\
 ({									\
-	typeof(part->dkstats->field) res = 0;				\
+	typeof((part)->dkstats->field) res = 0;				\
 	int i;								\
 	for_each_possible_cpu(i)					\
-		res += per_cpu_ptr(part->dkstats, i)->field;		\
+		res += per_cpu_ptr((part)->dkstats, i)->field;		\
 	res;								\
 })
 
@@ -284,109 +249,73 @@ static inline void part_stat_set_all(struct hd_struct *part, int value)
 				sizeof(struct disk_stats));
 }
 
-#else /* !CONFIG_SMP */
-#define disk_stat_lock()	({ rcu_read_lock(); 0; })
-#define disk_stat_unlock()	rcu_read_unlock()
-
-#define disk_stat_add(cpu, gendiskp, field, addnd)			\
-	(gendiskp->dkstats.field += addnd)
-#define disk_stat_read(gendiskp, field)	(gendiskp->dkstats.field)
-
-static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
+static inline int init_part_stats(struct hd_struct *part)
 {
-	memset(&gendiskp->dkstats, value, sizeof (struct disk_stats));
+	part->dkstats = alloc_percpu(struct disk_stats);
+	if (!part->dkstats)
+		return 0;
+	return 1;
 }
 
-#define part_stat_add(cpu, part, field, addnd)				\
-	(part->dkstats.field += addnd)
-
-#define all_stat_add(cpu, gendiskp, part, field, addnd, sector)		\
-({									\
-	if (part)							\
-		part_stat_add(cpu, part, field, addnd);			\
-	disk_stat_add(cpu, gendiskp, field, addnd);			\
-})
-
-#define part_stat_read(part, field)	(part->dkstats.field)
-
-static inline void part_stat_set_all(struct hd_struct *part, int value)
+static inline void free_part_stats(struct hd_struct *part)
 {
-	memset(&part->dkstats, value, sizeof(struct disk_stats));
+	free_percpu(part->dkstats);
 }
 
-#endif /* CONFIG_SMP */
-
-#define disk_stat_dec(cpu, gendiskp, field)				\
-	disk_stat_add(cpu, gendiskp, field, -1)
-#define disk_stat_inc(cpu, gendiskp, field)				\
-	disk_stat_add(cpu, gendiskp, field, 1)
-#define disk_stat_sub(cpu, gendiskp, field, subnd)			\
-	disk_stat_add(cpu, gendiskp, field, -subnd)
-
-#define part_stat_dec(cpu, gendiskp, field)				\
-	part_stat_add(cpu, gendiskp, field, -1)
-#define part_stat_inc(cpu, gendiskp, field)				\
-	part_stat_add(cpu, gendiskp, field, 1)
-#define part_stat_sub(cpu, gendiskp, field, subnd)			\
-	part_stat_add(cpu, gendiskp, field, -subnd)
+#else /* !CONFIG_SMP */
+#define part_stat_lock()	({ rcu_read_lock(); 0; })
+#define part_stat_unlock()	rcu_read_unlock()
 
-#define all_stat_dec(cpu, gendiskp, field, sector)			\
-	all_stat_add(cpu, gendiskp, field, -1, sector)
-#define all_stat_inc(cpu, gendiskp, part, field, sector)		\
-	all_stat_add(cpu, gendiskp, part, field, 1, sector)
-#define all_stat_sub(cpu, gendiskp, part, field, subnd, sector)		\
-	all_stat_add(cpu, gendiskp, part, field, -subnd, sector)
+#define __part_stat_add(cpu, part, field, addnd)				\
+	((part)->dkstats.field += addnd)
 
-/* Inlines to alloc and free disk stats in struct gendisk */
-#ifdef  CONFIG_SMP
-static inline int init_disk_stats(struct gendisk *disk)
-{
-	disk->dkstats = alloc_percpu(struct disk_stats);
-	if (!disk->dkstats)
-		return 0;
-	return 1;
-}
+#define part_stat_read(part, field)	((part)->dkstats.field)
 
-static inline void free_disk_stats(struct gendisk *disk)
+static inline void part_stat_set_all(struct hd_struct *part, int value)
 {
-	free_percpu(disk->dkstats);
+	memset(&part->dkstats, value, sizeof(struct disk_stats));
 }
 
 static inline int init_part_stats(struct hd_struct *part)
 {
-	part->dkstats = alloc_percpu(struct disk_stats);
-	if (!part->dkstats)
-		return 0;
 	return 1;
 }
 
 static inline void free_part_stats(struct hd_struct *part)
 {
-	free_percpu(part->dkstats);
 }
 
-#else	/* CONFIG_SMP */
-static inline int init_disk_stats(struct gendisk *disk)
-{
-	return 1;
-}
+#endif /* CONFIG_SMP */
 
-static inline void free_disk_stats(struct gendisk *disk)
-{
-}
+#define part_stat_add(cpu, part, field, addnd)	do {			\
+	__part_stat_add((cpu), (part), field, addnd);			\
+	if ((part)->partno)						\
+		__part_stat_add((cpu), &part_to_disk((part))->part0,	\
+				field, addnd);				\
+} while (0)
 
-static inline int init_part_stats(struct hd_struct *part)
+#define part_stat_dec(cpu, gendiskp, field)				\
+	part_stat_add(cpu, gendiskp, field, -1)
+#define part_stat_inc(cpu, gendiskp, field)				\
+	part_stat_add(cpu, gendiskp, field, 1)
+#define part_stat_sub(cpu, gendiskp, field, subnd)			\
+	part_stat_add(cpu, gendiskp, field, -subnd)
+
+static inline void part_inc_in_flight(struct hd_struct *part)
 {
-	return 1;
+	part->in_flight++;
+	if (part->partno)
+		part_to_disk(part)->part0.in_flight++;
 }
 
-static inline void free_part_stats(struct hd_struct *part)
+static inline void part_dec_in_flight(struct hd_struct *part)
 {
+	part->in_flight--;
+	if (part->partno)
+		part_to_disk(part)->part0.in_flight--;
 }
-#endif	/* CONFIG_SMP */
 
 /* drivers/block/ll_rw_blk.c */
-extern void disk_round_stats(int cpu, struct gendisk *disk);
 extern void part_round_stats(int cpu, struct hd_struct *part);
 
 /* drivers/block/genhd.c */
@@ -595,6 +524,8 @@ extern void blk_unregister_region(dev_t devt, unsigned long range);
 
 extern ssize_t part_size_show(struct device *dev,
 			      struct device_attribute *attr, char *buf);
+extern ssize_t part_stat_show(struct device *dev,
+			      struct device_attribute *attr, char *buf);
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 extern ssize_t part_fail_show(struct device *dev,
 			      struct device_attribute *attr, char *buf);