bcache: bcache now manages devices

Use bcache_get_dev(cache, path, flags) to open and
bcache_put_dev() to return the dev.

Devices are only really closed if nobody is holding the
dev, and the cache contains no blocks from the device.

Since it's really cheap to call bcache_dev_get(), you
should not hold onto the devices for long.  Just *_get_dev()
do your io, and *_put_dev().

If you want to guarantee your device is closed use
bcache_invalidate_dev().

Getting a dev in exclusive mode will open the device with
O_EXCL set.  If the device is already open, but without
O_EXCL set then it will be closed, all blocks invalidated
and reopened.  If the device is already open without EXCL,
and the device has another holder, then the EXCL get_dev will
fail.  See the unit tests for details.

The io engines both take a use_o_direct param.  This is just
intended for testing, where tmpfs can't handle O_DIRECT.  In
LVM we should have O_DIRECT on all the time.

io_engine_t.c has been removed, because the bcache_utils_t.c
provide far better coverage.

7 files changed, 704 insertions, 380 deletions

diff --git a/lib/device/bcache.c b/lib/device/bcache.c
index d476b0597..ad3167e75 100644
--- a/lib/device/bcache.c
+++ b/lib/device/bcache.c
@@ -136,6 +136,7 @@ struct async_engine {
 	io_context_t aio_context;
 	struct cb_set *cbs;
 	unsigned page_mask;
+	bool use_o_direct;
 };
 
 static struct async_engine *_to_async(struct io_engine *e)
@@ -158,21 +159,16 @@ static void _async_destroy(struct io_engine *ioe)
 	free(e);
 }
 
-static int _async_open(struct io_engine *e, const char *path, unsigned flags)
+// Used by both the async and sync engines
+static int _open_common(const char *path, int os_flags)
 {
-	int fd, os_flags = O_DIRECT | O_NOATIME;
-
-	if (flags & EF_READ_ONLY)
-		os_flags |= O_RDONLY;
-	else
-		os_flags |= O_RDWR;
+	int fd;
 
-	if (flags & EF_EXCL)
-		os_flags |= O_EXCL;
+	os_flags |= O_NOATIME;
 
-	fd = open(path, flags);
+	fd = open(path, os_flags);
 	if (fd < 0) {
-		if ((errno == EBUSY) && (flags & O_EXCL))
+		if ((errno == EBUSY) && (os_flags & O_EXCL))
 			log_error("Can't open %s exclusively.  Mounted filesystem?", path);
 		else
 			log_error("Couldn't open %s, errno = %d", path, errno);
@@ -181,6 +177,25 @@ static int _async_open(struct io_engine *e, const char *path, unsigned flags)
 	return fd;
 }
 
+static int _async_open(struct io_engine *ioe, const char *path, unsigned flags)
+{
+	struct async_engine *e = _to_async(ioe);
+	int os_flags = 0;
+
+	if (e->use_o_direct)
+		os_flags |= O_DIRECT;
+
+	if (flags & EF_READ_ONLY)
+		os_flags |= O_RDONLY;
+	else
+		os_flags |= O_RDWR;
+
+	if (flags & EF_EXCL)
+		os_flags |= O_EXCL;
+
+	return _open_common(path, os_flags);
+}
+
 static void _async_close(struct io_engine *e, int fd)
 {
 	close(fd);
@@ -278,7 +293,7 @@ static unsigned _async_max_io(struct io_engine *e)
 	return MAX_IO;
 }
 
-struct io_engine *create_async_io_engine(void)
+struct io_engine *create_async_io_engine(bool use_o_direct)
 {
 	int r;
 	struct async_engine *e = malloc(sizeof(*e));
@@ -309,6 +324,7 @@ struct io_engine *create_async_io_engine(void)
 	}
 
 	e->page_mask = sysconf(_SC_PAGESIZE) - 1;
+	e->use_o_direct = use_o_direct;
 
 	return &e->e;
 }
@@ -323,6 +339,7 @@ struct sync_io {
 struct sync_engine {
 	struct io_engine e;
 	struct dm_list complete;
+	bool use_o_direct;
 };
 
 static struct sync_engine *_to_sync(struct io_engine *e)
@@ -336,6 +353,25 @@ static void _sync_destroy(struct io_engine *ioe)
 	free(e);
 }
 
+static int _sync_open(struct io_engine *ioe, const char *path, unsigned flags)
+{
+	struct sync_engine *e = _to_sync(ioe);
+	int os_flags = 0;
+
+	if (e->use_o_direct)
+		os_flags |= O_DIRECT;
+
+	if (flags & EF_READ_ONLY)
+		os_flags |= O_RDONLY;
+	else
+		os_flags |= O_RDWR;
+
+	if (flags & EF_EXCL)
+		os_flags |= O_EXCL;
+
+	return _open_common(path, os_flags);
+}
+
 static bool _sync_issue(struct io_engine *ioe, enum dir d, int fd,
 			sector_t sb, sector_t se, void *data, void *context)
 {
@@ -403,7 +439,7 @@ static unsigned _sync_max_io(struct io_engine *e)
 	return 1;
 }
 
-struct io_engine *create_sync_io_engine(void)
+struct io_engine *create_sync_io_engine(bool use_o_direct)
 {
 	struct sync_engine *e = malloc(sizeof(*e));
 
@@ -411,11 +447,12 @@ struct io_engine *create_sync_io_engine(void)
 		return NULL;
 
 	e->e.destroy = _sync_destroy;
-	e->e.open = _async_open;
+	e->e.open = _sync_open;
 	e->e.close = _async_close;
 	e->e.issue = _sync_issue;
 	e->e.wait = _sync_wait;
 	e->e.max_io = _sync_max_io;
+	e->use_o_direct = use_o_direct;
 
 	dm_list_init(&e->complete);
 	return &e->e;
@@ -464,11 +501,17 @@ enum block_flags {
 };
 
 struct bcache_dev {
+	// The unit tests are relying on fd being the first element.
 	int fd;
 
-	// The reference count tracks users that are holding the dev, plus
+	struct bcache *cache;
+	char *path;
+	unsigned flags;
+
+	// The reference counts tracks users that are holding the dev, plus
 	// all the blocks on that device that are currently in the cache.
-	unsigned ref_count;
+	unsigned holders;
+	unsigned blocks;
 };
 
 struct bcache {
@@ -511,10 +554,11 @@ struct bcache {
 };
 
 //----------------------------------------------------------------
-//
+
 static void _free_dev(struct bcache *cache, struct bcache_dev *dev)
 {
 	cache->engine->close(cache->engine, dev->fd);
+	free(dev->path);
 	free(dev);
 }
 
@@ -523,6 +567,52 @@ static void _dev_dtr(void *context, union radix_value v)
         _free_dev(context, v.ptr);
 }
 
+static void _inc_holders(struct bcache_dev *dev)
+{
+	dev->holders++;
+}
+
+static void _inc_blocks(struct bcache_dev *dev)
+{
+	dev->blocks++;
+}
+
+static void _dev_maybe_close(struct bcache_dev *dev)
+{
+	if (dev->holders || dev->blocks)
+		return;
+
+	if (!radix_tree_remove(dev->cache->dev_tree,
+                                  (uint8_t *) dev->path,
+                                  (uint8_t *) dev->path + strlen(dev->path)))
+		log_error("couldn't remove bcache dev: %s", dev->path);
+}
+
+static void _dec_holders(struct bcache_dev *dev)
+{
+	if (!dev->holders)
+		log_error("internal error: holders refcount already at zero (%s)", dev->path);
+	else {
+		dev->holders--;
+		_dev_maybe_close(dev);
+	}
+}
+
+static void _dec_blocks(struct bcache_dev *dev)
+{
+	if (!dev->blocks)
+		log_error("internal error: blocks refcount already at zero (%s)", dev->path);
+	else {
+		dev->blocks--;
+		_dev_maybe_close(dev);
+	}
+}
+
+static bool _eflags(unsigned flags, unsigned flag)
+{
+	return flags & flag;
+}
+
 struct bcache_dev *bcache_get_dev(struct bcache *cache, const char *path, unsigned flags)
 {
 	union radix_value v;
@@ -530,7 +620,21 @@ struct bcache_dev *bcache_get_dev(struct bcache *cache, const char *path, unsign
 
 	if (radix_tree_lookup(cache->dev_tree, (uint8_t *) path, (uint8_t *) (path + strlen(path)), &v)) {
 		dev = v.ptr;
-		dev->ref_count++;
+		_inc_holders(dev);
+
+		if (_eflags(flags, EF_EXCL) && !_eflags(dev->flags, EF_EXCL)) {
+			if (dev->holders != 1) {
+				log_error("you can't update a bcache dev to exclusive with a concurrent holder (%s)",
+                                          dev->path);
+				_dec_holders(dev);
+				return NULL;
+			}
+
+			bcache_invalidate_dev(cache, dev);
+			_dec_holders(dev);
+			return bcache_get_dev(cache, path, flags);
+		}
+
 	} else {
 		dev = malloc(sizeof(*dev));
 		dev->fd = cache->engine->open(cache->engine, path, flags);
@@ -540,20 +644,36 @@ struct bcache_dev *bcache_get_dev(struct bcache *cache, const char *path, unsign
 			return NULL;
 		}
 
-		dev->ref_count = 1;
+		dev->path = strdup(path);
+		if (!dev->path) {
+			log_error("couldn't copy path when getting new device (%s)", path);
+			cache->engine->close(cache->engine, dev->fd);
+			free(dev);
+			return NULL;
+		}
+		dev->flags = flags;
+
+		dev->cache = cache;
+		dev->holders = 1;
+		dev->blocks = 0;
+
+
+		v.ptr = dev;
+		if (!radix_tree_insert(cache->dev_tree, (uint8_t *) path, (uint8_t *) (path + strlen(path)), v)) {
+			log_error("couldn't insert device into radix tree: %s", path);
+			cache->engine->close(cache->engine, dev->fd);
+			free(dev->path);
+			free(dev);
+			return NULL;
+		}
 	}
 
 	return dev;
 }
 
-void bcache_put_dev(struct bcache *cache, struct bcache_dev *dev)
+void bcache_put_dev(struct bcache_dev *dev)
 {
-	if (!dev->ref_count)
-		log_error("bcache_dev ref_count is already zero");
-
-	dev->ref_count--;
-	if (!dev->ref_count)
-		_free_dev(cache, dev);
+	_dec_holders(dev);
 }
 
 //----------------------------------------------------------------
@@ -838,6 +958,7 @@ static struct block *_new_block(struct bcache *cache, struct bcache_dev *dev, bl
 		dm_list_init(&b->list);
 		dm_list_init(&b->hash);
 		b->flags = 0;
+		_inc_blocks(dev);
 		b->dev = dev;
 		b->index = i;
 		b->ref_count = 0;
@@ -1024,19 +1145,59 @@ struct bcache *bcache_create(sector_t block_sectors, unsigned nr_cache_blocks,
 	return cache;
 }
 
+//----------------------------------------------------------------
+
+struct dev_iterator {
+	bool chastised;
+	struct radix_tree_iterator it;
+};
+
+static bool _check_dev(struct radix_tree_iterator *it,
+			 uint8_t *kb, uint8_t *ke, union radix_value v)
+{
+	struct dev_iterator *dit = container_of(it, struct dev_iterator, it);
+	struct bcache_dev *dev = v.ptr;
+
+	if (dev->holders) {
+		if (!dit->chastised) {
+			log_warn("Destroying a bcache whilst devices are still held:");
+			dit->chastised = true;
+		}
+
+		log_warn("    %s", dev->path);
+	}
+
+	return true;
+}
+
+static void _check_for_holders(struct bcache *cache)
+{
+	struct dev_iterator dit;
+
+	dit.chastised = false;
+	dit.it.visit = _check_dev;
+	radix_tree_iterate(cache->dev_tree, NULL, NULL, &dit.it);
+}
+
 void bcache_destroy(struct bcache *cache)
 {
 	if (cache->nr_locked)
 		log_warn("some blocks are still locked");
 
+	_check_for_holders(cache);
+
 	bcache_flush(cache);
 	_wait_all(cache);
+
 	_exit_free_list(cache);
 	radix_tree_destroy(cache->rtree);
+	radix_tree_destroy(cache->dev_tree);
 	cache->engine->destroy(cache->engine);
 	free(cache);
 }
 
+//----------------------------------------------------------------
+
 sector_t bcache_block_sectors(struct bcache *cache)
 {
 	return cache->block_sectors;
@@ -1073,6 +1234,7 @@ static void _recycle_block(struct bcache *cache, struct block *b)
 {
 	_unlink_block(b);
 	_block_remove(b);
+	_dec_blocks(b->dev);
 	_free_block(b);
 }
 
@@ -1228,6 +1390,7 @@ static bool _invalidate_v(struct radix_tree_iterator *it,
 	}
 
 	_unlink_block(b);
+	_dec_blocks(b->dev);
 	_free_block(b);
 
 	// We can't remove the block from the radix tree yet because
@@ -1254,5 +1417,20 @@ bool bcache_invalidate_dev(struct bcache *cache, struct bcache_dev *dev)
 	return it.success;
 }
 
+bool bcache_is_well_formed(struct bcache *cache)
+{
+	if (!radix_tree_is_well_formed(cache->rtree)) {
+		log_error("block tree is badly formed");
+		return false;
+	}
+
+	if (!radix_tree_is_well_formed(cache->dev_tree)) {
+		log_error("dev tree is badly formed");
+		return false;
+	}
+
+	return true;
+}
+
 //----------------------------------------------------------------
 
diff --git a/lib/device/bcache.h b/lib/device/bcache.h
index 3e0194501..168ab3140 100644
--- a/lib/device/bcache.h
+++ b/lib/device/bcache.h
@@ -18,8 +18,8 @@
 #include "device_mapper/all.h"
 
 #include <linux/fs.h>
-#include <stdint.h>
 #include <stdbool.h>
+#include <stdint.h>
 
 /*----------------------------------------------------------------*/
 
@@ -56,8 +56,8 @@ struct io_engine {
 	bool (*wait)(struct io_engine *e, io_complete_fn fn);
 };
 
-struct io_engine *create_async_io_engine(void);
-struct io_engine *create_sync_io_engine(void);
+struct io_engine *create_async_io_engine(bool use_o_direct);
+struct io_engine *create_sync_io_engine(bool use_o_direct);
 
 /*----------------------------------------------------------------*/
 
@@ -100,7 +100,7 @@ void bcache_destroy(struct bcache *cache);
 // - If blocks are in the cache that were acquired by a non exclusive holder,
 //   they will all be invalidated if a device is opened exclusively. 
 struct bcache_dev *bcache_get_dev(struct bcache *cache, const char *path, unsigned flags);
-void bcache_put_dev(struct bcache *cache, struct bcache_dev *dev);
+void bcache_put_dev(struct bcache_dev *dev);
 
 enum bcache_get_flags {
 	/*
@@ -167,6 +167,9 @@ bool bcache_flush_dev(struct bcache *cache, struct bcache_dev *dev);
 bool bcache_invalidate(struct bcache *cache, struct bcache_dev *dev, block_address index);
 bool bcache_invalidate_dev(struct bcache *cache, struct bcache_dev *dev);
 
+// For debug only
+bool bcache_is_well_formed(struct bcache *cache);
+
 //----------------------------------------------------------------
 // The next four functions are utilities written in terms of the above api.
  
diff --git a/test/unit/Makefile b/test/unit/Makefile
index 9155c4763..d586b0045 100644
--- a/test/unit/Makefile
+++ b/test/unit/Makefile
@@ -20,7 +20,6 @@ UNIT_SOURCE=\
 	test/unit/config_t.c \
 	test/unit/dmlist_t.c \
 	test/unit/dmstatus_t.c \
-	test/unit/io_engine_t.c \
 	test/unit/radix_tree_t.c \
 	test/unit/matcher_t.c \
 	test/unit/framework.c \
diff --git a/test/unit/bcache_t.c b/test/unit/bcache_t.c
index 5e0700572..74a0bdf6e 100644
--- a/test/unit/bcache_t.c
+++ b/test/unit/bcache_t.c
@@ -23,6 +23,13 @@
 
 #define SHOW_MOCK_CALLS 0
 
+//----------------------------------------------------------------
+// We're assuming the file descriptor is the first element of the
+// bcache_dev.
+struct bcache_dev {
+	int fd;
+};
+
 /*----------------------------------------------------------------
  * Mock engine
  *--------------------------------------------------------------*/
@@ -32,10 +39,13 @@ struct mock_engine {
 	struct dm_list issued_io;
 	unsigned max_io;
 	sector_t block_size;
+	int last_fd;
 };
 
 enum method {
 	E_DESTROY,
+	E_OPEN,
+	E_CLOSE,
 	E_ISSUE,
 	E_WAIT,
 	E_MAX_IO
@@ -47,10 +57,11 @@ struct mock_call {
 
 	bool match_args;
 	enum dir d;
-	int fd;
+	struct bcache_dev *dev;
 	block_address b;
 	bool issue_r;
 	bool wait_r;
+	unsigned engine_flags;
 };
 
 struct mock_io {
@@ -68,6 +79,10 @@ static const char *_show_method(enum method m)
 	switch (m) {
 	case E_DESTROY:
 		return "destroy()";
+	case E_OPEN:
+		return "open()";
+	case E_CLOSE:
+		return "close()";
 	case E_ISSUE:
 		return "issue()";
 	case E_WAIT:
@@ -87,13 +102,13 @@ static void _expect(struct mock_engine *e, enum method m)
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_read(struct mock_engine *e, int fd, block_address b)
+static void _expect_read(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_READ;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = true;
 	mc->wait_r = true;
@@ -110,71 +125,80 @@ static void _expect_read_any(struct mock_engine *e)
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_write(struct mock_engine *e, int fd, block_address b)
+static void _expect_write(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_WRITE;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = true;
 	mc->wait_r = true;
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_read_bad_issue(struct mock_engine *e, int fd, block_address b)
+static void _expect_read_bad_issue(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_READ;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = false;
 	mc->wait_r = true;
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_write_bad_issue(struct mock_engine *e, int fd, block_address b)
+static void _expect_write_bad_issue(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_WRITE;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = false;
 	mc->wait_r = true;
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_read_bad_wait(struct mock_engine *e, int fd, block_address b)
+static void _expect_read_bad_wait(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_READ;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = true;
 	mc->wait_r = false;
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
-static void _expect_write_bad_wait(struct mock_engine *e, int fd, block_address b)
+static void _expect_write_bad_wait(struct mock_engine *e, struct bcache_dev *dev, block_address b)
 {
 	struct mock_call *mc = malloc(sizeof(*mc));
 	mc->m = E_ISSUE;
 	mc->match_args = true;
 	mc->d = DIR_WRITE;
-	mc->fd = fd;
+	mc->dev = dev;
 	mc->b = b;
 	mc->issue_r = true;
 	mc->wait_r = false;
 	dm_list_add(&e->expected_calls, &mc->list);
 }
 
+static void _expect_open(struct mock_engine *e, unsigned eflags)
+{
+	struct mock_call *mc = malloc(sizeof(*mc));
+	mc->m = E_OPEN;
+	mc->match_args = true;
+	mc->engine_flags = eflags;
+	dm_list_add(&e->expected_calls, &mc->list);
+}
+
 static struct mock_call *_match_pop(struct mock_engine *e, enum method m)
 {
 
@@ -228,6 +252,27 @@ static void _mock_destroy(struct io_engine *e)
 	free(_to_mock(e));
 }
 
+static int _mock_open(struct io_engine *e, const char *path, unsigned flags)
+{
+	struct mock_engine *me = _to_mock(e);
+	struct mock_call *mc;
+
+	mc = _match_pop(me, E_OPEN);
+	if (mc->match_args)
+		T_ASSERT_EQUAL(mc->engine_flags, flags);
+	free(mc);
+	return me->last_fd++;
+}
+
+static void _mock_close(struct io_engine *e, int fd)
+{
+	struct mock_engine *me = _to_mock(e);
+	struct mock_call *mc;
+
+	mc = _match_pop(me, E_CLOSE);
+	free(mc);
+}
+
 static bool _mock_issue(struct io_engine *e, enum dir d, int fd,
 	      		sector_t sb, sector_t se, void *data, void *context)
 {
@@ -239,7 +284,7 @@ static bool _mock_issue(struct io_engine *e, enum dir d, int fd,
 	mc = _match_pop(me, E_ISSUE);
 	if (mc->match_args) {
 		T_ASSERT(d == mc->d);
-		T_ASSERT(fd == mc->fd);
+		T_ASSERT(fd == mc->dev->fd);
 		T_ASSERT(sb == mc->b * me->block_size);
 		T_ASSERT(se == (mc->b + 1) * me->block_size);
 	}
@@ -294,6 +339,8 @@ static struct mock_engine *_mock_create(unsigned max_io, sector_t block_size)
 	struct mock_engine *m = malloc(sizeof(*m));
 
 	m->e.destroy = _mock_destroy;
+	m->e.open = _mock_open;
+	m->e.close = _mock_close;
 	m->e.issue = _mock_issue;
 	m->e.wait = _mock_wait;
 	m->e.max_io = _mock_max_io;
@@ -302,6 +349,7 @@ static struct mock_engine *_mock_create(unsigned max_io, sector_t block_size)
 	m->block_size = block_size;
 	dm_list_init(&m->expected_calls);
 	dm_list_init(&m->issued_io);
+	m->last_fd = 2;
 
 	return m;
 }
@@ -420,184 +468,240 @@ static void test_get_triggers_read(void *context)
 {
 	struct fixture *f = context;
 
-	int fd = 17;   // arbitrary key
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct block *b;
 
-	_expect_read(f->me, fd, 0);
+	_expect(f->me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	T_ASSERT(dev);
+	_expect_read(f->me, dev, 0);
 	_expect(f->me, E_WAIT);
-	T_ASSERT(bcache_get(f->cache, fd, 0, 0, &b));
+	T_ASSERT(bcache_get(f->cache, dev, 0, 0, &b));
 	bcache_put(b);
 
-	_expect_read(f->me, fd, 1);
+	_expect_read(f->me, dev, 1);
 	_expect(f->me, E_WAIT);
-	T_ASSERT(bcache_get(f->cache, fd, 1, GF_DIRTY, &b));
-	_expect_write(f->me, fd, 1);
+	T_ASSERT(bcache_get(f->cache, dev, 1, GF_DIRTY, &b));
+	_expect_write(f->me, dev, 1);
 	_expect(f->me, E_WAIT);
 	bcache_put(b);
+
+	_expect(f->me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_repeated_reads_are_cached(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 
-	int fd = 17;   // arbitrary key
 	unsigned i;
 	struct block *b;
 
-	_expect_read(f->me, fd, 0);
+	_expect(f->me, E_OPEN);
+	dev = bcache_get_dev(f->cache, path, 0);
+	_expect_read(f->me, dev, 0);
 	_expect(f->me, E_WAIT);
 	for (i = 0; i < 100; i++) {
-		T_ASSERT(bcache_get(f->cache, fd, 0, 0, &b));
+		T_ASSERT(bcache_get(f->cache, dev, 0, 0, &b));
 		bcache_put(b);
 	}
+	_expect(f->me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_block_gets_evicted_with_many_reads(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	const unsigned nr_cache_blocks = 16;
 
-	int fd = 17;   // arbitrary key
 	unsigned i;
 	struct block *b;
 
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	fprintf(stderr, "1\n");
 	for (i = 0; i < nr_cache_blocks; i++) {
-		_expect_read(me, fd, i);
+		_expect_read(me, dev, i);
 		_expect(me, E_WAIT);
-		T_ASSERT(bcache_get(cache, fd, i, 0, &b));
+		T_ASSERT(bcache_get(cache, dev, i, 0, &b));
 		bcache_put(b);
 	}
 
+	fprintf(stderr, "2\n");
 	// Not enough cache blocks to hold this one
-	_expect_read(me, fd, nr_cache_blocks);
+	_expect_read(me, dev, nr_cache_blocks);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, nr_cache_blocks, 0, &b));
+	T_ASSERT(bcache_get(cache, dev, nr_cache_blocks, 0, &b));
 	bcache_put(b);
 
+	fprintf(stderr, "3\n");
 	// Now if we run through we should find one block has been
 	// evicted.  We go backwards because the oldest is normally
 	// evicted first.
 	_expect_read_any(me);
 	_expect(me, E_WAIT);
 	for (i = nr_cache_blocks; i; i--) {
-		T_ASSERT(bcache_get(cache, fd, i - 1, 0, &b));
+		T_ASSERT(bcache_get(cache, dev, i - 1, 0, &b));
 		bcache_put(b);
+		T_ASSERT(bcache_is_well_formed(cache));
 	}
+
+	fprintf(stderr, "4\n");
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_prefetch_issues_a_read(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	const unsigned nr_cache_blocks = 16;
 
-	int fd = 17;   // arbitrary key
 	unsigned i;
 	struct block *b;
 
+	_expect(me, E_OPEN);
+	dev = bcache_get_dev(f->cache, path, 0);
+
 	for (i = 0; i < nr_cache_blocks; i++) {
 		// prefetch should not wait
-		_expect_read(me, fd, i);
-		bcache_prefetch(cache, fd, i);
+		_expect_read(me, dev, i);
+		bcache_prefetch(cache, dev, i);
 	}
 	_no_outstanding_expectations(me);
 
 	for (i = 0; i < nr_cache_blocks; i++) {
 		_expect(me, E_WAIT);
-		T_ASSERT(bcache_get(cache, fd, i, 0, &b));
+		T_ASSERT(bcache_get(cache, dev, i, 0, &b));
 		bcache_put(b);
 	}
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_too_many_prefetches_does_not_trigger_a_wait(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 
 	const unsigned nr_cache_blocks = 16;
-	int fd = 17;   // arbitrary key
 	unsigned i;
 
+	_expect(me, E_OPEN);
+	dev = bcache_get_dev(f->cache, path, 0);
 	for (i = 0; i < 10 * nr_cache_blocks; i++) {
 		// prefetch should not wait
 		if (i < nr_cache_blocks)
-			_expect_read(me, fd, i);
-		bcache_prefetch(cache, fd, i);
+			_expect_read(me, dev, i);
+		bcache_prefetch(cache, dev, i);
 	}
 
 	// Destroy will wait for any in flight IO triggered by prefetches.
 	for (i = 0; i < nr_cache_blocks; i++)
 		_expect(me, E_WAIT);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_dirty_data_gets_written_back(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 
-	int fd = 17;   // arbitrary key
 	struct block *b;
 
+	_expect(f->me, E_OPEN);
+	dev = bcache_get_dev(f->cache, path, 0);
+
 	// Expect the read
-	_expect_read(me, fd, 0);
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, 0, GF_DIRTY, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, GF_DIRTY, &b));
 	bcache_put(b);
 
 	// Expect the write
-	_expect_write(me, fd, 0);
+	_expect_write(me, dev, 0);
 	_expect(me, E_WAIT);
+
+	bcache_put_dev(dev);
+	_expect(f->me, E_CLOSE);
 }
 
 static void test_zeroed_data_counts_as_dirty(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 
-	int fd = 17;   // arbitrary key
 	struct block *b;
 
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
 	// No read
-	T_ASSERT(bcache_get(cache, fd, 0, GF_ZERO, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, GF_ZERO, &b));
 	bcache_put(b);
 
 	// Expect the write
-	_expect_write(me, fd, 0);
+	_expect_write(me, dev, 0);
 	_expect(me, E_WAIT);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_flush_waits_for_all_dirty(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 
 	const unsigned count = 16;
-	int fd = 17;   // arbitrary key
 	unsigned i;
 	struct block *b;
 
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
 	for (i = 0; i < count; i++) {
 		if (i % 2) {
-			T_ASSERT(bcache_get(cache, fd, i, GF_ZERO, &b));
+			T_ASSERT(bcache_get(cache, dev, i, GF_ZERO, &b));
 		} else {
-			_expect_read(me, fd, i);
+			_expect_read(me, dev, i);
 			_expect(me, E_WAIT);
-			T_ASSERT(bcache_get(cache, fd, i, 0, &b));
+			T_ASSERT(bcache_get(cache, dev, i, 0, &b));
 		}
 		bcache_put(b);
 	}
 
 	for (i = 0; i < count; i++) {
 		if (i % 2)
-			_expect_write(me, fd, i);
+			_expect_write(me, dev, i);
 	}
 
 	for (i = 0; i < count; i++) {
@@ -607,207 +711,415 @@ static void test_flush_waits_for_all_dirty(void *context)
 
 	bcache_flush(cache);
 	_no_outstanding_expectations(me);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_multiple_files(void *context)
 {
-	static int _fds[] = {1, 128, 345, 678, 890};
+	static const char *_paths[] = {"/dev/dm-1", "/dev/dm-2", "/dev/dm-3", "/dev/dm-4"};
 
 	struct fixture *f = context;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
+	struct bcache_dev *dev;
 	struct block *b;
 	unsigned i;
 
-	for (i = 0; i < DM_ARRAY_SIZE(_fds); i++) {
-		_expect_read(me, _fds[i], 0);
+	for (i = 0; i < DM_ARRAY_SIZE(_paths); i++) {
+		_expect(me, E_OPEN);
+		dev = bcache_get_dev(cache, _paths[i], 0);
+		_expect_read(me, dev, 0);
 		_expect(me, E_WAIT);
 
-		T_ASSERT(bcache_get(cache, _fds[i], 0, 0, &b));
+		T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 		bcache_put(b);
+		bcache_put_dev(dev);
 	}
+
+	for (i = 0; i < DM_ARRAY_SIZE(_paths); i++)
+		_expect(me, E_CLOSE);
 }
 
 static void test_read_bad_issue(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
 
-	_expect_read_bad_issue(me, 17, 0);
-	T_ASSERT(!bcache_get(cache, 17, 0, 0, &b));
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	_expect_read_bad_issue(me, dev, 0);
+	T_ASSERT(!bcache_get(cache, dev, 0, 0, &b));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_read_bad_issue_intermittent(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	_expect_read_bad_issue(me, fd, 0);
-	T_ASSERT(!bcache_get(cache, fd, 0, 0, &b));
+	_expect(me, E_OPEN);
+	dev = bcache_get_dev(f->cache, path, 0);
+
+	_expect_read_bad_issue(me, dev, 0);
+	T_ASSERT(!bcache_get(cache, dev, 0, 0, &b));
 
-	_expect_read(me, fd, 0);
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 	bcache_put(b);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_read_bad_wait(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	_expect_read_bad_wait(me, fd, 0);
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	_expect_read_bad_wait(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(!bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(!bcache_get(cache, dev, 0, 0, &b));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_read_bad_wait_intermittent(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	_expect_read_bad_wait(me, fd, 0);
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	_expect_read_bad_wait(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(!bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(!bcache_get(cache, dev, 0, 0, &b));
 
-	_expect_read(me, fd, 0);
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 	bcache_put(b);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_write_bad_issue_stops_flush(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	T_ASSERT(bcache_get(cache, fd, 0, GF_ZERO, &b));
-	_expect_write_bad_issue(me, fd, 0);
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	T_ASSERT(bcache_get(cache, dev, 0, GF_ZERO, &b));
+	_expect_write_bad_issue(me, dev, 0);
 	bcache_put(b);
 	T_ASSERT(!bcache_flush(cache));
 
 	// we'll let it succeed the second time
-	_expect_write(me, fd, 0);
+	_expect_write(me, dev, 0);
 	_expect(me, E_WAIT);
 	T_ASSERT(bcache_flush(cache));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_write_bad_io_stops_flush(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	T_ASSERT(bcache_get(cache, fd, 0, GF_ZERO, &b));
-	_expect_write_bad_wait(me, fd, 0);
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	T_ASSERT(bcache_get(cache, dev, 0, GF_ZERO, &b));
+	_expect_write_bad_wait(me, dev, 0);
 	_expect(me, E_WAIT);
 	bcache_put(b);
 	T_ASSERT(!bcache_flush(cache));
 
 	// we'll let it succeed the second time
-	_expect_write(me, fd, 0);
+	_expect_write(me, dev, 0);
 	_expect(me, E_WAIT);
 	T_ASSERT(bcache_flush(cache));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_invalidate_not_present(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct bcache *cache = f->cache;
-	int fd = 17;
 
-	T_ASSERT(bcache_invalidate(cache, fd, 0));
+	_expect(f->me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+	T_ASSERT(bcache_invalidate(cache, dev, 0));
+	_expect(f->me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_invalidate_present(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	_expect_read(me, fd, 0);
+	_expect(f->me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 	bcache_put(b);
 
-	T_ASSERT(bcache_invalidate(cache, fd, 0));
+	T_ASSERT(bcache_invalidate(cache, dev, 0));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_invalidate_after_read_error(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	_expect_read_bad_issue(me, fd, 0);
-	T_ASSERT(!bcache_get(cache, fd, 0, 0, &b));
-	T_ASSERT(bcache_invalidate(cache, fd, 0));
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+	_expect_read_bad_issue(me, dev, 0);
+	T_ASSERT(!bcache_get(cache, dev, 0, 0, &b));
+	T_ASSERT(bcache_invalidate(cache, dev, 0));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_invalidate_after_write_error(void *context)
 {
 	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 	struct block *b;
-	int fd = 17;
 
-	T_ASSERT(bcache_get(cache, fd, 0, GF_ZERO, &b));
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+	T_ASSERT(bcache_get(cache, dev, 0, GF_ZERO, &b));
 	bcache_put(b);
 
 	// invalidate should fail if the write fails
-	_expect_write_bad_wait(me, fd, 0);
+	_expect_write_bad_wait(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(!bcache_invalidate(cache, fd, 0));
+	T_ASSERT(!bcache_invalidate(cache, dev, 0));
 
 	// and should succeed if the write does
-	_expect_write(me, fd, 0);
+	_expect_write(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_invalidate(cache, fd, 0));
+	T_ASSERT(bcache_invalidate(cache, dev, 0));
 
 	// a read is not required to get the block
-	_expect_read(me, fd, 0);
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
-	T_ASSERT(bcache_get(cache, fd, 0, 0, &b));
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 	bcache_put(b);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 static void test_invalidate_held_block(void *context)
 {
+	struct fixture *f = context;
+	const char *path = "/foo/bar/dev";
+	struct bcache_dev *dev;
+	struct mock_engine *me = f->me;
+	struct bcache *cache = f->cache;
+	struct block *b;
+
+	_expect(me, E_OPEN);
+ 	dev = bcache_get_dev(f->cache, path, 0);
+	T_ASSERT(bcache_get(cache, dev, 0, GF_ZERO, &b));
 
+	T_ASSERT(!bcache_invalidate(cache, dev, 0));
+
+	_expect_write(me, dev, 0);
+	_expect(me, E_WAIT);
+	bcache_put(b);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
+}
+
+//----------------------------------------------------------------
+
+static void test_concurrent_devs(void *context)
+{
+	struct fixture *f = context;
+	struct mock_engine *me = f->me;
+	struct bcache *cache = f->cache;
+
+	const char *path = "/dev/foo/bar";
+	struct bcache_dev *dev1, *dev2;
+
+	_expect(me, E_OPEN);
+	dev1 = bcache_get_dev(cache, path, 0);
+	dev2 = bcache_get_dev(cache, path, 0);
+
+	_expect(me, E_CLOSE);  // only one close
+
+	bcache_put_dev(dev1);
+	bcache_put_dev(dev2);
+}
+
+static void test_concurrent_devs_exclusive(void *context)
+{
+	struct fixture *f = context;
+	struct mock_engine *me = f->me;
+	struct bcache *cache = f->cache;
+
+	const char *path = "/dev/foo/bar";
+	struct bcache_dev *dev1, *dev2;
+
+	_expect(me, E_OPEN);
+	dev1 = bcache_get_dev(cache, path, EF_EXCL);
+	dev2 = bcache_get_dev(cache, path, EF_EXCL);
+
+	_expect(me, E_CLOSE);  // only one close
+
+	bcache_put_dev(dev1);
+	bcache_put_dev(dev2);
+}
+
+static void test_exclusive_flags_gets_passed_to_engine(void *context)
+{
+	struct fixture *f = context;
+	struct mock_engine *me = f->me;
+	struct bcache *cache = f->cache;
+
+	const char *path = "/dev/foo/bar";
+	struct bcache_dev *dev;
+
+	_expect_open(me, EF_EXCL);
+	dev = bcache_get_dev(cache, path, EF_EXCL);
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
+
+	_expect_open(me, EF_READ_ONLY);
+	dev = bcache_get_dev(cache, path, EF_READ_ONLY);
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
+
+	_expect_open(me, EF_EXCL | EF_READ_ONLY);
+	dev = bcache_get_dev(cache, path, EF_EXCL | EF_READ_ONLY);
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
+}
+
+static void test_reopen_exclusive_triggers_invalidate(void *context)
+{
 	struct fixture *f = context;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
+
+	const char *path = "/dev/foo/bar";
+	struct bcache_dev *dev;
 	struct block *b;
-	int fd = 17;
 
-	T_ASSERT(bcache_get(cache, fd, 0, GF_ZERO, &b));
+	_expect_open(me, 0);
+	dev = bcache_get_dev(cache, path, 0);
+	T_ASSERT(dev);
+	_expect_read(me, dev, 0);
+	_expect(me, E_WAIT);
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
+	bcache_put(b);
+	bcache_put_dev(dev);
+
+	_no_outstanding_expectations(me);
 
-	T_ASSERT(!bcache_invalidate(cache, fd, 0));
+	_expect(me, E_CLOSE);
+	_expect_open(me, EF_EXCL);
 
-	_expect_write(me, fd, 0);
+	dev = bcache_get_dev(cache, path, EF_EXCL);
+	T_ASSERT(dev);
+	_expect_read(me, dev, 0);
 	_expect(me, E_WAIT);
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 	bcache_put(b);
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
+}
+
+static void test_concurrent_reopen_excl_fails(void *context)
+{
+	struct fixture *f = context;
+	struct mock_engine *me = f->me;
+	struct bcache *cache = f->cache;
+
+	const char *path = "/dev/foo/bar";
+	struct bcache_dev *dev;
+	struct block *b;
+
+	_expect_open(me, 0);
+	dev = bcache_get_dev(cache, path, 0);
+	T_ASSERT(dev);
+	_expect_read(me, dev, 0);
+	_expect(me, E_WAIT);
+	T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
+	bcache_put(b);
+
+	_no_outstanding_expectations(me);
+
+	T_ASSERT(!bcache_get_dev(cache, path, EF_EXCL));
+
+	_expect(me, E_CLOSE);
+	bcache_put_dev(dev);
 }
 
 //----------------------------------------------------------------
@@ -815,6 +1127,8 @@ static void test_invalidate_held_block(void *context)
 
 static void _cycle(struct fixture *f, unsigned nr_cache_blocks)
 {
+	char buffer[64];
+	struct bcache_dev *dev;
 	struct mock_engine *me = f->me;
 	struct bcache *cache = f->cache;
 
@@ -822,18 +1136,25 @@ static void _cycle(struct fixture *f, unsigned nr_cache_blocks)
 	struct block *b;
 
 	for (i = 0; i < nr_cache_blocks; i++) {
+		snprintf(buffer, sizeof(buffer) - 1, "/dev/dm-%u", i);
+		_expect(me, E_OPEN);
+		dev = bcache_get_dev(f->cache, buffer, 0);
 		// prefetch should not wait
-		_expect_read(me, i, 0);
-		bcache_prefetch(cache, i, 0);
+		_expect_read(me, dev, 0);
+		bcache_prefetch(cache, dev, 0);
+		bcache_put_dev(dev);
 	}
 
 	// This double checks the reads occur in response to the prefetch
 	_no_outstanding_expectations(me);
 
 	for (i = 0; i < nr_cache_blocks; i++) {
+		snprintf(buffer, sizeof(buffer) - 1, "/dev/dm-%u", i);
+		dev = bcache_get_dev(f->cache, buffer, 0);
 		_expect(me, E_WAIT);
-		T_ASSERT(bcache_get(cache, i, 0, 0, &b));
+		T_ASSERT(bcache_get(cache, dev, 0, 0, &b));
 		bcache_put(b);
+		bcache_put_dev(dev);
 	}
 
 	_no_outstanding_expectations(me);
@@ -842,18 +1163,30 @@ static void _cycle(struct fixture *f, unsigned nr_cache_blocks)
 static void test_concurrent_reads_after_invalidate(void *context)
 {
 	struct fixture *f = context;
+	char buffer[64];
 	unsigned i, nr_cache_blocks = 16;
+	struct bcache_dev *dev;
 
 	_cycle(f, nr_cache_blocks);
-	for (i = 0; i < nr_cache_blocks; i++)
-        	bcache_invalidate_fd(f->cache, i);
+	for (i = 0; i < nr_cache_blocks; i++) {
+		snprintf(buffer, sizeof(buffer) - 1, "/dev/dm-%u", i);
+		dev = bcache_get_dev(f->cache, buffer, 0);
+        	bcache_invalidate_dev(f->cache, dev);
+        	_expect(f->me, E_CLOSE);
+        	bcache_put_dev(dev);
+        	_no_outstanding_expectations(f->me);
+	}
+
         _cycle(f, nr_cache_blocks);
+
+        for (i = 0; i < nr_cache_blocks; i++)
+	        _expect(f->me, E_CLOSE);
 }
 
 /*----------------------------------------------------------------
  * Top level
  *--------------------------------------------------------------*/
-#define T(path, desc, fn) register_test(ts, "/base/device/bcache/" path, desc, fn)
+#define T(path, desc, fn) register_test(ts, "/base/device/bcache/core/" path, desc, fn)
 
 static struct test_suite *_tiny_tests(void)
 {
@@ -900,6 +1233,11 @@ static struct test_suite *_small_tests(void)
 	T("invalidate-fails-in-held", "invalidating a held block fails", test_invalidate_held_block);
 	T("concurrent-reads-after-invalidate", "prefetch should still issue concurrent reads after invalidate",
           test_concurrent_reads_after_invalidate);
+	T("concurrent-devs", "a device may have more than one holder", test_concurrent_devs);
+	T("concurrent-devs-exclusive", "a device, opened exclusively, may have more than one holder", test_concurrent_devs_exclusive);
+	T("dev-flags-get-passed-to-engine", "EF_EXCL and EF_READ_ONLY get passed down", test_exclusive_flags_gets_passed_to_engine);
+	T("reopen-excl-invalidates", "reopening a dev EF_EXCL indicates you want to invalidate everything", test_reopen_exclusive_triggers_invalidate);
+	T("concurrent-reopen-excl-fails", "you can't reopen a dev EF_EXCL if there's already a holder", test_concurrent_reopen_excl_fails);
 
 	return ts;
 }
diff --git a/test/unit/bcache_utils_t.c b/test/unit/bcache_utils_t.c
index 66780ea21..8f07ed44a 100644
--- a/test/unit/bcache_utils_t.c
+++ b/test/unit/bcache_utils_t.c
@@ -34,9 +34,9 @@
 #define INIT_PATTERN 123
 
 struct fixture {
-	int fd;
 	char fname[32];
 	struct bcache *cache;
+	struct bcache_dev *dev;
 };
 
 static inline uint8_t _pattern_at(uint8_t pat, uint8_t byte)
@@ -49,57 +49,73 @@ static uint64_t byte(block_address b, uint64_t offset)
 	return b * T_BLOCK_SIZE + offset;
 }
 
-static void *_fix_init(struct io_engine *engine)
+// With testing in tmpfs directory O_DIRECT cannot be used
+// tmpfs has  f_fsid == 0  (unsure if this is best guess)
+static bool _use_o_direct_internal(void)
 {
-        uint8_t buffer[T_BLOCK_SIZE];
-        struct fixture *f = malloc(sizeof(*f));
-        unsigned b, i;
 	struct statvfs fsdata;
-	static int _runs_is_tmpfs = -1;
 
-	if (_runs_is_tmpfs == -1) {
-		// With testing in tmpfs directory O_DIRECT cannot be used
-		// tmpfs has  f_fsid == 0  (unsure if this is best guess)
-		_runs_is_tmpfs = (statvfs(".", &fsdata) == 0 && !fsdata.f_fsid) ? 1 : 0;
-		if (_runs_is_tmpfs)
+	if (statvfs(".", &fsdata))
+		// assume we can
+		return true;
+
+	return  fsdata.f_fsid;
+}
+
+static bool _use_o_direct(void)
+{
+	static bool latch = false;
+	static bool result;
+
+	if (!latch) {
+		latch = true;
+		result = _use_o_direct_internal();
+		if (!result)
 			printf("  Running test in tmpfs, *NOT* using O_DIRECT\n");
 	}
 
+	return result;
+}
+
+static void *_fix_init(struct io_engine *engine)
+{
+	int fd;
+        uint8_t buffer[T_BLOCK_SIZE];
+        struct fixture *f = malloc(sizeof(*f));
+        unsigned b, i;
+
         T_ASSERT(f);
 
         snprintf(f->fname, sizeof(f->fname), "unit-test-XXXXXX");
-	f->fd = mkstemp(f->fname);
-	T_ASSERT(f->fd >= 0);
+	fd = mkstemp(f->fname);
+	T_ASSERT(fd >= 0);
 
 	for (b = 0; b < NR_BLOCKS; b++) {
         	for (i = 0; i < sizeof(buffer); i++)
                 	buffer[i] = _pattern_at(INIT_PATTERN, byte(b, i));
-		T_ASSERT(write(f->fd, buffer, T_BLOCK_SIZE) > 0);
-	}
-
-	if (!_runs_is_tmpfs) {
-		close(f->fd);
-		// reopen with O_DIRECT
-		f->fd = open(f->fname, O_RDWR | O_DIRECT);
-		T_ASSERT(f->fd >= 0);
+		T_ASSERT(write(fd, buffer, T_BLOCK_SIZE) > 0);
 	}
+	close(fd);
 
 	f->cache = bcache_create(T_BLOCK_SIZE / 512, NR_BLOCKS, engine);
 	T_ASSERT(f->cache);
 
+	f->dev = bcache_get_dev(f->cache, f->fname, 0);
+	T_ASSERT(f->dev);
+
         return f;
 }
 
 static void *_async_init(void)
 {
-	struct io_engine *e = create_async_io_engine();
+	struct io_engine *e = create_async_io_engine(_use_o_direct());
 	T_ASSERT(e);
 	return _fix_init(e);
 }
 
 static void *_sync_init(void)
 {
-	struct io_engine *e = create_sync_io_engine();
+	struct io_engine *e = create_sync_io_engine(_use_o_direct());
 	T_ASSERT(e);
 	return _fix_init(e);
 }
@@ -108,8 +124,8 @@ static void _fix_exit(void *fixture)
 {
         struct fixture *f = fixture;
 
+	bcache_put_dev(f->dev);
 	bcache_destroy(f->cache);
-	close(f->fd);
 	unlink(f->fname);
         free(f);
 }
@@ -143,7 +159,7 @@ static void _verify(struct fixture *f, uint64_t byte_b, uint64_t byte_e, uint8_t
         	unsigned i;
         	size_t len2 = byte_e - byte_b;
 		uint8_t *buffer = malloc(len2);
-		T_ASSERT(bcache_read_bytes(f->cache, f->fd, byte_b, len2, buffer));
+		T_ASSERT(bcache_read_bytes(f->cache, f->dev, byte_b, len2, buffer));
 		for (i = 0; i < len; i++)
         		T_ASSERT_EQUAL(buffer[i], _pattern_at(pat, byte_b + i));
         	free(buffer);
@@ -151,7 +167,7 @@ static void _verify(struct fixture *f, uint64_t byte_b, uint64_t byte_e, uint8_t
 
 	// Verify again, driving bcache directly
 	for (; bb != be; bb++) {
-        	T_ASSERT(bcache_get(f->cache, f->fd, bb, 0, &b));
+        	T_ASSERT(bcache_get(f->cache, f->dev, bb, 0, &b));
 
 		blen = _min(T_BLOCK_SIZE - offset, len);
         	_verify_bytes(b, bb * T_BLOCK_SIZE, offset, blen, pat);
@@ -173,7 +189,7 @@ static void _verify_set(struct fixture *f, uint64_t byte_b, uint64_t byte_e, uin
 	uint64_t blen, len = byte_e - byte_b;
 
 	for (; bb != be; bb++) {
-        	T_ASSERT(bcache_get(f->cache, f->fd, bb, 0, &b));
+        	T_ASSERT(bcache_get(f->cache, f->dev, bb, 0, &b));
 
 		blen = _min(T_BLOCK_SIZE - offset, len);
 		for (i = 0; i < blen; i++)
@@ -201,30 +217,35 @@ static void _do_write(struct fixture *f, uint64_t byte_b, uint64_t byte_e, uint8
         for (i = 0; i < len; i++)
 		buffer[i] = _pattern_at(pat, byte_b + i);
 
-        T_ASSERT(bcache_write_bytes(f->cache, f->fd, byte_b, byte_e - byte_b, buffer));
+        T_ASSERT(bcache_write_bytes(f->cache, f->dev, byte_b, byte_e - byte_b, buffer));
 	free(buffer);
 }
 
 static void _do_zero(struct fixture *f, uint64_t byte_b, uint64_t byte_e)
 {
-	T_ASSERT(bcache_zero_bytes(f->cache, f->fd, byte_b, byte_e - byte_b));
+	T_ASSERT(bcache_zero_bytes(f->cache, f->dev, byte_b, byte_e - byte_b));
 }
 
 static void _do_set(struct fixture *f, uint64_t byte_b, uint64_t byte_e, uint8_t val)
 {
-	T_ASSERT(bcache_set_bytes(f->cache, f->fd, byte_b, byte_e - byte_b, val));
+	T_ASSERT(bcache_set_bytes(f->cache, f->dev, byte_b, byte_e - byte_b, val));
 }
 
 static void _reopen(struct fixture *f)
 {
         struct io_engine *engine;
 
+	bcache_put_dev(f->dev);
 	bcache_destroy(f->cache);
-	engine = create_async_io_engine();
+
+	engine = create_async_io_engine(_use_o_direct());
 	T_ASSERT(engine);
 
 	f->cache = bcache_create(T_BLOCK_SIZE / 512, NR_BLOCKS, engine);
 	T_ASSERT(f->cache);
+
+	f->dev = bcache_get_dev(f->cache, f->fname, 0);
+	T_ASSERT(f->cache);
 }
 
 //----------------------------------------------------------------
diff --git a/test/unit/io_engine_t.c b/test/unit/io_engine_t.c
deleted file mode 100644
index 1a4f638e5..000000000
--- a/test/unit/io_engine_t.c
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright (C) 2018 Red Hat, Inc. All rights reserved.
- *
- * This file is part of LVM2.
- *
- * This copyrighted material is made available to anyone wishing to use,
- * modify, copy, or redistribute it subject to the terms and conditions
- * of the GNU General Public License v.2.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#define _GNU_SOURCE
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include "lib/device/bcache.h"
-#include "framework.h"
-#include "units.h"
-
-//----------------------------------------------------------------
-
-#define SECTOR_SIZE 512
-#define BLOCK_SIZE_SECTORS 8
-#define NR_BLOCKS 64
-
-struct fixture {
-	struct io_engine *e;
-	uint8_t *data;
-
-	char fname[64];
-	int fd;
-};
-
-static void _fill_buffer(uint8_t *buffer, uint8_t seed, size_t count)
-{
-        unsigned i;
-        uint8_t b = seed;
-
-	for (i = 0; i < count; i++) {
-        	buffer[i] = b;
-        	b = ((b << 5) + b) + i;
-	}
-}
-
-static void _check_buffer(uint8_t *buffer, uint8_t seed, size_t count)
-{
-	unsigned i;
-	uint8_t b = seed;
-
-	for (i = 0; i < count; i++) {
-        	T_ASSERT_EQUAL(buffer[i], b);
-        	b = ((b << 5) + b) + i;
-	}
-}
-
-static void _print_buffer(const char *name, uint8_t *buffer, size_t count)
-{
-	unsigned col;
-
-	fprintf(stderr, "%s:\n", name);
-	while (count) {
-		for (col = 0; count && col < 20; col++) {
-        		fprintf(stderr, "%x, ", (unsigned) *buffer);
-        		col++;
-        		buffer++;
-        		count--;
-		}
-		fprintf(stderr, "\n");
-	}
-}
-
-static void *_fix_init(void)
-{
-        struct fixture *f = malloc(sizeof(*f));
-
-        T_ASSERT(f);
-        f->e = create_async_io_engine();
-        T_ASSERT(f->e);
-	if (posix_memalign((void **) &f->data, 4096, SECTOR_SIZE * BLOCK_SIZE_SECTORS))
-        	test_fail("posix_memalign failed");
-
-        snprintf(f->fname, sizeof(f->fname), "unit-test-XXXXXX");
-	f->fd = mkstemp(f->fname);
-	T_ASSERT(f->fd >= 0);
-
-	_fill_buffer(f->data, 123, SECTOR_SIZE * BLOCK_SIZE_SECTORS);
-
-	T_ASSERT(write(f->fd, f->data, SECTOR_SIZE * BLOCK_SIZE_SECTORS) > 0);
-	T_ASSERT(lseek(f->fd, 0, SEEK_SET) != -1);
-
-        return f;
-}
-
-static void _fix_exit(void *fixture)
-{
-        struct fixture *f = fixture;
-
-	close(f->fd);
-	unlink(f->fname);
-        free(f->data);
-        if (f->e)
-                f->e->destroy(f->e);
-        free(f);
-}
-
-static void _test_create(void *fixture)
-{
-	// empty
-}
-
-struct io {
-	bool completed;
-	int error;
-};
-
-static void _io_init(struct io *io)
-{
-	io->completed = false;
-	io->error = 0;
-}
-
-static void _complete_io(void *context, int io_error)
-{
-	struct io *io = context;
-	io->completed = true;
-	io->error = io_error;
-}
-
-static void _test_read(void *fixture)
-{
-	struct fixture *f = fixture;
-
-	struct io io;
-
-	_io_init(&io);
-	T_ASSERT(f->e->issue(f->e, DIR_READ, f->fd, 0, BLOCK_SIZE_SECTORS, f->data, &io));
-	T_ASSERT(f->e->wait(f->e, _complete_io));
-	T_ASSERT(io.completed);
-	T_ASSERT(!io.error);
-
-	_check_buffer(f->data, 123, sizeof(f->data));
-}
-
-static void _test_write(void *fixture)
-{
-	struct fixture *f = fixture;
-
-	struct io io;
-
-	_io_init(&io);
-	T_ASSERT(f->e->issue(f->e, DIR_WRITE, f->fd, 0, BLOCK_SIZE_SECTORS, f->data, &io));
-	T_ASSERT(f->e->wait(f->e, _complete_io));
-	T_ASSERT(io.completed);
-	T_ASSERT(!io.error);
-}
-
-static void _test_write_bytes(void *fixture)
-{
-	struct fixture *f = fixture;
-
-	unsigned offset = 345;
-	char buf_out[32];
-	char buf_in[32];
-	struct bcache *cache = bcache_create(8, BLOCK_SIZE_SECTORS, f->e);
-	T_ASSERT(cache);
-
-	// T_ASSERT(bcache_read_bytes(cache, f->fd, offset, sizeof(buf_in), buf_in));
-	_fill_buffer((uint8_t *) buf_out, 234, sizeof(buf_out));
-	T_ASSERT(bcache_write_bytes(cache, f->fd, offset, sizeof(buf_out), buf_out));
-	T_ASSERT(bcache_read_bytes(cache, f->fd, offset, sizeof(buf_in), buf_in));
-
-	_print_buffer("buf_out", (uint8_t *) buf_out, sizeof(buf_out));
-	_print_buffer("buf_in", (uint8_t *) buf_in, sizeof(buf_in));
-	T_ASSERT(!memcmp(buf_out, buf_in, sizeof(buf_out)));
-
-	bcache_destroy(cache);
-	f->e = NULL;   // already destroyed
-}
-
-//----------------------------------------------------------------
-
-#define T(path, desc, fn) register_test(ts, "/base/device/bcache/io-engine/" path, desc, fn)
-
-static struct test_suite *_tests(void)
-{
-        struct test_suite *ts = test_suite_create(_fix_init, _fix_exit);
-        if (!ts) {
-                fprintf(stderr, "out of memory\n");
-                exit(1);
-        }
-
-        T("create-destroy", "simple create/destroy", _test_create);
-        T("read", "read sanity check", _test_read);
-        T("write", "write sanity check", _test_write);
-        T("bcache-write-bytes", "test the utility fns", _test_write_bytes);
-
-        return ts;
-}
-
-void io_engine_tests(struct dm_list *all_tests)
-{
-	dm_list_add(all_tests, &_tests()->list);
-}
-
diff --git a/test/unit/units.h b/test/unit/units.h
index bc0db8d13..43834bd44 100644
--- a/test/unit/units.h
+++ b/test/unit/units.h
@@ -27,7 +27,6 @@ void bitset_tests(struct dm_list *suites);
 void config_tests(struct dm_list *suites);
 void dm_list_tests(struct dm_list *suites);
 void dm_status_tests(struct dm_list *suites);
-void io_engine_tests(struct dm_list *suites);
 void percent_tests(struct dm_list *suites);
 void radix_tree_tests(struct dm_list *suites);
 void regex_tests(struct dm_list *suites);
@@ -44,7 +43,6 @@ static inline void register_all_tests(struct dm_list *suites)
 	config_tests(suites);
 	dm_list_tests(suites);
 	dm_status_tests(suites);
-	io_engine_tests(suites);
 	percent_tests(suites);
 	radix_tree_tests(suites);
 	regex_tests(suites);