diff options
Diffstat (limited to 'drivers/virtio')
-rw-r--r-- | drivers/virtio/virtio.c | 6 | ||||
-rw-r--r-- | drivers/virtio/virtio_balloon.c | 2 | ||||
-rw-r--r-- | drivers/virtio/virtio_input.c | 2 | ||||
-rw-r--r-- | drivers/virtio/virtio_mem.c | 114 | ||||
-rw-r--r-- | drivers/virtio/virtio_pci_legacy.c | 2 | ||||
-rw-r--r-- | drivers/virtio/virtio_pci_legacy_dev.c | 4 | ||||
-rw-r--r-- | drivers/virtio/virtio_pci_modern_dev.c | 2 | ||||
-rw-r--r-- | drivers/virtio/virtio_ring.c | 4 | ||||
-rw-r--r-- | drivers/virtio/virtio_vdpa.c | 7 |
9 files changed, 95 insertions, 48 deletions
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 236081afe9a2..00ac9db792a4 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -204,6 +204,12 @@ int virtio_finalize_features(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(virtio_finalize_features); +void virtio_reset_device(struct virtio_device *dev) +{ + dev->config->reset(dev); +} +EXPORT_SYMBOL_GPL(virtio_reset_device); + static int virtio_dev_probe(struct device *_d) { int err, i; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index c22ff0117b46..f4c34a2a6b8e 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -1056,7 +1056,7 @@ static void remove_common(struct virtio_balloon *vb) return_free_pages_to_mm(vb, ULONG_MAX); /* Now we reset the device so we can clean up the queues. */ - vb->vdev->config->reset(vb->vdev); + virtio_reset_device(vb->vdev); vb->vdev->config->del_vqs(vb->vdev); } diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index ce51ae165943..3aa46703872d 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -347,7 +347,7 @@ static void virtinput_remove(struct virtio_device *vdev) spin_unlock_irqrestore(&vi->lock, flags); input_unregister_device(vi->idev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); while ((buf = virtqueue_detach_unused_buf(vi->sts)) != NULL) kfree(buf); vdev->config->del_vqs(vdev); diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 96e5a8782769..38becd8d578c 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -20,6 +20,7 @@ #include <linux/mutex.h> #include <linux/bitmap.h> #include <linux/lockdep.h> +#include <linux/log2.h> #include <acpi/acpi_numa.h> @@ -592,7 +593,7 @@ static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm) return -ENOMEM; mutex_lock(&vm->hotplug_mutex); - if (new_bitmap) + if (vm->sbm.sb_states) memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE); old_bitmap = vm->sbm.sb_states; @@ -1120,15 +1121,18 @@ static void virtio_mem_clear_fake_offline(unsigned long pfn, */ static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages) { - const unsigned long max_nr_pages = MAX_ORDER_NR_PAGES; + unsigned long order = MAX_ORDER - 1; unsigned long i; /* - * We are always called at least with MAX_ORDER_NR_PAGES - * granularity/alignment (e.g., the way subblocks work). All pages - * inside such a block are alike. + * We might get called for ranges that don't cover properly aligned + * MAX_ORDER - 1 pages; however, we can only online properly aligned + * pages with an order of MAX_ORDER - 1 at maximum. */ - for (i = 0; i < nr_pages; i += max_nr_pages) { + while (!IS_ALIGNED(pfn | nr_pages, 1 << order)) + order--; + + for (i = 0; i < nr_pages; i += 1 << order) { struct page *page = pfn_to_page(pfn + i); /* @@ -1138,14 +1142,12 @@ static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages) * alike. */ if (PageDirty(page)) { - virtio_mem_clear_fake_offline(pfn + i, max_nr_pages, - false); - generic_online_page(page, MAX_ORDER - 1); + virtio_mem_clear_fake_offline(pfn + i, 1 << order, false); + generic_online_page(page, order); } else { - virtio_mem_clear_fake_offline(pfn + i, max_nr_pages, - true); - free_contig_range(pfn + i, max_nr_pages); - adjust_managed_page_count(page, max_nr_pages); + virtio_mem_clear_fake_offline(pfn + i, 1 << order, true); + free_contig_range(pfn + i, 1 << order); + adjust_managed_page_count(page, 1 << order); } } } @@ -1228,28 +1230,46 @@ static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn, page_ref_inc(pfn_to_page(pfn + i)); } -static void virtio_mem_online_page_cb(struct page *page, unsigned int order) +static void virtio_mem_online_page(struct virtio_mem *vm, + struct page *page, unsigned int order) { - const unsigned long addr = page_to_phys(page); - unsigned long id, sb_id; - struct virtio_mem *vm; + const unsigned long start = page_to_phys(page); + const unsigned long end = start + PFN_PHYS(1 << order); + unsigned long addr, next, id, sb_id, count; bool do_online; - rcu_read_lock(); - list_for_each_entry_rcu(vm, &virtio_mem_devices, next) { - if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order))) - continue; + /* + * We can get called with any order up to MAX_ORDER - 1. If our + * subblock size is smaller than that and we have a mixture of plugged + * and unplugged subblocks within such a page, we have to process in + * smaller granularity. In that case we'll adjust the order exactly once + * within the loop. + */ + for (addr = start; addr < end; ) { + next = addr + PFN_PHYS(1 << order); if (vm->in_sbm) { - /* - * We exploit here that subblocks have at least - * MAX_ORDER_NR_PAGES size/alignment - so we cannot - * cross subblocks within one call. - */ id = virtio_mem_phys_to_mb_id(addr); sb_id = virtio_mem_phys_to_sb_id(vm, addr); - do_online = virtio_mem_sbm_test_sb_plugged(vm, id, - sb_id, 1); + count = virtio_mem_phys_to_sb_id(vm, next - 1) - sb_id + 1; + + if (virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, count)) { + /* Fully plugged. */ + do_online = true; + } else if (count == 1 || + virtio_mem_sbm_test_sb_unplugged(vm, id, sb_id, count)) { + /* Fully unplugged. */ + do_online = false; + } else { + /* + * Mixture, process sub-blocks instead. This + * will be at least the size of a pageblock. + * We'll run into this case exactly once. + */ + order = ilog2(vm->sbm.sb_size) - PAGE_SHIFT; + do_online = virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, 1); + continue; + } } else { /* * If the whole block is marked fake offline, keep @@ -1260,18 +1280,38 @@ static void virtio_mem_online_page_cb(struct page *page, unsigned int order) VIRTIO_MEM_BBM_BB_FAKE_OFFLINE; } + if (do_online) + generic_online_page(pfn_to_page(PFN_DOWN(addr)), order); + else + virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order, + false); + addr = next; + } +} + +static void virtio_mem_online_page_cb(struct page *page, unsigned int order) +{ + const unsigned long addr = page_to_phys(page); + struct virtio_mem *vm; + + rcu_read_lock(); + list_for_each_entry_rcu(vm, &virtio_mem_devices, next) { + /* + * Pages we're onlining will never cross memory blocks and, + * therefore, not virtio-mem devices. + */ + if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order))) + continue; + /* - * virtio_mem_set_fake_offline() might sleep, we don't need - * the device anymore. See virtio_mem_remove() how races + * virtio_mem_set_fake_offline() might sleep. We can safely + * drop the RCU lock at this point because the device + * cannot go away. See virtio_mem_remove() how races * between memory onlining and device removal are handled. */ rcu_read_unlock(); - if (do_online) - generic_online_page(page, order); - else - virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order, - false); + virtio_mem_online_page(vm, page, order); return; } rcu_read_unlock(); @@ -2438,8 +2478,6 @@ static int virtio_mem_init_hotplug(struct virtio_mem *vm) /* * We want subblocks to span at least MAX_ORDER_NR_PAGES and * pageblock_nr_pages pages. This: - * - Simplifies our page onlining code (virtio_mem_online_page_cb) - * and fake page onlining code (virtio_mem_fake_online). * - Is required for now for alloc_contig_range() to work reliably - * it doesn't properly handle smaller granularity on ZONE_NORMAL. */ @@ -2850,7 +2888,7 @@ static void virtio_mem_remove(struct virtio_device *vdev) virtio_mem_deinit_hotplug(vm); /* reset the device and cleanup the queues */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); kfree(vm); diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index b3f8128b7983..34141b9abe27 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -138,7 +138,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; if (q_pfn >> 32) { dev_err(&vp_dev->pci_dev->dev, - "platform bug: legacy virtio-mmio must not be used with RAM above 0x%llxGB\n", + "platform bug: legacy virtio-pci must not be used with RAM above 0x%llxGB\n", 0x1ULL << (32 + PAGE_SHIFT - 30)); err = -E2BIG; goto out_del_vq; diff --git a/drivers/virtio/virtio_pci_legacy_dev.c b/drivers/virtio/virtio_pci_legacy_dev.c index 9b97680dd02b..677d1f68bc9b 100644 --- a/drivers/virtio/virtio_pci_legacy_dev.c +++ b/drivers/virtio/virtio_pci_legacy_dev.c @@ -45,8 +45,10 @@ int vp_legacy_probe(struct virtio_pci_legacy_device *ldev) return rc; ldev->ioaddr = pci_iomap(pci_dev, 0, 0); - if (!ldev->ioaddr) + if (!ldev->ioaddr) { + rc = -EIO; goto err_iomap; + } ldev->isr = ldev->ioaddr + VIRTIO_PCI_ISR; diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index e11ed748e661..e8b3ff2b9fbc 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -345,7 +345,7 @@ err_map_common: EXPORT_SYMBOL_GPL(vp_modern_probe); /* - * vp_modern_probe: remove and cleanup the modern virtio pci device + * vp_modern_remove: remove and cleanup the modern virtio pci device * @mdev: the modern virtio-pci device */ void vp_modern_remove(struct virtio_pci_modern_device *mdev) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 028b05d44546..962f1477b1fa 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1197,8 +1197,10 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq, if (virtqueue_use_indirect(_vq, total_sg)) { err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs, in_sgs, data, gfp); - if (err != -ENOMEM) + if (err != -ENOMEM) { + END_USE(vq); return err; + } /* fall back on direct */ } diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index f85f860bc10b..7767a7f0119b 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -91,9 +91,8 @@ static u8 virtio_vdpa_get_status(struct virtio_device *vdev) static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); - const struct vdpa_config_ops *ops = vdpa->config; - return ops->set_status(vdpa, status); + return vdpa_set_status(vdpa, status); } static void virtio_vdpa_reset(struct virtio_device *vdev) @@ -308,7 +307,7 @@ static u64 virtio_vdpa_get_features(struct virtio_device *vdev) struct vdpa_device *vdpa = vd_get_vdpa(vdev); const struct vdpa_config_ops *ops = vdpa->config; - return ops->get_features(vdpa); + return ops->get_device_features(vdpa); } static int virtio_vdpa_finalize_features(struct virtio_device *vdev) @@ -318,7 +317,7 @@ static int virtio_vdpa_finalize_features(struct virtio_device *vdev) /* Give virtio_ring a chance to accept features. */ vring_transport_features(vdev); - return vdpa_set_features(vdpa, vdev->features); + return vdpa_set_features(vdpa, vdev->features, false); } static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) |