Lines Matching full:vm
285 static void virtio_mem_retry(struct virtio_mem *vm);
286 static int virtio_mem_create_resource(struct virtio_mem *vm);
287 static void virtio_mem_delete_resource(struct virtio_mem *vm);
293 static int register_virtio_mem_device(struct virtio_mem *vm) in register_virtio_mem_device() argument
302 list_add_rcu(&vm->next, &virtio_mem_devices); in register_virtio_mem_device()
312 static void unregister_virtio_mem_device(struct virtio_mem *vm) in unregister_virtio_mem_device() argument
316 list_del_rcu(&vm->next); in unregister_virtio_mem_device()
343 static unsigned long virtio_mem_phys_to_bb_id(struct virtio_mem *vm, in virtio_mem_phys_to_bb_id() argument
346 return addr / vm->bbm.bb_size; in virtio_mem_phys_to_bb_id()
352 static uint64_t virtio_mem_bb_id_to_phys(struct virtio_mem *vm, in virtio_mem_bb_id_to_phys() argument
355 return bb_id * vm->bbm.bb_size; in virtio_mem_bb_id_to_phys()
361 static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm, in virtio_mem_phys_to_sb_id() argument
367 return (addr - mb_addr) / vm->sbm.sb_size; in virtio_mem_phys_to_sb_id()
373 static void virtio_mem_bbm_set_bb_state(struct virtio_mem *vm, in virtio_mem_bbm_set_bb_state() argument
377 const unsigned long idx = bb_id - vm->bbm.first_bb_id; in virtio_mem_bbm_set_bb_state()
380 old_state = vm->bbm.bb_states[idx]; in virtio_mem_bbm_set_bb_state()
381 vm->bbm.bb_states[idx] = state; in virtio_mem_bbm_set_bb_state()
383 BUG_ON(vm->bbm.bb_count[old_state] == 0); in virtio_mem_bbm_set_bb_state()
384 vm->bbm.bb_count[old_state]--; in virtio_mem_bbm_set_bb_state()
385 vm->bbm.bb_count[state]++; in virtio_mem_bbm_set_bb_state()
391 static enum virtio_mem_bbm_bb_state virtio_mem_bbm_get_bb_state(struct virtio_mem *vm, in virtio_mem_bbm_get_bb_state() argument
394 return vm->bbm.bb_states[bb_id - vm->bbm.first_bb_id]; in virtio_mem_bbm_get_bb_state()
400 static int virtio_mem_bbm_bb_states_prepare_next_bb(struct virtio_mem *vm) in virtio_mem_bbm_bb_states_prepare_next_bb() argument
402 unsigned long old_bytes = vm->bbm.next_bb_id - vm->bbm.first_bb_id; in virtio_mem_bbm_bb_states_prepare_next_bb()
408 if (vm->bbm.bb_states && old_pages == new_pages) in virtio_mem_bbm_bb_states_prepare_next_bb()
415 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_bb_states_prepare_next_bb()
416 if (vm->bbm.bb_states) in virtio_mem_bbm_bb_states_prepare_next_bb()
417 memcpy(new_array, vm->bbm.bb_states, old_pages * PAGE_SIZE); in virtio_mem_bbm_bb_states_prepare_next_bb()
418 vfree(vm->bbm.bb_states); in virtio_mem_bbm_bb_states_prepare_next_bb()
419 vm->bbm.bb_states = new_array; in virtio_mem_bbm_bb_states_prepare_next_bb()
420 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_bb_states_prepare_next_bb()
426 for (_bb_id = vm->bbm.first_bb_id; \
427 _bb_id < vm->bbm.next_bb_id && _vm->bbm.bb_count[_state]; \
432 for (_bb_id = vm->bbm.next_bb_id - 1; \
433 _bb_id >= vm->bbm.first_bb_id && _vm->bbm.bb_count[_state]; \
440 static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm, in virtio_mem_sbm_set_mb_state() argument
443 const unsigned long idx = mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_set_mb_state()
446 old_state = vm->sbm.mb_states[idx]; in virtio_mem_sbm_set_mb_state()
447 vm->sbm.mb_states[idx] = state; in virtio_mem_sbm_set_mb_state()
449 BUG_ON(vm->sbm.mb_count[old_state] == 0); in virtio_mem_sbm_set_mb_state()
450 vm->sbm.mb_count[old_state]--; in virtio_mem_sbm_set_mb_state()
451 vm->sbm.mb_count[state]++; in virtio_mem_sbm_set_mb_state()
457 static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm, in virtio_mem_sbm_get_mb_state() argument
460 const unsigned long idx = mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_get_mb_state()
462 return vm->sbm.mb_states[idx]; in virtio_mem_sbm_get_mb_state()
468 static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm) in virtio_mem_sbm_mb_states_prepare_next_mb() argument
470 int old_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id); in virtio_mem_sbm_mb_states_prepare_next_mb()
471 int new_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id + 1); in virtio_mem_sbm_mb_states_prepare_next_mb()
474 if (vm->sbm.mb_states && old_pages == new_pages) in virtio_mem_sbm_mb_states_prepare_next_mb()
481 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_mb_states_prepare_next_mb()
482 if (vm->sbm.mb_states) in virtio_mem_sbm_mb_states_prepare_next_mb()
483 memcpy(new_array, vm->sbm.mb_states, old_pages * PAGE_SIZE); in virtio_mem_sbm_mb_states_prepare_next_mb()
484 vfree(vm->sbm.mb_states); in virtio_mem_sbm_mb_states_prepare_next_mb()
485 vm->sbm.mb_states = new_array; in virtio_mem_sbm_mb_states_prepare_next_mb()
486 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_mb_states_prepare_next_mb()
507 static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm, in virtio_mem_sbm_sb_state_bit_nr() argument
510 return (mb_id - vm->sbm.first_mb_id) * vm->sbm.sbs_per_mb + sb_id; in virtio_mem_sbm_sb_state_bit_nr()
518 static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm, in virtio_mem_sbm_set_sb_plugged() argument
522 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_set_sb_plugged()
524 __bitmap_set(vm->sbm.sb_states, bit, count); in virtio_mem_sbm_set_sb_plugged()
532 static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm, in virtio_mem_sbm_set_sb_unplugged() argument
536 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_set_sb_unplugged()
538 __bitmap_clear(vm->sbm.sb_states, bit, count); in virtio_mem_sbm_set_sb_unplugged()
544 static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm, in virtio_mem_sbm_test_sb_plugged() argument
548 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_test_sb_plugged()
551 return test_bit(bit, vm->sbm.sb_states); in virtio_mem_sbm_test_sb_plugged()
554 return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >= in virtio_mem_sbm_test_sb_plugged()
561 static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm, in virtio_mem_sbm_test_sb_unplugged() argument
565 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_test_sb_unplugged()
568 return find_next_bit(vm->sbm.sb_states, bit + count, bit) >= in virtio_mem_sbm_test_sb_unplugged()
573 * Find the first unplugged subblock. Returns vm->sbm.sbs_per_mb in case there is
576 static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm, in virtio_mem_sbm_first_unplugged_sb() argument
579 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0); in virtio_mem_sbm_first_unplugged_sb()
581 return find_next_zero_bit(vm->sbm.sb_states, in virtio_mem_sbm_first_unplugged_sb()
582 bit + vm->sbm.sbs_per_mb, bit) - bit; in virtio_mem_sbm_first_unplugged_sb()
588 static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm) in virtio_mem_sbm_sb_states_prepare_next_mb() argument
590 const unsigned long old_nb_mb = vm->sbm.next_mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_sb_states_prepare_next_mb()
591 const unsigned long old_nb_bits = old_nb_mb * vm->sbm.sbs_per_mb; in virtio_mem_sbm_sb_states_prepare_next_mb()
592 const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->sbm.sbs_per_mb; in virtio_mem_sbm_sb_states_prepare_next_mb()
597 if (vm->sbm.sb_states && old_pages == new_pages) in virtio_mem_sbm_sb_states_prepare_next_mb()
604 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_sb_states_prepare_next_mb()
605 if (vm->sbm.sb_states) in virtio_mem_sbm_sb_states_prepare_next_mb()
606 memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE); in virtio_mem_sbm_sb_states_prepare_next_mb()
608 old_bitmap = vm->sbm.sb_states; in virtio_mem_sbm_sb_states_prepare_next_mb()
609 vm->sbm.sb_states = new_bitmap; in virtio_mem_sbm_sb_states_prepare_next_mb()
610 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_sb_states_prepare_next_mb()
620 static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size) in virtio_mem_could_add_memory() argument
622 if (WARN_ON_ONCE(size > vm->offline_threshold)) in virtio_mem_could_add_memory()
625 return atomic64_read(&vm->offline_size) + size <= vm->offline_threshold; in virtio_mem_could_add_memory()
631 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
636 static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr, in virtio_mem_add_memory() argument
645 if (!vm->resource_name) { in virtio_mem_add_memory()
646 vm->resource_name = kstrdup_const("System RAM (virtio_mem)", in virtio_mem_add_memory()
648 if (!vm->resource_name) in virtio_mem_add_memory()
652 dev_dbg(&vm->vdev->dev, "adding memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_add_memory()
655 atomic64_add(size, &vm->offline_size); in virtio_mem_add_memory()
656 rc = add_memory_driver_managed(vm->mgid, addr, size, vm->resource_name, in virtio_mem_add_memory()
659 atomic64_sub(size, &vm->offline_size); in virtio_mem_add_memory()
660 dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc); in virtio_mem_add_memory()
672 static int virtio_mem_sbm_add_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_add_mb() argument
677 return virtio_mem_add_memory(vm, addr, size); in virtio_mem_sbm_add_mb()
683 static int virtio_mem_bbm_add_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_add_bb() argument
685 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_add_bb()
686 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_add_bb()
688 return virtio_mem_add_memory(vm, addr, size); in virtio_mem_bbm_add_bb()
695 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
700 static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr, in virtio_mem_remove_memory() argument
705 dev_dbg(&vm->vdev->dev, "removing memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_remove_memory()
709 atomic64_sub(size, &vm->offline_size); in virtio_mem_remove_memory()
714 virtio_mem_retry(vm); in virtio_mem_remove_memory()
716 dev_dbg(&vm->vdev->dev, "removing memory failed: %d\n", rc); in virtio_mem_remove_memory()
724 static int virtio_mem_sbm_remove_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_remove_mb() argument
729 return virtio_mem_remove_memory(vm, addr, size); in virtio_mem_sbm_remove_mb()
735 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
740 static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm, in virtio_mem_offline_and_remove_memory() argument
746 dev_dbg(&vm->vdev->dev, in virtio_mem_offline_and_remove_memory()
752 atomic64_sub(size, &vm->offline_size); in virtio_mem_offline_and_remove_memory()
757 virtio_mem_retry(vm); in virtio_mem_offline_and_remove_memory()
760 dev_dbg(&vm->vdev->dev, "offlining and removing memory failed: %d\n", rc); in virtio_mem_offline_and_remove_memory()
773 static int virtio_mem_sbm_offline_and_remove_mb(struct virtio_mem *vm, in virtio_mem_sbm_offline_and_remove_mb() argument
779 return virtio_mem_offline_and_remove_memory(vm, addr, size); in virtio_mem_sbm_offline_and_remove_mb()
788 static int virtio_mem_sbm_try_remove_unplugged_mb(struct virtio_mem *vm, in virtio_mem_sbm_try_remove_unplugged_mb() argument
797 if (!virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) in virtio_mem_sbm_try_remove_unplugged_mb()
801 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_try_remove_unplugged_mb()
802 rc = virtio_mem_sbm_offline_and_remove_mb(vm, mb_id); in virtio_mem_sbm_try_remove_unplugged_mb()
803 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_try_remove_unplugged_mb()
805 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_try_remove_unplugged_mb()
814 static int virtio_mem_bbm_offline_and_remove_bb(struct virtio_mem *vm, in virtio_mem_bbm_offline_and_remove_bb() argument
817 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_offline_and_remove_bb()
818 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_offline_and_remove_bb()
820 return virtio_mem_offline_and_remove_memory(vm, addr, size); in virtio_mem_bbm_offline_and_remove_bb()
826 static void virtio_mem_retry(struct virtio_mem *vm) in virtio_mem_retry() argument
830 spin_lock_irqsave(&vm->removal_lock, flags); in virtio_mem_retry()
831 if (!vm->removing) in virtio_mem_retry()
832 queue_work(system_freezable_wq, &vm->wq); in virtio_mem_retry()
833 spin_unlock_irqrestore(&vm->removal_lock, flags); in virtio_mem_retry()
836 static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id) in virtio_mem_translate_node_id() argument
841 if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM)) in virtio_mem_translate_node_id()
851 static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start, in virtio_mem_overlaps_range() argument
854 return start < vm->addr + vm->region_size && vm->addr < start + size; in virtio_mem_overlaps_range()
861 static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start, in virtio_mem_contains_range() argument
864 return start >= vm->addr && start + size <= vm->addr + vm->region_size; in virtio_mem_contains_range()
867 static int virtio_mem_sbm_notify_going_online(struct virtio_mem *vm, in virtio_mem_sbm_notify_going_online() argument
870 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_going_online()
877 dev_warn_ratelimited(&vm->vdev->dev, in virtio_mem_sbm_notify_going_online()
882 static void virtio_mem_sbm_notify_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_offline() argument
885 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_offline()
888 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_notify_offline()
893 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_notify_offline()
902 static void virtio_mem_sbm_notify_online(struct virtio_mem *vm, in virtio_mem_sbm_notify_online() argument
909 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_online()
924 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state); in virtio_mem_sbm_notify_online()
927 static void virtio_mem_sbm_notify_going_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_going_offline() argument
930 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size); in virtio_mem_sbm_notify_going_offline()
934 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) { in virtio_mem_sbm_notify_going_offline()
935 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_notify_going_offline()
938 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_notify_going_offline()
943 static void virtio_mem_sbm_notify_cancel_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_cancel_offline() argument
946 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size); in virtio_mem_sbm_notify_cancel_offline()
950 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) { in virtio_mem_sbm_notify_cancel_offline()
951 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_notify_cancel_offline()
954 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_notify_cancel_offline()
959 static void virtio_mem_bbm_notify_going_offline(struct virtio_mem *vm, in virtio_mem_bbm_notify_going_offline() argument
968 if (virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_notify_going_offline()
974 static void virtio_mem_bbm_notify_cancel_offline(struct virtio_mem *vm, in virtio_mem_bbm_notify_cancel_offline() argument
979 if (virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_notify_cancel_offline()
993 struct virtio_mem *vm = container_of(nb, struct virtio_mem, in virtio_mem_memory_notifier_cb() local
1001 if (!virtio_mem_overlaps_range(vm, start, size)) in virtio_mem_memory_notifier_cb()
1004 if (vm->in_sbm) { in virtio_mem_memory_notifier_cb()
1015 id = virtio_mem_phys_to_bb_id(vm, start); in virtio_mem_memory_notifier_cb()
1022 if (WARN_ON_ONCE(id != virtio_mem_phys_to_bb_id(vm, start + size - 1))) in virtio_mem_memory_notifier_cb()
1036 mutex_lock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1037 if (vm->removing) { in virtio_mem_memory_notifier_cb()
1039 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1042 vm->hotplug_active = true; in virtio_mem_memory_notifier_cb()
1043 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1044 virtio_mem_sbm_notify_going_offline(vm, id); in virtio_mem_memory_notifier_cb()
1046 virtio_mem_bbm_notify_going_offline(vm, id, in virtio_mem_memory_notifier_cb()
1051 mutex_lock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1052 if (vm->removing) { in virtio_mem_memory_notifier_cb()
1054 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1057 vm->hotplug_active = true; in virtio_mem_memory_notifier_cb()
1058 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1059 rc = virtio_mem_sbm_notify_going_online(vm, id); in virtio_mem_memory_notifier_cb()
1062 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1063 virtio_mem_sbm_notify_offline(vm, id); in virtio_mem_memory_notifier_cb()
1065 atomic64_add(size, &vm->offline_size); in virtio_mem_memory_notifier_cb()
1071 virtio_mem_retry(vm); in virtio_mem_memory_notifier_cb()
1073 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1074 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1077 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1078 virtio_mem_sbm_notify_online(vm, id, mhp->start_pfn); in virtio_mem_memory_notifier_cb()
1080 atomic64_sub(size, &vm->offline_size); in virtio_mem_memory_notifier_cb()
1087 if (!atomic_read(&vm->wq_active) && in virtio_mem_memory_notifier_cb()
1088 virtio_mem_could_add_memory(vm, vm->offline_threshold / 2)) in virtio_mem_memory_notifier_cb()
1089 virtio_mem_retry(vm); in virtio_mem_memory_notifier_cb()
1091 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1092 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1095 if (!vm->hotplug_active) in virtio_mem_memory_notifier_cb()
1097 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1098 virtio_mem_sbm_notify_cancel_offline(vm, id); in virtio_mem_memory_notifier_cb()
1100 virtio_mem_bbm_notify_cancel_offline(vm, id, in virtio_mem_memory_notifier_cb()
1103 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1104 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1107 if (!vm->hotplug_active) in virtio_mem_memory_notifier_cb()
1109 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1110 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1124 struct virtio_mem *vm = container_of(nb, struct virtio_mem, in virtio_mem_pm_notifier_cb() local
1130 * When restarting the VM, all memory is unplugged. Don't in virtio_mem_pm_notifier_cb()
1133 dev_err(&vm->vdev->dev, "hibernation is not supported.\n"); in virtio_mem_pm_notifier_cb()
1224 static int virtio_mem_fake_offline(struct virtio_mem *vm, unsigned long pfn, in virtio_mem_fake_offline() argument
1243 if (atomic_read(&vm->config_changed)) in virtio_mem_fake_offline()
1299 static void virtio_mem_online_page(struct virtio_mem *vm, in virtio_mem_online_page() argument
1317 if (vm->in_sbm) { in virtio_mem_online_page()
1319 sb_id = virtio_mem_phys_to_sb_id(vm, addr); in virtio_mem_online_page()
1320 count = virtio_mem_phys_to_sb_id(vm, next - 1) - sb_id + 1; in virtio_mem_online_page()
1322 if (virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, count)) { in virtio_mem_online_page()
1326 virtio_mem_sbm_test_sb_unplugged(vm, id, sb_id, count)) { in virtio_mem_online_page()
1335 order = ilog2(vm->sbm.sb_size) - PAGE_SHIFT; in virtio_mem_online_page()
1336 do_online = virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, 1); in virtio_mem_online_page()
1344 id = virtio_mem_phys_to_bb_id(vm, addr); in virtio_mem_online_page()
1345 do_online = virtio_mem_bbm_get_bb_state(vm, id) != in virtio_mem_online_page()
1361 struct virtio_mem *vm; in virtio_mem_online_page_cb() local
1364 list_for_each_entry_rcu(vm, &virtio_mem_devices, next) { in virtio_mem_online_page_cb()
1369 if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order))) in virtio_mem_online_page_cb()
1380 virtio_mem_online_page(vm, page, order); in virtio_mem_online_page_cb()
1389 static uint64_t virtio_mem_send_request(struct virtio_mem *vm, in virtio_mem_send_request() argument
1397 vm->req = *req; in virtio_mem_send_request()
1400 sg_init_one(&sg_req, &vm->req, sizeof(vm->req)); in virtio_mem_send_request()
1404 sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp)); in virtio_mem_send_request()
1407 rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL); in virtio_mem_send_request()
1411 virtqueue_kick(vm->vq); in virtio_mem_send_request()
1414 wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len)); in virtio_mem_send_request()
1416 return virtio16_to_cpu(vm->vdev, vm->resp.type); in virtio_mem_send_request()
1419 static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_plug_request() argument
1422 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_plug_request()
1424 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG), in virtio_mem_send_plug_request()
1425 .u.plug.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_plug_request()
1426 .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_plug_request()
1430 if (atomic_read(&vm->config_changed)) in virtio_mem_send_plug_request()
1433 dev_dbg(&vm->vdev->dev, "plugging memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_plug_request()
1436 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_plug_request()
1438 vm->plugged_size += size; in virtio_mem_send_plug_request()
1453 dev_dbg(&vm->vdev->dev, "plugging memory failed: %d\n", rc); in virtio_mem_send_plug_request()
1457 static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_unplug_request() argument
1460 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_unplug_request()
1462 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG), in virtio_mem_send_unplug_request()
1463 .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_unplug_request()
1464 .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_unplug_request()
1468 if (atomic_read(&vm->config_changed)) in virtio_mem_send_unplug_request()
1471 dev_dbg(&vm->vdev->dev, "unplugging memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_unplug_request()
1474 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_unplug_request()
1476 vm->plugged_size -= size; in virtio_mem_send_unplug_request()
1488 dev_dbg(&vm->vdev->dev, "unplugging memory failed: %d\n", rc); in virtio_mem_send_unplug_request()
1492 static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm) in virtio_mem_send_unplug_all_request() argument
1495 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL), in virtio_mem_send_unplug_all_request()
1499 dev_dbg(&vm->vdev->dev, "unplugging all memory"); in virtio_mem_send_unplug_all_request()
1501 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_unplug_all_request()
1503 vm->unplug_all_required = false; in virtio_mem_send_unplug_all_request()
1504 vm->plugged_size = 0; in virtio_mem_send_unplug_all_request()
1506 atomic_set(&vm->config_changed, 1); in virtio_mem_send_unplug_all_request()
1515 dev_dbg(&vm->vdev->dev, "unplugging all memory failed: %d\n", rc); in virtio_mem_send_unplug_all_request()
1523 static int virtio_mem_sbm_plug_sb(struct virtio_mem *vm, unsigned long mb_id, in virtio_mem_sbm_plug_sb() argument
1527 sb_id * vm->sbm.sb_size; in virtio_mem_sbm_plug_sb()
1528 const uint64_t size = count * vm->sbm.sb_size; in virtio_mem_sbm_plug_sb()
1531 rc = virtio_mem_send_plug_request(vm, addr, size); in virtio_mem_sbm_plug_sb()
1533 virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count); in virtio_mem_sbm_plug_sb()
1541 static int virtio_mem_sbm_unplug_sb(struct virtio_mem *vm, unsigned long mb_id, in virtio_mem_sbm_unplug_sb() argument
1545 sb_id * vm->sbm.sb_size; in virtio_mem_sbm_unplug_sb()
1546 const uint64_t size = count * vm->sbm.sb_size; in virtio_mem_sbm_unplug_sb()
1549 rc = virtio_mem_send_unplug_request(vm, addr, size); in virtio_mem_sbm_unplug_sb()
1551 virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_sb()
1560 static int virtio_mem_bbm_unplug_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_unplug_bb() argument
1562 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_unplug_bb()
1563 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_unplug_bb()
1565 return virtio_mem_send_unplug_request(vm, addr, size); in virtio_mem_bbm_unplug_bb()
1573 static int virtio_mem_bbm_plug_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_plug_bb() argument
1575 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_plug_bb()
1576 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_plug_bb()
1578 return virtio_mem_send_plug_request(vm, addr, size); in virtio_mem_bbm_plug_bb()
1590 static int virtio_mem_sbm_unplug_any_sb_raw(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_raw() argument
1596 sb_id = vm->sbm.sbs_per_mb - 1; in virtio_mem_sbm_unplug_any_sb_raw()
1600 virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_unplug_any_sb_raw()
1607 virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) { in virtio_mem_sbm_unplug_any_sb_raw()
1612 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_any_sb_raw()
1629 static int virtio_mem_sbm_unplug_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_unplug_mb() argument
1631 uint64_t nb_sb = vm->sbm.sbs_per_mb; in virtio_mem_sbm_unplug_mb()
1633 return virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, &nb_sb); in virtio_mem_sbm_unplug_mb()
1639 static int virtio_mem_sbm_prepare_next_mb(struct virtio_mem *vm, in virtio_mem_sbm_prepare_next_mb() argument
1644 if (vm->sbm.next_mb_id > vm->sbm.last_usable_mb_id) in virtio_mem_sbm_prepare_next_mb()
1648 rc = virtio_mem_sbm_mb_states_prepare_next_mb(vm); in virtio_mem_sbm_prepare_next_mb()
1653 rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm); in virtio_mem_sbm_prepare_next_mb()
1657 vm->sbm.mb_count[VIRTIO_MEM_SBM_MB_UNUSED]++; in virtio_mem_sbm_prepare_next_mb()
1658 *mb_id = vm->sbm.next_mb_id++; in virtio_mem_sbm_prepare_next_mb()
1668 static int virtio_mem_sbm_plug_and_add_mb(struct virtio_mem *vm, in virtio_mem_sbm_plug_and_add_mb() argument
1671 const int count = min_t(int, *nb_sb, vm->sbm.sbs_per_mb); in virtio_mem_sbm_plug_and_add_mb()
1681 rc = virtio_mem_sbm_plug_sb(vm, mb_id, 0, count); in virtio_mem_sbm_plug_and_add_mb()
1689 if (count == vm->sbm.sbs_per_mb) in virtio_mem_sbm_plug_and_add_mb()
1690 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_plug_and_add_mb()
1693 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_plug_and_add_mb()
1697 rc = virtio_mem_sbm_add_mb(vm, mb_id); in virtio_mem_sbm_plug_and_add_mb()
1701 if (virtio_mem_sbm_unplug_sb(vm, mb_id, 0, count)) in virtio_mem_sbm_plug_and_add_mb()
1703 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state); in virtio_mem_sbm_plug_and_add_mb()
1719 static int virtio_mem_sbm_plug_any_sb(struct virtio_mem *vm, in virtio_mem_sbm_plug_any_sb() argument
1722 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_plug_any_sb()
1731 sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id); in virtio_mem_sbm_plug_any_sb()
1732 if (sb_id >= vm->sbm.sbs_per_mb) in virtio_mem_sbm_plug_any_sb()
1736 sb_id + count < vm->sbm.sbs_per_mb && in virtio_mem_sbm_plug_any_sb()
1737 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1)) in virtio_mem_sbm_plug_any_sb()
1740 rc = virtio_mem_sbm_plug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_plug_any_sb()
1749 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_plug_any_sb()
1750 nr_pages = PFN_DOWN(count * vm->sbm.sb_size); in virtio_mem_sbm_plug_any_sb()
1754 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) in virtio_mem_sbm_plug_any_sb()
1755 virtio_mem_sbm_set_mb_state(vm, mb_id, old_state - 1); in virtio_mem_sbm_plug_any_sb()
1760 static int virtio_mem_sbm_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_sbm_plug_request() argument
1767 uint64_t nb_sb = diff / vm->sbm.sb_size; in virtio_mem_sbm_plug_request()
1775 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1778 virtio_mem_sbm_for_each_mb(vm, mb_id, mb_states[i]) { in virtio_mem_sbm_plug_request()
1779 rc = virtio_mem_sbm_plug_any_sb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1790 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1793 virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_UNUSED) { in virtio_mem_sbm_plug_request()
1794 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes())) in virtio_mem_sbm_plug_request()
1797 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1805 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes())) in virtio_mem_sbm_plug_request()
1808 rc = virtio_mem_sbm_prepare_next_mb(vm, &mb_id); in virtio_mem_sbm_plug_request()
1811 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1819 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1828 static int virtio_mem_bbm_plug_and_add_bb(struct virtio_mem *vm, in virtio_mem_bbm_plug_and_add_bb() argument
1833 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_plug_and_add_bb()
1837 rc = virtio_mem_bbm_plug_bb(vm, bb_id); in virtio_mem_bbm_plug_and_add_bb()
1840 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED); in virtio_mem_bbm_plug_and_add_bb()
1842 rc = virtio_mem_bbm_add_bb(vm, bb_id); in virtio_mem_bbm_plug_and_add_bb()
1844 if (!virtio_mem_bbm_unplug_bb(vm, bb_id)) in virtio_mem_bbm_plug_and_add_bb()
1845 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_plug_and_add_bb()
1849 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_plug_and_add_bb()
1859 static int virtio_mem_bbm_prepare_next_bb(struct virtio_mem *vm, in virtio_mem_bbm_prepare_next_bb() argument
1864 if (vm->bbm.next_bb_id > vm->bbm.last_usable_bb_id) in virtio_mem_bbm_prepare_next_bb()
1868 rc = virtio_mem_bbm_bb_states_prepare_next_bb(vm); in virtio_mem_bbm_prepare_next_bb()
1872 vm->bbm.bb_count[VIRTIO_MEM_BBM_BB_UNUSED]++; in virtio_mem_bbm_prepare_next_bb()
1873 *bb_id = vm->bbm.next_bb_id; in virtio_mem_bbm_prepare_next_bb()
1874 vm->bbm.next_bb_id++; in virtio_mem_bbm_prepare_next_bb()
1878 static int virtio_mem_bbm_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_bbm_plug_request() argument
1880 uint64_t nb_bb = diff / vm->bbm.bb_size; in virtio_mem_bbm_plug_request()
1888 virtio_mem_bbm_for_each_bb(vm, bb_id, VIRTIO_MEM_BBM_BB_UNUSED) { in virtio_mem_bbm_plug_request()
1889 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size)) in virtio_mem_bbm_plug_request()
1892 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id); in virtio_mem_bbm_plug_request()
1902 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size)) in virtio_mem_bbm_plug_request()
1905 rc = virtio_mem_bbm_prepare_next_bb(vm, &bb_id); in virtio_mem_bbm_plug_request()
1908 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id); in virtio_mem_bbm_plug_request()
1922 static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_plug_request() argument
1924 if (vm->in_sbm) in virtio_mem_plug_request()
1925 return virtio_mem_sbm_plug_request(vm, diff); in virtio_mem_plug_request()
1926 return virtio_mem_bbm_plug_request(vm, diff); in virtio_mem_plug_request()
1938 static int virtio_mem_sbm_unplug_any_sb_offline(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_offline() argument
1944 rc = virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb_offline()
1947 if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) in virtio_mem_sbm_unplug_any_sb_offline()
1948 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_any_sb_offline()
1953 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) { in virtio_mem_sbm_unplug_any_sb_offline()
1960 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_any_sb_offline()
1963 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_offline()
1964 rc = virtio_mem_sbm_remove_mb(vm, mb_id); in virtio_mem_sbm_unplug_any_sb_offline()
1966 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_offline()
1976 static int virtio_mem_sbm_unplug_sb_online(struct virtio_mem *vm, in virtio_mem_sbm_unplug_sb_online() argument
1980 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size) * count; in virtio_mem_sbm_unplug_sb_online()
1981 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_unplug_sb_online()
1986 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_unplug_sb_online()
1988 rc = virtio_mem_fake_offline(vm, start_pfn, nr_pages); in virtio_mem_sbm_unplug_sb_online()
1993 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_sb_online()
2002 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_sb_online()
2006 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_sb_online()
2024 static int virtio_mem_sbm_unplug_any_sb_online(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_online() argument
2031 if (*nb_sb >= vm->sbm.sbs_per_mb && in virtio_mem_sbm_unplug_any_sb_online()
2032 virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) { in virtio_mem_sbm_unplug_any_sb_online()
2033 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, 0, in virtio_mem_sbm_unplug_any_sb_online()
2034 vm->sbm.sbs_per_mb); in virtio_mem_sbm_unplug_any_sb_online()
2036 *nb_sb -= vm->sbm.sbs_per_mb; in virtio_mem_sbm_unplug_any_sb_online()
2043 for (sb_id = vm->sbm.sbs_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) { in virtio_mem_sbm_unplug_any_sb_online()
2046 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_unplug_any_sb_online()
2051 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, sb_id, 1); in virtio_mem_sbm_unplug_any_sb_online()
2060 rc = virtio_mem_sbm_try_remove_unplugged_mb(vm, mb_id); in virtio_mem_sbm_unplug_any_sb_online()
2062 vm->sbm.have_unplugged_mb = 1; in virtio_mem_sbm_unplug_any_sb_online()
2079 static int virtio_mem_sbm_unplug_any_sb(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb() argument
2083 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_unplug_any_sb()
2090 return virtio_mem_sbm_unplug_any_sb_online(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb()
2093 return virtio_mem_sbm_unplug_any_sb_offline(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb()
2098 static int virtio_mem_sbm_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_sbm_unplug_request() argument
2108 uint64_t nb_sb = diff / vm->sbm.sb_size; in virtio_mem_sbm_unplug_request()
2120 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2130 virtio_mem_sbm_for_each_mb_rev(vm, mb_id, mb_states[i]) { in virtio_mem_sbm_unplug_request()
2131 rc = virtio_mem_sbm_unplug_any_sb(vm, mb_id, &nb_sb); in virtio_mem_sbm_unplug_request()
2134 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2136 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2139 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2144 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2147 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2158 static int virtio_mem_bbm_offline_remove_and_unplug_bb(struct virtio_mem *vm, in virtio_mem_bbm_offline_remove_and_unplug_bb() argument
2161 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2162 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2168 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_offline_remove_and_unplug_bb()
2178 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2179 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_FAKE_OFFLINE); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2186 rc = virtio_mem_fake_offline(vm, pfn, PAGES_PER_SECTION); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2192 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2194 rc = virtio_mem_bbm_offline_and_remove_bb(vm, bb_id); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2196 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2200 rc = virtio_mem_bbm_unplug_bb(vm, bb_id); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2202 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_offline_remove_and_unplug_bb()
2205 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_offline_remove_and_unplug_bb()
2216 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2217 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2224 static bool virtio_mem_bbm_bb_is_offline(struct virtio_mem *vm, in virtio_mem_bbm_bb_is_offline() argument
2227 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_bb_is_offline()
2228 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_bb_is_offline()
2243 static bool virtio_mem_bbm_bb_is_movable(struct virtio_mem *vm, in virtio_mem_bbm_bb_is_movable() argument
2246 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_bb_is_movable()
2247 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_bb_is_movable()
2263 static int virtio_mem_bbm_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_bbm_unplug_request() argument
2265 uint64_t nb_bb = diff / vm->bbm.bb_size; in virtio_mem_bbm_unplug_request()
2277 virtio_mem_bbm_for_each_bb_rev(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED) { in virtio_mem_bbm_unplug_request()
2284 if (i == 0 && !virtio_mem_bbm_bb_is_offline(vm, bb_id)) in virtio_mem_bbm_unplug_request()
2286 if (i == 1 && !virtio_mem_bbm_bb_is_movable(vm, bb_id)) in virtio_mem_bbm_unplug_request()
2288 rc = virtio_mem_bbm_offline_remove_and_unplug_bb(vm, bb_id); in virtio_mem_bbm_unplug_request()
2306 static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_unplug_request() argument
2308 if (vm->in_sbm) in virtio_mem_unplug_request()
2309 return virtio_mem_sbm_unplug_request(vm, diff); in virtio_mem_unplug_request()
2310 return virtio_mem_bbm_unplug_request(vm, diff); in virtio_mem_unplug_request()
2318 static int virtio_mem_cleanup_pending_mb(struct virtio_mem *vm) in virtio_mem_cleanup_pending_mb() argument
2323 if (!vm->in_sbm) { in virtio_mem_cleanup_pending_mb()
2324 virtio_mem_bbm_for_each_bb(vm, id, in virtio_mem_cleanup_pending_mb()
2326 rc = virtio_mem_bbm_unplug_bb(vm, id); in virtio_mem_cleanup_pending_mb()
2329 virtio_mem_bbm_set_bb_state(vm, id, in virtio_mem_cleanup_pending_mb()
2335 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_PLUGGED) { in virtio_mem_cleanup_pending_mb()
2336 rc = virtio_mem_sbm_unplug_mb(vm, id); in virtio_mem_cleanup_pending_mb()
2339 virtio_mem_sbm_set_mb_state(vm, id, in virtio_mem_cleanup_pending_mb()
2343 if (!vm->sbm.have_unplugged_mb) in virtio_mem_cleanup_pending_mb()
2350 vm->sbm.have_unplugged_mb = false; in virtio_mem_cleanup_pending_mb()
2352 mutex_lock(&vm->hotplug_mutex); in virtio_mem_cleanup_pending_mb()
2353 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL) in virtio_mem_cleanup_pending_mb()
2354 rc |= virtio_mem_sbm_try_remove_unplugged_mb(vm, id); in virtio_mem_cleanup_pending_mb()
2355 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL) in virtio_mem_cleanup_pending_mb()
2356 rc |= virtio_mem_sbm_try_remove_unplugged_mb(vm, id); in virtio_mem_cleanup_pending_mb()
2357 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) in virtio_mem_cleanup_pending_mb()
2358 rc |= virtio_mem_sbm_try_remove_unplugged_mb(vm, id); in virtio_mem_cleanup_pending_mb()
2359 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_cleanup_pending_mb()
2362 vm->sbm.have_unplugged_mb = true; in virtio_mem_cleanup_pending_mb()
2370 static void virtio_mem_refresh_config(struct virtio_mem *vm) in virtio_mem_refresh_config() argument
2376 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size, in virtio_mem_refresh_config()
2378 if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size)) in virtio_mem_refresh_config()
2379 vm->plugged_size = new_plugged_size; in virtio_mem_refresh_config()
2382 virtio_cread_le(vm->vdev, struct virtio_mem_config, in virtio_mem_refresh_config()
2383 usable_region_size, &vm->usable_region_size); in virtio_mem_refresh_config()
2384 end_addr = min(vm->addr + vm->usable_region_size - 1, in virtio_mem_refresh_config()
2387 if (vm->in_sbm) { in virtio_mem_refresh_config()
2388 vm->sbm.last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr); in virtio_mem_refresh_config()
2390 vm->sbm.last_usable_mb_id--; in virtio_mem_refresh_config()
2392 vm->bbm.last_usable_bb_id = virtio_mem_phys_to_bb_id(vm, in virtio_mem_refresh_config()
2394 if (!IS_ALIGNED(end_addr + 1, vm->bbm.bb_size)) in virtio_mem_refresh_config()
2395 vm->bbm.last_usable_bb_id--; in virtio_mem_refresh_config()
2405 virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size, in virtio_mem_refresh_config()
2406 &vm->requested_size); in virtio_mem_refresh_config()
2408 dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size); in virtio_mem_refresh_config()
2409 dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size); in virtio_mem_refresh_config()
2417 struct virtio_mem *vm = container_of(work, struct virtio_mem, wq); in virtio_mem_run_wq() local
2421 if (unlikely(vm->in_kdump)) { in virtio_mem_run_wq()
2422 dev_warn_once(&vm->vdev->dev, in virtio_mem_run_wq()
2427 hrtimer_cancel(&vm->retry_timer); in virtio_mem_run_wq()
2429 if (vm->broken) in virtio_mem_run_wq()
2432 atomic_set(&vm->wq_active, 1); in virtio_mem_run_wq()
2437 if (unlikely(vm->unplug_all_required)) in virtio_mem_run_wq()
2438 rc = virtio_mem_send_unplug_all_request(vm); in virtio_mem_run_wq()
2440 if (atomic_read(&vm->config_changed)) { in virtio_mem_run_wq()
2441 atomic_set(&vm->config_changed, 0); in virtio_mem_run_wq()
2442 virtio_mem_refresh_config(vm); in virtio_mem_run_wq()
2447 rc = virtio_mem_cleanup_pending_mb(vm); in virtio_mem_run_wq()
2449 if (!rc && vm->requested_size != vm->plugged_size) { in virtio_mem_run_wq()
2450 if (vm->requested_size > vm->plugged_size) { in virtio_mem_run_wq()
2451 diff = vm->requested_size - vm->plugged_size; in virtio_mem_run_wq()
2452 rc = virtio_mem_plug_request(vm, diff); in virtio_mem_run_wq()
2454 diff = vm->plugged_size - vm->requested_size; in virtio_mem_run_wq()
2455 rc = virtio_mem_unplug_request(vm, diff); in virtio_mem_run_wq()
2463 if (!rc && vm->in_sbm && vm->sbm.have_unplugged_mb) in virtio_mem_run_wq()
2468 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; in virtio_mem_run_wq()
2488 hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms), in virtio_mem_run_wq()
2496 dev_err(&vm->vdev->dev, in virtio_mem_run_wq()
2498 vm->broken = true; in virtio_mem_run_wq()
2501 atomic_set(&vm->wq_active, 0); in virtio_mem_run_wq()
2506 struct virtio_mem *vm = container_of(timer, struct virtio_mem, in virtio_mem_timer_expired() local
2509 virtio_mem_retry(vm); in virtio_mem_timer_expired()
2510 vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2, in virtio_mem_timer_expired()
2517 struct virtio_mem *vm = vq->vdev->priv; in virtio_mem_handle_response() local
2519 wake_up(&vm->host_resp); in virtio_mem_handle_response()
2522 static int virtio_mem_init_vq(struct virtio_mem *vm) in virtio_mem_init_vq() argument
2526 vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response, in virtio_mem_init_vq()
2530 vm->vq = vq; in virtio_mem_init_vq()
2535 static int virtio_mem_init_hotplug(struct virtio_mem *vm) in virtio_mem_init_hotplug() argument
2542 if (!IS_ALIGNED(vm->addr, memory_block_size_bytes())) in virtio_mem_init_hotplug()
2543 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2545 if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes())) in virtio_mem_init_hotplug()
2546 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2548 if (vm->addr < pluggable_range.start || in virtio_mem_init_hotplug()
2549 vm->addr + vm->region_size - 1 > pluggable_range.end) in virtio_mem_init_hotplug()
2550 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2554 vm->offline_threshold = max_t(uint64_t, 2 * memory_block_size_bytes(), in virtio_mem_init_hotplug()
2562 sb_size = max_t(uint64_t, vm->device_block_size, sb_size); in virtio_mem_init_hotplug()
2566 vm->in_sbm = true; in virtio_mem_init_hotplug()
2567 vm->sbm.sb_size = sb_size; in virtio_mem_init_hotplug()
2568 vm->sbm.sbs_per_mb = memory_block_size_bytes() / in virtio_mem_init_hotplug()
2569 vm->sbm.sb_size; in virtio_mem_init_hotplug()
2572 addr = max_t(uint64_t, vm->addr, pluggable_range.start) + in virtio_mem_init_hotplug()
2574 vm->sbm.first_mb_id = virtio_mem_phys_to_mb_id(addr); in virtio_mem_init_hotplug()
2575 vm->sbm.next_mb_id = vm->sbm.first_mb_id; in virtio_mem_init_hotplug()
2578 vm->bbm.bb_size = max_t(uint64_t, vm->device_block_size, in virtio_mem_init_hotplug()
2583 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2585 } else if (bbm_block_size < vm->bbm.bb_size) { in virtio_mem_init_hotplug()
2586 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2589 vm->bbm.bb_size = bbm_block_size; in virtio_mem_init_hotplug()
2594 addr = max_t(uint64_t, vm->addr, pluggable_range.start) + in virtio_mem_init_hotplug()
2595 vm->bbm.bb_size - 1; in virtio_mem_init_hotplug()
2596 vm->bbm.first_bb_id = virtio_mem_phys_to_bb_id(vm, addr); in virtio_mem_init_hotplug()
2597 vm->bbm.next_bb_id = vm->bbm.first_bb_id; in virtio_mem_init_hotplug()
2600 vm->offline_threshold = max_t(uint64_t, 2 * vm->bbm.bb_size, in virtio_mem_init_hotplug()
2601 vm->offline_threshold); in virtio_mem_init_hotplug()
2604 dev_info(&vm->vdev->dev, "memory block size: 0x%lx", in virtio_mem_init_hotplug()
2606 if (vm->in_sbm) in virtio_mem_init_hotplug()
2607 dev_info(&vm->vdev->dev, "subblock size: 0x%llx", in virtio_mem_init_hotplug()
2608 (unsigned long long)vm->sbm.sb_size); in virtio_mem_init_hotplug()
2610 dev_info(&vm->vdev->dev, "big block size: 0x%llx", in virtio_mem_init_hotplug()
2611 (unsigned long long)vm->bbm.bb_size); in virtio_mem_init_hotplug()
2614 rc = virtio_mem_create_resource(vm); in virtio_mem_init_hotplug()
2619 if (vm->in_sbm) in virtio_mem_init_hotplug()
2622 unit_pages = PHYS_PFN(vm->bbm.bb_size); in virtio_mem_init_hotplug()
2623 rc = memory_group_register_dynamic(vm->nid, unit_pages); in virtio_mem_init_hotplug()
2626 vm->mgid = rc; in virtio_mem_init_hotplug()
2633 if (vm->plugged_size) { in virtio_mem_init_hotplug()
2634 vm->unplug_all_required = true; in virtio_mem_init_hotplug()
2635 dev_info(&vm->vdev->dev, "unplugging all memory is required\n"); in virtio_mem_init_hotplug()
2639 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb; in virtio_mem_init_hotplug()
2640 rc = register_memory_notifier(&vm->memory_notifier); in virtio_mem_init_hotplug()
2644 vm->pm_notifier.priority = INT_MAX; in virtio_mem_init_hotplug()
2645 vm->pm_notifier.notifier_call = virtio_mem_pm_notifier_cb; in virtio_mem_init_hotplug()
2646 rc = register_pm_notifier(&vm->pm_notifier); in virtio_mem_init_hotplug()
2649 rc = register_virtio_mem_device(vm); in virtio_mem_init_hotplug()
2653 virtio_device_ready(vm->vdev); in virtio_mem_init_hotplug()
2656 unregister_pm_notifier(&vm->pm_notifier); in virtio_mem_init_hotplug()
2658 unregister_memory_notifier(&vm->memory_notifier); in virtio_mem_init_hotplug()
2660 memory_group_unregister(vm->mgid); in virtio_mem_init_hotplug()
2662 virtio_mem_delete_resource(vm); in virtio_mem_init_hotplug()
2667 static int virtio_mem_send_state_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_state_request() argument
2670 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_state_request()
2672 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_STATE), in virtio_mem_send_state_request()
2673 .u.state.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_state_request()
2674 .u.state.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_state_request()
2678 dev_dbg(&vm->vdev->dev, "requesting state: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_state_request()
2681 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_state_request()
2683 return virtio16_to_cpu(vm->vdev, vm->resp.u.state.state); in virtio_mem_send_state_request()
2691 dev_dbg(&vm->vdev->dev, "requesting state failed: %d\n", rc); in virtio_mem_send_state_request()
2698 struct virtio_mem *vm = container_of(cb, struct virtio_mem, in virtio_mem_vmcore_pfn_is_ram() local
2704 if (!virtio_mem_contains_range(vm, addr, PAGE_SIZE)) in virtio_mem_vmcore_pfn_is_ram()
2706 if (!vm->plugged_size) in virtio_mem_vmcore_pfn_is_ram()
2713 mutex_lock(&vm->hotplug_mutex); in virtio_mem_vmcore_pfn_is_ram()
2715 addr = ALIGN_DOWN(addr, vm->device_block_size); in virtio_mem_vmcore_pfn_is_ram()
2716 if (addr != vm->last_block_addr) { in virtio_mem_vmcore_pfn_is_ram()
2717 rc = virtio_mem_send_state_request(vm, addr, in virtio_mem_vmcore_pfn_is_ram()
2718 vm->device_block_size); in virtio_mem_vmcore_pfn_is_ram()
2721 vm->last_block_plugged = true; in virtio_mem_vmcore_pfn_is_ram()
2723 vm->last_block_plugged = false; in virtio_mem_vmcore_pfn_is_ram()
2724 vm->last_block_addr = addr; in virtio_mem_vmcore_pfn_is_ram()
2727 is_ram = vm->last_block_plugged; in virtio_mem_vmcore_pfn_is_ram()
2728 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_vmcore_pfn_is_ram()
2733 static int virtio_mem_vmcore_add_device_ram(struct virtio_mem *vm, in virtio_mem_vmcore_add_device_ram() argument
2740 dev_err(&vm->vdev->dev, in virtio_mem_vmcore_add_device_ram()
2748 struct virtio_mem *vm = container_of(cb, struct virtio_mem, in virtio_mem_vmcore_get_device_ram() local
2750 const uint64_t device_start = vm->addr; in virtio_mem_vmcore_get_device_ram()
2751 const uint64_t device_end = vm->addr + vm->usable_region_size; in virtio_mem_vmcore_get_device_ram()
2756 if (!vm->plugged_size) in virtio_mem_vmcore_get_device_ram()
2761 vm->device_block_size); in virtio_mem_vmcore_get_device_ram()
2763 mutex_lock(&vm->hotplug_mutex); in virtio_mem_vmcore_get_device_ram()
2776 rc = virtio_mem_send_state_request(vm, cur_start, in virtio_mem_vmcore_get_device_ram()
2780 dev_err(&vm->vdev->dev, in virtio_mem_vmcore_get_device_ram()
2792 rc = virtio_mem_vmcore_add_device_ram(vm, &tmp_list, in virtio_mem_vmcore_get_device_ram()
2803 rc = virtio_mem_vmcore_add_device_ram(vm, &tmp_list, in virtio_mem_vmcore_get_device_ram()
2807 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_vmcore_get_device_ram()
2818 static int virtio_mem_init_kdump(struct virtio_mem *vm) in virtio_mem_init_kdump() argument
2821 virtio_device_ready(vm->vdev); in virtio_mem_init_kdump()
2823 dev_info(&vm->vdev->dev, "memory hot(un)plug disabled in kdump kernel\n"); in virtio_mem_init_kdump()
2824 vm->vmcore_cb.pfn_is_ram = virtio_mem_vmcore_pfn_is_ram; in virtio_mem_init_kdump()
2826 vm->vmcore_cb.get_device_ram = virtio_mem_vmcore_get_device_ram; in virtio_mem_init_kdump()
2828 register_vmcore_cb(&vm->vmcore_cb); in virtio_mem_init_kdump()
2831 dev_warn(&vm->vdev->dev, "disabled in kdump kernel without vmcore\n"); in virtio_mem_init_kdump()
2836 static int virtio_mem_init(struct virtio_mem *vm) in virtio_mem_init() argument
2840 if (!vm->vdev->config->get) { in virtio_mem_init()
2841 dev_err(&vm->vdev->dev, "config access disabled\n"); in virtio_mem_init()
2846 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size, in virtio_mem_init()
2847 &vm->plugged_size); in virtio_mem_init()
2848 virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size, in virtio_mem_init()
2849 &vm->device_block_size); in virtio_mem_init()
2850 virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id, in virtio_mem_init()
2852 vm->nid = virtio_mem_translate_node_id(vm, node_id); in virtio_mem_init()
2853 virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr); in virtio_mem_init()
2854 virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size, in virtio_mem_init()
2855 &vm->region_size); in virtio_mem_init()
2856 virtio_cread_le(vm->vdev, struct virtio_mem_config, usable_region_size, in virtio_mem_init()
2857 &vm->usable_region_size); in virtio_mem_init()
2860 if (vm->nid == NUMA_NO_NODE) in virtio_mem_init()
2861 vm->nid = memory_add_physaddr_to_nid(vm->addr); in virtio_mem_init()
2863 dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr); in virtio_mem_init()
2864 dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size); in virtio_mem_init()
2865 dev_info(&vm->vdev->dev, "device block size: 0x%llx", in virtio_mem_init()
2866 (unsigned long long)vm->device_block_size); in virtio_mem_init()
2867 if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA)) in virtio_mem_init()
2868 dev_info(&vm->vdev->dev, "nid: %d", vm->nid); in virtio_mem_init()
2874 if (vm->in_kdump) in virtio_mem_init()
2875 return virtio_mem_init_kdump(vm); in virtio_mem_init()
2876 return virtio_mem_init_hotplug(vm); in virtio_mem_init()
2879 static int virtio_mem_create_resource(struct virtio_mem *vm) in virtio_mem_create_resource() argument
2885 const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL); in virtio_mem_create_resource()
2891 vm->parent_resource = __request_mem_region(vm->addr, vm->region_size, in virtio_mem_create_resource()
2894 if (!vm->parent_resource) { in virtio_mem_create_resource()
2896 dev_warn(&vm->vdev->dev, "could not reserve device region\n"); in virtio_mem_create_resource()
2897 dev_info(&vm->vdev->dev, in virtio_mem_create_resource()
2903 vm->parent_resource->flags &= ~IORESOURCE_BUSY; in virtio_mem_create_resource()
2907 static void virtio_mem_delete_resource(struct virtio_mem *vm) in virtio_mem_delete_resource() argument
2911 if (!vm->parent_resource) in virtio_mem_delete_resource()
2914 name = vm->parent_resource->name; in virtio_mem_delete_resource()
2915 release_resource(vm->parent_resource); in virtio_mem_delete_resource()
2916 kfree(vm->parent_resource); in virtio_mem_delete_resource()
2918 vm->parent_resource = NULL; in virtio_mem_delete_resource()
2926 static bool virtio_mem_has_memory_added(struct virtio_mem *vm) in virtio_mem_has_memory_added() argument
2930 return walk_iomem_res_desc(IORES_DESC_NONE, flags, vm->addr, in virtio_mem_has_memory_added()
2931 vm->addr + vm->region_size, NULL, in virtio_mem_has_memory_added()
2937 struct virtio_mem *vm; in virtio_mem_probe() local
2943 vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL); in virtio_mem_probe()
2944 if (!vm) in virtio_mem_probe()
2947 init_waitqueue_head(&vm->host_resp); in virtio_mem_probe()
2948 vm->vdev = vdev; in virtio_mem_probe()
2949 INIT_WORK(&vm->wq, virtio_mem_run_wq); in virtio_mem_probe()
2950 mutex_init(&vm->hotplug_mutex); in virtio_mem_probe()
2951 INIT_LIST_HEAD(&vm->next); in virtio_mem_probe()
2952 spin_lock_init(&vm->removal_lock); in virtio_mem_probe()
2953 hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in virtio_mem_probe()
2954 vm->retry_timer.function = virtio_mem_timer_expired; in virtio_mem_probe()
2955 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; in virtio_mem_probe()
2956 vm->in_kdump = is_kdump_kernel(); in virtio_mem_probe()
2959 rc = virtio_mem_init_vq(vm); in virtio_mem_probe()
2964 rc = virtio_mem_init(vm); in virtio_mem_probe()
2969 if (!vm->in_kdump) { in virtio_mem_probe()
2970 atomic_set(&vm->config_changed, 1); in virtio_mem_probe()
2971 queue_work(system_freezable_wq, &vm->wq); in virtio_mem_probe()
2978 kfree(vm); in virtio_mem_probe()
2984 static void virtio_mem_deinit_hotplug(struct virtio_mem *vm) in virtio_mem_deinit_hotplug() argument
2993 mutex_lock(&vm->hotplug_mutex); in virtio_mem_deinit_hotplug()
2994 spin_lock_irq(&vm->removal_lock); in virtio_mem_deinit_hotplug()
2995 vm->removing = true; in virtio_mem_deinit_hotplug()
2996 spin_unlock_irq(&vm->removal_lock); in virtio_mem_deinit_hotplug()
2997 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_deinit_hotplug()
3000 cancel_work_sync(&vm->wq); in virtio_mem_deinit_hotplug()
3001 hrtimer_cancel(&vm->retry_timer); in virtio_mem_deinit_hotplug()
3003 if (vm->in_sbm) { in virtio_mem_deinit_hotplug()
3008 virtio_mem_sbm_for_each_mb(vm, mb_id, in virtio_mem_deinit_hotplug()
3010 rc = virtio_mem_sbm_remove_mb(vm, mb_id); in virtio_mem_deinit_hotplug()
3012 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_deinit_hotplug()
3023 unregister_virtio_mem_device(vm); in virtio_mem_deinit_hotplug()
3024 unregister_pm_notifier(&vm->pm_notifier); in virtio_mem_deinit_hotplug()
3025 unregister_memory_notifier(&vm->memory_notifier); in virtio_mem_deinit_hotplug()
3032 if (virtio_mem_has_memory_added(vm)) { in virtio_mem_deinit_hotplug()
3033 dev_warn(&vm->vdev->dev, in virtio_mem_deinit_hotplug()
3036 virtio_mem_delete_resource(vm); in virtio_mem_deinit_hotplug()
3037 kfree_const(vm->resource_name); in virtio_mem_deinit_hotplug()
3038 memory_group_unregister(vm->mgid); in virtio_mem_deinit_hotplug()
3042 if (vm->in_sbm) { in virtio_mem_deinit_hotplug()
3043 vfree(vm->sbm.mb_states); in virtio_mem_deinit_hotplug()
3044 vfree(vm->sbm.sb_states); in virtio_mem_deinit_hotplug()
3046 vfree(vm->bbm.bb_states); in virtio_mem_deinit_hotplug()
3050 static void virtio_mem_deinit_kdump(struct virtio_mem *vm) in virtio_mem_deinit_kdump() argument
3053 unregister_vmcore_cb(&vm->vmcore_cb); in virtio_mem_deinit_kdump()
3059 struct virtio_mem *vm = vdev->priv; in virtio_mem_remove() local
3061 if (vm->in_kdump) in virtio_mem_remove()
3062 virtio_mem_deinit_kdump(vm); in virtio_mem_remove()
3064 virtio_mem_deinit_hotplug(vm); in virtio_mem_remove()
3070 kfree(vm); in virtio_mem_remove()
3076 struct virtio_mem *vm = vdev->priv; in virtio_mem_config_changed() local
3078 if (unlikely(vm->in_kdump)) in virtio_mem_config_changed()
3081 atomic_set(&vm->config_changed, 1); in virtio_mem_config_changed()
3082 virtio_mem_retry(vm); in virtio_mem_config_changed()
3088 struct virtio_mem *vm = vdev->priv; in virtio_mem_freeze() local
3096 vm->plugged_size && in virtio_mem_freeze()
3097 !virtio_has_feature(vm->vdev, VIRTIO_MEM_F_PERSISTENT_SUSPEND)) { in virtio_mem_freeze()
3098 dev_err(&vm->vdev->dev, in virtio_mem_freeze()
3105 vm->vq = NULL; in virtio_mem_freeze()
3111 struct virtio_mem *vm = vdev->priv; in virtio_mem_restore() local
3114 ret = virtio_mem_init_vq(vm); in virtio_mem_restore()