Lines Matching refs:vb
140 struct virtio_balloon *vb = vq->vdev->priv;
142 wake_up(&vb->acked);
145 static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
150 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
153 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
157 wait_event(vb->acked, virtqueue_get_buf(vq, &len));
164 struct virtio_balloon *vb =
166 struct virtqueue *vq = vb->reporting_vq;
170 err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN);
183 wait_event(vb->acked, virtqueue_get_buf(vq, &unused));
188 static void set_page_pfns(struct virtio_balloon *vb,
200 pfns[i] = cpu_to_virtio32(vb->vdev,
204 static unsigned int fill_balloon(struct virtio_balloon *vb, size_t num)
212 num = min(num, ARRAY_SIZE(vb->pfns));
219 dev_info_ratelimited(&vb->vdev->dev,
230 mutex_lock(&vb->balloon_lock);
232 vb->num_pfns = 0;
235 balloon_page_enqueue(&vb->vb_dev_info, page);
237 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
238 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
239 if (!virtio_has_feature(vb->vdev,
242 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
245 num_allocated_pages = vb->num_pfns;
247 if (vb->num_pfns != 0)
248 tell_host(vb, vb->inflate_vq);
249 mutex_unlock(&vb->balloon_lock);
254 static void release_pages_balloon(struct virtio_balloon *vb,
260 if (!virtio_has_feature(vb->vdev,
268 static unsigned int leak_balloon(struct virtio_balloon *vb, size_t num)
272 struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
276 num = min(num, ARRAY_SIZE(vb->pfns));
278 mutex_lock(&vb->balloon_lock);
280 num = min(num, (size_t)vb->num_pages);
281 for (vb->num_pfns = 0; vb->num_pfns < num;
282 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
286 set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
288 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
291 num_freed_pages = vb->num_pfns;
297 if (vb->num_pfns != 0)
298 tell_host(vb, vb->deflate_vq);
299 release_pages_balloon(vb, &pages);
300 mutex_unlock(&vb->balloon_lock);
304 static inline void update_stat(struct virtio_balloon *vb, int idx,
308 vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
309 vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
314 static unsigned int update_balloon_stats(struct virtio_balloon *vb)
329 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
331 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
333 update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
334 update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
336 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
338 update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL,
342 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
344 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
346 update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
348 update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES,
365 struct virtio_balloon *vb = vq->vdev->priv;
367 spin_lock(&vb->stop_update_lock);
368 if (!vb->stop_update)
369 queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
370 spin_unlock(&vb->stop_update_lock);
373 static void stats_handle_request(struct virtio_balloon *vb)
379 num_stats = update_balloon_stats(vb);
381 vq = vb->stats_vq;
384 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
385 virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
389 static inline s64 towards_target(struct virtio_balloon *vb)
395 virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages,
403 return target - vb->num_pages;
407 static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb,
413 spin_lock_irq(&vb->free_page_list_lock);
415 page = balloon_page_pop(&vb->free_page_list);
421 vb->num_free_page_blocks -= num_returned;
422 spin_unlock_irq(&vb->free_page_list_lock);
427 static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb)
429 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
434 &vb->config_read_bitmap))
437 queue_work(vb->balloon_wq, &vb->report_free_page_work);
442 struct virtio_balloon *vb = vdev->priv;
445 spin_lock_irqsave(&vb->stop_update_lock, flags);
446 if (!vb->stop_update) {
448 &vb->update_balloon_size_work);
449 virtio_balloon_queue_free_page_work(vb);
451 spin_unlock_irqrestore(&vb->stop_update_lock, flags);
454 static void update_balloon_size(struct virtio_balloon *vb)
456 u32 actual = vb->num_pages;
459 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, actual,
465 struct virtio_balloon *vb;
467 vb = container_of(work, struct virtio_balloon,
469 stats_handle_request(vb);
474 struct virtio_balloon *vb;
477 vb = container_of(work, struct virtio_balloon,
479 diff = towards_target(vb);
485 diff -= fill_balloon(vb, diff);
487 diff += leak_balloon(vb, -diff);
488 update_balloon_size(vb);
494 static int init_vqs(struct virtio_balloon *vb)
516 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
521 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
526 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {
531 err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs,
536 vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE];
537 vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE];
538 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
541 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS];
547 num_stats = update_balloon_stats(vb);
549 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
550 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb,
553 dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n",
557 virtqueue_kick(vb->stats_vq);
560 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
561 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE];
563 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))
564 vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING];
569 static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb)
572 &vb->config_read_bitmap)) {
574 virtio_cread_le(vb->vdev, struct virtio_balloon_config,
576 &vb->cmd_id_received_cache);
579 return vb->cmd_id_received_cache;
582 static int send_cmd_id_start(struct virtio_balloon *vb)
585 struct virtqueue *vq = vb->free_page_vq;
592 vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
593 virtio_balloon_cmd_id_received(vb));
594 sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active));
595 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL);
601 static int send_cmd_id_stop(struct virtio_balloon *vb)
604 struct virtqueue *vq = vb->free_page_vq;
611 sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop));
612 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL);
618 static int get_free_page_and_send(struct virtio_balloon *vb)
620 struct virtqueue *vq = vb->free_page_vq;
650 spin_lock_irq(&vb->free_page_list_lock);
651 balloon_page_push(&vb->free_page_list, page);
652 vb->num_free_page_blocks++;
653 spin_unlock_irq(&vb->free_page_list_lock);
665 static int send_free_pages(struct virtio_balloon *vb)
675 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active);
677 virtio_balloon_cmd_id_received(vb)))
684 err = get_free_page_and_send(vb);
694 static void virtio_balloon_report_free_page(struct virtio_balloon *vb)
697 struct device *dev = &vb->vdev->dev;
700 err = send_cmd_id_start(vb);
704 err = send_free_pages(vb);
709 err = send_cmd_id_stop(vb);
716 struct virtio_balloon *vb = container_of(work, struct virtio_balloon,
720 cmd_id_received = virtio_balloon_cmd_id_received(vb);
723 return_free_pages_to_mm(vb, ULONG_MAX);
726 virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) {
727 virtio_balloon_report_free_page(vb);
744 * 1) insert newpage into vb->pages list and update the host about it;
745 * 2) update the host about the old page removed from vb->pages list;
753 struct virtio_balloon *vb = container_of(vb_dev_info,
765 if (!mutex_trylock(&vb->balloon_lock))
775 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) &&
787 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
788 set_page_pfns(vb, vb->pfns, newpage);
789 tell_host(vb, vb->inflate_vq);
795 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
796 set_page_pfns(vb, vb->pfns, page);
797 tell_host(vb, vb->deflate_vq);
799 mutex_unlock(&vb->balloon_lock);
807 static unsigned long shrink_free_pages(struct virtio_balloon *vb,
815 blocks_freed = return_free_pages_to_mm(vb, blocks_to_free);
823 struct virtio_balloon *vb = container_of(shrinker,
826 return shrink_free_pages(vb, sc->nr_to_scan);
832 struct virtio_balloon *vb = container_of(shrinker,
835 return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES;
841 struct virtio_balloon *vb = container_of(nb,
845 *freed += leak_balloon(vb, VIRTIO_BALLOON_OOM_NR_PAGES) /
847 update_balloon_size(vb);
852 static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb)
854 unregister_shrinker(&vb->shrinker);
857 static int virtio_balloon_register_shrinker(struct virtio_balloon *vb)
859 vb->shrinker.scan_objects = virtio_balloon_shrinker_scan;
860 vb->shrinker.count_objects = virtio_balloon_shrinker_count;
861 vb->shrinker.seeks = DEFAULT_SEEKS;
863 return register_shrinker(&vb->shrinker, "virtio-balloon");
868 struct virtio_balloon *vb;
877 vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL);
878 if (!vb) {
883 INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
884 INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
885 spin_lock_init(&vb->stop_update_lock);
886 mutex_init(&vb->balloon_lock);
887 init_waitqueue_head(&vb->acked);
888 vb->vdev = vdev;
890 balloon_devinfo_init(&vb->vb_dev_info);
892 err = init_vqs(vb);
897 vb->vb_dev_info.migratepage = virtballoon_migratepage;
904 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) {
908 vb->balloon_wq = alloc_workqueue("balloon-wq",
910 if (!vb->balloon_wq) {
914 INIT_WORK(&vb->report_free_page_work, report_free_page_func);
915 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP;
916 vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
918 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev,
920 spin_lock_init(&vb->free_page_list_lock);
921 INIT_LIST_HEAD(&vb->free_page_list);
926 err = virtio_balloon_register_shrinker(vb);
931 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) {
932 vb->oom_nb.notifier_call = virtio_balloon_oom_notify;
933 vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY;
934 err = register_oom_notifier(&vb->oom_nb);
955 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config,
959 vb->pr_dev_info.report = virtballoon_free_page_report;
960 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {
963 capacity = virtqueue_get_vring_size(vb->reporting_vq);
983 vb->pr_dev_info.order = 5;
986 err = page_reporting_register(&vb->pr_dev_info);
993 if (towards_target(vb))
998 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
999 unregister_oom_notifier(&vb->oom_nb);
1001 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1002 virtio_balloon_unregister_shrinker(vb);
1005 destroy_workqueue(vb->balloon_wq);
1009 kfree(vb);
1014 static void remove_common(struct virtio_balloon *vb)
1017 while (vb->num_pages)
1018 leak_balloon(vb, vb->num_pages);
1019 update_balloon_size(vb);
1022 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1023 return_free_pages_to_mm(vb, ULONG_MAX);
1026 virtio_reset_device(vb->vdev);
1028 vb->vdev->config->del_vqs(vb->vdev);
1033 struct virtio_balloon *vb = vdev->priv;
1035 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING))
1036 page_reporting_unregister(&vb->pr_dev_info);
1037 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
1038 unregister_oom_notifier(&vb->oom_nb);
1039 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
1040 virtio_balloon_unregister_shrinker(vb);
1041 spin_lock_irq(&vb->stop_update_lock);
1042 vb->stop_update = true;
1043 spin_unlock_irq(&vb->stop_update_lock);
1044 cancel_work_sync(&vb->update_balloon_size_work);
1045 cancel_work_sync(&vb->update_balloon_stats_work);
1048 cancel_work_sync(&vb->report_free_page_work);
1049 destroy_workqueue(vb->balloon_wq);
1052 remove_common(vb);
1053 kfree(vb);
1059 struct virtio_balloon *vb = vdev->priv;
1065 remove_common(vb);
1071 struct virtio_balloon *vb = vdev->priv;
1080 if (towards_target(vb))
1082 update_balloon_size(vb);