Lines Matching refs:dmirror
57 struct dmirror *dmirror;
64 struct dmirror {
120 struct dmirror *dmirror;
124 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
125 if (dmirror == NULL)
128 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
129 mutex_init(&dmirror->mutex);
130 xa_init(&dmirror->pt);
132 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
135 kfree(dmirror);
139 filp->private_data = dmirror;
145 struct dmirror *dmirror = filp->private_data;
147 mmu_interval_notifier_remove(&dmirror->notifier);
148 xa_destroy(&dmirror->pt);
149 kfree(dmirror);
160 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
186 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
194 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
205 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
207 xa_erase(&dmirror->pt, pfn);
214 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
221 range->migrate_pgmap_owner == dmirror->mdevice)
225 mutex_lock(&dmirror->mutex);
226 else if (!mutex_trylock(&dmirror->mutex))
230 dmirror_do_update(dmirror, range->start, range->end);
232 mutex_unlock(&dmirror->mutex);
240 static int dmirror_range_fault(struct dmirror *dmirror,
243 struct mm_struct *mm = dmirror->notifier.mm;
264 mutex_lock(&dmirror->mutex);
267 mutex_unlock(&dmirror->mutex);
273 ret = dmirror_do_fault(dmirror, range);
275 mutex_unlock(&dmirror->mutex);
280 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
283 struct mm_struct *mm = dmirror->notifier.mm;
287 .notifier = &dmirror->notifier,
292 .dev_private_owner = dmirror->mdevice,
304 ret = dmirror_range_fault(dmirror, &range);
313 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
326 entry = xa_load(&dmirror->pt, pfn);
342 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
359 mutex_lock(&dmirror->mutex);
360 ret = dmirror_do_read(dmirror, start, end, &bounce);
361 mutex_unlock(&dmirror->mutex);
366 ret = dmirror_fault(dmirror, start, end, false);
382 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
395 entry = xa_load(&dmirror->pt, pfn);
411 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
433 mutex_lock(&dmirror->mutex);
434 ret = dmirror_do_write(dmirror, start, end, &bounce);
435 mutex_unlock(&dmirror->mutex);
440 ret = dmirror_fault(dmirror, start, end, true);
574 struct dmirror *dmirror)
576 struct dmirror_device *mdevice = dmirror->mdevice;
612 rpage->zone_device_data = dmirror;
623 struct dmirror *dmirror)
632 mutex_lock(&dmirror->mutex);
653 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
655 mutex_unlock(&dmirror->mutex);
660 mutex_unlock(&dmirror->mutex);
664 static int dmirror_migrate(struct dmirror *dmirror,
669 struct mm_struct *mm = dmirror->notifier.mm;
704 args.pgmap_owner = dmirror->mdevice;
710 dmirror_migrate_alloc_and_copy(&args, dmirror);
712 dmirror_migrate_finalize_and_map(&args, dmirror);
722 mutex_lock(&dmirror->mutex);
723 ret = dmirror_do_read(dmirror, start, end, &bounce);
724 mutex_unlock(&dmirror->mutex);
740 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
757 if (dmirror->mdevice == dmirror_page_to_device(page))
781 struct dmirror *dmirror = dmi->dmirror;
784 mutex_lock(&dmirror->mutex);
785 else if (!mutex_trylock(&dmirror->mutex))
794 mutex_unlock(&dmirror->mutex);
802 static int dmirror_range_snapshot(struct dmirror *dmirror,
806 struct mm_struct *mm = dmirror->notifier.mm;
814 notifier.dmirror = dmirror;
840 mutex_lock(&dmirror->mutex);
843 mutex_unlock(&dmirror->mutex);
851 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
853 mutex_unlock(&dmirror->mutex);
859 static int dmirror_snapshot(struct dmirror *dmirror,
862 struct mm_struct *mm = dmirror->notifier.mm;
872 .dev_private_owner = dmirror->mdevice,
897 ret = dmirror_range_snapshot(dmirror, &range, perm);
921 struct dmirror *dmirror;
924 dmirror = filp->private_data;
925 if (!dmirror)
941 ret = dmirror_read(dmirror, &cmd);
945 ret = dmirror_write(dmirror, &cmd);
949 ret = dmirror_migrate(dmirror, &cmd);
953 ret = dmirror_snapshot(dmirror, &cmd);
1018 struct dmirror *dmirror)
1040 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1055 struct dmirror *dmirror;
1064 dmirror = rpage->zone_device_data;
1072 args.pgmap_owner = dmirror->mdevice;
1079 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);