1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2023 Advanced Micro Devices, Inc. */ 3 4#include <linux/anon_inodes.h> 5#include <linux/file.h> 6#include <linux/fs.h> 7#include <linux/highmem.h> 8#include <linux/vfio.h> 9#include <linux/vfio_pci_core.h> 10 11#include "vfio_dev.h" 12#include "cmds.h" 13 14static struct pds_vfio_lm_file * 15pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size) 16{ 17 struct pds_vfio_lm_file *lm_file = NULL; 18 unsigned long long npages; 19 struct page **pages; 20 void *page_mem; 21 const void *p; 22 23 if (!size) 24 return NULL; 25 26 /* Alloc file structure */ 27 lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL); 28 if (!lm_file) 29 return NULL; 30 31 /* Create file */ 32 lm_file->filep = 33 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags); 34 if (IS_ERR(lm_file->filep)) 35 goto out_free_file; 36 37 stream_open(lm_file->filep->f_inode, lm_file->filep); 38 mutex_init(&lm_file->lock); 39 40 /* prevent file from being released before we are done with it */ 41 get_file(lm_file->filep); 42 43 /* Allocate memory for file pages */ 44 npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE); 45 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL); 46 if (!pages) 47 goto out_put_file; 48 49 page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL); 50 if (!page_mem) 51 goto out_free_pages_array; 52 53 p = page_mem - offset_in_page(page_mem); 54 for (unsigned long long i = 0; i < npages; i++) { 55 if (is_vmalloc_addr(p)) 56 pages[i] = vmalloc_to_page(p); 57 else 58 pages[i] = kmap_to_page((void *)p); 59 if (!pages[i]) 60 goto out_free_page_mem; 61 62 p += PAGE_SIZE; 63 } 64 65 /* Create scatterlist of file pages to use for DMA mapping later */ 66 if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0, 67 size, GFP_KERNEL)) 68 goto out_free_page_mem; 69 70 lm_file->size = size; 71 lm_file->pages = pages; 72 lm_file->npages = npages; 73 lm_file->page_mem = page_mem; 74 lm_file->alloc_size = npages * PAGE_SIZE; 75 76 return lm_file; 77 78out_free_page_mem: 79 kvfree(page_mem); 80out_free_pages_array: 81 kfree(pages); 82out_put_file: 83 fput(lm_file->filep); 84 mutex_destroy(&lm_file->lock); 85out_free_file: 86 kfree(lm_file); 87 88 return NULL; 89} 90 91static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file) 92{ 93 mutex_lock(&lm_file->lock); 94 95 lm_file->size = 0; 96 lm_file->alloc_size = 0; 97 98 /* Free scatter list of file pages */ 99 sg_free_table(&lm_file->sg_table); 100 101 kvfree(lm_file->page_mem); 102 lm_file->page_mem = NULL; 103 kfree(lm_file->pages); 104 lm_file->pages = NULL; 105 106 mutex_unlock(&lm_file->lock); 107 108 /* allow file to be released since we are done with it */ 109 fput(lm_file->filep); 110} 111 112void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio) 113{ 114 if (!pds_vfio->save_file) 115 return; 116 117 pds_vfio_put_lm_file(pds_vfio->save_file); 118 pds_vfio->save_file = NULL; 119} 120 121void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio) 122{ 123 if (!pds_vfio->restore_file) 124 return; 125 126 pds_vfio_put_lm_file(pds_vfio->restore_file); 127 pds_vfio->restore_file = NULL; 128} 129 130static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file, 131 unsigned long offset) 132{ 133 unsigned long cur_offset = 0; 134 struct scatterlist *sg; 135 unsigned int i; 136 137 /* All accesses are sequential */ 138 if (offset < lm_file->last_offset || !lm_file->last_offset_sg) { 139 lm_file->last_offset = 0; 140 lm_file->last_offset_sg = lm_file->sg_table.sgl; 141 lm_file->sg_last_entry = 0; 142 } 143 144 cur_offset = lm_file->last_offset; 145 146 for_each_sg(lm_file->last_offset_sg, sg, 147 lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) { 148 if (offset < sg->length + cur_offset) { 149 lm_file->last_offset_sg = sg; 150 lm_file->sg_last_entry += i; 151 lm_file->last_offset = cur_offset; 152 return nth_page(sg_page(sg), 153 (offset - cur_offset) / PAGE_SIZE); 154 } 155 cur_offset += sg->length; 156 } 157 158 return NULL; 159} 160 161static int pds_vfio_release_file(struct inode *inode, struct file *filp) 162{ 163 struct pds_vfio_lm_file *lm_file = filp->private_data; 164 165 mutex_lock(&lm_file->lock); 166 lm_file->filep->f_pos = 0; 167 lm_file->size = 0; 168 mutex_unlock(&lm_file->lock); 169 mutex_destroy(&lm_file->lock); 170 kfree(lm_file); 171 172 return 0; 173} 174 175static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf, 176 size_t len, loff_t *pos) 177{ 178 struct pds_vfio_lm_file *lm_file = filp->private_data; 179 ssize_t done = 0; 180 181 if (pos) 182 return -ESPIPE; 183 pos = &filp->f_pos; 184 185 mutex_lock(&lm_file->lock); 186 if (*pos > lm_file->size) { 187 done = -EINVAL; 188 goto out_unlock; 189 } 190 191 len = min_t(size_t, lm_file->size - *pos, len); 192 while (len) { 193 size_t page_offset; 194 struct page *page; 195 size_t page_len; 196 u8 *from_buff; 197 int err; 198 199 page_offset = (*pos) % PAGE_SIZE; 200 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 201 if (!page) { 202 if (done == 0) 203 done = -EINVAL; 204 goto out_unlock; 205 } 206 207 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 208 from_buff = kmap_local_page(page); 209 err = copy_to_user(buf, from_buff + page_offset, page_len); 210 kunmap_local(from_buff); 211 if (err) { 212 done = -EFAULT; 213 goto out_unlock; 214 } 215 *pos += page_len; 216 len -= page_len; 217 done += page_len; 218 buf += page_len; 219 } 220 221out_unlock: 222 mutex_unlock(&lm_file->lock); 223 return done; 224} 225 226static const struct file_operations pds_vfio_save_fops = { 227 .owner = THIS_MODULE, 228 .read = pds_vfio_save_read, 229 .release = pds_vfio_release_file, 230 .llseek = no_llseek, 231}; 232 233static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio) 234{ 235 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 236 struct pds_vfio_lm_file *lm_file; 237 u64 size; 238 int err; 239 240 /* Get live migration state size in this state */ 241 err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size); 242 if (err) { 243 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err)); 244 return err; 245 } 246 247 dev_dbg(dev, "save status, size = %lld\n", size); 248 249 if (!size) { 250 dev_err(dev, "invalid state size\n"); 251 return -EIO; 252 } 253 254 lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size); 255 if (!lm_file) { 256 dev_err(dev, "failed to create save file\n"); 257 return -ENOENT; 258 } 259 260 dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n", 261 lm_file->size, lm_file->alloc_size, lm_file->npages); 262 263 pds_vfio->save_file = lm_file; 264 265 return 0; 266} 267 268static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf, 269 size_t len, loff_t *pos) 270{ 271 struct pds_vfio_lm_file *lm_file = filp->private_data; 272 loff_t requested_length; 273 ssize_t done = 0; 274 275 if (pos) 276 return -ESPIPE; 277 278 pos = &filp->f_pos; 279 280 if (*pos < 0 || 281 check_add_overflow((loff_t)len, *pos, &requested_length)) 282 return -EINVAL; 283 284 mutex_lock(&lm_file->lock); 285 286 while (len) { 287 size_t page_offset; 288 struct page *page; 289 size_t page_len; 290 u8 *to_buff; 291 int err; 292 293 page_offset = (*pos) % PAGE_SIZE; 294 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 295 if (!page) { 296 if (done == 0) 297 done = -EINVAL; 298 goto out_unlock; 299 } 300 301 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 302 to_buff = kmap_local_page(page); 303 err = copy_from_user(to_buff + page_offset, buf, page_len); 304 kunmap_local(to_buff); 305 if (err) { 306 done = -EFAULT; 307 goto out_unlock; 308 } 309 *pos += page_len; 310 len -= page_len; 311 done += page_len; 312 buf += page_len; 313 lm_file->size += page_len; 314 } 315out_unlock: 316 mutex_unlock(&lm_file->lock); 317 return done; 318} 319 320static const struct file_operations pds_vfio_restore_fops = { 321 .owner = THIS_MODULE, 322 .write = pds_vfio_restore_write, 323 .release = pds_vfio_release_file, 324 .llseek = no_llseek, 325}; 326 327static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio) 328{ 329 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 330 struct pds_vfio_lm_file *lm_file; 331 u64 size; 332 333 size = sizeof(union pds_lm_dev_state); 334 dev_dbg(dev, "restore status, size = %lld\n", size); 335 336 if (!size) { 337 dev_err(dev, "invalid state size"); 338 return -EIO; 339 } 340 341 lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size); 342 if (!lm_file) { 343 dev_err(dev, "failed to create restore file"); 344 return -ENOENT; 345 } 346 pds_vfio->restore_file = lm_file; 347 348 return 0; 349} 350 351struct file * 352pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio, 353 enum vfio_device_mig_state next) 354{ 355 enum vfio_device_mig_state cur = pds_vfio->state; 356 int err; 357 358 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) { 359 err = pds_vfio_get_save_file(pds_vfio); 360 if (err) 361 return ERR_PTR(err); 362 363 err = pds_vfio_get_lm_state_cmd(pds_vfio); 364 if (err) { 365 pds_vfio_put_save_file(pds_vfio); 366 return ERR_PTR(err); 367 } 368 369 return pds_vfio->save_file->filep; 370 } 371 372 if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) { 373 pds_vfio_put_save_file(pds_vfio); 374 pds_vfio_dirty_disable(pds_vfio, true); 375 return NULL; 376 } 377 378 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) { 379 err = pds_vfio_get_restore_file(pds_vfio); 380 if (err) 381 return ERR_PTR(err); 382 383 return pds_vfio->restore_file->filep; 384 } 385 386 if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) { 387 err = pds_vfio_set_lm_state_cmd(pds_vfio); 388 if (err) 389 return ERR_PTR(err); 390 391 pds_vfio_put_restore_file(pds_vfio); 392 return NULL; 393 } 394 395 if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 396 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, 397 PDS_LM_STA_IN_PROGRESS); 398 err = pds_vfio_suspend_device_cmd(pds_vfio, 399 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 400 if (err) 401 return ERR_PTR(err); 402 403 return NULL; 404 } 405 406 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) { 407 err = pds_vfio_resume_device_cmd(pds_vfio, 408 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 409 if (err) 410 return ERR_PTR(err); 411 412 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE); 413 return NULL; 414 } 415 416 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 417 err = pds_vfio_resume_device_cmd(pds_vfio, 418 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 419 if (err) 420 return ERR_PTR(err); 421 422 return NULL; 423 } 424 425 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) { 426 err = pds_vfio_suspend_device_cmd(pds_vfio, 427 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 428 if (err) 429 return ERR_PTR(err); 430 return NULL; 431 } 432 433 return ERR_PTR(-EINVAL); 434} 435