1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <rdma/ib_umem.h> 34#include <rdma/ib_umem_odp.h> 35#include <linux/kernel.h> 36 37#include "mlx5_ib.h" 38#include "cmd.h" 39#include "qp.h" 40 41#include <linux/mlx5/eq.h> 42 43/* Contains the details of a pagefault. */ 44struct mlx5_pagefault { 45 u32 bytes_committed; 46 u32 token; 47 u8 event_subtype; 48 u8 type; 49 union { 50 /* Initiator or send message responder pagefault details. */ 51 struct { 52 /* Received packet size, only valid for responders. */ 53 u32 packet_size; 54 /* 55 * Number of resource holding WQE, depends on type. 56 */ 57 u32 wq_num; 58 /* 59 * WQE index. Refers to either the send queue or 60 * receive queue, according to event_subtype. 61 */ 62 u16 wqe_index; 63 } wqe; 64 /* RDMA responder pagefault details */ 65 struct { 66 u32 r_key; 67 /* 68 * Received packet size, minimal size page fault 69 * resolution required for forward progress. 70 */ 71 u32 packet_size; 72 u32 rdma_op_len; 73 u64 rdma_va; 74 } rdma; 75 }; 76 77 struct mlx5_ib_pf_eq *eq; 78 struct work_struct work; 79}; 80 81#define MAX_PREFETCH_LEN (4*1024*1024U) 82 83/* Timeout in ms to wait for an active mmu notifier to complete when handling 84 * a pagefault. */ 85#define MMU_NOTIFIER_TIMEOUT 1000 86 87#define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 88#define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 89#define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 90#define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 91#define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 92 93#define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 94 95static u64 mlx5_imr_ksm_entries; 96 97static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, 98 struct mlx5_ib_mr *imr, int flags) 99{ 100 struct mlx5_klm *end = pklm + nentries; 101 102 if (flags & MLX5_IB_UPD_XLT_ZAP) { 103 for (; pklm != end; pklm++, idx++) { 104 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 105 pklm->key = cpu_to_be32(imr->dev->null_mkey); 106 pklm->va = 0; 107 } 108 return; 109 } 110 111 /* 112 * The locking here is pretty subtle. Ideally the implicit_children 113 * xarray would be protected by the umem_mutex, however that is not 114 * possible. Instead this uses a weaker update-then-lock pattern: 115 * 116 * srcu_read_lock() 117 * xa_store() 118 * mutex_lock(umem_mutex) 119 * mlx5_ib_update_xlt() 120 * mutex_unlock(umem_mutex) 121 * destroy lkey 122 * 123 * ie any change the xarray must be followed by the locked update_xlt 124 * before destroying. 125 * 126 * The umem_mutex provides the acquire/release semantic needed to make 127 * the xa_store() visible to a racing thread. While SRCU is not 128 * technically required, using it gives consistent use of the SRCU 129 * locking around the xarray. 130 */ 131 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); 132 lockdep_assert_held(&imr->dev->odp_srcu); 133 134 for (; pklm != end; pklm++, idx++) { 135 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); 136 137 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 138 if (mtt) { 139 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 140 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE); 141 } else { 142 pklm->key = cpu_to_be32(imr->dev->null_mkey); 143 pklm->va = 0; 144 } 145 } 146} 147 148static u64 umem_dma_to_mtt(dma_addr_t umem_dma) 149{ 150 u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK; 151 152 if (umem_dma & ODP_READ_ALLOWED_BIT) 153 mtt_entry |= MLX5_IB_MTT_READ; 154 if (umem_dma & ODP_WRITE_ALLOWED_BIT) 155 mtt_entry |= MLX5_IB_MTT_WRITE; 156 157 return mtt_entry; 158} 159 160static void populate_mtt(__be64 *pas, size_t idx, size_t nentries, 161 struct mlx5_ib_mr *mr, int flags) 162{ 163 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 164 dma_addr_t pa; 165 size_t i; 166 167 if (flags & MLX5_IB_UPD_XLT_ZAP) 168 return; 169 170 for (i = 0; i < nentries; i++) { 171 pa = odp->dma_list[idx + i]; 172 pas[i] = cpu_to_be64(umem_dma_to_mtt(pa)); 173 } 174} 175 176void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 177 struct mlx5_ib_mr *mr, int flags) 178{ 179 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 180 populate_klm(xlt, idx, nentries, mr, flags); 181 } else { 182 populate_mtt(xlt, idx, nentries, mr, flags); 183 } 184} 185 186static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) 187{ 188 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 189 190 /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */ 191 mutex_lock(&odp->umem_mutex); 192 if (odp->npages) { 193 mlx5_mr_cache_invalidate(mr); 194 ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp), 195 ib_umem_end(odp)); 196 WARN_ON(odp->npages); 197 } 198 odp->private = NULL; 199 mutex_unlock(&odp->umem_mutex); 200 201 if (!mr->cache_ent) { 202 mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey); 203 WARN_ON(mr->descs); 204 } 205} 206 207/* 208 * This must be called after the mr has been removed from implicit_children 209 * and the SRCU synchronized. NOTE: The MR does not necessarily have to be 210 * empty here, parallel page faults could have raced with the free process and 211 * added pages to it. 212 */ 213static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt) 214{ 215 struct mlx5_ib_mr *imr = mr->parent; 216 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 217 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 218 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 219 int srcu_key; 220 221 /* implicit_child_mr's are not allowed to have deferred work */ 222 WARN_ON(atomic_read(&mr->num_deferred_work)); 223 224 if (need_imr_xlt) { 225 srcu_key = srcu_read_lock(&mr->dev->odp_srcu); 226 mutex_lock(&odp_imr->umem_mutex); 227 mlx5_ib_update_xlt(mr->parent, idx, 1, 0, 228 MLX5_IB_UPD_XLT_INDIRECT | 229 MLX5_IB_UPD_XLT_ATOMIC); 230 mutex_unlock(&odp_imr->umem_mutex); 231 srcu_read_unlock(&mr->dev->odp_srcu, srcu_key); 232 } 233 234 dma_fence_odp_mr(mr); 235 236 mr->parent = NULL; 237 mlx5_mr_cache_free(mr->dev, mr); 238 ib_umem_odp_release(odp); 239 if (atomic_dec_and_test(&imr->num_deferred_work)) 240 wake_up(&imr->q_deferred_work); 241} 242 243static void free_implicit_child_mr_work(struct work_struct *work) 244{ 245 struct mlx5_ib_mr *mr = 246 container_of(work, struct mlx5_ib_mr, odp_destroy.work); 247 248 free_implicit_child_mr(mr, true); 249} 250 251static void free_implicit_child_mr_rcu(struct rcu_head *head) 252{ 253 struct mlx5_ib_mr *mr = 254 container_of(head, struct mlx5_ib_mr, odp_destroy.rcu); 255 256 /* Freeing a MR is a sleeping operation, so bounce to a work queue */ 257 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); 258 queue_work(system_unbound_wq, &mr->odp_destroy.work); 259} 260 261static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) 262{ 263 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 264 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 265 struct mlx5_ib_mr *imr = mr->parent; 266 267 xa_lock(&imr->implicit_children); 268 /* 269 * This can race with mlx5_ib_free_implicit_mr(), the first one to 270 * reach the xa lock wins the race and destroys the MR. 271 */ 272 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_ATOMIC) != 273 mr) 274 goto out_unlock; 275 276 atomic_inc(&imr->num_deferred_work); 277 call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu, 278 free_implicit_child_mr_rcu); 279 280out_unlock: 281 xa_unlock(&imr->implicit_children); 282} 283 284static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, 285 const struct mmu_notifier_range *range, 286 unsigned long cur_seq) 287{ 288 struct ib_umem_odp *umem_odp = 289 container_of(mni, struct ib_umem_odp, notifier); 290 struct mlx5_ib_mr *mr; 291 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 292 sizeof(struct mlx5_mtt)) - 1; 293 u64 idx = 0, blk_start_idx = 0; 294 u64 invalidations = 0; 295 unsigned long start; 296 unsigned long end; 297 int in_block = 0; 298 u64 addr; 299 300 if (!mmu_notifier_range_blockable(range)) 301 return false; 302 303 mutex_lock(&umem_odp->umem_mutex); 304 mmu_interval_set_seq(mni, cur_seq); 305 /* 306 * If npages is zero then umem_odp->private may not be setup yet. This 307 * does not complete until after the first page is mapped for DMA. 308 */ 309 if (!umem_odp->npages) 310 goto out; 311 mr = umem_odp->private; 312 313 start = max_t(u64, ib_umem_start(umem_odp), range->start); 314 end = min_t(u64, ib_umem_end(umem_odp), range->end); 315 316 /* 317 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 318 * while we are doing the invalidation, no page fault will attempt to 319 * overwrite the same MTTs. Concurent invalidations might race us, 320 * but they will write 0s as well, so no difference in the end result. 321 */ 322 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 323 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 324 /* 325 * Strive to write the MTTs in chunks, but avoid overwriting 326 * non-existing MTTs. The huristic here can be improved to 327 * estimate the cost of another UMR vs. the cost of bigger 328 * UMR. 329 */ 330 if (umem_odp->dma_list[idx] & 331 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 332 if (!in_block) { 333 blk_start_idx = idx; 334 in_block = 1; 335 } 336 337 /* Count page invalidations */ 338 invalidations += idx - blk_start_idx + 1; 339 } else { 340 u64 umr_offset = idx & umr_block_mask; 341 342 if (in_block && umr_offset == 0) { 343 mlx5_ib_update_xlt(mr, blk_start_idx, 344 idx - blk_start_idx, 0, 345 MLX5_IB_UPD_XLT_ZAP | 346 MLX5_IB_UPD_XLT_ATOMIC); 347 in_block = 0; 348 } 349 } 350 } 351 if (in_block) 352 mlx5_ib_update_xlt(mr, blk_start_idx, 353 idx - blk_start_idx + 1, 0, 354 MLX5_IB_UPD_XLT_ZAP | 355 MLX5_IB_UPD_XLT_ATOMIC); 356 357 mlx5_update_odp_stats(mr, invalidations, invalidations); 358 359 /* 360 * We are now sure that the device will not access the 361 * memory. We can safely unmap it, and mark it as dirty if 362 * needed. 363 */ 364 365 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 366 367 if (unlikely(!umem_odp->npages && mr->parent)) 368 destroy_unused_implicit_child_mr(mr); 369out: 370 mutex_unlock(&umem_odp->umem_mutex); 371 return true; 372} 373 374const struct mmu_interval_notifier_ops mlx5_mn_ops = { 375 .invalidate = mlx5_ib_invalidate_range, 376}; 377 378void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 379{ 380 struct ib_odp_caps *caps = &dev->odp_caps; 381 382 memset(caps, 0, sizeof(*caps)); 383 384 if (!MLX5_CAP_GEN(dev->mdev, pg) || 385 !mlx5_ib_can_load_pas_with_umr(dev, 0)) 386 return; 387 388 caps->general_caps = IB_ODP_SUPPORT; 389 390 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 391 dev->odp_max_size = U64_MAX; 392 else 393 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 394 395 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 396 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 397 398 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive)) 399 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 400 401 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 402 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 403 404 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 405 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 406 407 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 408 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 409 410 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 411 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 412 413 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 414 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 415 416 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive)) 417 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 418 419 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send)) 420 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 421 422 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive)) 423 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 424 425 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write)) 426 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 427 428 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read)) 429 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 430 431 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic)) 432 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 433 434 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive)) 435 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 436 437 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 438 MLX5_CAP_GEN(dev->mdev, null_mkey) && 439 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 440 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled)) 441 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 442} 443 444static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 445 struct mlx5_pagefault *pfault, 446 int error) 447{ 448 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 449 pfault->wqe.wq_num : pfault->token; 450 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {}; 451 int err; 452 453 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 454 MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type); 455 MLX5_SET(page_fault_resume_in, in, token, pfault->token); 456 MLX5_SET(page_fault_resume_in, in, wq_number, wq_num); 457 MLX5_SET(page_fault_resume_in, in, error, !!error); 458 459 err = mlx5_cmd_exec_in(dev->mdev, page_fault_resume, in); 460 if (err) 461 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 462 wq_num, err); 463} 464 465static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 466 unsigned long idx) 467{ 468 struct ib_umem_odp *odp; 469 struct mlx5_ib_mr *mr; 470 struct mlx5_ib_mr *ret; 471 int err; 472 473 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), 474 idx * MLX5_IMR_MTT_SIZE, 475 MLX5_IMR_MTT_SIZE, &mlx5_mn_ops); 476 if (IS_ERR(odp)) 477 return ERR_CAST(odp); 478 479 ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY, 480 imr->access_flags); 481 if (IS_ERR(mr)) 482 goto out_umem; 483 484 mr->ibmr.pd = imr->ibmr.pd; 485 mr->umem = &odp->umem; 486 mr->ibmr.lkey = mr->mmkey.key; 487 mr->ibmr.rkey = mr->mmkey.key; 488 mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE; 489 mr->parent = imr; 490 odp->private = mr; 491 492 err = mlx5_ib_update_xlt(mr, 0, 493 MLX5_IMR_MTT_ENTRIES, 494 PAGE_SHIFT, 495 MLX5_IB_UPD_XLT_ZAP | 496 MLX5_IB_UPD_XLT_ENABLE); 497 if (err) { 498 ret = ERR_PTR(err); 499 goto out_mr; 500 } 501 502 /* 503 * Once the store to either xarray completes any error unwind has to 504 * use synchronize_srcu(). Avoid this with xa_reserve() 505 */ 506 ret = xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, 507 GFP_KERNEL); 508 if (unlikely(ret)) { 509 if (xa_is_err(ret)) { 510 ret = ERR_PTR(xa_err(ret)); 511 goto out_mr; 512 } 513 /* 514 * Another thread beat us to creating the child mr, use 515 * theirs. 516 */ 517 goto out_mr; 518 } 519 520 mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr); 521 return mr; 522 523out_mr: 524 mlx5_mr_cache_free(imr->dev, mr); 525out_umem: 526 ib_umem_odp_release(odp); 527 return ret; 528} 529 530struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 531 struct ib_udata *udata, 532 int access_flags) 533{ 534 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 535 struct ib_umem_odp *umem_odp; 536 struct mlx5_ib_mr *imr; 537 int err; 538 539 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags); 540 if (IS_ERR(umem_odp)) 541 return ERR_CAST(umem_odp); 542 543 imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY, access_flags); 544 if (IS_ERR(imr)) { 545 err = PTR_ERR(imr); 546 goto out_umem; 547 } 548 549 imr->ibmr.pd = &pd->ibpd; 550 imr->mmkey.iova = 0; 551 imr->umem = &umem_odp->umem; 552 imr->ibmr.lkey = imr->mmkey.key; 553 imr->ibmr.rkey = imr->mmkey.key; 554 imr->umem = &umem_odp->umem; 555 imr->is_odp_implicit = true; 556 atomic_set(&imr->num_deferred_work, 0); 557 init_waitqueue_head(&imr->q_deferred_work); 558 xa_init(&imr->implicit_children); 559 560 err = mlx5_ib_update_xlt(imr, 0, 561 mlx5_imr_ksm_entries, 562 MLX5_KSM_PAGE_SHIFT, 563 MLX5_IB_UPD_XLT_INDIRECT | 564 MLX5_IB_UPD_XLT_ZAP | 565 MLX5_IB_UPD_XLT_ENABLE); 566 if (err) 567 goto out_mr; 568 569 err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key), 570 &imr->mmkey, GFP_KERNEL)); 571 if (err) 572 goto out_mr; 573 574 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); 575 return imr; 576out_mr: 577 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 578 mlx5_mr_cache_free(dev, imr); 579out_umem: 580 ib_umem_odp_release(umem_odp); 581 return ERR_PTR(err); 582} 583 584void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 585{ 586 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 587 struct mlx5_ib_dev *dev = imr->dev; 588 struct list_head destroy_list; 589 struct mlx5_ib_mr *mtt; 590 struct mlx5_ib_mr *tmp; 591 unsigned long idx; 592 593 INIT_LIST_HEAD(&destroy_list); 594 595 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key)); 596 /* 597 * This stops the SRCU protected page fault path from touching either 598 * the imr or any children. The page fault path can only reach the 599 * children xarray via the imr. 600 */ 601 synchronize_srcu(&dev->odp_srcu); 602 603 /* 604 * All work on the prefetch list must be completed, xa_erase() prevented 605 * new work from being created. 606 */ 607 wait_event(imr->q_deferred_work, !atomic_read(&imr->num_deferred_work)); 608 609 /* 610 * At this point it is forbidden for any other thread to enter 611 * pagefault_mr() on this imr. It is already forbidden to call 612 * pagefault_mr() on an implicit child. Due to this additions to 613 * implicit_children are prevented. 614 */ 615 616 /* 617 * Block destroy_unused_implicit_child_mr() from incrementing 618 * num_deferred_work. 619 */ 620 xa_lock(&imr->implicit_children); 621 xa_for_each (&imr->implicit_children, idx, mtt) { 622 __xa_erase(&imr->implicit_children, idx); 623 list_add(&mtt->odp_destroy.elm, &destroy_list); 624 } 625 xa_unlock(&imr->implicit_children); 626 627 /* 628 * Wait for any concurrent destroy_unused_implicit_child_mr() to 629 * complete. 630 */ 631 wait_event(imr->q_deferred_work, !atomic_read(&imr->num_deferred_work)); 632 633 /* 634 * Fence the imr before we destroy the children. This allows us to 635 * skip updating the XLT of the imr during destroy of the child mkey 636 * the imr points to. 637 */ 638 mlx5_mr_cache_invalidate(imr); 639 640 list_for_each_entry_safe (mtt, tmp, &destroy_list, odp_destroy.elm) 641 free_implicit_child_mr(mtt, false); 642 643 mlx5_mr_cache_free(dev, imr); 644 ib_umem_odp_release(odp_imr); 645} 646 647/** 648 * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR 649 * @mr: to fence 650 * 651 * On return no parallel threads will be touching this MR and no DMA will be 652 * active. 653 */ 654void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr) 655{ 656 /* Prevent new page faults and prefetch requests from succeeding */ 657 xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)); 658 659 /* Wait for all running page-fault handlers to finish. */ 660 synchronize_srcu(&mr->dev->odp_srcu); 661 662 wait_event(mr->q_deferred_work, !atomic_read(&mr->num_deferred_work)); 663 664 dma_fence_odp_mr(mr); 665} 666 667#define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 668#define MLX5_PF_FLAGS_SNAPSHOT BIT(2) 669#define MLX5_PF_FLAGS_ENABLE BIT(3) 670static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, 671 u64 user_va, size_t bcnt, u32 *bytes_mapped, 672 u32 flags) 673{ 674 int page_shift, ret, np; 675 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 676 u64 access_mask; 677 u64 start_idx; 678 bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); 679 u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; 680 681 if (flags & MLX5_PF_FLAGS_ENABLE) 682 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 683 684 page_shift = odp->page_shift; 685 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 686 access_mask = ODP_READ_ALLOWED_BIT; 687 688 if (odp->umem.writable && !downgrade) 689 access_mask |= ODP_WRITE_ALLOWED_BIT; 690 691 np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); 692 if (np < 0) 693 return np; 694 695 /* 696 * No need to check whether the MTTs really belong to this MR, since 697 * ib_umem_odp_map_dma_and_lock already checks this. 698 */ 699 ret = mlx5_ib_update_xlt(mr, start_idx, np, page_shift, xlt_flags); 700 mutex_unlock(&odp->umem_mutex); 701 702 if (ret < 0) { 703 if (ret != -EAGAIN) 704 mlx5_ib_err(mr->dev, 705 "Failed to update mkey page tables\n"); 706 goto out; 707 } 708 709 if (bytes_mapped) { 710 u32 new_mappings = (np << page_shift) - 711 (user_va - round_down(user_va, 1 << page_shift)); 712 713 *bytes_mapped += min_t(u32, new_mappings, bcnt); 714 } 715 716 return np << (page_shift - PAGE_SHIFT); 717 718out: 719 return ret; 720} 721 722static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, 723 struct ib_umem_odp *odp_imr, u64 user_va, 724 size_t bcnt, u32 *bytes_mapped, u32 flags) 725{ 726 unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT; 727 unsigned long upd_start_idx = end_idx + 1; 728 unsigned long upd_len = 0; 729 unsigned long npages = 0; 730 int err; 731 int ret; 732 733 if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE || 734 mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt)) 735 return -EFAULT; 736 737 /* Fault each child mr that intersects with our interval. */ 738 while (bcnt) { 739 unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT; 740 struct ib_umem_odp *umem_odp; 741 struct mlx5_ib_mr *mtt; 742 u64 len; 743 744 mtt = xa_load(&imr->implicit_children, idx); 745 if (unlikely(!mtt)) { 746 mtt = implicit_get_child_mr(imr, idx); 747 if (IS_ERR(mtt)) { 748 ret = PTR_ERR(mtt); 749 goto out; 750 } 751 upd_start_idx = min(upd_start_idx, idx); 752 upd_len = idx - upd_start_idx + 1; 753 } 754 755 umem_odp = to_ib_umem_odp(mtt->umem); 756 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - 757 user_va; 758 759 ret = pagefault_real_mr(mtt, umem_odp, user_va, len, 760 bytes_mapped, flags); 761 if (ret < 0) 762 goto out; 763 user_va += len; 764 bcnt -= len; 765 npages += ret; 766 } 767 768 ret = npages; 769 770 /* 771 * Any time the implicit_children are changed we must perform an 772 * update of the xlt before exiting to ensure the HW and the 773 * implicit_children remains synchronized. 774 */ 775out: 776 if (likely(!upd_len)) 777 return ret; 778 779 /* 780 * Notice this is not strictly ordered right, the KSM is updated after 781 * the implicit_children is updated, so a parallel page fault could 782 * see a MR that is not yet visible in the KSM. This is similar to a 783 * parallel page fault seeing a MR that is being concurrently removed 784 * from the KSM. Both of these improbable situations are resolved 785 * safely by resuming the HW and then taking another page fault. The 786 * next pagefault handler will see the new information. 787 */ 788 mutex_lock(&odp_imr->umem_mutex); 789 err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0, 790 MLX5_IB_UPD_XLT_INDIRECT | 791 MLX5_IB_UPD_XLT_ATOMIC); 792 mutex_unlock(&odp_imr->umem_mutex); 793 if (err) { 794 mlx5_ib_err(imr->dev, "Failed to update PAS\n"); 795 return err; 796 } 797 return ret; 798} 799 800/* 801 * Returns: 802 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are 803 * not accessible, or the MR is no longer valid. 804 * -EAGAIN/-ENOMEM: The operation should be retried 805 * 806 * -EINVAL/others: General internal malfunction 807 * >0: Number of pages mapped 808 */ 809static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, 810 u32 *bytes_mapped, u32 flags) 811{ 812 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 813 814 lockdep_assert_held(&mr->dev->odp_srcu); 815 if (unlikely(io_virt < mr->mmkey.iova)) 816 return -EFAULT; 817 818 if (!odp->is_implicit_odp) { 819 u64 user_va; 820 821 if (check_add_overflow(io_virt - mr->mmkey.iova, 822 (u64)odp->umem.address, &user_va)) 823 return -EFAULT; 824 if (unlikely(user_va >= ib_umem_end(odp) || 825 ib_umem_end(odp) - user_va < bcnt)) 826 return -EFAULT; 827 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, 828 flags); 829 } 830 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, 831 flags); 832} 833 834int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, bool enable) 835{ 836 u32 flags = MLX5_PF_FLAGS_SNAPSHOT; 837 int ret; 838 839 if (enable) 840 flags |= MLX5_PF_FLAGS_ENABLE; 841 842 ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), 843 mr->umem->address, mr->umem->length, NULL, 844 flags); 845 return ret >= 0 ? 0 : ret; 846} 847 848struct pf_frame { 849 struct pf_frame *next; 850 u32 key; 851 u64 io_virt; 852 size_t bcnt; 853 int depth; 854}; 855 856static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key) 857{ 858 if (!mmkey) 859 return false; 860 if (mmkey->type == MLX5_MKEY_MW) 861 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 862 return mmkey->key == key; 863} 864 865static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey) 866{ 867 struct mlx5_ib_mw *mw; 868 struct mlx5_ib_devx_mr *devx_mr; 869 870 if (mmkey->type == MLX5_MKEY_MW) { 871 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 872 return mw->ndescs; 873 } 874 875 devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr, 876 mmkey); 877 return devx_mr->ndescs; 878} 879 880/* 881 * Handle a single data segment in a page-fault WQE or RDMA region. 882 * 883 * Returns number of OS pages retrieved on success. The caller may continue to 884 * the next data segment. 885 * Can return the following error codes: 886 * -EAGAIN to designate a temporary error. The caller will abort handling the 887 * page fault and resolve it. 888 * -EFAULT when there's an error mapping the requested pages. The caller will 889 * abort the page fault handling. 890 */ 891static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 892 struct ib_pd *pd, u32 key, 893 u64 io_virt, size_t bcnt, 894 u32 *bytes_committed, 895 u32 *bytes_mapped) 896{ 897 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 898 struct pf_frame *head = NULL, *frame; 899 struct mlx5_core_mkey *mmkey; 900 struct mlx5_ib_mr *mr; 901 struct mlx5_klm *pklm; 902 u32 *out = NULL; 903 size_t offset; 904 int ndescs; 905 906 srcu_key = srcu_read_lock(&dev->odp_srcu); 907 908 io_virt += *bytes_committed; 909 bcnt -= *bytes_committed; 910 911next_mr: 912 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); 913 if (!mmkey) { 914 mlx5_ib_dbg( 915 dev, 916 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 917 key); 918 if (bytes_mapped) 919 *bytes_mapped += bcnt; 920 /* 921 * The user could specify a SGL with multiple lkeys and only 922 * some of them are ODP. Treat the non-ODP ones as fully 923 * faulted. 924 */ 925 ret = 0; 926 goto srcu_unlock; 927 } 928 if (!mkey_is_eq(mmkey, key)) { 929 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 930 ret = -EFAULT; 931 goto srcu_unlock; 932 } 933 934 switch (mmkey->type) { 935 case MLX5_MKEY_MR: 936 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 937 938 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0); 939 if (ret < 0) 940 goto srcu_unlock; 941 942 mlx5_update_odp_stats(mr, faults, ret); 943 944 npages += ret; 945 ret = 0; 946 break; 947 948 case MLX5_MKEY_MW: 949 case MLX5_MKEY_INDIRECT_DEVX: 950 ndescs = get_indirect_num_descs(mmkey); 951 952 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 953 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 954 ret = -EFAULT; 955 goto srcu_unlock; 956 } 957 958 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 959 sizeof(*pklm) * (ndescs - 2); 960 961 if (outlen > cur_outlen) { 962 kfree(out); 963 out = kzalloc(outlen, GFP_KERNEL); 964 if (!out) { 965 ret = -ENOMEM; 966 goto srcu_unlock; 967 } 968 cur_outlen = outlen; 969 } 970 971 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 972 bsf0_klm0_pas_mtt0_1); 973 974 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); 975 if (ret) 976 goto srcu_unlock; 977 978 offset = io_virt - MLX5_GET64(query_mkey_out, out, 979 memory_key_mkey_entry.start_addr); 980 981 for (i = 0; bcnt && i < ndescs; i++, pklm++) { 982 if (offset >= be32_to_cpu(pklm->bcount)) { 983 offset -= be32_to_cpu(pklm->bcount); 984 continue; 985 } 986 987 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 988 if (!frame) { 989 ret = -ENOMEM; 990 goto srcu_unlock; 991 } 992 993 frame->key = be32_to_cpu(pklm->key); 994 frame->io_virt = be64_to_cpu(pklm->va) + offset; 995 frame->bcnt = min_t(size_t, bcnt, 996 be32_to_cpu(pklm->bcount) - offset); 997 frame->depth = depth + 1; 998 frame->next = head; 999 head = frame; 1000 1001 bcnt -= frame->bcnt; 1002 offset = 0; 1003 } 1004 break; 1005 1006 default: 1007 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 1008 ret = -EFAULT; 1009 goto srcu_unlock; 1010 } 1011 1012 if (head) { 1013 frame = head; 1014 head = frame->next; 1015 1016 key = frame->key; 1017 io_virt = frame->io_virt; 1018 bcnt = frame->bcnt; 1019 depth = frame->depth; 1020 kfree(frame); 1021 1022 goto next_mr; 1023 } 1024 1025srcu_unlock: 1026 while (head) { 1027 frame = head; 1028 head = frame->next; 1029 kfree(frame); 1030 } 1031 kfree(out); 1032 1033 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1034 *bytes_committed = 0; 1035 return ret ? ret : npages; 1036} 1037 1038/** 1039 * Parse a series of data segments for page fault handling. 1040 * 1041 * @pfault contains page fault information. 1042 * @wqe points at the first data segment in the WQE. 1043 * @wqe_end points after the end of the WQE. 1044 * @bytes_mapped receives the number of bytes that the function was able to 1045 * map. This allows the caller to decide intelligently whether 1046 * enough memory was mapped to resolve the page fault 1047 * successfully (e.g. enough for the next MTU, or the entire 1048 * WQE). 1049 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 1050 * the committed bytes). 1051 * 1052 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 1053 * negative error code. 1054 */ 1055static int pagefault_data_segments(struct mlx5_ib_dev *dev, 1056 struct mlx5_pagefault *pfault, 1057 void *wqe, 1058 void *wqe_end, u32 *bytes_mapped, 1059 u32 *total_wqe_bytes, bool receive_queue) 1060{ 1061 int ret = 0, npages = 0; 1062 u64 io_virt; 1063 u32 key; 1064 u32 byte_count; 1065 size_t bcnt; 1066 int inline_segment; 1067 1068 if (bytes_mapped) 1069 *bytes_mapped = 0; 1070 if (total_wqe_bytes) 1071 *total_wqe_bytes = 0; 1072 1073 while (wqe < wqe_end) { 1074 struct mlx5_wqe_data_seg *dseg = wqe; 1075 1076 io_virt = be64_to_cpu(dseg->addr); 1077 key = be32_to_cpu(dseg->lkey); 1078 byte_count = be32_to_cpu(dseg->byte_count); 1079 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1080 bcnt = byte_count & ~MLX5_INLINE_SEG; 1081 1082 if (inline_segment) { 1083 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 1084 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 1085 16); 1086 } else { 1087 wqe += sizeof(*dseg); 1088 } 1089 1090 /* receive WQE end of sg list. */ 1091 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 1092 io_virt == 0) 1093 break; 1094 1095 if (!inline_segment && total_wqe_bytes) { 1096 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 1097 pfault->bytes_committed); 1098 } 1099 1100 /* A zero length data segment designates a length of 2GB. */ 1101 if (bcnt == 0) 1102 bcnt = 1U << 31; 1103 1104 if (inline_segment || bcnt <= pfault->bytes_committed) { 1105 pfault->bytes_committed -= 1106 min_t(size_t, bcnt, 1107 pfault->bytes_committed); 1108 continue; 1109 } 1110 1111 ret = pagefault_single_data_segment(dev, NULL, key, 1112 io_virt, bcnt, 1113 &pfault->bytes_committed, 1114 bytes_mapped); 1115 if (ret < 0) 1116 break; 1117 npages += ret; 1118 } 1119 1120 return ret < 0 ? ret : npages; 1121} 1122 1123/* 1124 * Parse initiator WQE. Advances the wqe pointer to point at the 1125 * scatter-gather list, and set wqe_end to the end of the WQE. 1126 */ 1127static int mlx5_ib_mr_initiator_pfault_handler( 1128 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1129 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1130{ 1131 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1132 u16 wqe_index = pfault->wqe.wqe_index; 1133 struct mlx5_base_av *av; 1134 unsigned ds, opcode; 1135 u32 qpn = qp->trans_qp.base.mqp.qpn; 1136 1137 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1138 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1139 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1140 ds, wqe_length); 1141 return -EFAULT; 1142 } 1143 1144 if (ds == 0) { 1145 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1146 wqe_index, qpn); 1147 return -EFAULT; 1148 } 1149 1150 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1151 *wqe += sizeof(*ctrl); 1152 1153 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1154 MLX5_WQE_CTRL_OPCODE_MASK; 1155 1156 if (qp->ibqp.qp_type == IB_QPT_XRC_INI) 1157 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1158 1159 if (qp->type == IB_QPT_UD || qp->type == MLX5_IB_QPT_DCI) { 1160 av = *wqe; 1161 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1162 *wqe += sizeof(struct mlx5_av); 1163 else 1164 *wqe += sizeof(struct mlx5_base_av); 1165 } 1166 1167 switch (opcode) { 1168 case MLX5_OPCODE_RDMA_WRITE: 1169 case MLX5_OPCODE_RDMA_WRITE_IMM: 1170 case MLX5_OPCODE_RDMA_READ: 1171 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1172 break; 1173 case MLX5_OPCODE_ATOMIC_CS: 1174 case MLX5_OPCODE_ATOMIC_FA: 1175 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1176 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1177 break; 1178 } 1179 1180 return 0; 1181} 1182 1183/* 1184 * Parse responder WQE and set wqe_end to the end of the WQE. 1185 */ 1186static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1187 struct mlx5_ib_srq *srq, 1188 void **wqe, void **wqe_end, 1189 int wqe_length) 1190{ 1191 int wqe_size = 1 << srq->msrq.wqe_shift; 1192 1193 if (wqe_size > wqe_length) { 1194 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1195 return -EFAULT; 1196 } 1197 1198 *wqe_end = *wqe + wqe_size; 1199 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1200 1201 return 0; 1202} 1203 1204static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1205 struct mlx5_ib_qp *qp, 1206 void *wqe, void **wqe_end, 1207 int wqe_length) 1208{ 1209 struct mlx5_ib_wq *wq = &qp->rq; 1210 int wqe_size = 1 << wq->wqe_shift; 1211 1212 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) { 1213 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1214 return -EFAULT; 1215 } 1216 1217 if (wqe_size > wqe_length) { 1218 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1219 return -EFAULT; 1220 } 1221 1222 *wqe_end = wqe + wqe_size; 1223 1224 return 0; 1225} 1226 1227static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1228 u32 wq_num, int pf_type) 1229{ 1230 struct mlx5_core_rsc_common *common = NULL; 1231 struct mlx5_core_srq *srq; 1232 1233 switch (pf_type) { 1234 case MLX5_WQE_PF_TYPE_RMP: 1235 srq = mlx5_cmd_get_srq(dev, wq_num); 1236 if (srq) 1237 common = &srq->common; 1238 break; 1239 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1240 case MLX5_WQE_PF_TYPE_RESP: 1241 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1242 common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP); 1243 break; 1244 default: 1245 break; 1246 } 1247 1248 return common; 1249} 1250 1251static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1252{ 1253 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1254 1255 return to_mibqp(mqp); 1256} 1257 1258static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1259{ 1260 struct mlx5_core_srq *msrq = 1261 container_of(res, struct mlx5_core_srq, common); 1262 1263 return to_mibsrq(msrq); 1264} 1265 1266static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1267 struct mlx5_pagefault *pfault) 1268{ 1269 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1270 u16 wqe_index = pfault->wqe.wqe_index; 1271 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1272 u32 bytes_mapped, total_wqe_bytes; 1273 struct mlx5_core_rsc_common *res; 1274 int resume_with_error = 1; 1275 struct mlx5_ib_qp *qp; 1276 size_t bytes_copied; 1277 int ret = 0; 1278 1279 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1280 if (!res) { 1281 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1282 return; 1283 } 1284 1285 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1286 res->res != MLX5_RES_XSRQ) { 1287 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1288 pfault->type); 1289 goto resolve_page_fault; 1290 } 1291 1292 wqe_start = (void *)__get_free_page(GFP_KERNEL); 1293 if (!wqe_start) { 1294 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1295 goto resolve_page_fault; 1296 } 1297 1298 wqe = wqe_start; 1299 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1300 if (qp && sq) { 1301 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1302 &bytes_copied); 1303 if (ret) 1304 goto read_user; 1305 ret = mlx5_ib_mr_initiator_pfault_handler( 1306 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1307 } else if (qp && !sq) { 1308 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1309 &bytes_copied); 1310 if (ret) 1311 goto read_user; 1312 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1313 dev, qp, wqe, &wqe_end, bytes_copied); 1314 } else if (!qp) { 1315 struct mlx5_ib_srq *srq = res_to_srq(res); 1316 1317 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1318 &bytes_copied); 1319 if (ret) 1320 goto read_user; 1321 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1322 dev, srq, &wqe, &wqe_end, bytes_copied); 1323 } 1324 1325 if (ret < 0 || wqe >= wqe_end) 1326 goto resolve_page_fault; 1327 1328 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1329 &total_wqe_bytes, !sq); 1330 if (ret == -EAGAIN) 1331 goto out; 1332 1333 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1334 goto resolve_page_fault; 1335 1336out: 1337 ret = 0; 1338 resume_with_error = 0; 1339 1340read_user: 1341 if (ret) 1342 mlx5_ib_err( 1343 dev, 1344 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n", 1345 ret, wqe_index, pfault->token); 1346 1347resolve_page_fault: 1348 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1349 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1350 pfault->wqe.wq_num, resume_with_error, 1351 pfault->type); 1352 mlx5_core_res_put(res); 1353 free_page((unsigned long)wqe_start); 1354} 1355 1356static int pages_in_range(u64 address, u32 length) 1357{ 1358 return (ALIGN(address + length, PAGE_SIZE) - 1359 (address & PAGE_MASK)) >> PAGE_SHIFT; 1360} 1361 1362static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1363 struct mlx5_pagefault *pfault) 1364{ 1365 u64 address; 1366 u32 length; 1367 u32 prefetch_len = pfault->bytes_committed; 1368 int prefetch_activated = 0; 1369 u32 rkey = pfault->rdma.r_key; 1370 int ret; 1371 1372 /* The RDMA responder handler handles the page fault in two parts. 1373 * First it brings the necessary pages for the current packet 1374 * (and uses the pfault context), and then (after resuming the QP) 1375 * prefetches more pages. The second operation cannot use the pfault 1376 * context and therefore uses the dummy_pfault context allocated on 1377 * the stack */ 1378 pfault->rdma.rdma_va += pfault->bytes_committed; 1379 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1380 pfault->rdma.rdma_op_len); 1381 pfault->bytes_committed = 0; 1382 1383 address = pfault->rdma.rdma_va; 1384 length = pfault->rdma.rdma_op_len; 1385 1386 /* For some operations, the hardware cannot tell the exact message 1387 * length, and in those cases it reports zero. Use prefetch 1388 * logic. */ 1389 if (length == 0) { 1390 prefetch_activated = 1; 1391 length = pfault->rdma.packet_size; 1392 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1393 } 1394 1395 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1396 &pfault->bytes_committed, NULL); 1397 if (ret == -EAGAIN) { 1398 /* We're racing with an invalidation, don't prefetch */ 1399 prefetch_activated = 0; 1400 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1401 mlx5_ib_page_fault_resume(dev, pfault, 1); 1402 if (ret != -ENOENT) 1403 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1404 ret, pfault->token, pfault->type); 1405 return; 1406 } 1407 1408 mlx5_ib_page_fault_resume(dev, pfault, 0); 1409 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1410 pfault->token, pfault->type, 1411 prefetch_activated); 1412 1413 /* At this point, there might be a new pagefault already arriving in 1414 * the eq, switch to the dummy pagefault for the rest of the 1415 * processing. We're still OK with the objects being alive as the 1416 * work-queue is being fenced. */ 1417 1418 if (prefetch_activated) { 1419 u32 bytes_committed = 0; 1420 1421 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1422 prefetch_len, 1423 &bytes_committed, NULL); 1424 if (ret < 0 && ret != -EAGAIN) { 1425 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1426 ret, pfault->token, address, prefetch_len); 1427 } 1428 } 1429} 1430 1431static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1432{ 1433 u8 event_subtype = pfault->event_subtype; 1434 1435 switch (event_subtype) { 1436 case MLX5_PFAULT_SUBTYPE_WQE: 1437 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1438 break; 1439 case MLX5_PFAULT_SUBTYPE_RDMA: 1440 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1441 break; 1442 default: 1443 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1444 event_subtype); 1445 mlx5_ib_page_fault_resume(dev, pfault, 1); 1446 } 1447} 1448 1449static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1450{ 1451 struct mlx5_pagefault *pfault = container_of(work, 1452 struct mlx5_pagefault, 1453 work); 1454 struct mlx5_ib_pf_eq *eq = pfault->eq; 1455 1456 mlx5_ib_pfault(eq->dev, pfault); 1457 mempool_free(pfault, eq->pool); 1458} 1459 1460static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1461{ 1462 struct mlx5_eqe_page_fault *pf_eqe; 1463 struct mlx5_pagefault *pfault; 1464 struct mlx5_eqe *eqe; 1465 int cc = 0; 1466 1467 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1468 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1469 if (!pfault) { 1470 schedule_work(&eq->work); 1471 break; 1472 } 1473 1474 pf_eqe = &eqe->data.page_fault; 1475 pfault->event_subtype = eqe->sub_type; 1476 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); 1477 1478 mlx5_ib_dbg(eq->dev, 1479 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", 1480 eqe->sub_type, pfault->bytes_committed); 1481 1482 switch (eqe->sub_type) { 1483 case MLX5_PFAULT_SUBTYPE_RDMA: 1484 /* RDMA based event */ 1485 pfault->type = 1486 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1487 pfault->token = 1488 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1489 MLX5_24BIT_MASK; 1490 pfault->rdma.r_key = 1491 be32_to_cpu(pf_eqe->rdma.r_key); 1492 pfault->rdma.packet_size = 1493 be16_to_cpu(pf_eqe->rdma.packet_length); 1494 pfault->rdma.rdma_op_len = 1495 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1496 pfault->rdma.rdma_va = 1497 be64_to_cpu(pf_eqe->rdma.rdma_va); 1498 mlx5_ib_dbg(eq->dev, 1499 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", 1500 pfault->type, pfault->token, 1501 pfault->rdma.r_key); 1502 mlx5_ib_dbg(eq->dev, 1503 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1504 pfault->rdma.rdma_op_len, 1505 pfault->rdma.rdma_va); 1506 break; 1507 1508 case MLX5_PFAULT_SUBTYPE_WQE: 1509 /* WQE based event */ 1510 pfault->type = 1511 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1512 pfault->token = 1513 be32_to_cpu(pf_eqe->wqe.token); 1514 pfault->wqe.wq_num = 1515 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1516 MLX5_24BIT_MASK; 1517 pfault->wqe.wqe_index = 1518 be16_to_cpu(pf_eqe->wqe.wqe_index); 1519 pfault->wqe.packet_size = 1520 be16_to_cpu(pf_eqe->wqe.packet_length); 1521 mlx5_ib_dbg(eq->dev, 1522 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1523 pfault->type, pfault->token, 1524 pfault->wqe.wq_num, 1525 pfault->wqe.wqe_index); 1526 break; 1527 1528 default: 1529 mlx5_ib_warn(eq->dev, 1530 "Unsupported page fault event sub-type: 0x%02hhx\n", 1531 eqe->sub_type); 1532 /* Unsupported page faults should still be 1533 * resolved by the page fault handler 1534 */ 1535 } 1536 1537 pfault->eq = eq; 1538 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1539 queue_work(eq->wq, &pfault->work); 1540 1541 cc = mlx5_eq_update_cc(eq->core, ++cc); 1542 } 1543 1544 mlx5_eq_update_ci(eq->core, cc, 1); 1545} 1546 1547static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1548 void *data) 1549{ 1550 struct mlx5_ib_pf_eq *eq = 1551 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1552 unsigned long flags; 1553 1554 if (spin_trylock_irqsave(&eq->lock, flags)) { 1555 mlx5_ib_eq_pf_process(eq); 1556 spin_unlock_irqrestore(&eq->lock, flags); 1557 } else { 1558 schedule_work(&eq->work); 1559 } 1560 1561 return IRQ_HANDLED; 1562} 1563 1564/* mempool_refill() was proposed but unfortunately wasn't accepted 1565 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1566 * Cheap workaround. 1567 */ 1568static void mempool_refill(mempool_t *pool) 1569{ 1570 while (pool->curr_nr < pool->min_nr) 1571 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1572} 1573 1574static void mlx5_ib_eq_pf_action(struct work_struct *work) 1575{ 1576 struct mlx5_ib_pf_eq *eq = 1577 container_of(work, struct mlx5_ib_pf_eq, work); 1578 1579 mempool_refill(eq->pool); 1580 1581 spin_lock_irq(&eq->lock); 1582 mlx5_ib_eq_pf_process(eq); 1583 spin_unlock_irq(&eq->lock); 1584} 1585 1586enum { 1587 MLX5_IB_NUM_PF_EQE = 0x1000, 1588 MLX5_IB_NUM_PF_DRAIN = 64, 1589}; 1590 1591static int 1592mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1593{ 1594 struct mlx5_eq_param param = {}; 1595 int err; 1596 1597 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1598 spin_lock_init(&eq->lock); 1599 eq->dev = dev; 1600 1601 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1602 sizeof(struct mlx5_pagefault)); 1603 if (!eq->pool) 1604 return -ENOMEM; 1605 1606 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1607 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1608 MLX5_NUM_CMD_EQE); 1609 if (!eq->wq) { 1610 err = -ENOMEM; 1611 goto err_mempool; 1612 } 1613 1614 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1615 param = (struct mlx5_eq_param) { 1616 .irq_index = 0, 1617 .nent = MLX5_IB_NUM_PF_EQE, 1618 }; 1619 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1620 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1621 if (IS_ERR(eq->core)) { 1622 err = PTR_ERR(eq->core); 1623 goto err_wq; 1624 } 1625 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1626 if (err) { 1627 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1628 goto err_eq; 1629 } 1630 1631 return 0; 1632err_eq: 1633 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1634err_wq: 1635 destroy_workqueue(eq->wq); 1636err_mempool: 1637 mempool_destroy(eq->pool); 1638 return err; 1639} 1640 1641static int 1642mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1643{ 1644 int err; 1645 1646 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1647 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1648 cancel_work_sync(&eq->work); 1649 destroy_workqueue(eq->wq); 1650 mempool_destroy(eq->pool); 1651 1652 return err; 1653} 1654 1655void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1656{ 1657 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1658 return; 1659 1660 switch (ent->order - 2) { 1661 case MLX5_IMR_MTT_CACHE_ENTRY: 1662 ent->page = PAGE_SHIFT; 1663 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1664 sizeof(struct mlx5_mtt) / 1665 MLX5_IB_UMR_OCTOWORD; 1666 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1667 ent->limit = 0; 1668 break; 1669 1670 case MLX5_IMR_KSM_CACHE_ENTRY: 1671 ent->page = MLX5_KSM_PAGE_SHIFT; 1672 ent->xlt = mlx5_imr_ksm_entries * 1673 sizeof(struct mlx5_klm) / 1674 MLX5_IB_UMR_OCTOWORD; 1675 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1676 ent->limit = 0; 1677 break; 1678 } 1679} 1680 1681static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1682 .advise_mr = mlx5_ib_advise_mr, 1683}; 1684 1685int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1686{ 1687 int ret = 0; 1688 1689 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1690 return ret; 1691 1692 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1693 1694 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1695 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1696 if (ret) { 1697 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1698 return ret; 1699 } 1700 } 1701 1702 ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1703 1704 return ret; 1705} 1706 1707void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1708{ 1709 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1710 return; 1711 1712 mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1713} 1714 1715int mlx5_ib_odp_init(void) 1716{ 1717 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1718 MLX5_IMR_MTT_BITS); 1719 1720 return 0; 1721} 1722 1723struct prefetch_mr_work { 1724 struct work_struct work; 1725 u32 pf_flags; 1726 u32 num_sge; 1727 struct { 1728 u64 io_virt; 1729 struct mlx5_ib_mr *mr; 1730 size_t length; 1731 } frags[]; 1732}; 1733 1734static void destroy_prefetch_work(struct prefetch_mr_work *work) 1735{ 1736 u32 i; 1737 1738 for (i = 0; i < work->num_sge; ++i) 1739 if (atomic_dec_and_test(&work->frags[i].mr->num_deferred_work)) 1740 wake_up(&work->frags[i].mr->q_deferred_work); 1741 kvfree(work); 1742} 1743 1744static struct mlx5_ib_mr * 1745get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, 1746 u32 lkey) 1747{ 1748 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1749 struct mlx5_core_mkey *mmkey; 1750 struct ib_umem_odp *odp; 1751 struct mlx5_ib_mr *mr; 1752 1753 lockdep_assert_held(&dev->odp_srcu); 1754 1755 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); 1756 if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR) 1757 return NULL; 1758 1759 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1760 1761 if (mr->ibmr.pd != pd) 1762 return NULL; 1763 1764 odp = to_ib_umem_odp(mr->umem); 1765 1766 /* prefetch with write-access must be supported by the MR */ 1767 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1768 !odp->umem.writable) 1769 return NULL; 1770 1771 return mr; 1772} 1773 1774static void mlx5_ib_prefetch_mr_work(struct work_struct *w) 1775{ 1776 struct prefetch_mr_work *work = 1777 container_of(w, struct prefetch_mr_work, work); 1778 struct mlx5_ib_dev *dev; 1779 u32 bytes_mapped = 0; 1780 int srcu_key; 1781 int ret; 1782 u32 i; 1783 1784 /* We rely on IB/core that work is executed if we have num_sge != 0 only. */ 1785 WARN_ON(!work->num_sge); 1786 dev = work->frags[0].mr->dev; 1787 /* SRCU should be held when calling to mlx5_odp_populate_xlt() */ 1788 srcu_key = srcu_read_lock(&dev->odp_srcu); 1789 for (i = 0; i < work->num_sge; ++i) { 1790 ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, 1791 work->frags[i].length, &bytes_mapped, 1792 work->pf_flags); 1793 if (ret <= 0) 1794 continue; 1795 mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret); 1796 } 1797 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1798 1799 destroy_prefetch_work(work); 1800} 1801 1802static bool init_prefetch_work(struct ib_pd *pd, 1803 enum ib_uverbs_advise_mr_advice advice, 1804 u32 pf_flags, struct prefetch_mr_work *work, 1805 struct ib_sge *sg_list, u32 num_sge) 1806{ 1807 u32 i; 1808 1809 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 1810 work->pf_flags = pf_flags; 1811 1812 for (i = 0; i < num_sge; ++i) { 1813 work->frags[i].io_virt = sg_list[i].addr; 1814 work->frags[i].length = sg_list[i].length; 1815 work->frags[i].mr = 1816 get_prefetchable_mr(pd, advice, sg_list[i].lkey); 1817 if (!work->frags[i].mr) { 1818 work->num_sge = i; 1819 return false; 1820 } 1821 1822 /* Keep the MR pointer will valid outside the SRCU */ 1823 atomic_inc(&work->frags[i].mr->num_deferred_work); 1824 } 1825 work->num_sge = num_sge; 1826 return true; 1827} 1828 1829static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, 1830 enum ib_uverbs_advise_mr_advice advice, 1831 u32 pf_flags, struct ib_sge *sg_list, 1832 u32 num_sge) 1833{ 1834 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1835 u32 bytes_mapped = 0; 1836 int srcu_key; 1837 int ret = 0; 1838 u32 i; 1839 1840 srcu_key = srcu_read_lock(&dev->odp_srcu); 1841 for (i = 0; i < num_sge; ++i) { 1842 struct mlx5_ib_mr *mr; 1843 1844 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 1845 if (!mr) { 1846 ret = -ENOENT; 1847 goto out; 1848 } 1849 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, 1850 &bytes_mapped, pf_flags); 1851 if (ret < 0) 1852 goto out; 1853 mlx5_update_odp_stats(mr, prefetch, ret); 1854 } 1855 ret = 0; 1856 1857out: 1858 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1859 return ret; 1860} 1861 1862int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1863 enum ib_uverbs_advise_mr_advice advice, 1864 u32 flags, struct ib_sge *sg_list, u32 num_sge) 1865{ 1866 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1867 u32 pf_flags = 0; 1868 struct prefetch_mr_work *work; 1869 int srcu_key; 1870 1871 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 1872 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 1873 1874 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 1875 pf_flags |= MLX5_PF_FLAGS_SNAPSHOT; 1876 1877 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 1878 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, 1879 num_sge); 1880 1881 work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL); 1882 if (!work) 1883 return -ENOMEM; 1884 1885 srcu_key = srcu_read_lock(&dev->odp_srcu); 1886 if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) { 1887 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1888 destroy_prefetch_work(work); 1889 return -EINVAL; 1890 } 1891 queue_work(system_unbound_wq, &work->work); 1892 srcu_read_unlock(&dev->odp_srcu, srcu_key); 1893 return 0; 1894} 1895