1/* 2 * Copyright(c) 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48#include <linux/slab.h> 49#include <linux/vmalloc.h> 50#include <rdma/ib_umem.h> 51#include <rdma/rdma_vt.h> 52#include "vt.h" 53#include "mr.h" 54#include "trace.h" 55 56/** 57 * rvt_driver_mr_init - Init MR resources per driver 58 * @rdi: rvt dev struct 59 * 60 * Do any intilization needed when a driver registers with rdmavt. 61 * 62 * Return: 0 on success or errno on failure 63 */ 64int rvt_driver_mr_init(struct rvt_dev_info *rdi) 65{ 66 unsigned int lkey_table_size = rdi->dparms.lkey_table_size; 67 unsigned lk_tab_size; 68 int i; 69 70 /* 71 * The top hfi1_lkey_table_size bits are used to index the 72 * table. The lower 8 bits can be owned by the user (copied from 73 * the LKEY). The remaining bits act as a generation number or tag. 74 */ 75 if (!lkey_table_size) 76 return -EINVAL; 77 78 spin_lock_init(&rdi->lkey_table.lock); 79 80 /* ensure generation is at least 4 bits */ 81 if (lkey_table_size > RVT_MAX_LKEY_TABLE_BITS) { 82 rvt_pr_warn(rdi, "lkey bits %u too large, reduced to %u\n", 83 lkey_table_size, RVT_MAX_LKEY_TABLE_BITS); 84 rdi->dparms.lkey_table_size = RVT_MAX_LKEY_TABLE_BITS; 85 lkey_table_size = rdi->dparms.lkey_table_size; 86 } 87 rdi->lkey_table.max = 1 << lkey_table_size; 88 rdi->lkey_table.shift = 32 - lkey_table_size; 89 lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table); 90 rdi->lkey_table.table = (struct rvt_mregion __rcu **) 91 vmalloc_node(lk_tab_size, rdi->dparms.node); 92 if (!rdi->lkey_table.table) 93 return -ENOMEM; 94 95 RCU_INIT_POINTER(rdi->dma_mr, NULL); 96 for (i = 0; i < rdi->lkey_table.max; i++) 97 RCU_INIT_POINTER(rdi->lkey_table.table[i], NULL); 98 99 rdi->dparms.props.max_mr = rdi->lkey_table.max; 100 return 0; 101} 102 103/** 104 *rvt_mr_exit: clean up MR 105 *@rdi: rvt dev structure 106 * 107 * called when drivers have unregistered or perhaps failed to register with us 108 */ 109void rvt_mr_exit(struct rvt_dev_info *rdi) 110{ 111 if (rdi->dma_mr) 112 rvt_pr_err(rdi, "DMA MR not null!\n"); 113 114 vfree(rdi->lkey_table.table); 115} 116 117static void rvt_deinit_mregion(struct rvt_mregion *mr) 118{ 119 int i = mr->mapsz; 120 121 mr->mapsz = 0; 122 while (i) 123 kfree(mr->map[--i]); 124 percpu_ref_exit(&mr->refcount); 125} 126 127static void __rvt_mregion_complete(struct percpu_ref *ref) 128{ 129 struct rvt_mregion *mr = container_of(ref, struct rvt_mregion, 130 refcount); 131 132 complete(&mr->comp); 133} 134 135static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd, 136 int count, unsigned int percpu_flags) 137{ 138 int m, i = 0; 139 struct rvt_dev_info *dev = ib_to_rvt(pd->device); 140 141 mr->mapsz = 0; 142 m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ; 143 for (; i < m; i++) { 144 mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL, 145 dev->dparms.node); 146 if (!mr->map[i]) 147 goto bail; 148 mr->mapsz++; 149 } 150 init_completion(&mr->comp); 151 /* count returning the ptr to user */ 152 if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete, 153 percpu_flags, GFP_KERNEL)) 154 goto bail; 155 156 atomic_set(&mr->lkey_invalid, 0); 157 mr->pd = pd; 158 mr->max_segs = count; 159 return 0; 160bail: 161 rvt_deinit_mregion(mr); 162 return -ENOMEM; 163} 164 165/** 166 * rvt_alloc_lkey - allocate an lkey 167 * @mr: memory region that this lkey protects 168 * @dma_region: 0->normal key, 1->restricted DMA key 169 * 170 * Returns 0 if successful, otherwise returns -errno. 171 * 172 * Increments mr reference count as required. 173 * 174 * Sets the lkey field mr for non-dma regions. 175 * 176 */ 177static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region) 178{ 179 unsigned long flags; 180 u32 r; 181 u32 n; 182 int ret = 0; 183 struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device); 184 struct rvt_lkey_table *rkt = &dev->lkey_table; 185 186 rvt_get_mr(mr); 187 spin_lock_irqsave(&rkt->lock, flags); 188 189 /* special case for dma_mr lkey == 0 */ 190 if (dma_region) { 191 struct rvt_mregion *tmr; 192 193 tmr = rcu_access_pointer(dev->dma_mr); 194 if (!tmr) { 195 mr->lkey_published = 1; 196 /* Insure published written first */ 197 rcu_assign_pointer(dev->dma_mr, mr); 198 rvt_get_mr(mr); 199 } 200 goto success; 201 } 202 203 /* Find the next available LKEY */ 204 r = rkt->next; 205 n = r; 206 for (;;) { 207 if (!rcu_access_pointer(rkt->table[r])) 208 break; 209 r = (r + 1) & (rkt->max - 1); 210 if (r == n) 211 goto bail; 212 } 213 rkt->next = (r + 1) & (rkt->max - 1); 214 /* 215 * Make sure lkey is never zero which is reserved to indicate an 216 * unrestricted LKEY. 217 */ 218 rkt->gen++; 219 /* 220 * bits are capped to ensure enough bits for generation number 221 */ 222 mr->lkey = (r << (32 - dev->dparms.lkey_table_size)) | 223 ((((1 << (24 - dev->dparms.lkey_table_size)) - 1) & rkt->gen) 224 << 8); 225 if (mr->lkey == 0) { 226 mr->lkey |= 1 << 8; 227 rkt->gen++; 228 } 229 mr->lkey_published = 1; 230 /* Insure published written first */ 231 rcu_assign_pointer(rkt->table[r], mr); 232success: 233 spin_unlock_irqrestore(&rkt->lock, flags); 234out: 235 return ret; 236bail: 237 rvt_put_mr(mr); 238 spin_unlock_irqrestore(&rkt->lock, flags); 239 ret = -ENOMEM; 240 goto out; 241} 242 243/** 244 * rvt_free_lkey - free an lkey 245 * @mr: mr to free from tables 246 */ 247static void rvt_free_lkey(struct rvt_mregion *mr) 248{ 249 unsigned long flags; 250 u32 lkey = mr->lkey; 251 u32 r; 252 struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device); 253 struct rvt_lkey_table *rkt = &dev->lkey_table; 254 int freed = 0; 255 256 spin_lock_irqsave(&rkt->lock, flags); 257 if (!lkey) { 258 if (mr->lkey_published) { 259 mr->lkey_published = 0; 260 /* insure published is written before pointer */ 261 rcu_assign_pointer(dev->dma_mr, NULL); 262 rvt_put_mr(mr); 263 } 264 } else { 265 if (!mr->lkey_published) 266 goto out; 267 r = lkey >> (32 - dev->dparms.lkey_table_size); 268 mr->lkey_published = 0; 269 /* insure published is written before pointer */ 270 rcu_assign_pointer(rkt->table[r], NULL); 271 } 272 freed++; 273out: 274 spin_unlock_irqrestore(&rkt->lock, flags); 275 if (freed) 276 percpu_ref_kill(&mr->refcount); 277} 278 279static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd) 280{ 281 struct rvt_mr *mr; 282 int rval = -ENOMEM; 283 int m; 284 285 /* Allocate struct plus pointers to first level page tables. */ 286 m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ; 287 mr = kzalloc(struct_size(mr, mr.map, m), GFP_KERNEL); 288 if (!mr) 289 goto bail; 290 291 rval = rvt_init_mregion(&mr->mr, pd, count, 0); 292 if (rval) 293 goto bail; 294 /* 295 * ib_reg_phys_mr() will initialize mr->ibmr except for 296 * lkey and rkey. 297 */ 298 rval = rvt_alloc_lkey(&mr->mr, 0); 299 if (rval) 300 goto bail_mregion; 301 mr->ibmr.lkey = mr->mr.lkey; 302 mr->ibmr.rkey = mr->mr.lkey; 303done: 304 return mr; 305 306bail_mregion: 307 rvt_deinit_mregion(&mr->mr); 308bail: 309 kfree(mr); 310 mr = ERR_PTR(rval); 311 goto done; 312} 313 314static void __rvt_free_mr(struct rvt_mr *mr) 315{ 316 rvt_free_lkey(&mr->mr); 317 rvt_deinit_mregion(&mr->mr); 318 kfree(mr); 319} 320 321/** 322 * rvt_get_dma_mr - get a DMA memory region 323 * @pd: protection domain for this memory region 324 * @acc: access flags 325 * 326 * Return: the memory region on success, otherwise returns an errno. 327 */ 328struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc) 329{ 330 struct rvt_mr *mr; 331 struct ib_mr *ret; 332 int rval; 333 334 if (ibpd_to_rvtpd(pd)->user) 335 return ERR_PTR(-EPERM); 336 337 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 338 if (!mr) { 339 ret = ERR_PTR(-ENOMEM); 340 goto bail; 341 } 342 343 rval = rvt_init_mregion(&mr->mr, pd, 0, 0); 344 if (rval) { 345 ret = ERR_PTR(rval); 346 goto bail; 347 } 348 349 rval = rvt_alloc_lkey(&mr->mr, 1); 350 if (rval) { 351 ret = ERR_PTR(rval); 352 goto bail_mregion; 353 } 354 355 mr->mr.access_flags = acc; 356 ret = &mr->ibmr; 357done: 358 return ret; 359 360bail_mregion: 361 rvt_deinit_mregion(&mr->mr); 362bail: 363 kfree(mr); 364 goto done; 365} 366 367/** 368 * rvt_reg_user_mr - register a userspace memory region 369 * @pd: protection domain for this memory region 370 * @start: starting userspace address 371 * @length: length of region to register 372 * @mr_access_flags: access flags for this memory region 373 * @udata: unused by the driver 374 * 375 * Return: the memory region on success, otherwise returns an errno. 376 */ 377struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 378 u64 virt_addr, int mr_access_flags, 379 struct ib_udata *udata) 380{ 381 struct rvt_mr *mr; 382 struct ib_umem *umem; 383 struct sg_page_iter sg_iter; 384 int n, m; 385 struct ib_mr *ret; 386 387 if (length == 0) 388 return ERR_PTR(-EINVAL); 389 390 umem = ib_umem_get(pd->device, start, length, mr_access_flags); 391 if (IS_ERR(umem)) 392 return (void *)umem; 393 394 n = ib_umem_num_pages(umem); 395 396 mr = __rvt_alloc_mr(n, pd); 397 if (IS_ERR(mr)) { 398 ret = (struct ib_mr *)mr; 399 goto bail_umem; 400 } 401 402 mr->mr.user_base = start; 403 mr->mr.iova = virt_addr; 404 mr->mr.length = length; 405 mr->mr.offset = ib_umem_offset(umem); 406 mr->mr.access_flags = mr_access_flags; 407 mr->umem = umem; 408 409 mr->mr.page_shift = PAGE_SHIFT; 410 m = 0; 411 n = 0; 412 for_each_sg_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 413 void *vaddr; 414 415 vaddr = page_address(sg_page_iter_page(&sg_iter)); 416 if (!vaddr) { 417 ret = ERR_PTR(-EINVAL); 418 goto bail_inval; 419 } 420 mr->mr.map[m]->segs[n].vaddr = vaddr; 421 mr->mr.map[m]->segs[n].length = PAGE_SIZE; 422 trace_rvt_mr_user_seg(&mr->mr, m, n, vaddr, PAGE_SIZE); 423 if (++n == RVT_SEGSZ) { 424 m++; 425 n = 0; 426 } 427 } 428 return &mr->ibmr; 429 430bail_inval: 431 __rvt_free_mr(mr); 432 433bail_umem: 434 ib_umem_release(umem); 435 436 return ret; 437} 438 439/** 440 * rvt_dereg_clean_qp_cb - callback from iterator 441 * @qp - the qp 442 * @v - the mregion (as u64) 443 * 444 * This routine fields the callback for all QPs and 445 * for QPs in the same PD as the MR will call the 446 * rvt_qp_mr_clean() to potentially cleanup references. 447 */ 448static void rvt_dereg_clean_qp_cb(struct rvt_qp *qp, u64 v) 449{ 450 struct rvt_mregion *mr = (struct rvt_mregion *)v; 451 452 /* skip PDs that are not ours */ 453 if (mr->pd != qp->ibqp.pd) 454 return; 455 rvt_qp_mr_clean(qp, mr->lkey); 456} 457 458/** 459 * rvt_dereg_clean_qps - find QPs for reference cleanup 460 * @mr - the MR that is being deregistered 461 * 462 * This routine iterates RC QPs looking for references 463 * to the lkey noted in mr. 464 */ 465static void rvt_dereg_clean_qps(struct rvt_mregion *mr) 466{ 467 struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device); 468 469 rvt_qp_iter(rdi, (u64)mr, rvt_dereg_clean_qp_cb); 470} 471 472/** 473 * rvt_check_refs - check references 474 * @mr - the megion 475 * @t - the caller identification 476 * 477 * This routine checks MRs holding a reference during 478 * when being de-registered. 479 * 480 * If the count is non-zero, the code calls a clean routine then 481 * waits for the timeout for the count to zero. 482 */ 483static int rvt_check_refs(struct rvt_mregion *mr, const char *t) 484{ 485 unsigned long timeout; 486 struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device); 487 488 if (mr->lkey) { 489 /* avoid dma mr */ 490 rvt_dereg_clean_qps(mr); 491 /* @mr was indexed on rcu protected @lkey_table */ 492 synchronize_rcu(); 493 } 494 495 timeout = wait_for_completion_timeout(&mr->comp, 5 * HZ); 496 if (!timeout) { 497 rvt_pr_err(rdi, 498 "%s timeout mr %p pd %p lkey %x refcount %ld\n", 499 t, mr, mr->pd, mr->lkey, 500 atomic_long_read(&mr->refcount.data->count)); 501 rvt_get_mr(mr); 502 return -EBUSY; 503 } 504 return 0; 505} 506 507/** 508 * rvt_mr_has_lkey - is MR 509 * @mr - the mregion 510 * @lkey - the lkey 511 */ 512bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey) 513{ 514 return mr && lkey == mr->lkey; 515} 516 517/** 518 * rvt_ss_has_lkey - is mr in sge tests 519 * @ss - the sge state 520 * @lkey 521 * 522 * This code tests for an MR in the indicated 523 * sge state. 524 */ 525bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey) 526{ 527 int i; 528 bool rval = false; 529 530 if (!ss->num_sge) 531 return rval; 532 /* first one */ 533 rval = rvt_mr_has_lkey(ss->sge.mr, lkey); 534 /* any others */ 535 for (i = 0; !rval && i < ss->num_sge - 1; i++) 536 rval = rvt_mr_has_lkey(ss->sg_list[i].mr, lkey); 537 return rval; 538} 539 540/** 541 * rvt_dereg_mr - unregister and free a memory region 542 * @ibmr: the memory region to free 543 * 544 * 545 * Note that this is called to free MRs created by rvt_get_dma_mr() 546 * or rvt_reg_user_mr(). 547 * 548 * Returns 0 on success. 549 */ 550int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 551{ 552 struct rvt_mr *mr = to_imr(ibmr); 553 int ret; 554 555 rvt_free_lkey(&mr->mr); 556 557 rvt_put_mr(&mr->mr); /* will set completion if last */ 558 ret = rvt_check_refs(&mr->mr, __func__); 559 if (ret) 560 goto out; 561 rvt_deinit_mregion(&mr->mr); 562 ib_umem_release(mr->umem); 563 kfree(mr); 564out: 565 return ret; 566} 567 568/** 569 * rvt_alloc_mr - Allocate a memory region usable with the 570 * @pd: protection domain for this memory region 571 * @mr_type: mem region type 572 * @max_num_sg: Max number of segments allowed 573 * 574 * Return: the memory region on success, otherwise return an errno. 575 */ 576struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 577 u32 max_num_sg) 578{ 579 struct rvt_mr *mr; 580 581 if (mr_type != IB_MR_TYPE_MEM_REG) 582 return ERR_PTR(-EINVAL); 583 584 mr = __rvt_alloc_mr(max_num_sg, pd); 585 if (IS_ERR(mr)) 586 return (struct ib_mr *)mr; 587 588 return &mr->ibmr; 589} 590 591/** 592 * rvt_set_page - page assignment function called by ib_sg_to_pages 593 * @ibmr: memory region 594 * @addr: dma address of mapped page 595 * 596 * Return: 0 on success 597 */ 598static int rvt_set_page(struct ib_mr *ibmr, u64 addr) 599{ 600 struct rvt_mr *mr = to_imr(ibmr); 601 u32 ps = 1 << mr->mr.page_shift; 602 u32 mapped_segs = mr->mr.length >> mr->mr.page_shift; 603 int m, n; 604 605 if (unlikely(mapped_segs == mr->mr.max_segs)) 606 return -ENOMEM; 607 608 m = mapped_segs / RVT_SEGSZ; 609 n = mapped_segs % RVT_SEGSZ; 610 mr->mr.map[m]->segs[n].vaddr = (void *)addr; 611 mr->mr.map[m]->segs[n].length = ps; 612 mr->mr.length += ps; 613 trace_rvt_mr_page_seg(&mr->mr, m, n, (void *)addr, ps); 614 615 return 0; 616} 617 618/** 619 * rvt_map_mr_sg - map sg list and set it the memory region 620 * @ibmr: memory region 621 * @sg: dma mapped scatterlist 622 * @sg_nents: number of entries in sg 623 * @sg_offset: offset in bytes into sg 624 * 625 * Overwrite rvt_mr length with mr length calculated by ib_sg_to_pages. 626 * 627 * Return: number of sg elements mapped to the memory region 628 */ 629int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 630 int sg_nents, unsigned int *sg_offset) 631{ 632 struct rvt_mr *mr = to_imr(ibmr); 633 int ret; 634 635 mr->mr.length = 0; 636 mr->mr.page_shift = PAGE_SHIFT; 637 ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rvt_set_page); 638 mr->mr.user_base = ibmr->iova; 639 mr->mr.iova = ibmr->iova; 640 mr->mr.offset = ibmr->iova - (u64)mr->mr.map[0]->segs[0].vaddr; 641 mr->mr.length = (size_t)ibmr->length; 642 trace_rvt_map_mr_sg(ibmr, sg_nents, sg_offset); 643 return ret; 644} 645 646/** 647 * rvt_fast_reg_mr - fast register physical MR 648 * @qp: the queue pair where the work request comes from 649 * @ibmr: the memory region to be registered 650 * @key: updated key for this memory region 651 * @access: access flags for this memory region 652 * 653 * Returns 0 on success. 654 */ 655int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key, 656 int access) 657{ 658 struct rvt_mr *mr = to_imr(ibmr); 659 660 if (qp->ibqp.pd != mr->mr.pd) 661 return -EACCES; 662 663 /* not applicable to dma MR or user MR */ 664 if (!mr->mr.lkey || mr->umem) 665 return -EINVAL; 666 667 if ((key & 0xFFFFFF00) != (mr->mr.lkey & 0xFFFFFF00)) 668 return -EINVAL; 669 670 ibmr->lkey = key; 671 ibmr->rkey = key; 672 mr->mr.lkey = key; 673 mr->mr.access_flags = access; 674 mr->mr.iova = ibmr->iova; 675 atomic_set(&mr->mr.lkey_invalid, 0); 676 677 return 0; 678} 679EXPORT_SYMBOL(rvt_fast_reg_mr); 680 681/** 682 * rvt_invalidate_rkey - invalidate an MR rkey 683 * @qp: queue pair associated with the invalidate op 684 * @rkey: rkey to invalidate 685 * 686 * Returns 0 on success. 687 */ 688int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey) 689{ 690 struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device); 691 struct rvt_lkey_table *rkt = &dev->lkey_table; 692 struct rvt_mregion *mr; 693 694 if (rkey == 0) 695 return -EINVAL; 696 697 rcu_read_lock(); 698 mr = rcu_dereference( 699 rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]); 700 if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd)) 701 goto bail; 702 703 atomic_set(&mr->lkey_invalid, 1); 704 rcu_read_unlock(); 705 return 0; 706 707bail: 708 rcu_read_unlock(); 709 return -EINVAL; 710} 711EXPORT_SYMBOL(rvt_invalidate_rkey); 712 713/** 714 * rvt_sge_adjacent - is isge compressible 715 * @last_sge: last outgoing SGE written 716 * @sge: SGE to check 717 * 718 * If adjacent will update last_sge to add length. 719 * 720 * Return: true if isge is adjacent to last sge 721 */ 722static inline bool rvt_sge_adjacent(struct rvt_sge *last_sge, 723 struct ib_sge *sge) 724{ 725 if (last_sge && sge->lkey == last_sge->mr->lkey && 726 ((uint64_t)(last_sge->vaddr + last_sge->length) == sge->addr)) { 727 if (sge->lkey) { 728 if (unlikely((sge->addr - last_sge->mr->user_base + 729 sge->length > last_sge->mr->length))) 730 return false; /* overrun, caller will catch */ 731 } else { 732 last_sge->length += sge->length; 733 } 734 last_sge->sge_length += sge->length; 735 trace_rvt_sge_adjacent(last_sge, sge); 736 return true; 737 } 738 return false; 739} 740 741/** 742 * rvt_lkey_ok - check IB SGE for validity and initialize 743 * @rkt: table containing lkey to check SGE against 744 * @pd: protection domain 745 * @isge: outgoing internal SGE 746 * @last_sge: last outgoing SGE written 747 * @sge: SGE to check 748 * @acc: access flags 749 * 750 * Check the IB SGE for validity and initialize our internal version 751 * of it. 752 * 753 * Increments the reference count when a new sge is stored. 754 * 755 * Return: 0 if compressed, 1 if added , otherwise returns -errno. 756 */ 757int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd, 758 struct rvt_sge *isge, struct rvt_sge *last_sge, 759 struct ib_sge *sge, int acc) 760{ 761 struct rvt_mregion *mr; 762 unsigned n, m; 763 size_t off; 764 765 /* 766 * We use LKEY == zero for kernel virtual addresses 767 * (see rvt_get_dma_mr()). 768 */ 769 if (sge->lkey == 0) { 770 struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device); 771 772 if (pd->user) 773 return -EINVAL; 774 if (rvt_sge_adjacent(last_sge, sge)) 775 return 0; 776 rcu_read_lock(); 777 mr = rcu_dereference(dev->dma_mr); 778 if (!mr) 779 goto bail; 780 rvt_get_mr(mr); 781 rcu_read_unlock(); 782 783 isge->mr = mr; 784 isge->vaddr = (void *)sge->addr; 785 isge->length = sge->length; 786 isge->sge_length = sge->length; 787 isge->m = 0; 788 isge->n = 0; 789 goto ok; 790 } 791 if (rvt_sge_adjacent(last_sge, sge)) 792 return 0; 793 rcu_read_lock(); 794 mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]); 795 if (!mr) 796 goto bail; 797 rvt_get_mr(mr); 798 if (!READ_ONCE(mr->lkey_published)) 799 goto bail_unref; 800 801 if (unlikely(atomic_read(&mr->lkey_invalid) || 802 mr->lkey != sge->lkey || mr->pd != &pd->ibpd)) 803 goto bail_unref; 804 805 off = sge->addr - mr->user_base; 806 if (unlikely(sge->addr < mr->user_base || 807 off + sge->length > mr->length || 808 (mr->access_flags & acc) != acc)) 809 goto bail_unref; 810 rcu_read_unlock(); 811 812 off += mr->offset; 813 if (mr->page_shift) { 814 /* 815 * page sizes are uniform power of 2 so no loop is necessary 816 * entries_spanned_by_off is the number of times the loop below 817 * would have executed. 818 */ 819 size_t entries_spanned_by_off; 820 821 entries_spanned_by_off = off >> mr->page_shift; 822 off -= (entries_spanned_by_off << mr->page_shift); 823 m = entries_spanned_by_off / RVT_SEGSZ; 824 n = entries_spanned_by_off % RVT_SEGSZ; 825 } else { 826 m = 0; 827 n = 0; 828 while (off >= mr->map[m]->segs[n].length) { 829 off -= mr->map[m]->segs[n].length; 830 n++; 831 if (n >= RVT_SEGSZ) { 832 m++; 833 n = 0; 834 } 835 } 836 } 837 isge->mr = mr; 838 isge->vaddr = mr->map[m]->segs[n].vaddr + off; 839 isge->length = mr->map[m]->segs[n].length - off; 840 isge->sge_length = sge->length; 841 isge->m = m; 842 isge->n = n; 843ok: 844 trace_rvt_sge_new(isge, sge); 845 return 1; 846bail_unref: 847 rvt_put_mr(mr); 848bail: 849 rcu_read_unlock(); 850 return -EINVAL; 851} 852EXPORT_SYMBOL(rvt_lkey_ok); 853 854/** 855 * rvt_rkey_ok - check the IB virtual address, length, and RKEY 856 * @qp: qp for validation 857 * @sge: SGE state 858 * @len: length of data 859 * @vaddr: virtual address to place data 860 * @rkey: rkey to check 861 * @acc: access flags 862 * 863 * Return: 1 if successful, otherwise 0. 864 * 865 * increments the reference count upon success 866 */ 867int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, 868 u32 len, u64 vaddr, u32 rkey, int acc) 869{ 870 struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device); 871 struct rvt_lkey_table *rkt = &dev->lkey_table; 872 struct rvt_mregion *mr; 873 unsigned n, m; 874 size_t off; 875 876 /* 877 * We use RKEY == zero for kernel virtual addresses 878 * (see rvt_get_dma_mr()). 879 */ 880 rcu_read_lock(); 881 if (rkey == 0) { 882 struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd); 883 struct rvt_dev_info *rdi = ib_to_rvt(pd->ibpd.device); 884 885 if (pd->user) 886 goto bail; 887 mr = rcu_dereference(rdi->dma_mr); 888 if (!mr) 889 goto bail; 890 rvt_get_mr(mr); 891 rcu_read_unlock(); 892 893 sge->mr = mr; 894 sge->vaddr = (void *)vaddr; 895 sge->length = len; 896 sge->sge_length = len; 897 sge->m = 0; 898 sge->n = 0; 899 goto ok; 900 } 901 902 mr = rcu_dereference(rkt->table[rkey >> rkt->shift]); 903 if (!mr) 904 goto bail; 905 rvt_get_mr(mr); 906 /* insure mr read is before test */ 907 if (!READ_ONCE(mr->lkey_published)) 908 goto bail_unref; 909 if (unlikely(atomic_read(&mr->lkey_invalid) || 910 mr->lkey != rkey || qp->ibqp.pd != mr->pd)) 911 goto bail_unref; 912 913 off = vaddr - mr->iova; 914 if (unlikely(vaddr < mr->iova || off + len > mr->length || 915 (mr->access_flags & acc) == 0)) 916 goto bail_unref; 917 rcu_read_unlock(); 918 919 off += mr->offset; 920 if (mr->page_shift) { 921 /* 922 * page sizes are uniform power of 2 so no loop is necessary 923 * entries_spanned_by_off is the number of times the loop below 924 * would have executed. 925 */ 926 size_t entries_spanned_by_off; 927 928 entries_spanned_by_off = off >> mr->page_shift; 929 off -= (entries_spanned_by_off << mr->page_shift); 930 m = entries_spanned_by_off / RVT_SEGSZ; 931 n = entries_spanned_by_off % RVT_SEGSZ; 932 } else { 933 m = 0; 934 n = 0; 935 while (off >= mr->map[m]->segs[n].length) { 936 off -= mr->map[m]->segs[n].length; 937 n++; 938 if (n >= RVT_SEGSZ) { 939 m++; 940 n = 0; 941 } 942 } 943 } 944 sge->mr = mr; 945 sge->vaddr = mr->map[m]->segs[n].vaddr + off; 946 sge->length = mr->map[m]->segs[n].length - off; 947 sge->sge_length = len; 948 sge->m = m; 949 sge->n = n; 950ok: 951 return 1; 952bail_unref: 953 rvt_put_mr(mr); 954bail: 955 rcu_read_unlock(); 956 return 0; 957} 958EXPORT_SYMBOL(rvt_rkey_ok); 959