1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2020, Intel Corporation. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 35#include <linux/kref.h> 36#include <linux/random.h> 37#include <linux/debugfs.h> 38#include <linux/export.h> 39#include <linux/delay.h> 40#include <linux/dma-buf.h> 41#include <linux/dma-resv.h> 42#include <rdma/ib_umem_odp.h> 43#include "dm.h" 44#include "mlx5_ib.h" 45#include "umr.h" 46 47enum { 48 MAX_PENDING_REG_MR = 8, 49}; 50 51#define MLX5_UMR_ALIGN 2048 52 53static void 54create_mkey_callback(int status, struct mlx5_async_work *context); 55static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, 56 u64 iova, int access_flags, 57 unsigned int page_size, bool populate); 58 59static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, 60 struct ib_pd *pd) 61{ 62 struct mlx5_ib_dev *dev = to_mdev(pd->device); 63 64 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 65 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 66 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 67 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 68 MLX5_SET(mkc, mkc, lr, 1); 69 70 if (acc & IB_ACCESS_RELAXED_ORDERING) { 71 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) 72 MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); 73 74 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) || 75 (MLX5_CAP_GEN(dev->mdev, 76 relaxed_ordering_read_pci_enabled) && 77 pcie_relaxed_ordering_enabled(dev->mdev->pdev))) 78 MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); 79 } 80 81 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 82 MLX5_SET(mkc, mkc, qpn, 0xffffff); 83 MLX5_SET64(mkc, mkc, start_addr, start_addr); 84} 85 86static void assign_mkey_variant(struct mlx5_ib_dev *dev, u32 *mkey, u32 *in) 87{ 88 u8 key = atomic_inc_return(&dev->mkey_var); 89 void *mkc; 90 91 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 92 MLX5_SET(mkc, mkc, mkey_7_0, key); 93 *mkey = key; 94} 95 96static int mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, 97 struct mlx5_ib_mkey *mkey, u32 *in, int inlen) 98{ 99 int ret; 100 101 assign_mkey_variant(dev, &mkey->key, in); 102 ret = mlx5_core_create_mkey(dev->mdev, &mkey->key, in, inlen); 103 if (!ret) 104 init_waitqueue_head(&mkey->wait); 105 106 return ret; 107} 108 109static int mlx5_ib_create_mkey_cb(struct mlx5r_async_create_mkey *async_create) 110{ 111 struct mlx5_ib_dev *dev = async_create->ent->dev; 112 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 113 size_t outlen = MLX5_ST_SZ_BYTES(create_mkey_out); 114 115 MLX5_SET(create_mkey_in, async_create->in, opcode, 116 MLX5_CMD_OP_CREATE_MKEY); 117 assign_mkey_variant(dev, &async_create->mkey, async_create->in); 118 return mlx5_cmd_exec_cb(&dev->async_ctx, async_create->in, inlen, 119 async_create->out, outlen, create_mkey_callback, 120 &async_create->cb_work); 121} 122 123static int mkey_cache_max_order(struct mlx5_ib_dev *dev); 124static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent); 125 126static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 127{ 128 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))); 129 130 return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key); 131} 132 133static void create_mkey_warn(struct mlx5_ib_dev *dev, int status, void *out) 134{ 135 if (status == -ENXIO) /* core driver is not available */ 136 return; 137 138 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 139 if (status != -EREMOTEIO) /* driver specific failure */ 140 return; 141 142 /* Failed in FW, print cmd out failure details */ 143 mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out); 144} 145 146static int push_mkey_locked(struct mlx5_cache_ent *ent, bool limit_pendings, 147 void *to_store) 148{ 149 XA_STATE(xas, &ent->mkeys, 0); 150 void *curr; 151 152 if (limit_pendings && 153 (ent->reserved - ent->stored) > MAX_PENDING_REG_MR) 154 return -EAGAIN; 155 156 while (1) { 157 /* 158 * This is cmpxchg (NULL, XA_ZERO_ENTRY) however this version 159 * doesn't transparently unlock. Instead we set the xas index to 160 * the current value of reserved every iteration. 161 */ 162 xas_set(&xas, ent->reserved); 163 curr = xas_load(&xas); 164 if (!curr) { 165 if (to_store && ent->stored == ent->reserved) 166 xas_store(&xas, to_store); 167 else 168 xas_store(&xas, XA_ZERO_ENTRY); 169 if (xas_valid(&xas)) { 170 ent->reserved++; 171 if (to_store) { 172 if (ent->stored != ent->reserved) 173 __xa_store(&ent->mkeys, 174 ent->stored, 175 to_store, 176 GFP_KERNEL); 177 ent->stored++; 178 queue_adjust_cache_locked(ent); 179 WRITE_ONCE(ent->dev->cache.last_add, 180 jiffies); 181 } 182 } 183 } 184 xa_unlock_irq(&ent->mkeys); 185 186 /* 187 * Notice xas_nomem() must always be called as it cleans 188 * up any cached allocation. 189 */ 190 if (!xas_nomem(&xas, GFP_KERNEL)) 191 break; 192 xa_lock_irq(&ent->mkeys); 193 } 194 xa_lock_irq(&ent->mkeys); 195 if (xas_error(&xas)) 196 return xas_error(&xas); 197 if (WARN_ON(curr)) 198 return -EINVAL; 199 return 0; 200} 201 202static int push_mkey(struct mlx5_cache_ent *ent, bool limit_pendings, 203 void *to_store) 204{ 205 int ret; 206 207 xa_lock_irq(&ent->mkeys); 208 ret = push_mkey_locked(ent, limit_pendings, to_store); 209 xa_unlock_irq(&ent->mkeys); 210 return ret; 211} 212 213static void undo_push_reserve_mkey(struct mlx5_cache_ent *ent) 214{ 215 void *old; 216 217 ent->reserved--; 218 old = __xa_erase(&ent->mkeys, ent->reserved); 219 WARN_ON(old); 220} 221 222static void push_to_reserved(struct mlx5_cache_ent *ent, u32 mkey) 223{ 224 void *old; 225 226 old = __xa_store(&ent->mkeys, ent->stored, xa_mk_value(mkey), 0); 227 WARN_ON(old); 228 ent->stored++; 229} 230 231static u32 pop_stored_mkey(struct mlx5_cache_ent *ent) 232{ 233 void *old, *xa_mkey; 234 235 ent->stored--; 236 ent->reserved--; 237 238 if (ent->stored == ent->reserved) { 239 xa_mkey = __xa_erase(&ent->mkeys, ent->stored); 240 WARN_ON(!xa_mkey); 241 return (u32)xa_to_value(xa_mkey); 242 } 243 244 xa_mkey = __xa_store(&ent->mkeys, ent->stored, XA_ZERO_ENTRY, 245 GFP_KERNEL); 246 WARN_ON(!xa_mkey || xa_is_err(xa_mkey)); 247 old = __xa_erase(&ent->mkeys, ent->reserved); 248 WARN_ON(old); 249 return (u32)xa_to_value(xa_mkey); 250} 251 252static void create_mkey_callback(int status, struct mlx5_async_work *context) 253{ 254 struct mlx5r_async_create_mkey *mkey_out = 255 container_of(context, struct mlx5r_async_create_mkey, cb_work); 256 struct mlx5_cache_ent *ent = mkey_out->ent; 257 struct mlx5_ib_dev *dev = ent->dev; 258 unsigned long flags; 259 260 if (status) { 261 create_mkey_warn(dev, status, mkey_out->out); 262 kfree(mkey_out); 263 xa_lock_irqsave(&ent->mkeys, flags); 264 undo_push_reserve_mkey(ent); 265 WRITE_ONCE(dev->fill_delay, 1); 266 xa_unlock_irqrestore(&ent->mkeys, flags); 267 mod_timer(&dev->delay_timer, jiffies + HZ); 268 return; 269 } 270 271 mkey_out->mkey |= mlx5_idx_to_mkey( 272 MLX5_GET(create_mkey_out, mkey_out->out, mkey_index)); 273 WRITE_ONCE(dev->cache.last_add, jiffies); 274 275 xa_lock_irqsave(&ent->mkeys, flags); 276 push_to_reserved(ent, mkey_out->mkey); 277 /* If we are doing fill_to_high_water then keep going. */ 278 queue_adjust_cache_locked(ent); 279 xa_unlock_irqrestore(&ent->mkeys, flags); 280 kfree(mkey_out); 281} 282 283static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs) 284{ 285 int ret = 0; 286 287 switch (access_mode) { 288 case MLX5_MKC_ACCESS_MODE_MTT: 289 ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD / 290 sizeof(struct mlx5_mtt)); 291 break; 292 case MLX5_MKC_ACCESS_MODE_KSM: 293 ret = DIV_ROUND_UP(ndescs, MLX5_IB_UMR_OCTOWORD / 294 sizeof(struct mlx5_klm)); 295 break; 296 default: 297 WARN_ON(1); 298 } 299 return ret; 300} 301 302static void set_cache_mkc(struct mlx5_cache_ent *ent, void *mkc) 303{ 304 set_mkc_access_pd_addr_fields(mkc, ent->rb_key.access_flags, 0, 305 ent->dev->umrc.pd); 306 MLX5_SET(mkc, mkc, free, 1); 307 MLX5_SET(mkc, mkc, umr_en, 1); 308 MLX5_SET(mkc, mkc, access_mode_1_0, ent->rb_key.access_mode & 0x3); 309 MLX5_SET(mkc, mkc, access_mode_4_2, 310 (ent->rb_key.access_mode >> 2) & 0x7); 311 312 MLX5_SET(mkc, mkc, translations_octword_size, 313 get_mkc_octo_size(ent->rb_key.access_mode, 314 ent->rb_key.ndescs)); 315 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 316} 317 318/* Asynchronously schedule new MRs to be populated in the cache. */ 319static int add_keys(struct mlx5_cache_ent *ent, unsigned int num) 320{ 321 struct mlx5r_async_create_mkey *async_create; 322 void *mkc; 323 int err = 0; 324 int i; 325 326 for (i = 0; i < num; i++) { 327 async_create = kzalloc(sizeof(struct mlx5r_async_create_mkey), 328 GFP_KERNEL); 329 if (!async_create) 330 return -ENOMEM; 331 mkc = MLX5_ADDR_OF(create_mkey_in, async_create->in, 332 memory_key_mkey_entry); 333 set_cache_mkc(ent, mkc); 334 async_create->ent = ent; 335 336 err = push_mkey(ent, true, NULL); 337 if (err) 338 goto free_async_create; 339 340 err = mlx5_ib_create_mkey_cb(async_create); 341 if (err) { 342 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err); 343 goto err_undo_reserve; 344 } 345 } 346 347 return 0; 348 349err_undo_reserve: 350 xa_lock_irq(&ent->mkeys); 351 undo_push_reserve_mkey(ent); 352 xa_unlock_irq(&ent->mkeys); 353free_async_create: 354 kfree(async_create); 355 return err; 356} 357 358/* Synchronously create a MR in the cache */ 359static int create_cache_mkey(struct mlx5_cache_ent *ent, u32 *mkey) 360{ 361 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 362 void *mkc; 363 u32 *in; 364 int err; 365 366 in = kzalloc(inlen, GFP_KERNEL); 367 if (!in) 368 return -ENOMEM; 369 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 370 set_cache_mkc(ent, mkc); 371 372 err = mlx5_core_create_mkey(ent->dev->mdev, mkey, in, inlen); 373 if (err) 374 goto free_in; 375 376 WRITE_ONCE(ent->dev->cache.last_add, jiffies); 377free_in: 378 kfree(in); 379 return err; 380} 381 382static void remove_cache_mr_locked(struct mlx5_cache_ent *ent) 383{ 384 u32 mkey; 385 386 lockdep_assert_held(&ent->mkeys.xa_lock); 387 if (!ent->stored) 388 return; 389 mkey = pop_stored_mkey(ent); 390 xa_unlock_irq(&ent->mkeys); 391 mlx5_core_destroy_mkey(ent->dev->mdev, mkey); 392 xa_lock_irq(&ent->mkeys); 393} 394 395static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target, 396 bool limit_fill) 397 __acquires(&ent->mkeys) __releases(&ent->mkeys) 398{ 399 int err; 400 401 lockdep_assert_held(&ent->mkeys.xa_lock); 402 403 while (true) { 404 if (limit_fill) 405 target = ent->limit * 2; 406 if (target == ent->reserved) 407 return 0; 408 if (target > ent->reserved) { 409 u32 todo = target - ent->reserved; 410 411 xa_unlock_irq(&ent->mkeys); 412 err = add_keys(ent, todo); 413 if (err == -EAGAIN) 414 usleep_range(3000, 5000); 415 xa_lock_irq(&ent->mkeys); 416 if (err) { 417 if (err != -EAGAIN) 418 return err; 419 } else 420 return 0; 421 } else { 422 remove_cache_mr_locked(ent); 423 } 424 } 425} 426 427static ssize_t size_write(struct file *filp, const char __user *buf, 428 size_t count, loff_t *pos) 429{ 430 struct mlx5_cache_ent *ent = filp->private_data; 431 u32 target; 432 int err; 433 434 err = kstrtou32_from_user(buf, count, 0, &target); 435 if (err) 436 return err; 437 438 /* 439 * Target is the new value of total_mrs the user requests, however we 440 * cannot free MRs that are in use. Compute the target value for stored 441 * mkeys. 442 */ 443 xa_lock_irq(&ent->mkeys); 444 if (target < ent->in_use) { 445 err = -EINVAL; 446 goto err_unlock; 447 } 448 target = target - ent->in_use; 449 if (target < ent->limit || target > ent->limit*2) { 450 err = -EINVAL; 451 goto err_unlock; 452 } 453 err = resize_available_mrs(ent, target, false); 454 if (err) 455 goto err_unlock; 456 xa_unlock_irq(&ent->mkeys); 457 458 return count; 459 460err_unlock: 461 xa_unlock_irq(&ent->mkeys); 462 return err; 463} 464 465static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 466 loff_t *pos) 467{ 468 struct mlx5_cache_ent *ent = filp->private_data; 469 char lbuf[20]; 470 int err; 471 472 err = snprintf(lbuf, sizeof(lbuf), "%ld\n", ent->stored + ent->in_use); 473 if (err < 0) 474 return err; 475 476 return simple_read_from_buffer(buf, count, pos, lbuf, err); 477} 478 479static const struct file_operations size_fops = { 480 .owner = THIS_MODULE, 481 .open = simple_open, 482 .write = size_write, 483 .read = size_read, 484}; 485 486static ssize_t limit_write(struct file *filp, const char __user *buf, 487 size_t count, loff_t *pos) 488{ 489 struct mlx5_cache_ent *ent = filp->private_data; 490 u32 var; 491 int err; 492 493 err = kstrtou32_from_user(buf, count, 0, &var); 494 if (err) 495 return err; 496 497 /* 498 * Upon set we immediately fill the cache to high water mark implied by 499 * the limit. 500 */ 501 xa_lock_irq(&ent->mkeys); 502 ent->limit = var; 503 err = resize_available_mrs(ent, 0, true); 504 xa_unlock_irq(&ent->mkeys); 505 if (err) 506 return err; 507 return count; 508} 509 510static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 511 loff_t *pos) 512{ 513 struct mlx5_cache_ent *ent = filp->private_data; 514 char lbuf[20]; 515 int err; 516 517 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 518 if (err < 0) 519 return err; 520 521 return simple_read_from_buffer(buf, count, pos, lbuf, err); 522} 523 524static const struct file_operations limit_fops = { 525 .owner = THIS_MODULE, 526 .open = simple_open, 527 .write = limit_write, 528 .read = limit_read, 529}; 530 531static bool someone_adding(struct mlx5_mkey_cache *cache) 532{ 533 struct mlx5_cache_ent *ent; 534 struct rb_node *node; 535 bool ret; 536 537 mutex_lock(&cache->rb_lock); 538 for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) { 539 ent = rb_entry(node, struct mlx5_cache_ent, node); 540 xa_lock_irq(&ent->mkeys); 541 ret = ent->stored < ent->limit; 542 xa_unlock_irq(&ent->mkeys); 543 if (ret) { 544 mutex_unlock(&cache->rb_lock); 545 return true; 546 } 547 } 548 mutex_unlock(&cache->rb_lock); 549 return false; 550} 551 552/* 553 * Check if the bucket is outside the high/low water mark and schedule an async 554 * update. The cache refill has hysteresis, once the low water mark is hit it is 555 * refilled up to the high mark. 556 */ 557static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent) 558{ 559 lockdep_assert_held(&ent->mkeys.xa_lock); 560 561 if (ent->disabled || READ_ONCE(ent->dev->fill_delay) || ent->is_tmp) 562 return; 563 if (ent->stored < ent->limit) { 564 ent->fill_to_high_water = true; 565 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); 566 } else if (ent->fill_to_high_water && 567 ent->reserved < 2 * ent->limit) { 568 /* 569 * Once we start populating due to hitting a low water mark 570 * continue until we pass the high water mark. 571 */ 572 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); 573 } else if (ent->stored == 2 * ent->limit) { 574 ent->fill_to_high_water = false; 575 } else if (ent->stored > 2 * ent->limit) { 576 /* Queue deletion of excess entries */ 577 ent->fill_to_high_water = false; 578 if (ent->stored != ent->reserved) 579 queue_delayed_work(ent->dev->cache.wq, &ent->dwork, 580 msecs_to_jiffies(1000)); 581 else 582 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); 583 } 584} 585 586static void __cache_work_func(struct mlx5_cache_ent *ent) 587{ 588 struct mlx5_ib_dev *dev = ent->dev; 589 struct mlx5_mkey_cache *cache = &dev->cache; 590 int err; 591 592 xa_lock_irq(&ent->mkeys); 593 if (ent->disabled) 594 goto out; 595 596 if (ent->fill_to_high_water && ent->reserved < 2 * ent->limit && 597 !READ_ONCE(dev->fill_delay)) { 598 xa_unlock_irq(&ent->mkeys); 599 err = add_keys(ent, 1); 600 xa_lock_irq(&ent->mkeys); 601 if (ent->disabled) 602 goto out; 603 if (err) { 604 /* 605 * EAGAIN only happens if there are pending MRs, so we 606 * will be rescheduled when storing them. The only 607 * failure path here is ENOMEM. 608 */ 609 if (err != -EAGAIN) { 610 mlx5_ib_warn( 611 dev, 612 "add keys command failed, err %d\n", 613 err); 614 queue_delayed_work(cache->wq, &ent->dwork, 615 msecs_to_jiffies(1000)); 616 } 617 } 618 } else if (ent->stored > 2 * ent->limit) { 619 bool need_delay; 620 621 /* 622 * The remove_cache_mr() logic is performed as garbage 623 * collection task. Such task is intended to be run when no 624 * other active processes are running. 625 * 626 * The need_resched() will return TRUE if there are user tasks 627 * to be activated in near future. 628 * 629 * In such case, we don't execute remove_cache_mr() and postpone 630 * the garbage collection work to try to run in next cycle, in 631 * order to free CPU resources to other tasks. 632 */ 633 xa_unlock_irq(&ent->mkeys); 634 need_delay = need_resched() || someone_adding(cache) || 635 !time_after(jiffies, 636 READ_ONCE(cache->last_add) + 300 * HZ); 637 xa_lock_irq(&ent->mkeys); 638 if (ent->disabled) 639 goto out; 640 if (need_delay) { 641 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 642 goto out; 643 } 644 remove_cache_mr_locked(ent); 645 queue_adjust_cache_locked(ent); 646 } 647out: 648 xa_unlock_irq(&ent->mkeys); 649} 650 651static void delayed_cache_work_func(struct work_struct *work) 652{ 653 struct mlx5_cache_ent *ent; 654 655 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 656 __cache_work_func(ent); 657} 658 659static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1, 660 struct mlx5r_cache_rb_key key2) 661{ 662 int res; 663 664 res = key1.ats - key2.ats; 665 if (res) 666 return res; 667 668 res = key1.access_mode - key2.access_mode; 669 if (res) 670 return res; 671 672 res = key1.access_flags - key2.access_flags; 673 if (res) 674 return res; 675 676 /* 677 * keep ndescs the last in the compare table since the find function 678 * searches for an exact match on all properties and only closest 679 * match in size. 680 */ 681 return key1.ndescs - key2.ndescs; 682} 683 684static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache, 685 struct mlx5_cache_ent *ent) 686{ 687 struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL; 688 struct mlx5_cache_ent *cur; 689 int cmp; 690 691 /* Figure out where to put new node */ 692 while (*new) { 693 cur = rb_entry(*new, struct mlx5_cache_ent, node); 694 parent = *new; 695 cmp = cache_ent_key_cmp(cur->rb_key, ent->rb_key); 696 if (cmp > 0) 697 new = &((*new)->rb_left); 698 if (cmp < 0) 699 new = &((*new)->rb_right); 700 if (cmp == 0) { 701 mutex_unlock(&cache->rb_lock); 702 return -EEXIST; 703 } 704 } 705 706 /* Add new node and rebalance tree. */ 707 rb_link_node(&ent->node, parent, new); 708 rb_insert_color(&ent->node, &cache->rb_root); 709 710 return 0; 711} 712 713static struct mlx5_cache_ent * 714mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev, 715 struct mlx5r_cache_rb_key rb_key) 716{ 717 struct rb_node *node = dev->cache.rb_root.rb_node; 718 struct mlx5_cache_ent *cur, *smallest = NULL; 719 int cmp; 720 721 /* 722 * Find the smallest ent with order >= requested_order. 723 */ 724 while (node) { 725 cur = rb_entry(node, struct mlx5_cache_ent, node); 726 cmp = cache_ent_key_cmp(cur->rb_key, rb_key); 727 if (cmp > 0) { 728 smallest = cur; 729 node = node->rb_left; 730 } 731 if (cmp < 0) 732 node = node->rb_right; 733 if (cmp == 0) 734 return cur; 735 } 736 737 return (smallest && 738 smallest->rb_key.access_mode == rb_key.access_mode && 739 smallest->rb_key.access_flags == rb_key.access_flags && 740 smallest->rb_key.ats == rb_key.ats) ? 741 smallest : 742 NULL; 743} 744 745static struct mlx5_ib_mr *_mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 746 struct mlx5_cache_ent *ent, 747 int access_flags) 748{ 749 struct mlx5_ib_mr *mr; 750 int err; 751 752 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 753 if (!mr) 754 return ERR_PTR(-ENOMEM); 755 756 xa_lock_irq(&ent->mkeys); 757 ent->in_use++; 758 759 if (!ent->stored) { 760 queue_adjust_cache_locked(ent); 761 ent->miss++; 762 xa_unlock_irq(&ent->mkeys); 763 err = create_cache_mkey(ent, &mr->mmkey.key); 764 if (err) { 765 xa_lock_irq(&ent->mkeys); 766 ent->in_use--; 767 xa_unlock_irq(&ent->mkeys); 768 kfree(mr); 769 return ERR_PTR(err); 770 } 771 } else { 772 mr->mmkey.key = pop_stored_mkey(ent); 773 queue_adjust_cache_locked(ent); 774 xa_unlock_irq(&ent->mkeys); 775 } 776 mr->mmkey.cache_ent = ent; 777 mr->mmkey.type = MLX5_MKEY_MR; 778 init_waitqueue_head(&mr->mmkey.wait); 779 return mr; 780} 781 782static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, 783 int access_flags) 784{ 785 int ret = 0; 786 787 if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) && 788 MLX5_CAP_GEN(dev->mdev, atomic) && 789 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) 790 ret |= IB_ACCESS_REMOTE_ATOMIC; 791 792 if ((access_flags & IB_ACCESS_RELAXED_ORDERING) && 793 MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) && 794 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) 795 ret |= IB_ACCESS_RELAXED_ORDERING; 796 797 if ((access_flags & IB_ACCESS_RELAXED_ORDERING) && 798 (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) || 799 MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled)) && 800 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) 801 ret |= IB_ACCESS_RELAXED_ORDERING; 802 803 return ret; 804} 805 806struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 807 int access_flags, int access_mode, 808 int ndescs) 809{ 810 struct mlx5r_cache_rb_key rb_key = { 811 .ndescs = ndescs, 812 .access_mode = access_mode, 813 .access_flags = get_unchangeable_access_flags(dev, access_flags) 814 }; 815 struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key); 816 817 if (!ent) 818 return ERR_PTR(-EOPNOTSUPP); 819 820 return _mlx5_mr_cache_alloc(dev, ent, access_flags); 821} 822 823static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent) 824{ 825 u32 mkey; 826 827 cancel_delayed_work(&ent->dwork); 828 xa_lock_irq(&ent->mkeys); 829 while (ent->stored) { 830 mkey = pop_stored_mkey(ent); 831 xa_unlock_irq(&ent->mkeys); 832 mlx5_core_destroy_mkey(dev->mdev, mkey); 833 xa_lock_irq(&ent->mkeys); 834 } 835 xa_unlock_irq(&ent->mkeys); 836} 837 838static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 839{ 840 if (!mlx5_debugfs_root || dev->is_rep) 841 return; 842 843 debugfs_remove_recursive(dev->cache.fs_root); 844 dev->cache.fs_root = NULL; 845} 846 847static void mlx5_mkey_cache_debugfs_add_ent(struct mlx5_ib_dev *dev, 848 struct mlx5_cache_ent *ent) 849{ 850 int order = order_base_2(ent->rb_key.ndescs); 851 struct dentry *dir; 852 853 if (!mlx5_debugfs_root || dev->is_rep) 854 return; 855 856 if (ent->rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM) 857 order = MLX5_IMR_KSM_CACHE_ENTRY + 2; 858 859 sprintf(ent->name, "%d", order); 860 dir = debugfs_create_dir(ent->name, dev->cache.fs_root); 861 debugfs_create_file("size", 0600, dir, ent, &size_fops); 862 debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 863 debugfs_create_ulong("cur", 0400, dir, &ent->stored); 864 debugfs_create_u32("miss", 0600, dir, &ent->miss); 865} 866 867static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev) 868{ 869 struct dentry *dbg_root = mlx5_debugfs_get_dev_root(dev->mdev); 870 struct mlx5_mkey_cache *cache = &dev->cache; 871 872 if (!mlx5_debugfs_root || dev->is_rep) 873 return; 874 875 cache->fs_root = debugfs_create_dir("mr_cache", dbg_root); 876} 877 878static void delay_time_func(struct timer_list *t) 879{ 880 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 881 882 WRITE_ONCE(dev->fill_delay, 0); 883} 884 885struct mlx5_cache_ent * 886mlx5r_cache_create_ent_locked(struct mlx5_ib_dev *dev, 887 struct mlx5r_cache_rb_key rb_key, 888 bool persistent_entry) 889{ 890 struct mlx5_cache_ent *ent; 891 int order; 892 int ret; 893 894 ent = kzalloc(sizeof(*ent), GFP_KERNEL); 895 if (!ent) 896 return ERR_PTR(-ENOMEM); 897 898 xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ); 899 ent->rb_key = rb_key; 900 ent->dev = dev; 901 ent->is_tmp = !persistent_entry; 902 903 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 904 905 ret = mlx5_cache_ent_insert(&dev->cache, ent); 906 if (ret) { 907 kfree(ent); 908 return ERR_PTR(ret); 909 } 910 911 if (persistent_entry) { 912 if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM) 913 order = MLX5_IMR_KSM_CACHE_ENTRY; 914 else 915 order = order_base_2(rb_key.ndescs) - 2; 916 917 if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && 918 !dev->is_rep && mlx5_core_is_pf(dev->mdev) && 919 mlx5r_umr_can_load_pas(dev, 0)) 920 ent->limit = dev->mdev->profile.mr_cache[order].limit; 921 else 922 ent->limit = 0; 923 924 mlx5_mkey_cache_debugfs_add_ent(dev, ent); 925 } else { 926 mod_delayed_work(ent->dev->cache.wq, 927 &ent->dev->cache.remove_ent_dwork, 928 msecs_to_jiffies(30 * 1000)); 929 } 930 931 return ent; 932} 933 934static void remove_ent_work_func(struct work_struct *work) 935{ 936 struct mlx5_mkey_cache *cache; 937 struct mlx5_cache_ent *ent; 938 struct rb_node *cur; 939 940 cache = container_of(work, struct mlx5_mkey_cache, 941 remove_ent_dwork.work); 942 mutex_lock(&cache->rb_lock); 943 cur = rb_last(&cache->rb_root); 944 while (cur) { 945 ent = rb_entry(cur, struct mlx5_cache_ent, node); 946 cur = rb_prev(cur); 947 mutex_unlock(&cache->rb_lock); 948 949 xa_lock_irq(&ent->mkeys); 950 if (!ent->is_tmp) { 951 xa_unlock_irq(&ent->mkeys); 952 mutex_lock(&cache->rb_lock); 953 continue; 954 } 955 xa_unlock_irq(&ent->mkeys); 956 957 clean_keys(ent->dev, ent); 958 mutex_lock(&cache->rb_lock); 959 } 960 mutex_unlock(&cache->rb_lock); 961} 962 963int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) 964{ 965 struct mlx5_mkey_cache *cache = &dev->cache; 966 struct rb_root *root = &dev->cache.rb_root; 967 struct mlx5r_cache_rb_key rb_key = { 968 .access_mode = MLX5_MKC_ACCESS_MODE_MTT, 969 }; 970 struct mlx5_cache_ent *ent; 971 struct rb_node *node; 972 int ret; 973 int i; 974 975 mutex_init(&dev->slow_path_mutex); 976 mutex_init(&dev->cache.rb_lock); 977 dev->cache.rb_root = RB_ROOT; 978 INIT_DELAYED_WORK(&dev->cache.remove_ent_dwork, remove_ent_work_func); 979 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 980 if (!cache->wq) { 981 mlx5_ib_warn(dev, "failed to create work queue\n"); 982 return -ENOMEM; 983 } 984 985 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 986 timer_setup(&dev->delay_timer, delay_time_func, 0); 987 mlx5_mkey_cache_debugfs_init(dev); 988 mutex_lock(&cache->rb_lock); 989 for (i = 0; i <= mkey_cache_max_order(dev); i++) { 990 rb_key.ndescs = 1 << (i + 2); 991 ent = mlx5r_cache_create_ent_locked(dev, rb_key, true); 992 if (IS_ERR(ent)) { 993 ret = PTR_ERR(ent); 994 goto err; 995 } 996 } 997 998 ret = mlx5_odp_init_mkey_cache(dev); 999 if (ret) 1000 goto err; 1001 1002 mutex_unlock(&cache->rb_lock); 1003 for (node = rb_first(root); node; node = rb_next(node)) { 1004 ent = rb_entry(node, struct mlx5_cache_ent, node); 1005 xa_lock_irq(&ent->mkeys); 1006 queue_adjust_cache_locked(ent); 1007 xa_unlock_irq(&ent->mkeys); 1008 } 1009 1010 return 0; 1011 1012err: 1013 mutex_unlock(&cache->rb_lock); 1014 mlx5_mkey_cache_debugfs_cleanup(dev); 1015 mlx5_ib_warn(dev, "failed to create mkey cache entry\n"); 1016 return ret; 1017} 1018 1019void mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) 1020{ 1021 struct rb_root *root = &dev->cache.rb_root; 1022 struct mlx5_cache_ent *ent; 1023 struct rb_node *node; 1024 1025 if (!dev->cache.wq) 1026 return; 1027 1028 mutex_lock(&dev->cache.rb_lock); 1029 cancel_delayed_work(&dev->cache.remove_ent_dwork); 1030 for (node = rb_first(root); node; node = rb_next(node)) { 1031 ent = rb_entry(node, struct mlx5_cache_ent, node); 1032 xa_lock_irq(&ent->mkeys); 1033 ent->disabled = true; 1034 xa_unlock_irq(&ent->mkeys); 1035 cancel_delayed_work(&ent->dwork); 1036 } 1037 mutex_unlock(&dev->cache.rb_lock); 1038 1039 /* 1040 * After all entries are disabled and will not reschedule on WQ, 1041 * flush it and all async commands. 1042 */ 1043 flush_workqueue(dev->cache.wq); 1044 1045 mlx5_mkey_cache_debugfs_cleanup(dev); 1046 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 1047 1048 /* At this point all entries are disabled and have no concurrent work. */ 1049 mutex_lock(&dev->cache.rb_lock); 1050 node = rb_first(root); 1051 while (node) { 1052 ent = rb_entry(node, struct mlx5_cache_ent, node); 1053 node = rb_next(node); 1054 clean_keys(dev, ent); 1055 rb_erase(&ent->node, root); 1056 kfree(ent); 1057 } 1058 mutex_unlock(&dev->cache.rb_lock); 1059 1060 destroy_workqueue(dev->cache.wq); 1061 del_timer_sync(&dev->delay_timer); 1062} 1063 1064struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 1065{ 1066 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1067 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1068 struct mlx5_ib_mr *mr; 1069 void *mkc; 1070 u32 *in; 1071 int err; 1072 1073 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1074 if (!mr) 1075 return ERR_PTR(-ENOMEM); 1076 1077 in = kzalloc(inlen, GFP_KERNEL); 1078 if (!in) { 1079 err = -ENOMEM; 1080 goto err_free; 1081 } 1082 1083 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1084 1085 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 1086 MLX5_SET(mkc, mkc, length64, 1); 1087 set_mkc_access_pd_addr_fields(mkc, acc | IB_ACCESS_RELAXED_ORDERING, 0, 1088 pd); 1089 1090 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1091 if (err) 1092 goto err_in; 1093 1094 kfree(in); 1095 mr->mmkey.type = MLX5_MKEY_MR; 1096 mr->ibmr.lkey = mr->mmkey.key; 1097 mr->ibmr.rkey = mr->mmkey.key; 1098 mr->umem = NULL; 1099 1100 return &mr->ibmr; 1101 1102err_in: 1103 kfree(in); 1104 1105err_free: 1106 kfree(mr); 1107 1108 return ERR_PTR(err); 1109} 1110 1111static int get_octo_len(u64 addr, u64 len, int page_shift) 1112{ 1113 u64 page_size = 1ULL << page_shift; 1114 u64 offset; 1115 int npages; 1116 1117 offset = addr & (page_size - 1); 1118 npages = ALIGN(len + offset, page_size) >> page_shift; 1119 return (npages + 1) / 2; 1120} 1121 1122static int mkey_cache_max_order(struct mlx5_ib_dev *dev) 1123{ 1124 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 1125 return MKEY_CACHE_LAST_STD_ENTRY; 1126 return MLX5_MAX_UMR_SHIFT; 1127} 1128 1129static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 1130 u64 length, int access_flags, u64 iova) 1131{ 1132 mr->ibmr.lkey = mr->mmkey.key; 1133 mr->ibmr.rkey = mr->mmkey.key; 1134 mr->ibmr.length = length; 1135 mr->ibmr.device = &dev->ib_dev; 1136 mr->ibmr.iova = iova; 1137 mr->access_flags = access_flags; 1138} 1139 1140static unsigned int mlx5_umem_dmabuf_default_pgsz(struct ib_umem *umem, 1141 u64 iova) 1142{ 1143 /* 1144 * The alignment of iova has already been checked upon entering 1145 * UVERBS_METHOD_REG_DMABUF_MR 1146 */ 1147 umem->iova = iova; 1148 return PAGE_SIZE; 1149} 1150 1151static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, 1152 struct ib_umem *umem, u64 iova, 1153 int access_flags) 1154{ 1155 struct mlx5r_cache_rb_key rb_key = { 1156 .access_mode = MLX5_MKC_ACCESS_MODE_MTT, 1157 }; 1158 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1159 struct mlx5_cache_ent *ent; 1160 struct mlx5_ib_mr *mr; 1161 unsigned int page_size; 1162 1163 if (umem->is_dmabuf) 1164 page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova); 1165 else 1166 page_size = mlx5_umem_find_best_pgsz(umem, mkc, log_page_size, 1167 0, iova); 1168 if (WARN_ON(!page_size)) 1169 return ERR_PTR(-EINVAL); 1170 1171 rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size); 1172 rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags); 1173 rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags); 1174 ent = mkey_cache_ent_from_rb_key(dev, rb_key); 1175 /* 1176 * If the MR can't come from the cache then synchronously create an uncached 1177 * one. 1178 */ 1179 if (!ent) { 1180 mutex_lock(&dev->slow_path_mutex); 1181 mr = reg_create(pd, umem, iova, access_flags, page_size, false); 1182 mutex_unlock(&dev->slow_path_mutex); 1183 if (IS_ERR(mr)) 1184 return mr; 1185 mr->mmkey.rb_key = rb_key; 1186 return mr; 1187 } 1188 1189 mr = _mlx5_mr_cache_alloc(dev, ent, access_flags); 1190 if (IS_ERR(mr)) 1191 return mr; 1192 1193 mr->ibmr.pd = pd; 1194 mr->umem = umem; 1195 mr->page_shift = order_base_2(page_size); 1196 set_mr_fields(dev, mr, umem->length, access_flags, iova); 1197 1198 return mr; 1199} 1200 1201/* 1202 * If ibmr is NULL it will be allocated by reg_create. 1203 * Else, the given ibmr will be used. 1204 */ 1205static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, 1206 u64 iova, int access_flags, 1207 unsigned int page_size, bool populate) 1208{ 1209 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1210 struct mlx5_ib_mr *mr; 1211 __be64 *pas; 1212 void *mkc; 1213 int inlen; 1214 u32 *in; 1215 int err; 1216 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1217 1218 if (!page_size) 1219 return ERR_PTR(-EINVAL); 1220 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1221 if (!mr) 1222 return ERR_PTR(-ENOMEM); 1223 1224 mr->ibmr.pd = pd; 1225 mr->access_flags = access_flags; 1226 mr->page_shift = order_base_2(page_size); 1227 1228 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1229 if (populate) 1230 inlen += sizeof(*pas) * 1231 roundup(ib_umem_num_dma_blocks(umem, page_size), 2); 1232 in = kvzalloc(inlen, GFP_KERNEL); 1233 if (!in) { 1234 err = -ENOMEM; 1235 goto err_1; 1236 } 1237 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1238 if (populate) { 1239 if (WARN_ON(access_flags & IB_ACCESS_ON_DEMAND)) { 1240 err = -EINVAL; 1241 goto err_2; 1242 } 1243 mlx5_ib_populate_pas(umem, 1UL << mr->page_shift, pas, 1244 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1245 } 1246 1247 /* The pg_access bit allows setting the access flags 1248 * in the page list submitted with the command. 1249 */ 1250 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1251 1252 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1253 set_mkc_access_pd_addr_fields(mkc, access_flags, iova, 1254 populate ? pd : dev->umrc.pd); 1255 MLX5_SET(mkc, mkc, free, !populate); 1256 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1257 MLX5_SET(mkc, mkc, umr_en, 1); 1258 1259 MLX5_SET64(mkc, mkc, len, umem->length); 1260 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1261 MLX5_SET(mkc, mkc, translations_octword_size, 1262 get_octo_len(iova, umem->length, mr->page_shift)); 1263 MLX5_SET(mkc, mkc, log_page_size, mr->page_shift); 1264 if (mlx5_umem_needs_ats(dev, umem, access_flags)) 1265 MLX5_SET(mkc, mkc, ma_translation_mode, 1); 1266 if (populate) { 1267 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1268 get_octo_len(iova, umem->length, mr->page_shift)); 1269 } 1270 1271 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1272 if (err) { 1273 mlx5_ib_warn(dev, "create mkey failed\n"); 1274 goto err_2; 1275 } 1276 mr->mmkey.type = MLX5_MKEY_MR; 1277 mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift); 1278 mr->umem = umem; 1279 set_mr_fields(dev, mr, umem->length, access_flags, iova); 1280 kvfree(in); 1281 1282 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1283 1284 return mr; 1285 1286err_2: 1287 kvfree(in); 1288err_1: 1289 kfree(mr); 1290 return ERR_PTR(err); 1291} 1292 1293static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr, 1294 u64 length, int acc, int mode) 1295{ 1296 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1297 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1298 struct mlx5_ib_mr *mr; 1299 void *mkc; 1300 u32 *in; 1301 int err; 1302 1303 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1304 if (!mr) 1305 return ERR_PTR(-ENOMEM); 1306 1307 in = kzalloc(inlen, GFP_KERNEL); 1308 if (!in) { 1309 err = -ENOMEM; 1310 goto err_free; 1311 } 1312 1313 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1314 1315 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3); 1316 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 1317 MLX5_SET64(mkc, mkc, len, length); 1318 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd); 1319 1320 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1321 if (err) 1322 goto err_in; 1323 1324 kfree(in); 1325 1326 set_mr_fields(dev, mr, length, acc, start_addr); 1327 1328 return &mr->ibmr; 1329 1330err_in: 1331 kfree(in); 1332 1333err_free: 1334 kfree(mr); 1335 1336 return ERR_PTR(err); 1337} 1338 1339int mlx5_ib_advise_mr(struct ib_pd *pd, 1340 enum ib_uverbs_advise_mr_advice advice, 1341 u32 flags, 1342 struct ib_sge *sg_list, 1343 u32 num_sge, 1344 struct uverbs_attr_bundle *attrs) 1345{ 1346 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1347 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1348 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 1349 return -EOPNOTSUPP; 1350 1351 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1352 sg_list, num_sge); 1353} 1354 1355struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1356 struct ib_dm_mr_attr *attr, 1357 struct uverbs_attr_bundle *attrs) 1358{ 1359 struct mlx5_ib_dm *mdm = to_mdm(dm); 1360 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev; 1361 u64 start_addr = mdm->dev_addr + attr->offset; 1362 int mode; 1363 1364 switch (mdm->type) { 1365 case MLX5_IB_UAPI_DM_TYPE_MEMIC: 1366 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS) 1367 return ERR_PTR(-EINVAL); 1368 1369 mode = MLX5_MKC_ACCESS_MODE_MEMIC; 1370 start_addr -= pci_resource_start(dev->pdev, 0); 1371 break; 1372 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 1373 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 1374 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM: 1375 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS) 1376 return ERR_PTR(-EINVAL); 1377 1378 mode = MLX5_MKC_ACCESS_MODE_SW_ICM; 1379 break; 1380 default: 1381 return ERR_PTR(-EINVAL); 1382 } 1383 1384 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length, 1385 attr->access_flags, mode); 1386} 1387 1388static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, 1389 u64 iova, int access_flags) 1390{ 1391 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1392 struct mlx5_ib_mr *mr = NULL; 1393 bool xlt_with_umr; 1394 int err; 1395 1396 xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem->length); 1397 if (xlt_with_umr) { 1398 mr = alloc_cacheable_mr(pd, umem, iova, access_flags); 1399 } else { 1400 unsigned int page_size = mlx5_umem_find_best_pgsz( 1401 umem, mkc, log_page_size, 0, iova); 1402 1403 mutex_lock(&dev->slow_path_mutex); 1404 mr = reg_create(pd, umem, iova, access_flags, page_size, true); 1405 mutex_unlock(&dev->slow_path_mutex); 1406 } 1407 if (IS_ERR(mr)) { 1408 ib_umem_release(umem); 1409 return ERR_CAST(mr); 1410 } 1411 1412 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1413 1414 atomic_add(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); 1415 1416 if (xlt_with_umr) { 1417 /* 1418 * If the MR was created with reg_create then it will be 1419 * configured properly but left disabled. It is safe to go ahead 1420 * and configure it again via UMR while enabling it. 1421 */ 1422 err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); 1423 if (err) { 1424 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1425 return ERR_PTR(err); 1426 } 1427 } 1428 return &mr->ibmr; 1429} 1430 1431static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, 1432 u64 iova, int access_flags, 1433 struct ib_udata *udata) 1434{ 1435 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1436 struct ib_umem_odp *odp; 1437 struct mlx5_ib_mr *mr; 1438 int err; 1439 1440 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1441 return ERR_PTR(-EOPNOTSUPP); 1442 1443 err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); 1444 if (err) 1445 return ERR_PTR(err); 1446 if (!start && length == U64_MAX) { 1447 if (iova != 0) 1448 return ERR_PTR(-EINVAL); 1449 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1450 return ERR_PTR(-EINVAL); 1451 1452 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags); 1453 if (IS_ERR(mr)) 1454 return ERR_CAST(mr); 1455 return &mr->ibmr; 1456 } 1457 1458 /* ODP requires xlt update via umr to work. */ 1459 if (!mlx5r_umr_can_load_pas(dev, length)) 1460 return ERR_PTR(-EINVAL); 1461 1462 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags, 1463 &mlx5_mn_ops); 1464 if (IS_ERR(odp)) 1465 return ERR_CAST(odp); 1466 1467 mr = alloc_cacheable_mr(pd, &odp->umem, iova, access_flags); 1468 if (IS_ERR(mr)) { 1469 ib_umem_release(&odp->umem); 1470 return ERR_CAST(mr); 1471 } 1472 xa_init(&mr->implicit_children); 1473 1474 odp->private = mr; 1475 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); 1476 if (err) 1477 goto err_dereg_mr; 1478 1479 err = mlx5_ib_init_odp_mr(mr); 1480 if (err) 1481 goto err_dereg_mr; 1482 return &mr->ibmr; 1483 1484err_dereg_mr: 1485 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1486 return ERR_PTR(err); 1487} 1488 1489struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1490 u64 iova, int access_flags, 1491 struct ib_udata *udata) 1492{ 1493 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1494 struct ib_umem *umem; 1495 1496 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1497 return ERR_PTR(-EOPNOTSUPP); 1498 1499 mlx5_ib_dbg(dev, "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n", 1500 start, iova, length, access_flags); 1501 1502 if (access_flags & IB_ACCESS_ON_DEMAND) 1503 return create_user_odp_mr(pd, start, length, iova, access_flags, 1504 udata); 1505 umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); 1506 if (IS_ERR(umem)) 1507 return ERR_CAST(umem); 1508 return create_real_mr(pd, umem, iova, access_flags); 1509} 1510 1511static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) 1512{ 1513 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 1514 struct mlx5_ib_mr *mr = umem_dmabuf->private; 1515 1516 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 1517 1518 if (!umem_dmabuf->sgt) 1519 return; 1520 1521 mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); 1522 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 1523} 1524 1525static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = { 1526 .allow_peer2peer = 1, 1527 .move_notify = mlx5_ib_dmabuf_invalidate_cb, 1528}; 1529 1530struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset, 1531 u64 length, u64 virt_addr, 1532 int fd, int access_flags, 1533 struct ib_udata *udata) 1534{ 1535 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1536 struct mlx5_ib_mr *mr = NULL; 1537 struct ib_umem_dmabuf *umem_dmabuf; 1538 int err; 1539 1540 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || 1541 !IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1542 return ERR_PTR(-EOPNOTSUPP); 1543 1544 mlx5_ib_dbg(dev, 1545 "offset 0x%llx, virt_addr 0x%llx, length 0x%llx, fd %d, access_flags 0x%x\n", 1546 offset, virt_addr, length, fd, access_flags); 1547 1548 /* dmabuf requires xlt update via umr to work. */ 1549 if (!mlx5r_umr_can_load_pas(dev, length)) 1550 return ERR_PTR(-EINVAL); 1551 1552 umem_dmabuf = ib_umem_dmabuf_get(&dev->ib_dev, offset, length, fd, 1553 access_flags, 1554 &mlx5_ib_dmabuf_attach_ops); 1555 if (IS_ERR(umem_dmabuf)) { 1556 mlx5_ib_dbg(dev, "umem_dmabuf get failed (%ld)\n", 1557 PTR_ERR(umem_dmabuf)); 1558 return ERR_CAST(umem_dmabuf); 1559 } 1560 1561 mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr, 1562 access_flags); 1563 if (IS_ERR(mr)) { 1564 ib_umem_release(&umem_dmabuf->umem); 1565 return ERR_CAST(mr); 1566 } 1567 1568 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1569 1570 atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); 1571 umem_dmabuf->private = mr; 1572 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); 1573 if (err) 1574 goto err_dereg_mr; 1575 1576 err = mlx5_ib_init_dmabuf_mr(mr); 1577 if (err) 1578 goto err_dereg_mr; 1579 return &mr->ibmr; 1580 1581err_dereg_mr: 1582 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1583 return ERR_PTR(err); 1584} 1585 1586/* 1587 * True if the change in access flags can be done via UMR, only some access 1588 * flags can be updated. 1589 */ 1590static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev, 1591 unsigned int current_access_flags, 1592 unsigned int target_access_flags) 1593{ 1594 unsigned int diffs = current_access_flags ^ target_access_flags; 1595 1596 if (diffs & ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | 1597 IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING)) 1598 return false; 1599 return mlx5r_umr_can_reconfig(dev, current_access_flags, 1600 target_access_flags); 1601} 1602 1603static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr, 1604 struct ib_umem *new_umem, 1605 int new_access_flags, u64 iova, 1606 unsigned long *page_size) 1607{ 1608 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1609 1610 /* We only track the allocated sizes of MRs from the cache */ 1611 if (!mr->mmkey.cache_ent) 1612 return false; 1613 if (!mlx5r_umr_can_load_pas(dev, new_umem->length)) 1614 return false; 1615 1616 *page_size = 1617 mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova); 1618 if (WARN_ON(!*page_size)) 1619 return false; 1620 return (mr->mmkey.cache_ent->rb_key.ndescs) >= 1621 ib_umem_num_dma_blocks(new_umem, *page_size); 1622} 1623 1624static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, 1625 int access_flags, int flags, struct ib_umem *new_umem, 1626 u64 iova, unsigned long page_size) 1627{ 1628 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1629 int upd_flags = MLX5_IB_UPD_XLT_ADDR | MLX5_IB_UPD_XLT_ENABLE; 1630 struct ib_umem *old_umem = mr->umem; 1631 int err; 1632 1633 /* 1634 * To keep everything simple the MR is revoked before we start to mess 1635 * with it. This ensure the change is atomic relative to any use of the 1636 * MR. 1637 */ 1638 err = mlx5r_umr_revoke_mr(mr); 1639 if (err) 1640 return err; 1641 1642 if (flags & IB_MR_REREG_PD) { 1643 mr->ibmr.pd = pd; 1644 upd_flags |= MLX5_IB_UPD_XLT_PD; 1645 } 1646 if (flags & IB_MR_REREG_ACCESS) { 1647 mr->access_flags = access_flags; 1648 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1649 } 1650 1651 mr->ibmr.iova = iova; 1652 mr->ibmr.length = new_umem->length; 1653 mr->page_shift = order_base_2(page_size); 1654 mr->umem = new_umem; 1655 err = mlx5r_umr_update_mr_pas(mr, upd_flags); 1656 if (err) { 1657 /* 1658 * The MR is revoked at this point so there is no issue to free 1659 * new_umem. 1660 */ 1661 mr->umem = old_umem; 1662 return err; 1663 } 1664 1665 atomic_sub(ib_umem_num_pages(old_umem), &dev->mdev->priv.reg_pages); 1666 ib_umem_release(old_umem); 1667 atomic_add(ib_umem_num_pages(new_umem), &dev->mdev->priv.reg_pages); 1668 return 0; 1669} 1670 1671struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1672 u64 length, u64 iova, int new_access_flags, 1673 struct ib_pd *new_pd, 1674 struct ib_udata *udata) 1675{ 1676 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1677 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1678 int err; 1679 1680 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1681 return ERR_PTR(-EOPNOTSUPP); 1682 1683 mlx5_ib_dbg( 1684 dev, 1685 "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n", 1686 start, iova, length, new_access_flags); 1687 1688 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 1689 return ERR_PTR(-EOPNOTSUPP); 1690 1691 if (!(flags & IB_MR_REREG_ACCESS)) 1692 new_access_flags = mr->access_flags; 1693 if (!(flags & IB_MR_REREG_PD)) 1694 new_pd = ib_mr->pd; 1695 1696 if (!(flags & IB_MR_REREG_TRANS)) { 1697 struct ib_umem *umem; 1698 1699 /* Fast path for PD/access change */ 1700 if (can_use_umr_rereg_access(dev, mr->access_flags, 1701 new_access_flags)) { 1702 err = mlx5r_umr_rereg_pd_access(mr, new_pd, 1703 new_access_flags); 1704 if (err) 1705 return ERR_PTR(err); 1706 return NULL; 1707 } 1708 /* DM or ODP MR's don't have a normal umem so we can't re-use it */ 1709 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) 1710 goto recreate; 1711 1712 /* 1713 * Only one active MR can refer to a umem at one time, revoke 1714 * the old MR before assigning the umem to the new one. 1715 */ 1716 err = mlx5r_umr_revoke_mr(mr); 1717 if (err) 1718 return ERR_PTR(err); 1719 umem = mr->umem; 1720 mr->umem = NULL; 1721 atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); 1722 1723 return create_real_mr(new_pd, umem, mr->ibmr.iova, 1724 new_access_flags); 1725 } 1726 1727 /* 1728 * DM doesn't have a PAS list so we can't re-use it, odp/dmabuf does 1729 * but the logic around releasing the umem is different 1730 */ 1731 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) 1732 goto recreate; 1733 1734 if (!(new_access_flags & IB_ACCESS_ON_DEMAND) && 1735 can_use_umr_rereg_access(dev, mr->access_flags, new_access_flags)) { 1736 struct ib_umem *new_umem; 1737 unsigned long page_size; 1738 1739 new_umem = ib_umem_get(&dev->ib_dev, start, length, 1740 new_access_flags); 1741 if (IS_ERR(new_umem)) 1742 return ERR_CAST(new_umem); 1743 1744 /* Fast path for PAS change */ 1745 if (can_use_umr_rereg_pas(mr, new_umem, new_access_flags, iova, 1746 &page_size)) { 1747 err = umr_rereg_pas(mr, new_pd, new_access_flags, flags, 1748 new_umem, iova, page_size); 1749 if (err) { 1750 ib_umem_release(new_umem); 1751 return ERR_PTR(err); 1752 } 1753 return NULL; 1754 } 1755 return create_real_mr(new_pd, new_umem, iova, new_access_flags); 1756 } 1757 1758 /* 1759 * Everything else has no state we can preserve, just create a new MR 1760 * from scratch 1761 */ 1762recreate: 1763 return mlx5_ib_reg_user_mr(new_pd, start, length, iova, 1764 new_access_flags, udata); 1765} 1766 1767static int 1768mlx5_alloc_priv_descs(struct ib_device *device, 1769 struct mlx5_ib_mr *mr, 1770 int ndescs, 1771 int desc_size) 1772{ 1773 struct mlx5_ib_dev *dev = to_mdev(device); 1774 struct device *ddev = &dev->mdev->pdev->dev; 1775 int size = ndescs * desc_size; 1776 int add_size; 1777 int ret; 1778 1779 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1780 if (is_power_of_2(MLX5_UMR_ALIGN) && add_size) { 1781 int end = max_t(int, MLX5_UMR_ALIGN, roundup_pow_of_two(size)); 1782 1783 add_size = min_t(int, end - size, add_size); 1784 } 1785 1786 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1787 if (!mr->descs_alloc) 1788 return -ENOMEM; 1789 1790 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1791 1792 mr->desc_map = dma_map_single(ddev, mr->descs, size, DMA_TO_DEVICE); 1793 if (dma_mapping_error(ddev, mr->desc_map)) { 1794 ret = -ENOMEM; 1795 goto err; 1796 } 1797 1798 return 0; 1799err: 1800 kfree(mr->descs_alloc); 1801 1802 return ret; 1803} 1804 1805static void 1806mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1807{ 1808 if (!mr->umem && mr->descs) { 1809 struct ib_device *device = mr->ibmr.device; 1810 int size = mr->max_descs * mr->desc_size; 1811 struct mlx5_ib_dev *dev = to_mdev(device); 1812 1813 dma_unmap_single(&dev->mdev->pdev->dev, mr->desc_map, size, 1814 DMA_TO_DEVICE); 1815 kfree(mr->descs_alloc); 1816 mr->descs = NULL; 1817 } 1818} 1819 1820static int cache_ent_find_and_store(struct mlx5_ib_dev *dev, 1821 struct mlx5_ib_mr *mr) 1822{ 1823 struct mlx5_mkey_cache *cache = &dev->cache; 1824 struct mlx5_cache_ent *ent; 1825 int ret; 1826 1827 if (mr->mmkey.cache_ent) { 1828 xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1829 mr->mmkey.cache_ent->in_use--; 1830 goto end; 1831 } 1832 1833 mutex_lock(&cache->rb_lock); 1834 ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key); 1835 if (ent) { 1836 if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) { 1837 if (ent->disabled) { 1838 mutex_unlock(&cache->rb_lock); 1839 return -EOPNOTSUPP; 1840 } 1841 mr->mmkey.cache_ent = ent; 1842 xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1843 mutex_unlock(&cache->rb_lock); 1844 goto end; 1845 } 1846 } 1847 1848 ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false); 1849 mutex_unlock(&cache->rb_lock); 1850 if (IS_ERR(ent)) 1851 return PTR_ERR(ent); 1852 1853 mr->mmkey.cache_ent = ent; 1854 xa_lock_irq(&mr->mmkey.cache_ent->mkeys); 1855 1856end: 1857 ret = push_mkey_locked(mr->mmkey.cache_ent, false, 1858 xa_mk_value(mr->mmkey.key)); 1859 xa_unlock_irq(&mr->mmkey.cache_ent->mkeys); 1860 return ret; 1861} 1862 1863int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1864{ 1865 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1866 struct mlx5_ib_dev *dev = to_mdev(ibmr->device); 1867 int rc; 1868 1869 /* 1870 * Any async use of the mr must hold the refcount, once the refcount 1871 * goes to zero no other thread, such as ODP page faults, prefetch, any 1872 * UMR activity, etc can touch the mkey. Thus it is safe to destroy it. 1873 */ 1874 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && 1875 refcount_read(&mr->mmkey.usecount) != 0 && 1876 xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))) 1877 mlx5r_deref_wait_odp_mkey(&mr->mmkey); 1878 1879 if (ibmr->type == IB_MR_TYPE_INTEGRITY) { 1880 xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), 1881 mr->sig, NULL, GFP_KERNEL); 1882 1883 if (mr->mtt_mr) { 1884 rc = mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); 1885 if (rc) 1886 return rc; 1887 mr->mtt_mr = NULL; 1888 } 1889 if (mr->klm_mr) { 1890 rc = mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); 1891 if (rc) 1892 return rc; 1893 mr->klm_mr = NULL; 1894 } 1895 1896 if (mlx5_core_destroy_psv(dev->mdev, 1897 mr->sig->psv_memory.psv_idx)) 1898 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1899 mr->sig->psv_memory.psv_idx); 1900 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) 1901 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1902 mr->sig->psv_wire.psv_idx); 1903 kfree(mr->sig); 1904 mr->sig = NULL; 1905 } 1906 1907 /* Stop DMA */ 1908 if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length)) 1909 if (mlx5r_umr_revoke_mr(mr) || 1910 cache_ent_find_and_store(dev, mr)) 1911 mr->mmkey.cache_ent = NULL; 1912 1913 if (!mr->mmkey.cache_ent) { 1914 rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); 1915 if (rc) 1916 return rc; 1917 } 1918 1919 if (mr->umem) { 1920 bool is_odp = is_odp_mr(mr); 1921 1922 if (!is_odp) 1923 atomic_sub(ib_umem_num_pages(mr->umem), 1924 &dev->mdev->priv.reg_pages); 1925 ib_umem_release(mr->umem); 1926 if (is_odp) 1927 mlx5_ib_free_odp_mr(mr); 1928 } 1929 1930 if (!mr->mmkey.cache_ent) 1931 mlx5_free_priv_descs(mr); 1932 1933 kfree(mr); 1934 return 0; 1935} 1936 1937static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, 1938 int access_mode, int page_shift) 1939{ 1940 void *mkc; 1941 1942 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1943 1944 /* This is only used from the kernel, so setting the PD is OK. */ 1945 set_mkc_access_pd_addr_fields(mkc, IB_ACCESS_RELAXED_ORDERING, 0, pd); 1946 MLX5_SET(mkc, mkc, free, 1); 1947 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1948 MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); 1949 MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); 1950 MLX5_SET(mkc, mkc, umr_en, 1); 1951 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1952} 1953 1954static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1955 int ndescs, int desc_size, int page_shift, 1956 int access_mode, u32 *in, int inlen) 1957{ 1958 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1959 int err; 1960 1961 mr->access_mode = access_mode; 1962 mr->desc_size = desc_size; 1963 mr->max_descs = ndescs; 1964 1965 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); 1966 if (err) 1967 return err; 1968 1969 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); 1970 1971 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1972 if (err) 1973 goto err_free_descs; 1974 1975 mr->mmkey.type = MLX5_MKEY_MR; 1976 mr->ibmr.lkey = mr->mmkey.key; 1977 mr->ibmr.rkey = mr->mmkey.key; 1978 1979 return 0; 1980 1981err_free_descs: 1982 mlx5_free_priv_descs(mr); 1983 return err; 1984} 1985 1986static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, 1987 u32 max_num_sg, u32 max_num_meta_sg, 1988 int desc_size, int access_mode) 1989{ 1990 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1991 int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); 1992 int page_shift = 0; 1993 struct mlx5_ib_mr *mr; 1994 u32 *in; 1995 int err; 1996 1997 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1998 if (!mr) 1999 return ERR_PTR(-ENOMEM); 2000 2001 mr->ibmr.pd = pd; 2002 mr->ibmr.device = pd->device; 2003 2004 in = kzalloc(inlen, GFP_KERNEL); 2005 if (!in) { 2006 err = -ENOMEM; 2007 goto err_free; 2008 } 2009 2010 if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) 2011 page_shift = PAGE_SHIFT; 2012 2013 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, 2014 access_mode, in, inlen); 2015 if (err) 2016 goto err_free_in; 2017 2018 mr->umem = NULL; 2019 kfree(in); 2020 2021 return mr; 2022 2023err_free_in: 2024 kfree(in); 2025err_free: 2026 kfree(mr); 2027 return ERR_PTR(err); 2028} 2029 2030static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2031 int ndescs, u32 *in, int inlen) 2032{ 2033 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), 2034 PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in, 2035 inlen); 2036} 2037 2038static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2039 int ndescs, u32 *in, int inlen) 2040{ 2041 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), 2042 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 2043} 2044 2045static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2046 int max_num_sg, int max_num_meta_sg, 2047 u32 *in, int inlen) 2048{ 2049 struct mlx5_ib_dev *dev = to_mdev(pd->device); 2050 u32 psv_index[2]; 2051 void *mkc; 2052 int err; 2053 2054 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 2055 if (!mr->sig) 2056 return -ENOMEM; 2057 2058 /* create mem & wire PSVs */ 2059 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); 2060 if (err) 2061 goto err_free_sig; 2062 2063 mr->sig->psv_memory.psv_idx = psv_index[0]; 2064 mr->sig->psv_wire.psv_idx = psv_index[1]; 2065 2066 mr->sig->sig_status_checked = true; 2067 mr->sig->sig_err_exists = false; 2068 /* Next UMR, Arm SIGERR */ 2069 ++mr->sig->sigerr_count; 2070 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 2071 sizeof(struct mlx5_klm), 2072 MLX5_MKC_ACCESS_MODE_KLMS); 2073 if (IS_ERR(mr->klm_mr)) { 2074 err = PTR_ERR(mr->klm_mr); 2075 goto err_destroy_psv; 2076 } 2077 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 2078 sizeof(struct mlx5_mtt), 2079 MLX5_MKC_ACCESS_MODE_MTT); 2080 if (IS_ERR(mr->mtt_mr)) { 2081 err = PTR_ERR(mr->mtt_mr); 2082 goto err_free_klm_mr; 2083 } 2084 2085 /* Set bsf descriptors for mkey */ 2086 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2087 MLX5_SET(mkc, mkc, bsf_en, 1); 2088 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 2089 2090 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, 2091 MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 2092 if (err) 2093 goto err_free_mtt_mr; 2094 2095 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), 2096 mr->sig, GFP_KERNEL)); 2097 if (err) 2098 goto err_free_descs; 2099 return 0; 2100 2101err_free_descs: 2102 destroy_mkey(dev, mr); 2103 mlx5_free_priv_descs(mr); 2104err_free_mtt_mr: 2105 mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); 2106 mr->mtt_mr = NULL; 2107err_free_klm_mr: 2108 mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); 2109 mr->klm_mr = NULL; 2110err_destroy_psv: 2111 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) 2112 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 2113 mr->sig->psv_memory.psv_idx); 2114 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) 2115 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 2116 mr->sig->psv_wire.psv_idx); 2117err_free_sig: 2118 kfree(mr->sig); 2119 2120 return err; 2121} 2122 2123static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, 2124 enum ib_mr_type mr_type, u32 max_num_sg, 2125 u32 max_num_meta_sg) 2126{ 2127 struct mlx5_ib_dev *dev = to_mdev(pd->device); 2128 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 2129 int ndescs = ALIGN(max_num_sg, 4); 2130 struct mlx5_ib_mr *mr; 2131 u32 *in; 2132 int err; 2133 2134 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2135 if (!mr) 2136 return ERR_PTR(-ENOMEM); 2137 2138 in = kzalloc(inlen, GFP_KERNEL); 2139 if (!in) { 2140 err = -ENOMEM; 2141 goto err_free; 2142 } 2143 2144 mr->ibmr.device = pd->device; 2145 mr->umem = NULL; 2146 2147 switch (mr_type) { 2148 case IB_MR_TYPE_MEM_REG: 2149 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); 2150 break; 2151 case IB_MR_TYPE_SG_GAPS: 2152 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); 2153 break; 2154 case IB_MR_TYPE_INTEGRITY: 2155 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, 2156 max_num_meta_sg, in, inlen); 2157 break; 2158 default: 2159 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 2160 err = -EINVAL; 2161 } 2162 2163 if (err) 2164 goto err_free_in; 2165 2166 kfree(in); 2167 2168 return &mr->ibmr; 2169 2170err_free_in: 2171 kfree(in); 2172err_free: 2173 kfree(mr); 2174 return ERR_PTR(err); 2175} 2176 2177struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2178 u32 max_num_sg) 2179{ 2180 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); 2181} 2182 2183struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, 2184 u32 max_num_sg, u32 max_num_meta_sg) 2185{ 2186 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, 2187 max_num_meta_sg); 2188} 2189 2190int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2191{ 2192 struct mlx5_ib_dev *dev = to_mdev(ibmw->device); 2193 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 2194 struct mlx5_ib_mw *mw = to_mmw(ibmw); 2195 unsigned int ndescs; 2196 u32 *in = NULL; 2197 void *mkc; 2198 int err; 2199 struct mlx5_ib_alloc_mw req = {}; 2200 struct { 2201 __u32 comp_mask; 2202 __u32 response_length; 2203 } resp = {}; 2204 2205 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 2206 if (err) 2207 return err; 2208 2209 if (req.comp_mask || req.reserved1 || req.reserved2) 2210 return -EOPNOTSUPP; 2211 2212 if (udata->inlen > sizeof(req) && 2213 !ib_is_udata_cleared(udata, sizeof(req), 2214 udata->inlen - sizeof(req))) 2215 return -EOPNOTSUPP; 2216 2217 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 2218 2219 in = kzalloc(inlen, GFP_KERNEL); 2220 if (!in) 2221 return -ENOMEM; 2222 2223 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2224 2225 MLX5_SET(mkc, mkc, free, 1); 2226 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 2227 MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn); 2228 MLX5_SET(mkc, mkc, umr_en, 1); 2229 MLX5_SET(mkc, mkc, lr, 1); 2230 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 2231 MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2))); 2232 MLX5_SET(mkc, mkc, qpn, 0xffffff); 2233 2234 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen); 2235 if (err) 2236 goto free; 2237 2238 mw->mmkey.type = MLX5_MKEY_MW; 2239 ibmw->rkey = mw->mmkey.key; 2240 mw->mmkey.ndescs = ndescs; 2241 2242 resp.response_length = 2243 min(offsetofend(typeof(resp), response_length), udata->outlen); 2244 if (resp.response_length) { 2245 err = ib_copy_to_udata(udata, &resp, resp.response_length); 2246 if (err) 2247 goto free_mkey; 2248 } 2249 2250 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { 2251 err = mlx5r_store_odp_mkey(dev, &mw->mmkey); 2252 if (err) 2253 goto free_mkey; 2254 } 2255 2256 kfree(in); 2257 return 0; 2258 2259free_mkey: 2260 mlx5_core_destroy_mkey(dev->mdev, mw->mmkey.key); 2261free: 2262 kfree(in); 2263 return err; 2264} 2265 2266int mlx5_ib_dealloc_mw(struct ib_mw *mw) 2267{ 2268 struct mlx5_ib_dev *dev = to_mdev(mw->device); 2269 struct mlx5_ib_mw *mmw = to_mmw(mw); 2270 2271 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && 2272 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key))) 2273 /* 2274 * pagefault_single_data_segment() may be accessing mmw 2275 * if the user bound an ODP MR to this MW. 2276 */ 2277 mlx5r_deref_wait_odp_mkey(&mmw->mmkey); 2278 2279 return mlx5_core_destroy_mkey(dev->mdev, mmw->mmkey.key); 2280} 2281 2282int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 2283 struct ib_mr_status *mr_status) 2284{ 2285 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 2286 int ret = 0; 2287 2288 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 2289 pr_err("Invalid status check mask\n"); 2290 ret = -EINVAL; 2291 goto done; 2292 } 2293 2294 mr_status->fail_status = 0; 2295 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 2296 if (!mmr->sig) { 2297 ret = -EINVAL; 2298 pr_err("signature status check requested on a non-signature enabled MR\n"); 2299 goto done; 2300 } 2301 2302 mmr->sig->sig_status_checked = true; 2303 if (!mmr->sig->sig_err_exists) 2304 goto done; 2305 2306 if (ibmr->lkey == mmr->sig->err_item.key) 2307 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 2308 sizeof(mr_status->sig_err)); 2309 else { 2310 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 2311 mr_status->sig_err.sig_err_offset = 0; 2312 mr_status->sig_err.key = mmr->sig->err_item.key; 2313 } 2314 2315 mmr->sig->sig_err_exists = false; 2316 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 2317 } 2318 2319done: 2320 return ret; 2321} 2322 2323static int 2324mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2325 int data_sg_nents, unsigned int *data_sg_offset, 2326 struct scatterlist *meta_sg, int meta_sg_nents, 2327 unsigned int *meta_sg_offset) 2328{ 2329 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2330 unsigned int sg_offset = 0; 2331 int n = 0; 2332 2333 mr->meta_length = 0; 2334 if (data_sg_nents == 1) { 2335 n++; 2336 mr->mmkey.ndescs = 1; 2337 if (data_sg_offset) 2338 sg_offset = *data_sg_offset; 2339 mr->data_length = sg_dma_len(data_sg) - sg_offset; 2340 mr->data_iova = sg_dma_address(data_sg) + sg_offset; 2341 if (meta_sg_nents == 1) { 2342 n++; 2343 mr->meta_ndescs = 1; 2344 if (meta_sg_offset) 2345 sg_offset = *meta_sg_offset; 2346 else 2347 sg_offset = 0; 2348 mr->meta_length = sg_dma_len(meta_sg) - sg_offset; 2349 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; 2350 } 2351 ibmr->length = mr->data_length + mr->meta_length; 2352 } 2353 2354 return n; 2355} 2356 2357static int 2358mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 2359 struct scatterlist *sgl, 2360 unsigned short sg_nents, 2361 unsigned int *sg_offset_p, 2362 struct scatterlist *meta_sgl, 2363 unsigned short meta_sg_nents, 2364 unsigned int *meta_sg_offset_p) 2365{ 2366 struct scatterlist *sg = sgl; 2367 struct mlx5_klm *klms = mr->descs; 2368 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 2369 u32 lkey = mr->ibmr.pd->local_dma_lkey; 2370 int i, j = 0; 2371 2372 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 2373 mr->ibmr.length = 0; 2374 2375 for_each_sg(sgl, sg, sg_nents, i) { 2376 if (unlikely(i >= mr->max_descs)) 2377 break; 2378 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 2379 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 2380 klms[i].key = cpu_to_be32(lkey); 2381 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2382 2383 sg_offset = 0; 2384 } 2385 2386 if (sg_offset_p) 2387 *sg_offset_p = sg_offset; 2388 2389 mr->mmkey.ndescs = i; 2390 mr->data_length = mr->ibmr.length; 2391 2392 if (meta_sg_nents) { 2393 sg = meta_sgl; 2394 sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0; 2395 for_each_sg(meta_sgl, sg, meta_sg_nents, j) { 2396 if (unlikely(i + j >= mr->max_descs)) 2397 break; 2398 klms[i + j].va = cpu_to_be64(sg_dma_address(sg) + 2399 sg_offset); 2400 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - 2401 sg_offset); 2402 klms[i + j].key = cpu_to_be32(lkey); 2403 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2404 2405 sg_offset = 0; 2406 } 2407 if (meta_sg_offset_p) 2408 *meta_sg_offset_p = sg_offset; 2409 2410 mr->meta_ndescs = j; 2411 mr->meta_length = mr->ibmr.length - mr->data_length; 2412 } 2413 2414 return i + j; 2415} 2416 2417static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 2418{ 2419 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2420 __be64 *descs; 2421 2422 if (unlikely(mr->mmkey.ndescs == mr->max_descs)) 2423 return -ENOMEM; 2424 2425 descs = mr->descs; 2426 descs[mr->mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2427 2428 return 0; 2429} 2430 2431static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr) 2432{ 2433 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2434 __be64 *descs; 2435 2436 if (unlikely(mr->mmkey.ndescs + mr->meta_ndescs == mr->max_descs)) 2437 return -ENOMEM; 2438 2439 descs = mr->descs; 2440 descs[mr->mmkey.ndescs + mr->meta_ndescs++] = 2441 cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2442 2443 return 0; 2444} 2445 2446static int 2447mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2448 int data_sg_nents, unsigned int *data_sg_offset, 2449 struct scatterlist *meta_sg, int meta_sg_nents, 2450 unsigned int *meta_sg_offset) 2451{ 2452 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2453 struct mlx5_ib_mr *pi_mr = mr->mtt_mr; 2454 int n; 2455 2456 pi_mr->mmkey.ndescs = 0; 2457 pi_mr->meta_ndescs = 0; 2458 pi_mr->meta_length = 0; 2459 2460 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2461 pi_mr->desc_size * pi_mr->max_descs, 2462 DMA_TO_DEVICE); 2463 2464 pi_mr->ibmr.page_size = ibmr->page_size; 2465 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, 2466 mlx5_set_page); 2467 if (n != data_sg_nents) 2468 return n; 2469 2470 pi_mr->data_iova = pi_mr->ibmr.iova; 2471 pi_mr->data_length = pi_mr->ibmr.length; 2472 pi_mr->ibmr.length = pi_mr->data_length; 2473 ibmr->length = pi_mr->data_length; 2474 2475 if (meta_sg_nents) { 2476 u64 page_mask = ~((u64)ibmr->page_size - 1); 2477 u64 iova = pi_mr->data_iova; 2478 2479 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, 2480 meta_sg_offset, mlx5_set_page_pi); 2481 2482 pi_mr->meta_length = pi_mr->ibmr.length; 2483 /* 2484 * PI address for the HW is the offset of the metadata address 2485 * relative to the first data page address. 2486 * It equals to first data page address + size of data pages + 2487 * metadata offset at the first metadata page 2488 */ 2489 pi_mr->pi_iova = (iova & page_mask) + 2490 pi_mr->mmkey.ndescs * ibmr->page_size + 2491 (pi_mr->ibmr.iova & ~page_mask); 2492 /* 2493 * In order to use one MTT MR for data and metadata, we register 2494 * also the gaps between the end of the data and the start of 2495 * the metadata (the sig MR will verify that the HW will access 2496 * to right addresses). This mapping is safe because we use 2497 * internal mkey for the registration. 2498 */ 2499 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; 2500 pi_mr->ibmr.iova = iova; 2501 ibmr->length += pi_mr->meta_length; 2502 } 2503 2504 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2505 pi_mr->desc_size * pi_mr->max_descs, 2506 DMA_TO_DEVICE); 2507 2508 return n; 2509} 2510 2511static int 2512mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2513 int data_sg_nents, unsigned int *data_sg_offset, 2514 struct scatterlist *meta_sg, int meta_sg_nents, 2515 unsigned int *meta_sg_offset) 2516{ 2517 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2518 struct mlx5_ib_mr *pi_mr = mr->klm_mr; 2519 int n; 2520 2521 pi_mr->mmkey.ndescs = 0; 2522 pi_mr->meta_ndescs = 0; 2523 pi_mr->meta_length = 0; 2524 2525 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2526 pi_mr->desc_size * pi_mr->max_descs, 2527 DMA_TO_DEVICE); 2528 2529 n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, 2530 meta_sg, meta_sg_nents, meta_sg_offset); 2531 2532 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2533 pi_mr->desc_size * pi_mr->max_descs, 2534 DMA_TO_DEVICE); 2535 2536 /* This is zero-based memory region */ 2537 pi_mr->data_iova = 0; 2538 pi_mr->ibmr.iova = 0; 2539 pi_mr->pi_iova = pi_mr->data_length; 2540 ibmr->length = pi_mr->ibmr.length; 2541 2542 return n; 2543} 2544 2545int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2546 int data_sg_nents, unsigned int *data_sg_offset, 2547 struct scatterlist *meta_sg, int meta_sg_nents, 2548 unsigned int *meta_sg_offset) 2549{ 2550 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2551 struct mlx5_ib_mr *pi_mr = NULL; 2552 int n; 2553 2554 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); 2555 2556 mr->mmkey.ndescs = 0; 2557 mr->data_length = 0; 2558 mr->data_iova = 0; 2559 mr->meta_ndescs = 0; 2560 mr->pi_iova = 0; 2561 /* 2562 * As a performance optimization, if possible, there is no need to 2563 * perform UMR operation to register the data/metadata buffers. 2564 * First try to map the sg lists to PA descriptors with local_dma_lkey. 2565 * Fallback to UMR only in case of a failure. 2566 */ 2567 n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2568 data_sg_offset, meta_sg, meta_sg_nents, 2569 meta_sg_offset); 2570 if (n == data_sg_nents + meta_sg_nents) 2571 goto out; 2572 /* 2573 * As a performance optimization, if possible, there is no need to map 2574 * the sg lists to KLM descriptors. First try to map the sg lists to MTT 2575 * descriptors and fallback to KLM only in case of a failure. 2576 * It's more efficient for the HW to work with MTT descriptors 2577 * (especially in high load). 2578 * Use KLM (indirect access) only if it's mandatory. 2579 */ 2580 pi_mr = mr->mtt_mr; 2581 n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2582 data_sg_offset, meta_sg, meta_sg_nents, 2583 meta_sg_offset); 2584 if (n == data_sg_nents + meta_sg_nents) 2585 goto out; 2586 2587 pi_mr = mr->klm_mr; 2588 n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2589 data_sg_offset, meta_sg, meta_sg_nents, 2590 meta_sg_offset); 2591 if (unlikely(n != data_sg_nents + meta_sg_nents)) 2592 return -ENOMEM; 2593 2594out: 2595 /* This is zero-based memory region */ 2596 ibmr->iova = 0; 2597 mr->pi_mr = pi_mr; 2598 if (pi_mr) 2599 ibmr->sig_attrs->meta_length = pi_mr->meta_length; 2600 else 2601 ibmr->sig_attrs->meta_length = mr->meta_length; 2602 2603 return 0; 2604} 2605 2606int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2607 unsigned int *sg_offset) 2608{ 2609 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2610 int n; 2611 2612 mr->mmkey.ndescs = 0; 2613 2614 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 2615 mr->desc_size * mr->max_descs, 2616 DMA_TO_DEVICE); 2617 2618 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 2619 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0, 2620 NULL); 2621 else 2622 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 2623 mlx5_set_page); 2624 2625 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 2626 mr->desc_size * mr->max_descs, 2627 DMA_TO_DEVICE); 2628 2629 return n; 2630} 2631