1/* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/platform_device.h> 35#include "hns_roce_device.h" 36#include "hns_roce_hem.h" 37#include "hns_roce_common.h" 38 39#define HEM_INDEX_BUF BIT(0) 40#define HEM_INDEX_L0 BIT(1) 41#define HEM_INDEX_L1 BIT(2) 42struct hns_roce_hem_index { 43 u64 buf; 44 u64 l0; 45 u64 l1; 46 u32 inited; /* indicate which index is available */ 47}; 48 49bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 50{ 51 int hop_num = 0; 52 53 switch (type) { 54 case HEM_TYPE_QPC: 55 hop_num = hr_dev->caps.qpc_hop_num; 56 break; 57 case HEM_TYPE_MTPT: 58 hop_num = hr_dev->caps.mpt_hop_num; 59 break; 60 case HEM_TYPE_CQC: 61 hop_num = hr_dev->caps.cqc_hop_num; 62 break; 63 case HEM_TYPE_SRQC: 64 hop_num = hr_dev->caps.srqc_hop_num; 65 break; 66 case HEM_TYPE_SCCC: 67 hop_num = hr_dev->caps.sccc_hop_num; 68 break; 69 case HEM_TYPE_QPC_TIMER: 70 hop_num = hr_dev->caps.qpc_timer_hop_num; 71 break; 72 case HEM_TYPE_CQC_TIMER: 73 hop_num = hr_dev->caps.cqc_timer_hop_num; 74 break; 75 default: 76 return false; 77 } 78 79 return hop_num ? true : false; 80} 81 82static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, 83 u32 bt_chunk_num, u64 hem_max_num) 84{ 85 u64 start_idx = round_down(hem_idx, bt_chunk_num); 86 u64 check_max_num = start_idx + bt_chunk_num; 87 u64 i; 88 89 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) 90 if (i != hem_idx && hem[i]) 91 return false; 92 93 return true; 94} 95 96static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) 97{ 98 u64 start_idx = round_down(ba_idx, bt_chunk_num); 99 int i; 100 101 for (i = 0; i < bt_chunk_num; i++) 102 if (i != ba_idx && bt[start_idx + i]) 103 return false; 104 105 return true; 106} 107 108static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 109{ 110 if (check_whether_bt_num_3(table_type, hop_num)) 111 return 3; 112 else if (check_whether_bt_num_2(table_type, hop_num)) 113 return 2; 114 else if (check_whether_bt_num_1(table_type, hop_num)) 115 return 1; 116 else 117 return 0; 118} 119 120static int get_hem_table_config(struct hns_roce_dev *hr_dev, 121 struct hns_roce_hem_mhop *mhop, 122 u32 type) 123{ 124 struct device *dev = hr_dev->dev; 125 126 switch (type) { 127 case HEM_TYPE_QPC: 128 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 129 + PAGE_SHIFT); 130 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 131 + PAGE_SHIFT); 132 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 133 mhop->hop_num = hr_dev->caps.qpc_hop_num; 134 break; 135 case HEM_TYPE_MTPT: 136 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 137 + PAGE_SHIFT); 138 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 139 + PAGE_SHIFT); 140 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 141 mhop->hop_num = hr_dev->caps.mpt_hop_num; 142 break; 143 case HEM_TYPE_CQC: 144 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 145 + PAGE_SHIFT); 146 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 147 + PAGE_SHIFT); 148 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 149 mhop->hop_num = hr_dev->caps.cqc_hop_num; 150 break; 151 case HEM_TYPE_SCCC: 152 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 153 + PAGE_SHIFT); 154 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 155 + PAGE_SHIFT); 156 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 157 mhop->hop_num = hr_dev->caps.sccc_hop_num; 158 break; 159 case HEM_TYPE_QPC_TIMER: 160 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 161 + PAGE_SHIFT); 162 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 163 + PAGE_SHIFT); 164 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 165 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 166 break; 167 case HEM_TYPE_CQC_TIMER: 168 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 169 + PAGE_SHIFT); 170 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 171 + PAGE_SHIFT); 172 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 173 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 174 break; 175 case HEM_TYPE_SRQC: 176 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 177 + PAGE_SHIFT); 178 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 179 + PAGE_SHIFT); 180 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 181 mhop->hop_num = hr_dev->caps.srqc_hop_num; 182 break; 183 default: 184 dev_err(dev, "table %u not support multi-hop addressing!\n", 185 type); 186 return -EINVAL; 187 } 188 189 return 0; 190} 191 192int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 193 struct hns_roce_hem_table *table, unsigned long *obj, 194 struct hns_roce_hem_mhop *mhop) 195{ 196 struct device *dev = hr_dev->dev; 197 u32 chunk_ba_num; 198 u32 chunk_size; 199 u32 table_idx; 200 u32 bt_num; 201 202 if (get_hem_table_config(hr_dev, mhop, table->type)) 203 return -EINVAL; 204 205 if (!obj) 206 return 0; 207 208 /* 209 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 210 * MTT/CQE alloc hem for bt pages. 211 */ 212 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 213 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 214 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 215 mhop->bt_chunk_size; 216 table_idx = (*obj & (table->num_obj - 1)) / 217 (chunk_size / table->obj_size); 218 switch (bt_num) { 219 case 3: 220 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 221 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 222 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 223 break; 224 case 2: 225 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 226 mhop->l0_idx = table_idx / chunk_ba_num; 227 break; 228 case 1: 229 mhop->l0_idx = table_idx; 230 break; 231 default: 232 dev_err(dev, "table %u not support hop_num = %u!\n", 233 table->type, mhop->hop_num); 234 return -EINVAL; 235 } 236 if (mhop->l0_idx >= mhop->ba_l0_num) 237 mhop->l0_idx %= mhop->ba_l0_num; 238 239 return 0; 240} 241 242static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 243 int npages, 244 unsigned long hem_alloc_size, 245 gfp_t gfp_mask) 246{ 247 struct hns_roce_hem_chunk *chunk = NULL; 248 struct hns_roce_hem *hem; 249 struct scatterlist *mem; 250 int order; 251 void *buf; 252 253 WARN_ON(gfp_mask & __GFP_HIGHMEM); 254 255 hem = kmalloc(sizeof(*hem), 256 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 257 if (!hem) 258 return NULL; 259 260 INIT_LIST_HEAD(&hem->chunk_list); 261 262 order = get_order(hem_alloc_size); 263 264 while (npages > 0) { 265 if (!chunk) { 266 chunk = kmalloc(sizeof(*chunk), 267 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 268 if (!chunk) 269 goto fail; 270 271 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 272 chunk->npages = 0; 273 chunk->nsg = 0; 274 memset(chunk->buf, 0, sizeof(chunk->buf)); 275 list_add_tail(&chunk->list, &hem->chunk_list); 276 } 277 278 while (1 << order > npages) 279 --order; 280 281 /* 282 * Alloc memory one time. If failed, don't alloc small block 283 * memory, directly return fail. 284 */ 285 mem = &chunk->mem[chunk->npages]; 286 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 287 &sg_dma_address(mem), gfp_mask); 288 if (!buf) 289 goto fail; 290 291 chunk->buf[chunk->npages] = buf; 292 sg_dma_len(mem) = PAGE_SIZE << order; 293 294 ++chunk->npages; 295 ++chunk->nsg; 296 npages -= 1 << order; 297 } 298 299 return hem; 300 301fail: 302 hns_roce_free_hem(hr_dev, hem); 303 return NULL; 304} 305 306void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 307{ 308 struct hns_roce_hem_chunk *chunk, *tmp; 309 int i; 310 311 if (!hem) 312 return; 313 314 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 315 for (i = 0; i < chunk->npages; ++i) 316 dma_free_coherent(hr_dev->dev, 317 sg_dma_len(&chunk->mem[i]), 318 chunk->buf[i], 319 sg_dma_address(&chunk->mem[i])); 320 kfree(chunk); 321 } 322 323 kfree(hem); 324} 325 326static int calc_hem_config(struct hns_roce_dev *hr_dev, 327 struct hns_roce_hem_table *table, unsigned long obj, 328 struct hns_roce_hem_mhop *mhop, 329 struct hns_roce_hem_index *index) 330{ 331 struct ib_device *ibdev = &hr_dev->ib_dev; 332 unsigned long mhop_obj = obj; 333 u32 l0_idx, l1_idx, l2_idx; 334 u32 chunk_ba_num; 335 u32 bt_num; 336 int ret; 337 338 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); 339 if (ret) 340 return ret; 341 342 l0_idx = mhop->l0_idx; 343 l1_idx = mhop->l1_idx; 344 l2_idx = mhop->l2_idx; 345 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 346 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 347 switch (bt_num) { 348 case 3: 349 index->l1 = l0_idx * chunk_ba_num + l1_idx; 350 index->l0 = l0_idx; 351 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + 352 l1_idx * chunk_ba_num + l2_idx; 353 break; 354 case 2: 355 index->l0 = l0_idx; 356 index->buf = l0_idx * chunk_ba_num + l1_idx; 357 break; 358 case 1: 359 index->buf = l0_idx; 360 break; 361 default: 362 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n", 363 table->type, mhop->hop_num); 364 return -EINVAL; 365 } 366 367 if (unlikely(index->buf >= table->num_hem)) { 368 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n", 369 table->type, index->buf, table->num_hem); 370 return -EINVAL; 371 } 372 373 return 0; 374} 375 376static void free_mhop_hem(struct hns_roce_dev *hr_dev, 377 struct hns_roce_hem_table *table, 378 struct hns_roce_hem_mhop *mhop, 379 struct hns_roce_hem_index *index) 380{ 381 u32 bt_size = mhop->bt_chunk_size; 382 struct device *dev = hr_dev->dev; 383 384 if (index->inited & HEM_INDEX_BUF) { 385 hns_roce_free_hem(hr_dev, table->hem[index->buf]); 386 table->hem[index->buf] = NULL; 387 } 388 389 if (index->inited & HEM_INDEX_L1) { 390 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], 391 table->bt_l1_dma_addr[index->l1]); 392 table->bt_l1[index->l1] = NULL; 393 } 394 395 if (index->inited & HEM_INDEX_L0) { 396 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], 397 table->bt_l0_dma_addr[index->l0]); 398 table->bt_l0[index->l0] = NULL; 399 } 400} 401 402static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, 403 struct hns_roce_hem_table *table, 404 struct hns_roce_hem_mhop *mhop, 405 struct hns_roce_hem_index *index) 406{ 407 u32 bt_size = mhop->bt_chunk_size; 408 struct device *dev = hr_dev->dev; 409 struct hns_roce_hem_iter iter; 410 gfp_t flag; 411 u64 bt_ba; 412 u32 size; 413 int ret; 414 415 /* alloc L1 BA's chunk */ 416 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || 417 check_whether_bt_num_2(table->type, mhop->hop_num)) && 418 !table->bt_l0[index->l0]) { 419 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, 420 &table->bt_l0_dma_addr[index->l0], 421 GFP_KERNEL); 422 if (!table->bt_l0[index->l0]) { 423 ret = -ENOMEM; 424 goto out; 425 } 426 index->inited |= HEM_INDEX_L0; 427 } 428 429 /* alloc L2 BA's chunk */ 430 if (check_whether_bt_num_3(table->type, mhop->hop_num) && 431 !table->bt_l1[index->l1]) { 432 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, 433 &table->bt_l1_dma_addr[index->l1], 434 GFP_KERNEL); 435 if (!table->bt_l1[index->l1]) { 436 ret = -ENOMEM; 437 goto err_alloc_hem; 438 } 439 index->inited |= HEM_INDEX_L1; 440 *(table->bt_l0[index->l0] + mhop->l1_idx) = 441 table->bt_l1_dma_addr[index->l1]; 442 } 443 444 /* 445 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 446 * alloc bt space chunk for MTT/CQE. 447 */ 448 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; 449 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; 450 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, 451 size, flag); 452 if (!table->hem[index->buf]) { 453 ret = -ENOMEM; 454 goto err_alloc_hem; 455 } 456 457 index->inited |= HEM_INDEX_BUF; 458 hns_roce_hem_first(table->hem[index->buf], &iter); 459 bt_ba = hns_roce_hem_addr(&iter); 460 if (table->type < HEM_TYPE_MTT) { 461 if (mhop->hop_num == 2) 462 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; 463 else if (mhop->hop_num == 1) 464 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 465 } else if (mhop->hop_num == 2) { 466 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 467 } 468 469 return 0; 470err_alloc_hem: 471 free_mhop_hem(hr_dev, table, mhop, index); 472out: 473 return ret; 474} 475 476static int set_mhop_hem(struct hns_roce_dev *hr_dev, 477 struct hns_roce_hem_table *table, unsigned long obj, 478 struct hns_roce_hem_mhop *mhop, 479 struct hns_roce_hem_index *index) 480{ 481 struct ib_device *ibdev = &hr_dev->ib_dev; 482 int step_idx; 483 int ret = 0; 484 485 if (index->inited & HEM_INDEX_L0) { 486 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); 487 if (ret) { 488 ibdev_err(ibdev, "set HEM step 0 failed!\n"); 489 goto out; 490 } 491 } 492 493 if (index->inited & HEM_INDEX_L1) { 494 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); 495 if (ret) { 496 ibdev_err(ibdev, "set HEM step 1 failed!\n"); 497 goto out; 498 } 499 } 500 501 if (index->inited & HEM_INDEX_BUF) { 502 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) 503 step_idx = 0; 504 else 505 step_idx = mhop->hop_num; 506 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); 507 if (ret) 508 ibdev_err(ibdev, "set HEM step last failed!\n"); 509 } 510out: 511 return ret; 512} 513 514static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 515 struct hns_roce_hem_table *table, 516 unsigned long obj) 517{ 518 struct ib_device *ibdev = &hr_dev->ib_dev; 519 struct hns_roce_hem_index index = {}; 520 struct hns_roce_hem_mhop mhop = {}; 521 int ret; 522 523 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 524 if (ret) { 525 ibdev_err(ibdev, "calc hem config failed!\n"); 526 return ret; 527 } 528 529 mutex_lock(&table->mutex); 530 if (table->hem[index.buf]) { 531 refcount_inc(&table->hem[index.buf]->refcount); 532 goto out; 533 } 534 535 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); 536 if (ret) { 537 ibdev_err(ibdev, "alloc mhop hem failed!\n"); 538 goto out; 539 } 540 541 /* set HEM base address to hardware */ 542 if (table->type < HEM_TYPE_MTT) { 543 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); 544 if (ret) { 545 ibdev_err(ibdev, "set HEM address to HW failed!\n"); 546 goto err_alloc; 547 } 548 } 549 550 refcount_set(&table->hem[index.buf]->refcount, 1); 551 goto out; 552 553err_alloc: 554 free_mhop_hem(hr_dev, table, &mhop, &index); 555out: 556 mutex_unlock(&table->mutex); 557 return ret; 558} 559 560int hns_roce_table_get(struct hns_roce_dev *hr_dev, 561 struct hns_roce_hem_table *table, unsigned long obj) 562{ 563 struct device *dev = hr_dev->dev; 564 unsigned long i; 565 int ret = 0; 566 567 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 568 return hns_roce_table_mhop_get(hr_dev, table, obj); 569 570 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / 571 table->obj_size); 572 573 mutex_lock(&table->mutex); 574 575 if (table->hem[i]) { 576 refcount_inc(&table->hem[i]->refcount); 577 goto out; 578 } 579 580 table->hem[i] = hns_roce_alloc_hem(hr_dev, 581 table->table_chunk_size >> PAGE_SHIFT, 582 table->table_chunk_size, 583 (table->lowmem ? GFP_KERNEL : 584 GFP_HIGHUSER) | __GFP_NOWARN); 585 if (!table->hem[i]) { 586 ret = -ENOMEM; 587 goto out; 588 } 589 590 /* Set HEM base address(128K/page, pa) to Hardware */ 591 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); 592 if (ret) { 593 hns_roce_free_hem(hr_dev, table->hem[i]); 594 table->hem[i] = NULL; 595 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", 596 ret); 597 goto out; 598 } 599 600 refcount_set(&table->hem[i]->refcount, 1); 601out: 602 mutex_unlock(&table->mutex); 603 return ret; 604} 605 606static void clear_mhop_hem(struct hns_roce_dev *hr_dev, 607 struct hns_roce_hem_table *table, unsigned long obj, 608 struct hns_roce_hem_mhop *mhop, 609 struct hns_roce_hem_index *index) 610{ 611 struct ib_device *ibdev = &hr_dev->ib_dev; 612 u32 hop_num = mhop->hop_num; 613 u32 chunk_ba_num; 614 int step_idx; 615 616 index->inited = HEM_INDEX_BUF; 617 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 618 if (check_whether_bt_num_2(table->type, hop_num)) { 619 if (hns_roce_check_hem_null(table->hem, index->buf, 620 chunk_ba_num, table->num_hem)) 621 index->inited |= HEM_INDEX_L0; 622 } else if (check_whether_bt_num_3(table->type, hop_num)) { 623 if (hns_roce_check_hem_null(table->hem, index->buf, 624 chunk_ba_num, table->num_hem)) { 625 index->inited |= HEM_INDEX_L1; 626 if (hns_roce_check_bt_null(table->bt_l1, index->l1, 627 chunk_ba_num)) 628 index->inited |= HEM_INDEX_L0; 629 } 630 } 631 632 if (table->type < HEM_TYPE_MTT) { 633 if (hop_num == HNS_ROCE_HOP_NUM_0) 634 step_idx = 0; 635 else 636 step_idx = hop_num; 637 638 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) 639 ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num); 640 641 if (index->inited & HEM_INDEX_L1) 642 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 643 ibdev_warn(ibdev, "failed to clear HEM step 1.\n"); 644 645 if (index->inited & HEM_INDEX_L0) 646 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 647 ibdev_warn(ibdev, "failed to clear HEM step 0.\n"); 648 } 649} 650 651static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 652 struct hns_roce_hem_table *table, 653 unsigned long obj, 654 int check_refcount) 655{ 656 struct ib_device *ibdev = &hr_dev->ib_dev; 657 struct hns_roce_hem_index index = {}; 658 struct hns_roce_hem_mhop mhop = {}; 659 int ret; 660 661 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 662 if (ret) { 663 ibdev_err(ibdev, "calc hem config failed!\n"); 664 return; 665 } 666 667 if (!check_refcount) 668 mutex_lock(&table->mutex); 669 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, 670 &table->mutex)) 671 return; 672 673 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); 674 free_mhop_hem(hr_dev, table, &mhop, &index); 675 676 mutex_unlock(&table->mutex); 677} 678 679void hns_roce_table_put(struct hns_roce_dev *hr_dev, 680 struct hns_roce_hem_table *table, unsigned long obj) 681{ 682 struct device *dev = hr_dev->dev; 683 unsigned long i; 684 685 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 686 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 687 return; 688 } 689 690 i = (obj & (table->num_obj - 1)) / 691 (table->table_chunk_size / table->obj_size); 692 693 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, 694 &table->mutex)) 695 return; 696 697 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) 698 dev_warn(dev, "failed to clear HEM base address.\n"); 699 700 hns_roce_free_hem(hr_dev, table->hem[i]); 701 table->hem[i] = NULL; 702 703 mutex_unlock(&table->mutex); 704} 705 706void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 707 struct hns_roce_hem_table *table, 708 unsigned long obj, dma_addr_t *dma_handle) 709{ 710 struct hns_roce_hem_chunk *chunk; 711 struct hns_roce_hem_mhop mhop; 712 struct hns_roce_hem *hem; 713 unsigned long mhop_obj = obj; 714 unsigned long obj_per_chunk; 715 unsigned long idx_offset; 716 int offset, dma_offset; 717 void *addr = NULL; 718 u32 hem_idx = 0; 719 int length; 720 int i, j; 721 722 if (!table->lowmem) 723 return NULL; 724 725 mutex_lock(&table->mutex); 726 727 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 728 obj_per_chunk = table->table_chunk_size / table->obj_size; 729 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; 730 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; 731 dma_offset = offset = idx_offset * table->obj_size; 732 } else { 733 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 734 735 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) 736 goto out; 737 /* mtt mhop */ 738 i = mhop.l0_idx; 739 j = mhop.l1_idx; 740 if (mhop.hop_num == 2) 741 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; 742 else if (mhop.hop_num == 1 || 743 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 744 hem_idx = i; 745 746 hem = table->hem[hem_idx]; 747 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % 748 mhop.bt_chunk_size; 749 if (mhop.hop_num == 2) 750 dma_offset = offset = 0; 751 } 752 753 if (!hem) 754 goto out; 755 756 list_for_each_entry(chunk, &hem->chunk_list, list) { 757 for (i = 0; i < chunk->npages; ++i) { 758 length = sg_dma_len(&chunk->mem[i]); 759 if (dma_handle && dma_offset >= 0) { 760 if (length > (u32)dma_offset) 761 *dma_handle = sg_dma_address( 762 &chunk->mem[i]) + dma_offset; 763 dma_offset -= length; 764 } 765 766 if (length > (u32)offset) { 767 addr = chunk->buf[i] + offset; 768 goto out; 769 } 770 offset -= length; 771 } 772 } 773 774out: 775 mutex_unlock(&table->mutex); 776 return addr; 777} 778 779int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 780 struct hns_roce_hem_table *table, u32 type, 781 unsigned long obj_size, unsigned long nobj, 782 int use_lowmem) 783{ 784 unsigned long obj_per_chunk; 785 unsigned long num_hem; 786 787 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 788 table->table_chunk_size = hr_dev->caps.chunk_sz; 789 obj_per_chunk = table->table_chunk_size / obj_size; 790 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 791 792 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 793 if (!table->hem) 794 return -ENOMEM; 795 } else { 796 struct hns_roce_hem_mhop mhop = {}; 797 unsigned long buf_chunk_size; 798 unsigned long bt_chunk_size; 799 unsigned long bt_chunk_num; 800 unsigned long num_bt_l0 = 0; 801 u32 hop_num; 802 803 if (get_hem_table_config(hr_dev, &mhop, type)) 804 return -EINVAL; 805 806 buf_chunk_size = mhop.buf_chunk_size; 807 bt_chunk_size = mhop.bt_chunk_size; 808 num_bt_l0 = mhop.ba_l0_num; 809 hop_num = mhop.hop_num; 810 811 obj_per_chunk = buf_chunk_size / obj_size; 812 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 813 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; 814 if (type >= HEM_TYPE_MTT) 815 num_bt_l0 = bt_chunk_num; 816 817 table->hem = kcalloc(num_hem, sizeof(*table->hem), 818 GFP_KERNEL); 819 if (!table->hem) 820 goto err_kcalloc_hem_buf; 821 822 if (check_whether_bt_num_3(type, hop_num)) { 823 unsigned long num_bt_l1; 824 825 num_bt_l1 = (num_hem + bt_chunk_num - 1) / 826 bt_chunk_num; 827 table->bt_l1 = kcalloc(num_bt_l1, 828 sizeof(*table->bt_l1), 829 GFP_KERNEL); 830 if (!table->bt_l1) 831 goto err_kcalloc_bt_l1; 832 833 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 834 sizeof(*table->bt_l1_dma_addr), 835 GFP_KERNEL); 836 837 if (!table->bt_l1_dma_addr) 838 goto err_kcalloc_l1_dma; 839 } 840 841 if (check_whether_bt_num_2(type, hop_num) || 842 check_whether_bt_num_3(type, hop_num)) { 843 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 844 GFP_KERNEL); 845 if (!table->bt_l0) 846 goto err_kcalloc_bt_l0; 847 848 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 849 sizeof(*table->bt_l0_dma_addr), 850 GFP_KERNEL); 851 if (!table->bt_l0_dma_addr) 852 goto err_kcalloc_l0_dma; 853 } 854 } 855 856 table->type = type; 857 table->num_hem = num_hem; 858 table->num_obj = nobj; 859 table->obj_size = obj_size; 860 table->lowmem = use_lowmem; 861 mutex_init(&table->mutex); 862 863 return 0; 864 865err_kcalloc_l0_dma: 866 kfree(table->bt_l0); 867 table->bt_l0 = NULL; 868 869err_kcalloc_bt_l0: 870 kfree(table->bt_l1_dma_addr); 871 table->bt_l1_dma_addr = NULL; 872 873err_kcalloc_l1_dma: 874 kfree(table->bt_l1); 875 table->bt_l1 = NULL; 876 877err_kcalloc_bt_l1: 878 kfree(table->hem); 879 table->hem = NULL; 880 881err_kcalloc_hem_buf: 882 return -ENOMEM; 883} 884 885static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 886 struct hns_roce_hem_table *table) 887{ 888 struct hns_roce_hem_mhop mhop; 889 u32 buf_chunk_size; 890 u64 obj; 891 int i; 892 893 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 894 return; 895 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 896 mhop.bt_chunk_size; 897 898 for (i = 0; i < table->num_hem; ++i) { 899 obj = i * buf_chunk_size / table->obj_size; 900 if (table->hem[i]) 901 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 902 } 903 904 kfree(table->hem); 905 table->hem = NULL; 906 kfree(table->bt_l1); 907 table->bt_l1 = NULL; 908 kfree(table->bt_l1_dma_addr); 909 table->bt_l1_dma_addr = NULL; 910 kfree(table->bt_l0); 911 table->bt_l0 = NULL; 912 kfree(table->bt_l0_dma_addr); 913 table->bt_l0_dma_addr = NULL; 914} 915 916void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 917 struct hns_roce_hem_table *table) 918{ 919 struct device *dev = hr_dev->dev; 920 unsigned long i; 921 922 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 923 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 924 return; 925 } 926 927 for (i = 0; i < table->num_hem; ++i) 928 if (table->hem[i]) { 929 if (hr_dev->hw->clear_hem(hr_dev, table, 930 i * table->table_chunk_size / table->obj_size, 0)) 931 dev_err(dev, "Clear HEM base address failed.\n"); 932 933 hns_roce_free_hem(hr_dev, table->hem[i]); 934 } 935 936 kfree(table->hem); 937} 938 939void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 940{ 941 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) 942 hns_roce_cleanup_hem_table(hr_dev, 943 &hr_dev->srq_table.table); 944 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 945 if (hr_dev->caps.qpc_timer_entry_sz) 946 hns_roce_cleanup_hem_table(hr_dev, 947 &hr_dev->qpc_timer_table); 948 if (hr_dev->caps.cqc_timer_entry_sz) 949 hns_roce_cleanup_hem_table(hr_dev, 950 &hr_dev->cqc_timer_table); 951 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 952 hns_roce_cleanup_hem_table(hr_dev, 953 &hr_dev->qp_table.sccc_table); 954 if (hr_dev->caps.trrl_entry_sz) 955 hns_roce_cleanup_hem_table(hr_dev, 956 &hr_dev->qp_table.trrl_table); 957 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 958 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 959 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 960} 961 962struct roce_hem_item { 963 struct list_head list; /* link all hems in the same bt level */ 964 struct list_head sibling; /* link all hems in last hop for mtt */ 965 void *addr; 966 dma_addr_t dma_addr; 967 size_t count; /* max ba numbers */ 968 int start; /* start buf offset in this hem */ 969 int end; /* end buf offset in this hem */ 970}; 971 972static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, 973 int start, int end, 974 int count, bool exist_bt, 975 int bt_level) 976{ 977 struct roce_hem_item *hem; 978 979 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 980 if (!hem) 981 return NULL; 982 983 if (exist_bt) { 984 hem->addr = dma_alloc_coherent(hr_dev->dev, 985 count * BA_BYTE_LEN, 986 &hem->dma_addr, GFP_KERNEL); 987 if (!hem->addr) { 988 kfree(hem); 989 return NULL; 990 } 991 } 992 993 hem->count = count; 994 hem->start = start; 995 hem->end = end; 996 INIT_LIST_HEAD(&hem->list); 997 INIT_LIST_HEAD(&hem->sibling); 998 999 return hem; 1000} 1001 1002static void hem_list_free_item(struct hns_roce_dev *hr_dev, 1003 struct roce_hem_item *hem, bool exist_bt) 1004{ 1005 if (exist_bt) 1006 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, 1007 hem->addr, hem->dma_addr); 1008 kfree(hem); 1009} 1010 1011static void hem_list_free_all(struct hns_roce_dev *hr_dev, 1012 struct list_head *head, bool exist_bt) 1013{ 1014 struct roce_hem_item *hem, *temp_hem; 1015 1016 list_for_each_entry_safe(hem, temp_hem, head, list) { 1017 list_del(&hem->list); 1018 hem_list_free_item(hr_dev, hem, exist_bt); 1019 } 1020} 1021 1022static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, 1023 u64 table_addr) 1024{ 1025 *(u64 *)(base_addr) = table_addr; 1026} 1027 1028/* assign L0 table address to hem from root bt */ 1029static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 1030 struct roce_hem_item *hem, void *cpu_addr, 1031 u64 phy_addr) 1032{ 1033 hem->addr = cpu_addr; 1034 hem->dma_addr = (dma_addr_t)phy_addr; 1035} 1036 1037static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, 1038 int offset) 1039{ 1040 return (hem->start <= offset && offset <= hem->end); 1041} 1042 1043static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1044 int page_offset) 1045{ 1046 struct roce_hem_item *hem, *temp_hem; 1047 struct roce_hem_item *found = NULL; 1048 1049 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1050 if (hem_list_page_is_in_range(hem, page_offset)) { 1051 found = hem; 1052 break; 1053 } 1054 } 1055 1056 return found; 1057} 1058 1059static bool hem_list_is_bottom_bt(int hopnum, int bt_level) 1060{ 1061 /* 1062 * hopnum base address table levels 1063 * 0 L0(buf) 1064 * 1 L0 -> buf 1065 * 2 L0 -> L1 -> buf 1066 * 3 L0 -> L1 -> L2 -> buf 1067 */ 1068 return bt_level >= (hopnum ? hopnum - 1 : hopnum); 1069} 1070 1071/** 1072 * calc base address entries num 1073 * @hopnum: num of mutihop addressing 1074 * @bt_level: base address table level 1075 * @unit: ba entries per bt page 1076 */ 1077static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) 1078{ 1079 u32 step; 1080 int max; 1081 int i; 1082 1083 if (hopnum <= bt_level) 1084 return 0; 1085 /* 1086 * hopnum bt_level range 1087 * 1 0 unit 1088 * ------------ 1089 * 2 0 unit * unit 1090 * 2 1 unit 1091 * ------------ 1092 * 3 0 unit * unit * unit 1093 * 3 1 unit * unit 1094 * 3 2 unit 1095 */ 1096 step = 1; 1097 max = hopnum - bt_level; 1098 for (i = 0; i < max; i++) 1099 step = step * unit; 1100 1101 return step; 1102} 1103 1104/** 1105 * calc the root ba entries which could cover all regions 1106 * @regions: buf region array 1107 * @region_cnt: array size of @regions 1108 * @unit: ba entries per bt page 1109 */ 1110int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, 1111 int region_cnt, int unit) 1112{ 1113 struct hns_roce_buf_region *r; 1114 int total = 0; 1115 int step; 1116 int i; 1117 1118 for (i = 0; i < region_cnt; i++) { 1119 r = (struct hns_roce_buf_region *)®ions[i]; 1120 if (r->hopnum > 1) { 1121 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1122 if (step > 0) 1123 total += (r->count + step - 1) / step; 1124 } else { 1125 total += r->count; 1126 } 1127 } 1128 1129 return total; 1130} 1131 1132static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, 1133 const struct hns_roce_buf_region *r, int unit, 1134 int offset, struct list_head *mid_bt, 1135 struct list_head *btm_bt) 1136{ 1137 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1138 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1139 struct roce_hem_item *cur, *pre; 1140 const int hopnum = r->hopnum; 1141 int start_aligned; 1142 int distance; 1143 int ret = 0; 1144 int max_ofs; 1145 int level; 1146 u32 step; 1147 int end; 1148 1149 if (hopnum <= 1) 1150 return 0; 1151 1152 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { 1153 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); 1154 return -EINVAL; 1155 } 1156 1157 if (offset < r->offset) { 1158 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", 1159 offset, r->offset); 1160 return -EINVAL; 1161 } 1162 1163 distance = offset - r->offset; 1164 max_ofs = r->offset + r->count - 1; 1165 for (level = 0; level < hopnum; level++) 1166 INIT_LIST_HEAD(&temp_list[level]); 1167 1168 /* config L1 bt to last bt and link them to corresponding parent */ 1169 for (level = 1; level < hopnum; level++) { 1170 cur = hem_list_search_item(&mid_bt[level], offset); 1171 if (cur) { 1172 hem_ptrs[level] = cur; 1173 continue; 1174 } 1175 1176 step = hem_list_calc_ba_range(hopnum, level, unit); 1177 if (step < 1) { 1178 ret = -EINVAL; 1179 goto err_exit; 1180 } 1181 1182 start_aligned = (distance / step) * step + r->offset; 1183 end = min_t(int, start_aligned + step - 1, max_ofs); 1184 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, 1185 true, level); 1186 if (!cur) { 1187 ret = -ENOMEM; 1188 goto err_exit; 1189 } 1190 hem_ptrs[level] = cur; 1191 list_add(&cur->list, &temp_list[level]); 1192 if (hem_list_is_bottom_bt(hopnum, level)) 1193 list_add(&cur->sibling, &temp_list[0]); 1194 1195 /* link bt to parent bt */ 1196 if (level > 1) { 1197 pre = hem_ptrs[level - 1]; 1198 step = (cur->start - pre->start) / step * BA_BYTE_LEN; 1199 hem_list_link_bt(hr_dev, pre->addr + step, 1200 cur->dma_addr); 1201 } 1202 } 1203 1204 list_splice(&temp_list[0], btm_bt); 1205 for (level = 1; level < hopnum; level++) 1206 list_splice(&temp_list[level], &mid_bt[level]); 1207 1208 return 0; 1209 1210err_exit: 1211 for (level = 1; level < hopnum; level++) 1212 hem_list_free_all(hr_dev, &temp_list[level], true); 1213 1214 return ret; 1215} 1216 1217static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1218 struct hns_roce_hem_list *hem_list, int unit, 1219 const struct hns_roce_buf_region *regions, 1220 int region_cnt) 1221{ 1222 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; 1223 struct roce_hem_item *hem, *temp_hem, *root_hem; 1224 const struct hns_roce_buf_region *r; 1225 struct list_head temp_root; 1226 struct list_head temp_btm; 1227 void *cpu_base; 1228 u64 phy_base; 1229 int ret = 0; 1230 int ba_num; 1231 int offset; 1232 int total; 1233 int step; 1234 int i; 1235 1236 r = ®ions[0]; 1237 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); 1238 if (root_hem) 1239 return 0; 1240 1241 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); 1242 if (ba_num < 1) 1243 return -ENOMEM; 1244 1245 INIT_LIST_HEAD(&temp_root); 1246 offset = r->offset; 1247 /* indicate to last region */ 1248 r = ®ions[region_cnt - 1]; 1249 root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1250 ba_num, true, 0); 1251 if (!root_hem) 1252 return -ENOMEM; 1253 list_add(&root_hem->list, &temp_root); 1254 1255 hem_list->root_ba = root_hem->dma_addr; 1256 1257 INIT_LIST_HEAD(&temp_btm); 1258 for (i = 0; i < region_cnt; i++) 1259 INIT_LIST_HEAD(&temp_list[i]); 1260 1261 total = 0; 1262 for (i = 0; i < region_cnt && total < ba_num; i++) { 1263 r = ®ions[i]; 1264 if (!r->count) 1265 continue; 1266 1267 /* all regions's mid[x][0] shared the root_bt's trunk */ 1268 cpu_base = root_hem->addr + total * BA_BYTE_LEN; 1269 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; 1270 1271 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1272 * which's address share to all regions. 1273 */ 1274 if (hem_list_is_bottom_bt(r->hopnum, 0)) { 1275 hem = hem_list_alloc_item(hr_dev, r->offset, 1276 r->offset + r->count - 1, 1277 r->count, false, 0); 1278 if (!hem) { 1279 ret = -ENOMEM; 1280 goto err_exit; 1281 } 1282 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1283 list_add(&hem->list, &temp_list[i]); 1284 list_add(&hem->sibling, &temp_btm); 1285 total += r->count; 1286 } else { 1287 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1288 if (step < 1) { 1289 ret = -EINVAL; 1290 goto err_exit; 1291 } 1292 /* if exist mid bt, link L1 to L0 */ 1293 list_for_each_entry_safe(hem, temp_hem, 1294 &hem_list->mid_bt[i][1], list) { 1295 offset = (hem->start - r->offset) / step * 1296 BA_BYTE_LEN; 1297 hem_list_link_bt(hr_dev, cpu_base + offset, 1298 hem->dma_addr); 1299 total++; 1300 } 1301 } 1302 } 1303 1304 list_splice(&temp_btm, &hem_list->btm_bt); 1305 list_splice(&temp_root, &hem_list->root_bt); 1306 for (i = 0; i < region_cnt; i++) 1307 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); 1308 1309 return 0; 1310 1311err_exit: 1312 for (i = 0; i < region_cnt; i++) 1313 hem_list_free_all(hr_dev, &temp_list[i], false); 1314 1315 hem_list_free_all(hr_dev, &temp_root, true); 1316 1317 return ret; 1318} 1319 1320/* construct the base address table and link them by address hop config */ 1321int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, 1322 struct hns_roce_hem_list *hem_list, 1323 const struct hns_roce_buf_region *regions, 1324 int region_cnt, unsigned int bt_pg_shift) 1325{ 1326 const struct hns_roce_buf_region *r; 1327 int ofs, end; 1328 int unit; 1329 int ret; 1330 int i; 1331 1332 if (region_cnt > HNS_ROCE_MAX_BT_REGION) { 1333 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", 1334 region_cnt); 1335 return -EINVAL; 1336 } 1337 1338 unit = (1 << bt_pg_shift) / BA_BYTE_LEN; 1339 for (i = 0; i < region_cnt; i++) { 1340 r = ®ions[i]; 1341 if (!r->count) 1342 continue; 1343 1344 end = r->offset + r->count; 1345 for (ofs = r->offset; ofs < end; ofs += unit) { 1346 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, 1347 hem_list->mid_bt[i], 1348 &hem_list->btm_bt); 1349 if (ret) { 1350 dev_err(hr_dev->dev, 1351 "alloc hem trunk fail ret=%d!\n", ret); 1352 goto err_alloc; 1353 } 1354 } 1355 } 1356 1357 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, 1358 region_cnt); 1359 if (ret) 1360 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); 1361 else 1362 return 0; 1363 1364err_alloc: 1365 hns_roce_hem_list_release(hr_dev, hem_list); 1366 1367 return ret; 1368} 1369 1370void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, 1371 struct hns_roce_hem_list *hem_list) 1372{ 1373 int i, j; 1374 1375 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1376 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1377 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], 1378 j != 0); 1379 1380 hem_list_free_all(hr_dev, &hem_list->root_bt, true); 1381 INIT_LIST_HEAD(&hem_list->btm_bt); 1382 hem_list->root_ba = 0; 1383} 1384 1385void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) 1386{ 1387 int i, j; 1388 1389 INIT_LIST_HEAD(&hem_list->root_bt); 1390 INIT_LIST_HEAD(&hem_list->btm_bt); 1391 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1392 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1393 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); 1394} 1395 1396void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, 1397 struct hns_roce_hem_list *hem_list, 1398 int offset, int *mtt_cnt, u64 *phy_addr) 1399{ 1400 struct list_head *head = &hem_list->btm_bt; 1401 struct roce_hem_item *hem, *temp_hem; 1402 void *cpu_base = NULL; 1403 u64 phy_base = 0; 1404 int nr = 0; 1405 1406 list_for_each_entry_safe(hem, temp_hem, head, sibling) { 1407 if (hem_list_page_is_in_range(hem, offset)) { 1408 nr = offset - hem->start; 1409 cpu_base = hem->addr + nr * BA_BYTE_LEN; 1410 phy_base = hem->dma_addr + nr * BA_BYTE_LEN; 1411 nr = hem->end + 1 - offset; 1412 break; 1413 } 1414 } 1415 1416 if (mtt_cnt) 1417 *mtt_cnt = nr; 1418 1419 if (phy_addr) 1420 *phy_addr = phy_base; 1421 1422 return cpu_base; 1423} 1424