1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6#include <linux/bitfield.h> 7#include <linux/highmem.h> 8 9#include "ivpu_drv.h" 10#include "ivpu_hw.h" 11#include "ivpu_mmu.h" 12#include "ivpu_mmu_context.h" 13 14#define IVPU_MMU_VPU_ADDRESS_MASK GENMASK(47, 12) 15#define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39) 16#define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30) 17#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21) 18#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12) 19#define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0)) 20#define IVPU_MMU_ENTRY_FLAG_CONT BIT(52) 21#define IVPU_MMU_ENTRY_FLAG_NG BIT(11) 22#define IVPU_MMU_ENTRY_FLAG_AF BIT(10) 23#define IVPU_MMU_ENTRY_FLAG_USER BIT(6) 24#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2) 25#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1) 26#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0) 27 28#define IVPU_MMU_PAGE_SIZE SZ_4K 29#define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16) 30#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE) 31#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE) 32#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE) 33#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE) 34#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64)) 35 36#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000 37#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID) 38#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK) 39#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \ 40 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID) 41 42static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) 43{ 44 dma_addr_t pgd_dma; 45 46 pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, 47 GFP_KERNEL); 48 if (!pgtable->pgd_dma_ptr) 49 return -ENOMEM; 50 51 pgtable->pgd_dma = pgd_dma; 52 53 return 0; 54} 55 56static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr) 57{ 58 if (cpu_addr) 59 dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr, 60 dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK); 61} 62 63static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) 64{ 65 int pgd_idx, pud_idx, pmd_idx; 66 dma_addr_t pud_dma, pmd_dma, pte_dma; 67 u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr; 68 69 for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) { 70 pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; 71 pud_dma = pgtable->pgd_dma_ptr[pgd_idx]; 72 73 if (!pud_dma_ptr) 74 continue; 75 76 for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) { 77 pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; 78 pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx]; 79 80 if (!pmd_dma_ptr) 81 continue; 82 83 for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) { 84 pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; 85 pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx]; 86 87 ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma); 88 } 89 90 kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]); 91 ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma); 92 } 93 94 kfree(pgtable->pmd_ptrs[pgd_idx]); 95 kfree(pgtable->pte_ptrs[pgd_idx]); 96 ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma); 97 } 98 99 ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma); 100} 101 102static u64* 103ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx) 104{ 105 u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; 106 dma_addr_t pud_dma; 107 108 if (pud_dma_ptr) 109 return pud_dma_ptr; 110 111 pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL); 112 if (!pud_dma_ptr) 113 return NULL; 114 115 drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]); 116 pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); 117 if (!pgtable->pmd_ptrs[pgd_idx]) 118 goto err_free_pud_dma_ptr; 119 120 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]); 121 pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); 122 if (!pgtable->pte_ptrs[pgd_idx]) 123 goto err_free_pmd_ptrs; 124 125 pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr; 126 pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID; 127 128 return pud_dma_ptr; 129 130err_free_pmd_ptrs: 131 kfree(pgtable->pmd_ptrs[pgd_idx]); 132 133err_free_pud_dma_ptr: 134 ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma); 135 return NULL; 136} 137 138static u64* 139ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, 140 int pud_idx) 141{ 142 u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; 143 dma_addr_t pmd_dma; 144 145 if (pmd_dma_ptr) 146 return pmd_dma_ptr; 147 148 pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL); 149 if (!pmd_dma_ptr) 150 return NULL; 151 152 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]); 153 pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL); 154 if (!pgtable->pte_ptrs[pgd_idx][pud_idx]) 155 goto err_free_pmd_dma_ptr; 156 157 pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr; 158 pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID; 159 160 return pmd_dma_ptr; 161 162err_free_pmd_dma_ptr: 163 ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma); 164 return NULL; 165} 166 167static u64* 168ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, 169 int pgd_idx, int pud_idx, int pmd_idx) 170{ 171 u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx]; 172 dma_addr_t pte_dma; 173 174 if (pte_dma_ptr) 175 return pte_dma_ptr; 176 177 pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL); 178 if (!pte_dma_ptr) 179 return NULL; 180 181 pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr; 182 pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID; 183 184 return pte_dma_ptr; 185} 186 187static int 188ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 189 u64 vpu_addr, dma_addr_t dma_addr, u64 prot) 190{ 191 u64 *pte; 192 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 193 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); 194 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 195 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); 196 197 /* Allocate PUD - second level page table if needed */ 198 if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx)) 199 return -ENOMEM; 200 201 /* Allocate PMD - third level page table if needed */ 202 if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx)) 203 return -ENOMEM; 204 205 /* Allocate PTE - fourth level page table if needed */ 206 pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx); 207 if (!pte) 208 return -ENOMEM; 209 210 /* Update PTE */ 211 pte[pte_idx] = dma_addr | prot; 212 213 return 0; 214} 215 216static int 217ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr, 218 dma_addr_t dma_addr, u64 prot) 219{ 220 size_t size = IVPU_MMU_CONT_PAGES_SIZE; 221 222 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size)); 223 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size)); 224 225 prot |= IVPU_MMU_ENTRY_FLAG_CONT; 226 227 while (size) { 228 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); 229 230 if (ret) 231 return ret; 232 233 size -= IVPU_MMU_PAGE_SIZE; 234 vpu_addr += IVPU_MMU_PAGE_SIZE; 235 dma_addr += IVPU_MMU_PAGE_SIZE; 236 } 237 238 return 0; 239} 240 241static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr) 242{ 243 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 244 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); 245 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 246 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr); 247 248 /* Update PTE with dummy physical address and clear flags */ 249 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID; 250} 251 252static void 253ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) 254{ 255 struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable; 256 u64 end_addr = vpu_addr + size; 257 258 /* Align to PMD entry (2 MB) */ 259 vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1); 260 261 while (vpu_addr < end_addr) { 262 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr); 263 u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE; 264 265 while (vpu_addr < end_addr && vpu_addr < pud_end) { 266 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr); 267 u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE; 268 269 while (vpu_addr < end_addr && vpu_addr < pmd_end) { 270 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr); 271 272 clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx], 273 IVPU_MMU_PGTABLE_SIZE); 274 vpu_addr += IVPU_MMU_PTE_MAP_SIZE; 275 } 276 clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx], 277 IVPU_MMU_PGTABLE_SIZE); 278 } 279 clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE); 280 } 281 clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE); 282} 283 284static int 285ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 286 u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot) 287{ 288 int map_size; 289 int ret; 290 291 while (size) { 292 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE && 293 IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) { 294 ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot); 295 map_size = IVPU_MMU_CONT_PAGES_SIZE; 296 } else { 297 ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot); 298 map_size = IVPU_MMU_PAGE_SIZE; 299 } 300 301 if (ret) 302 return ret; 303 304 vpu_addr += map_size; 305 dma_addr += map_size; 306 size -= map_size; 307 } 308 309 return 0; 310} 311 312static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size) 313{ 314 while (size) { 315 ivpu_mmu_context_unmap_page(ctx, vpu_addr); 316 vpu_addr += IVPU_MMU_PAGE_SIZE; 317 size -= IVPU_MMU_PAGE_SIZE; 318 } 319} 320 321int 322ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 323 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent) 324{ 325 struct scatterlist *sg; 326 int ret; 327 u64 prot; 328 u64 i; 329 330 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) 331 return -EINVAL; 332 333 if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK) 334 return -EINVAL; 335 336 prot = IVPU_MMU_ENTRY_MAPPED; 337 if (llc_coherent) 338 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT; 339 340 mutex_lock(&ctx->lock); 341 342 for_each_sgtable_dma_sg(sgt, sg, i) { 343 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset; 344 size_t size = sg_dma_len(sg) + sg->offset; 345 346 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot); 347 if (ret) { 348 ivpu_err(vdev, "Failed to map context pages\n"); 349 mutex_unlock(&ctx->lock); 350 return ret; 351 } 352 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size); 353 vpu_addr += size; 354 } 355 356 mutex_unlock(&ctx->lock); 357 358 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); 359 if (ret) 360 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); 361 return ret; 362} 363 364void 365ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 366 u64 vpu_addr, struct sg_table *sgt) 367{ 368 struct scatterlist *sg; 369 int ret; 370 u64 i; 371 372 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE)) 373 ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr); 374 375 mutex_lock(&ctx->lock); 376 377 for_each_sgtable_dma_sg(sgt, sg, i) { 378 size_t size = sg_dma_len(sg) + sg->offset; 379 380 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size); 381 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size); 382 vpu_addr += size; 383 } 384 385 mutex_unlock(&ctx->lock); 386 387 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id); 388 if (ret) 389 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret); 390} 391 392int 393ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx, 394 const struct ivpu_addr_range *range, 395 u64 size, struct drm_mm_node *node) 396{ 397 lockdep_assert_held(&ctx->lock); 398 399 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) { 400 if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0, 401 range->start, range->end, DRM_MM_INSERT_BEST)) 402 return 0; 403 } 404 405 return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0, 406 range->start, range->end, DRM_MM_INSERT_BEST); 407} 408 409void 410ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node) 411{ 412 lockdep_assert_held(&ctx->lock); 413 414 drm_mm_remove_node(node); 415} 416 417static int 418ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id) 419{ 420 u64 start, end; 421 int ret; 422 423 mutex_init(&ctx->lock); 424 INIT_LIST_HEAD(&ctx->bo_list); 425 426 ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable); 427 if (ret) 428 return ret; 429 430 if (!context_id) { 431 start = vdev->hw->ranges.global.start; 432 end = vdev->hw->ranges.shave.end; 433 } else { 434 start = vdev->hw->ranges.user.start; 435 end = vdev->hw->ranges.dma.end; 436 } 437 438 drm_mm_init(&ctx->mm, start, end - start); 439 ctx->id = context_id; 440 441 return 0; 442} 443 444static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 445{ 446 if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr)) 447 return; 448 449 mutex_destroy(&ctx->lock); 450 ivpu_mmu_pgtables_free(vdev, &ctx->pgtable); 451 drm_mm_takedown(&ctx->mm); 452 453 ctx->pgtable.pgd_dma_ptr = NULL; 454 ctx->pgtable.pgd_dma = 0; 455} 456 457int ivpu_mmu_global_context_init(struct ivpu_device *vdev) 458{ 459 return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID); 460} 461 462void ivpu_mmu_global_context_fini(struct ivpu_device *vdev) 463{ 464 return ivpu_mmu_context_fini(vdev, &vdev->gctx); 465} 466 467void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid) 468{ 469 struct ivpu_file_priv *file_priv; 470 471 xa_lock(&vdev->context_xa); 472 473 file_priv = xa_load(&vdev->context_xa, ssid); 474 if (file_priv) 475 file_priv->has_mmu_faults = true; 476 477 xa_unlock(&vdev->context_xa); 478} 479 480int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id) 481{ 482 int ret; 483 484 drm_WARN_ON(&vdev->drm, !ctx_id); 485 486 ret = ivpu_mmu_context_init(vdev, ctx, ctx_id); 487 if (ret) { 488 ivpu_err(vdev, "Failed to initialize context: %d\n", ret); 489 return ret; 490 } 491 492 ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable); 493 if (ret) { 494 ivpu_err(vdev, "Failed to set page table: %d\n", ret); 495 goto err_context_fini; 496 } 497 498 return 0; 499 500err_context_fini: 501 ivpu_mmu_context_fini(vdev, ctx); 502 return ret; 503} 504 505void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 506{ 507 drm_WARN_ON(&vdev->drm, !ctx->id); 508 509 ivpu_mmu_clear_pgtable(vdev, ctx->id); 510 ivpu_mmu_context_fini(vdev, ctx); 511} 512