1/* 2 * Copyright 2013 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * on the rights to use, copy, modify, merge, publish, distribute, sub 9 * license, and/or sell copies of the Software, and to permit persons to whom 10 * the Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 22 * USE OR OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 25#include "si_pipe.h" 26#include "sid.h" 27#include "si_build_pm4.h" 28 29/* Set this if you want the ME to wait until CP DMA is done. 30 * It should be set on the last CP DMA packet. */ 31#define CP_DMA_SYNC (1 << 0) 32 33/* Set this if the source data was used as a destination in a previous CP DMA 34 * packet. It's for preventing a read-after-write (RAW) hazard between two 35 * CP DMA packets. */ 36#define CP_DMA_RAW_WAIT (1 << 1) 37#define CP_DMA_DST_IS_GDS (1 << 2) 38#define CP_DMA_CLEAR (1 << 3) 39#define CP_DMA_PFP_SYNC_ME (1 << 4) 40#define CP_DMA_SRC_IS_GDS (1 << 5) 41 42/* The max number of bytes that can be copied per packet. */ 43static inline unsigned cp_dma_max_byte_count(struct si_context *sctx) 44{ 45 unsigned max = 46 sctx->gfx_level >= GFX11 ? 32767 : 47 sctx->gfx_level >= GFX9 ? S_415_BYTE_COUNT_GFX9(~0u) : S_415_BYTE_COUNT_GFX6(~0u); 48 49 /* make it aligned for optimal performance */ 50 return max & ~(SI_CPDMA_ALIGNMENT - 1); 51} 52 53/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear 54 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit 55 * clear value. 56 */ 57static void si_emit_cp_dma(struct si_context *sctx, struct radeon_cmdbuf *cs, uint64_t dst_va, 58 uint64_t src_va, unsigned size, unsigned flags, 59 enum si_cache_policy cache_policy) 60{ 61 uint32_t header = 0, command = 0; 62 63 assert(size <= cp_dma_max_byte_count(sctx)); 64 assert(sctx->gfx_level != GFX6 || cache_policy == L2_BYPASS); 65 66 if (sctx->gfx_level >= GFX9) 67 command |= S_415_BYTE_COUNT_GFX9(size); 68 else 69 command |= S_415_BYTE_COUNT_GFX6(size); 70 71 /* Sync flags. */ 72 if (flags & CP_DMA_SYNC) 73 header |= S_411_CP_SYNC(1); 74 75 if (flags & CP_DMA_RAW_WAIT) 76 command |= S_415_RAW_WAIT(1); 77 78 /* Src and dst flags. */ 79 if (sctx->gfx_level >= GFX9 && !(flags & CP_DMA_CLEAR) && src_va == dst_va) { 80 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */ 81 } else if (flags & CP_DMA_DST_IS_GDS) { 82 header |= S_411_DST_SEL(V_411_GDS); 83 /* GDS increments the address, not CP. */ 84 command |= S_415_DAS(V_415_REGISTER) | S_415_DAIC(V_415_NO_INCREMENT); 85 } else if (sctx->gfx_level >= GFX7 && cache_policy != L2_BYPASS) { 86 header |= 87 S_411_DST_SEL(V_411_DST_ADDR_TC_L2) | S_500_DST_CACHE_POLICY(cache_policy == L2_STREAM); 88 } 89 90 if (flags & CP_DMA_CLEAR) { 91 header |= S_411_SRC_SEL(V_411_DATA); 92 } else if (flags & CP_DMA_SRC_IS_GDS) { 93 header |= S_411_SRC_SEL(V_411_GDS); 94 /* Both of these are required for GDS. It does increment the address. */ 95 command |= S_415_SAS(V_415_REGISTER) | S_415_SAIC(V_415_NO_INCREMENT); 96 } else if (sctx->gfx_level >= GFX7 && cache_policy != L2_BYPASS) { 97 header |= 98 S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2) | S_500_SRC_CACHE_POLICY(cache_policy == L2_STREAM); 99 } 100 101 radeon_begin(cs); 102 103 if (sctx->gfx_level >= GFX7) { 104 radeon_emit(PKT3(PKT3_DMA_DATA, 5, 0)); 105 radeon_emit(header); 106 radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ 107 radeon_emit(src_va >> 32); /* SRC_ADDR_HI [31:0] */ 108 radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ 109 radeon_emit(dst_va >> 32); /* DST_ADDR_HI [31:0] */ 110 radeon_emit(command); 111 } else { 112 header |= S_411_SRC_ADDR_HI(src_va >> 32); 113 114 radeon_emit(PKT3(PKT3_CP_DMA, 4, 0)); 115 radeon_emit(src_va); /* SRC_ADDR_LO [31:0] */ 116 radeon_emit(header); /* SRC_ADDR_HI [15:0] + flags. */ 117 radeon_emit(dst_va); /* DST_ADDR_LO [31:0] */ 118 radeon_emit((dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */ 119 radeon_emit(command); 120 } 121 122 /* CP DMA is executed in ME, but index buffers are read by PFP. 123 * This ensures that ME (CP DMA) is idle before PFP starts fetching 124 * indices. If we wanted to execute CP DMA in PFP, this packet 125 * should precede it. 126 */ 127 if (sctx->has_graphics && flags & CP_DMA_PFP_SYNC_ME) { 128 radeon_emit(PKT3(PKT3_PFP_SYNC_ME, 0, 0)); 129 radeon_emit(0); 130 } 131 radeon_end(); 132} 133 134void si_cp_dma_wait_for_idle(struct si_context *sctx, struct radeon_cmdbuf *cs) 135{ 136 /* Issue a dummy DMA that copies zero bytes. 137 * 138 * The DMA engine will see that there's no work to do and skip this 139 * DMA request, however, the CP will see the sync flag and still wait 140 * for all DMAs to complete. 141 */ 142 si_emit_cp_dma(sctx, cs, 0, 0, 0, CP_DMA_SYNC, L2_BYPASS); 143} 144 145static void si_cp_dma_prepare(struct si_context *sctx, struct pipe_resource *dst, 146 struct pipe_resource *src, unsigned byte_count, 147 uint64_t remaining_size, unsigned user_flags, enum si_coherency coher, 148 bool *is_first, unsigned *packet_flags) 149{ 150 /* Count memory usage in so that need_cs_space can take it into account. */ 151 if (dst) 152 si_context_add_resource_size(sctx, dst); 153 if (src) 154 si_context_add_resource_size(sctx, src); 155 156 if (!(user_flags & SI_OP_CPDMA_SKIP_CHECK_CS_SPACE)) 157 si_need_gfx_cs_space(sctx, 0); 158 159 /* This must be done after need_cs_space. */ 160 if (dst) 161 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, si_resource(dst), 162 RADEON_USAGE_WRITE | RADEON_PRIO_CP_DMA); 163 if (src) 164 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, si_resource(src), 165 RADEON_USAGE_READ | RADEON_PRIO_CP_DMA); 166 167 /* Flush the caches for the first copy only. 168 * Also wait for the previous CP DMA operations. 169 */ 170 if (*is_first && sctx->flags) 171 sctx->emit_cache_flush(sctx, &sctx->gfx_cs); 172 173 if (user_flags & SI_OP_SYNC_CPDMA_BEFORE && *is_first && !(*packet_flags & CP_DMA_CLEAR)) 174 *packet_flags |= CP_DMA_RAW_WAIT; 175 176 *is_first = false; 177 178 /* Do the synchronization after the last dma, so that all data 179 * is written to memory. 180 */ 181 if (user_flags & SI_OP_SYNC_AFTER && byte_count == remaining_size) { 182 *packet_flags |= CP_DMA_SYNC; 183 184 if (coher == SI_COHERENCY_SHADER) 185 *packet_flags |= CP_DMA_PFP_SYNC_ME; 186 } 187} 188 189void si_cp_dma_clear_buffer(struct si_context *sctx, struct radeon_cmdbuf *cs, 190 struct pipe_resource *dst, uint64_t offset, uint64_t size, 191 unsigned value, unsigned user_flags, enum si_coherency coher, 192 enum si_cache_policy cache_policy) 193{ 194 struct si_resource *sdst = si_resource(dst); 195 uint64_t va = (sdst ? sdst->gpu_address : 0) + offset; 196 bool is_first = true; 197 198 assert(size && size % 4 == 0); 199 200 if (user_flags & SI_OP_SYNC_GE_BEFORE) 201 sctx->flags |= SI_CONTEXT_VS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 202 203 if (user_flags & SI_OP_SYNC_CS_BEFORE) 204 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 205 206 if (user_flags & SI_OP_SYNC_PS_BEFORE) 207 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 208 209 /* Mark the buffer range of destination as valid (initialized), 210 * so that transfer_map knows it should wait for the GPU when mapping 211 * that range. */ 212 if (sdst) { 213 util_range_add(dst, &sdst->valid_buffer_range, offset, offset + size); 214 215 if (!(user_flags & SI_OP_SKIP_CACHE_INV_BEFORE)) 216 sctx->flags |= si_get_flush_flags(sctx, coher, cache_policy); 217 } 218 219 while (size) { 220 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx)); 221 unsigned dma_flags = CP_DMA_CLEAR | (sdst ? 0 : CP_DMA_DST_IS_GDS); 222 223 si_cp_dma_prepare(sctx, dst, NULL, byte_count, size, user_flags, coher, &is_first, 224 &dma_flags); 225 226 /* Emit the clear packet. */ 227 si_emit_cp_dma(sctx, cs, va, value, byte_count, dma_flags, cache_policy); 228 229 size -= byte_count; 230 va += byte_count; 231 } 232 233 if (sdst && cache_policy != L2_BYPASS) 234 sdst->TC_L2_dirty = true; 235 236 /* If it's not a framebuffer fast clear... */ 237 if (coher == SI_COHERENCY_SHADER) 238 sctx->num_cp_dma_calls++; 239} 240 241/** 242 * Realign the CP DMA engine. This must be done after a copy with an unaligned 243 * size. 244 * 245 * \param size Remaining size to the CP DMA alignment. 246 */ 247static void si_cp_dma_realign_engine(struct si_context *sctx, unsigned size, unsigned user_flags, 248 enum si_coherency coher, enum si_cache_policy cache_policy, 249 bool *is_first) 250{ 251 uint64_t va; 252 unsigned dma_flags = 0; 253 unsigned scratch_size = SI_CPDMA_ALIGNMENT * 2; 254 255 assert(size < SI_CPDMA_ALIGNMENT); 256 257 /* Use the scratch buffer as the dummy buffer. The 3D engine should be 258 * idle at this point. 259 */ 260 if (!sctx->scratch_buffer || sctx->scratch_buffer->b.b.width0 < scratch_size) { 261 si_resource_reference(&sctx->scratch_buffer, NULL); 262 sctx->scratch_buffer = si_aligned_buffer_create(&sctx->screen->b, 263 PIPE_RESOURCE_FLAG_UNMAPPABLE | SI_RESOURCE_FLAG_DRIVER_INTERNAL | 264 SI_RESOURCE_FLAG_DISCARDABLE, 265 PIPE_USAGE_DEFAULT, scratch_size, 256); 266 if (!sctx->scratch_buffer) 267 return; 268 269 si_mark_atom_dirty(sctx, &sctx->atoms.s.scratch_state); 270 } 271 272 si_cp_dma_prepare(sctx, &sctx->scratch_buffer->b.b, &sctx->scratch_buffer->b.b, size, size, 273 user_flags, coher, is_first, &dma_flags); 274 275 va = sctx->scratch_buffer->gpu_address; 276 si_emit_cp_dma(sctx, &sctx->gfx_cs, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags, cache_policy); 277} 278 279/** 280 * Do memcpy between buffers using CP DMA. 281 * If src or dst is NULL, it means read or write GDS, respectively. 282 * 283 * \param user_flags bitmask of SI_CPDMA_* 284 */ 285void si_cp_dma_copy_buffer(struct si_context *sctx, struct pipe_resource *dst, 286 struct pipe_resource *src, uint64_t dst_offset, uint64_t src_offset, 287 unsigned size, unsigned user_flags, enum si_coherency coher, 288 enum si_cache_policy cache_policy) 289{ 290 uint64_t main_dst_offset, main_src_offset; 291 unsigned skipped_size = 0; 292 unsigned realign_size = 0; 293 unsigned gds_flags = (dst ? 0 : CP_DMA_DST_IS_GDS) | (src ? 0 : CP_DMA_SRC_IS_GDS); 294 bool is_first = true; 295 296 assert(size); 297 298 if (dst) { 299 /* Skip this for the L2 prefetch. */ 300 if (dst != src || dst_offset != src_offset) { 301 /* Mark the buffer range of destination as valid (initialized), 302 * so that transfer_map knows it should wait for the GPU when mapping 303 * that range. */ 304 util_range_add(dst, &si_resource(dst)->valid_buffer_range, dst_offset, dst_offset + size); 305 } 306 307 dst_offset += si_resource(dst)->gpu_address; 308 } 309 if (src) 310 src_offset += si_resource(src)->gpu_address; 311 312 /* The workarounds aren't needed on Fiji and beyond. */ 313 if (sctx->family <= CHIP_CARRIZO || sctx->family == CHIP_STONEY) { 314 /* If the size is not aligned, we must add a dummy copy at the end 315 * just to align the internal counter. Otherwise, the DMA engine 316 * would slow down by an order of magnitude for following copies. 317 */ 318 if (size % SI_CPDMA_ALIGNMENT) 319 realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT); 320 321 /* If the copy begins unaligned, we must start copying from the next 322 * aligned block and the skipped part should be copied after everything 323 * else has been copied. Only the src alignment matters, not dst. 324 * 325 * GDS doesn't need the source address to be aligned. 326 */ 327 if (src && src_offset % SI_CPDMA_ALIGNMENT) { 328 skipped_size = SI_CPDMA_ALIGNMENT - (src_offset % SI_CPDMA_ALIGNMENT); 329 /* The main part will be skipped if the size is too small. */ 330 skipped_size = MIN2(skipped_size, size); 331 size -= skipped_size; 332 } 333 } 334 335 /* TMZ handling */ 336 if (unlikely(radeon_uses_secure_bos(sctx->ws))) { 337 bool secure = src && (si_resource(src)->flags & RADEON_FLAG_ENCRYPTED); 338 assert(!secure || (!dst || (si_resource(dst)->flags & RADEON_FLAG_ENCRYPTED))); 339 if (secure != sctx->ws->cs_is_secure(&sctx->gfx_cs)) { 340 si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW | 341 RADEON_FLUSH_TOGGLE_SECURE_SUBMISSION, NULL); 342 } 343 } 344 345 if (user_flags & SI_OP_SYNC_GE_BEFORE) 346 sctx->flags |= SI_CONTEXT_VS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 347 348 if (user_flags & SI_OP_SYNC_CS_BEFORE) 349 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 350 351 if (user_flags & SI_OP_SYNC_PS_BEFORE) 352 sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_PFP_SYNC_ME; 353 354 if ((dst || src) && !(user_flags & SI_OP_SKIP_CACHE_INV_BEFORE)) 355 sctx->flags |= si_get_flush_flags(sctx, coher, cache_policy); 356 357 /* This is the main part doing the copying. Src is always aligned. */ 358 main_dst_offset = dst_offset + skipped_size; 359 main_src_offset = src_offset + skipped_size; 360 361 while (size) { 362 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(sctx)); 363 unsigned dma_flags = gds_flags; 364 365 si_cp_dma_prepare(sctx, dst, src, byte_count, size + skipped_size + realign_size, user_flags, 366 coher, &is_first, &dma_flags); 367 368 si_emit_cp_dma(sctx, &sctx->gfx_cs, main_dst_offset, main_src_offset, byte_count, dma_flags, 369 cache_policy); 370 371 size -= byte_count; 372 main_src_offset += byte_count; 373 main_dst_offset += byte_count; 374 } 375 376 /* Copy the part we skipped because src wasn't aligned. */ 377 if (skipped_size) { 378 unsigned dma_flags = gds_flags; 379 380 si_cp_dma_prepare(sctx, dst, src, skipped_size, skipped_size + realign_size, user_flags, 381 coher, &is_first, &dma_flags); 382 383 si_emit_cp_dma(sctx, &sctx->gfx_cs, dst_offset, src_offset, skipped_size, dma_flags, 384 cache_policy); 385 } 386 387 /* Finally, realign the engine if the size wasn't aligned. */ 388 if (realign_size) { 389 si_cp_dma_realign_engine(sctx, realign_size, user_flags, coher, cache_policy, &is_first); 390 } 391 392 if (dst && cache_policy != L2_BYPASS) 393 si_resource(dst)->TC_L2_dirty = true; 394 395 /* If it's not a prefetch or GDS copy... */ 396 if (dst && src && (dst != src || dst_offset != src_offset)) 397 sctx->num_cp_dma_calls++; 398} 399 400void si_test_gds(struct si_context *sctx) 401{ 402 struct pipe_context *ctx = &sctx->b; 403 struct pipe_resource *src, *dst; 404 unsigned r[4] = {}; 405 unsigned offset = debug_get_num_option("OFFSET", 16); 406 407 src = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16); 408 dst = pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_DEFAULT, 16); 409 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, src, 0, 4, 0xabcdef01, SI_OP_SYNC_BEFORE_AFTER, 410 SI_COHERENCY_SHADER, L2_BYPASS); 411 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, src, 4, 4, 0x23456789, SI_OP_SYNC_BEFORE_AFTER, 412 SI_COHERENCY_SHADER, L2_BYPASS); 413 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, src, 8, 4, 0x87654321, SI_OP_SYNC_BEFORE_AFTER, 414 SI_COHERENCY_SHADER, L2_BYPASS); 415 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, src, 12, 4, 0xfedcba98, SI_OP_SYNC_BEFORE_AFTER, 416 SI_COHERENCY_SHADER, L2_BYPASS); 417 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, dst, 0, 16, 0xdeadbeef, SI_OP_SYNC_BEFORE_AFTER, 418 SI_COHERENCY_SHADER, L2_BYPASS); 419 420 si_cp_dma_copy_buffer(sctx, NULL, src, offset, 0, 16, SI_OP_SYNC_BEFORE_AFTER, 421 SI_COHERENCY_NONE, L2_BYPASS); 422 si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, SI_OP_SYNC_BEFORE_AFTER, 423 SI_COHERENCY_NONE, L2_BYPASS); 424 425 pipe_buffer_read(ctx, dst, 0, sizeof(r), r); 426 printf("GDS copy = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3], 427 r[0] == 0xabcdef01 && r[1] == 0x23456789 && r[2] == 0x87654321 && r[3] == 0xfedcba98 428 ? "pass" 429 : "fail"); 430 431 si_cp_dma_clear_buffer(sctx, &sctx->gfx_cs, NULL, offset, 16, 0xc1ea4146, 432 SI_OP_SYNC_BEFORE_AFTER, SI_COHERENCY_NONE, L2_BYPASS); 433 si_cp_dma_copy_buffer(sctx, dst, NULL, 0, offset, 16, SI_OP_SYNC_BEFORE_AFTER, 434 SI_COHERENCY_NONE, L2_BYPASS); 435 436 pipe_buffer_read(ctx, dst, 0, sizeof(r), r); 437 printf("GDS clear = %08x %08x %08x %08x -> %s\n", r[0], r[1], r[2], r[3], 438 r[0] == 0xc1ea4146 && r[1] == 0xc1ea4146 && r[2] == 0xc1ea4146 && r[3] == 0xc1ea4146 439 ? "pass" 440 : "fail"); 441 442 pipe_resource_reference(&src, NULL); 443 pipe_resource_reference(&dst, NULL); 444 exit(0); 445} 446 447void si_cp_write_data(struct si_context *sctx, struct si_resource *buf, unsigned offset, 448 unsigned size, unsigned dst_sel, unsigned engine, const void *data) 449{ 450 struct radeon_cmdbuf *cs = &sctx->gfx_cs; 451 452 assert(offset % 4 == 0); 453 assert(size % 4 == 0); 454 455 if (sctx->gfx_level == GFX6 && dst_sel == V_370_MEM) 456 dst_sel = V_370_MEM_GRBM; 457 458 radeon_add_to_buffer_list(sctx, cs, buf, RADEON_USAGE_WRITE | RADEON_PRIO_CP_DMA); 459 uint64_t va = buf->gpu_address + offset; 460 461 radeon_begin(cs); 462 radeon_emit(PKT3(PKT3_WRITE_DATA, 2 + size / 4, 0)); 463 radeon_emit(S_370_DST_SEL(dst_sel) | S_370_WR_CONFIRM(1) | S_370_ENGINE_SEL(engine)); 464 radeon_emit(va); 465 radeon_emit(va >> 32); 466 radeon_emit_array((const uint32_t *)data, size / 4); 467 radeon_end(); 468} 469 470void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs, unsigned dst_sel, 471 struct si_resource *dst, unsigned dst_offset, unsigned src_sel, 472 struct si_resource *src, unsigned src_offset) 473{ 474 /* cs can point to the compute IB, which has the buffer list in gfx_cs. */ 475 if (dst) { 476 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, dst, RADEON_USAGE_WRITE | RADEON_PRIO_CP_DMA); 477 } 478 if (src) { 479 radeon_add_to_buffer_list(sctx, &sctx->gfx_cs, src, RADEON_USAGE_READ | RADEON_PRIO_CP_DMA); 480 } 481 482 uint64_t dst_va = (dst ? dst->gpu_address : 0ull) + dst_offset; 483 uint64_t src_va = (src ? src->gpu_address : 0ull) + src_offset; 484 485 radeon_begin(cs); 486 radeon_emit(PKT3(PKT3_COPY_DATA, 4, 0)); 487 radeon_emit(COPY_DATA_SRC_SEL(src_sel) | COPY_DATA_DST_SEL(dst_sel) | COPY_DATA_WR_CONFIRM); 488 radeon_emit(src_va); 489 radeon_emit(src_va >> 32); 490 radeon_emit(dst_va); 491 radeon_emit(dst_va >> 32); 492 radeon_end(); 493} 494