1/* 2 * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Rob Clark <robclark@freedesktop.org> 25 */ 26 27#include "util/hash_table.h" 28#include "util/list.h" 29#include "util/set.h" 30#include "util/u_string.h" 31 32#include "freedreno_batch.h" 33#include "freedreno_context.h" 34#include "freedreno_fence.h" 35#include "freedreno_query_hw.h" 36#include "freedreno_resource.h" 37 38static struct fd_ringbuffer * 39alloc_ring(struct fd_batch *batch, unsigned sz, enum fd_ringbuffer_flags flags) 40{ 41 struct fd_context *ctx = batch->ctx; 42 43 /* if kernel is too old to support unlimited # of cmd buffers, we 44 * have no option but to allocate large worst-case sizes so that 45 * we don't need to grow the ringbuffer. Performance is likely to 46 * suffer, but there is no good alternative. 47 * 48 * Otherwise if supported, allocate a growable ring with initial 49 * size of zero. 50 */ 51 if ((fd_device_version(ctx->screen->dev) >= FD_VERSION_UNLIMITED_CMDS) && 52 !FD_DBG(NOGROW)) { 53 flags |= FD_RINGBUFFER_GROWABLE; 54 sz = 0; 55 } 56 57 return fd_submit_new_ringbuffer(batch->submit, sz, flags); 58} 59 60static void 61batch_init(struct fd_batch *batch) 62{ 63 struct fd_context *ctx = batch->ctx; 64 65 batch->submit = fd_submit_new(ctx->pipe); 66 if (batch->nondraw) { 67 batch->gmem = alloc_ring(batch, 0x1000, FD_RINGBUFFER_PRIMARY); 68 batch->draw = alloc_ring(batch, 0x100000, 0); 69 } else { 70 batch->gmem = alloc_ring(batch, 0x100000, FD_RINGBUFFER_PRIMARY); 71 batch->draw = alloc_ring(batch, 0x100000, 0); 72 73 /* a6xx+ re-uses draw rb for both draw and binning pass: */ 74 if (ctx->screen->gen < 6) { 75 batch->binning = alloc_ring(batch, 0x100000, 0); 76 } 77 } 78 79 batch->in_fence_fd = -1; 80 batch->fence = NULL; 81 82 /* Work around problems on earlier gens with submit merging, etc, 83 * by always creating a fence to request that the submit is flushed 84 * immediately: 85 */ 86 if (ctx->screen->gen < 6) 87 batch->fence = fd_fence_create(batch); 88 89 batch->cleared = 0; 90 batch->fast_cleared = 0; 91 batch->invalidated = 0; 92 batch->restore = batch->resolve = 0; 93 batch->needs_flush = false; 94 batch->flushed = false; 95 batch->gmem_reason = 0; 96 batch->num_draws = 0; 97 batch->num_vertices = 0; 98 batch->num_bins_per_pipe = 0; 99 batch->prim_strm_bits = 0; 100 batch->draw_strm_bits = 0; 101 102 fd_reset_wfi(batch); 103 104 util_dynarray_init(&batch->draw_patches, NULL); 105 util_dynarray_init(&batch->fb_read_patches, NULL); 106 107 if (is_a2xx(ctx->screen)) { 108 util_dynarray_init(&batch->shader_patches, NULL); 109 util_dynarray_init(&batch->gmem_patches, NULL); 110 } 111 112 if (is_a3xx(ctx->screen)) 113 util_dynarray_init(&batch->rbrc_patches, NULL); 114 115 assert(batch->resources->entries == 0); 116 117 util_dynarray_init(&batch->samples, NULL); 118 119 u_trace_init(&batch->trace, &ctx->trace_context); 120 batch->last_timestamp_cmd = NULL; 121} 122 123struct fd_batch * 124fd_batch_create(struct fd_context *ctx, bool nondraw) 125{ 126 struct fd_batch *batch = CALLOC_STRUCT(fd_batch); 127 128 if (!batch) 129 return NULL; 130 131 DBG("%p", batch); 132 133 pipe_reference_init(&batch->reference, 1); 134 batch->ctx = ctx; 135 batch->nondraw = nondraw; 136 137 simple_mtx_init(&batch->submit_lock, mtx_plain); 138 139 batch->resources = 140 _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); 141 142 batch_init(batch); 143 144 return batch; 145} 146 147static void 148cleanup_submit(struct fd_batch *batch) 149{ 150 if (!batch->submit) 151 return; 152 153 fd_ringbuffer_del(batch->draw); 154 fd_ringbuffer_del(batch->gmem); 155 156 if (batch->binning) { 157 fd_ringbuffer_del(batch->binning); 158 batch->binning = NULL; 159 } 160 161 if (batch->prologue) { 162 fd_ringbuffer_del(batch->prologue); 163 batch->prologue = NULL; 164 } 165 166 if (batch->epilogue) { 167 fd_ringbuffer_del(batch->epilogue); 168 batch->epilogue = NULL; 169 } 170 171 if (batch->tile_setup) { 172 fd_ringbuffer_del(batch->tile_setup); 173 batch->tile_setup = NULL; 174 } 175 176 if (batch->tile_fini) { 177 fd_ringbuffer_del(batch->tile_fini); 178 batch->tile_fini = NULL; 179 } 180 181 fd_submit_del(batch->submit); 182 batch->submit = NULL; 183} 184 185static void 186batch_fini(struct fd_batch *batch) 187{ 188 DBG("%p", batch); 189 190 pipe_resource_reference(&batch->query_buf, NULL); 191 192 if (batch->in_fence_fd != -1) 193 close(batch->in_fence_fd); 194 195 /* in case batch wasn't flushed but fence was created: */ 196 if (batch->fence) 197 fd_fence_set_batch(batch->fence, NULL); 198 199 fd_fence_ref(&batch->fence, NULL); 200 201 cleanup_submit(batch); 202 203 util_dynarray_fini(&batch->draw_patches); 204 util_dynarray_fini(&batch->fb_read_patches); 205 206 if (is_a2xx(batch->ctx->screen)) { 207 util_dynarray_fini(&batch->shader_patches); 208 util_dynarray_fini(&batch->gmem_patches); 209 } 210 211 if (is_a3xx(batch->ctx->screen)) 212 util_dynarray_fini(&batch->rbrc_patches); 213 214 while (batch->samples.size > 0) { 215 struct fd_hw_sample *samp = 216 util_dynarray_pop(&batch->samples, struct fd_hw_sample *); 217 fd_hw_sample_reference(batch->ctx, &samp, NULL); 218 } 219 util_dynarray_fini(&batch->samples); 220 221 u_trace_fini(&batch->trace); 222} 223 224static void 225batch_flush_dependencies(struct fd_batch *batch) assert_dt 226{ 227 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 228 struct fd_batch *dep; 229 230 foreach_batch (dep, cache, batch->dependents_mask) { 231 fd_batch_flush(dep); 232 fd_batch_reference(&dep, NULL); 233 } 234 235 batch->dependents_mask = 0; 236} 237 238static void 239batch_reset_dependencies(struct fd_batch *batch) 240{ 241 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 242 struct fd_batch *dep; 243 244 foreach_batch (dep, cache, batch->dependents_mask) { 245 fd_batch_reference(&dep, NULL); 246 } 247 248 batch->dependents_mask = 0; 249} 250 251static void 252batch_reset_resources(struct fd_batch *batch) 253{ 254 fd_screen_assert_locked(batch->ctx->screen); 255 256 set_foreach (batch->resources, entry) { 257 struct fd_resource *rsc = (struct fd_resource *)entry->key; 258 _mesa_set_remove(batch->resources, entry); 259 assert(rsc->track->batch_mask & (1 << batch->idx)); 260 rsc->track->batch_mask &= ~(1 << batch->idx); 261 if (rsc->track->write_batch == batch) 262 fd_batch_reference_locked(&rsc->track->write_batch, NULL); 263 } 264} 265 266static void 267batch_reset(struct fd_batch *batch) assert_dt 268{ 269 DBG("%p", batch); 270 271 batch_reset_dependencies(batch); 272 273 fd_screen_lock(batch->ctx->screen); 274 batch_reset_resources(batch); 275 fd_screen_unlock(batch->ctx->screen); 276 277 batch_fini(batch); 278 batch_init(batch); 279} 280 281void 282fd_batch_reset(struct fd_batch *batch) 283{ 284 if (batch->needs_flush) 285 batch_reset(batch); 286} 287 288void 289__fd_batch_destroy(struct fd_batch *batch) 290{ 291 struct fd_context *ctx = batch->ctx; 292 293 DBG("%p", batch); 294 295 fd_screen_assert_locked(batch->ctx->screen); 296 297 fd_bc_invalidate_batch(batch, true); 298 299 batch_reset_resources(batch); 300 assert(batch->resources->entries == 0); 301 _mesa_set_destroy(batch->resources, NULL); 302 303 fd_screen_unlock(ctx->screen); 304 batch_reset_dependencies(batch); 305 assert(batch->dependents_mask == 0); 306 307 util_copy_framebuffer_state(&batch->framebuffer, NULL); 308 batch_fini(batch); 309 310 simple_mtx_destroy(&batch->submit_lock); 311 312 free(batch->key); 313 free(batch); 314 fd_screen_lock(ctx->screen); 315} 316 317void 318__fd_batch_describe(char *buf, const struct fd_batch *batch) 319{ 320 sprintf(buf, "fd_batch<%u>", batch->seqno); 321} 322 323/* Get per-batch prologue */ 324struct fd_ringbuffer * 325fd_batch_get_prologue(struct fd_batch *batch) 326{ 327 if (!batch->prologue) 328 batch->prologue = alloc_ring(batch, 0x1000, 0); 329 return batch->prologue; 330} 331 332/* Only called from fd_batch_flush() */ 333static void 334batch_flush(struct fd_batch *batch) assert_dt 335{ 336 DBG("%p: needs_flush=%d", batch, batch->needs_flush); 337 338 if (!fd_batch_lock_submit(batch)) 339 return; 340 341 batch->needs_flush = false; 342 343 /* close out the draw cmds by making sure any active queries are 344 * paused: 345 */ 346 fd_batch_finish_queries(batch); 347 348 batch_flush_dependencies(batch); 349 350 fd_screen_lock(batch->ctx->screen); 351 batch_reset_resources(batch); 352 /* NOTE: remove=false removes the batch from the hashtable, so future 353 * lookups won't cache-hit a flushed batch, but leaves the weak reference 354 * to the batch to avoid having multiple batches with same batch->idx, as 355 * that causes all sorts of hilarity. 356 */ 357 fd_bc_invalidate_batch(batch, false); 358 batch->flushed = true; 359 360 if (batch == batch->ctx->batch) 361 fd_batch_reference_locked(&batch->ctx->batch, NULL); 362 363 fd_screen_unlock(batch->ctx->screen); 364 365 if (batch->fence) 366 fd_fence_ref(&batch->ctx->last_fence, batch->fence); 367 368 fd_gmem_render_tiles(batch); 369 370 assert(batch->reference.count > 0); 371 372 cleanup_submit(batch); 373 fd_batch_unlock_submit(batch); 374} 375 376/* NOTE: could drop the last ref to batch 377 */ 378void 379fd_batch_flush(struct fd_batch *batch) 380{ 381 struct fd_batch *tmp = NULL; 382 383 /* NOTE: we need to hold an extra ref across the body of flush, 384 * since the last ref to this batch could be dropped when cleaning 385 * up used_resources 386 */ 387 fd_batch_reference(&tmp, batch); 388 batch_flush(tmp); 389 fd_batch_reference(&tmp, NULL); 390} 391 392/* find a batches dependents mask, including recursive dependencies: */ 393static uint32_t 394recursive_dependents_mask(struct fd_batch *batch) 395{ 396 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 397 struct fd_batch *dep; 398 uint32_t dependents_mask = batch->dependents_mask; 399 400 foreach_batch (dep, cache, batch->dependents_mask) 401 dependents_mask |= recursive_dependents_mask(dep); 402 403 return dependents_mask; 404} 405 406void 407fd_batch_add_dep(struct fd_batch *batch, struct fd_batch *dep) 408{ 409 fd_screen_assert_locked(batch->ctx->screen); 410 411 if (batch->dependents_mask & (1 << dep->idx)) 412 return; 413 414 /* a loop should not be possible */ 415 assert(!((1 << batch->idx) & recursive_dependents_mask(dep))); 416 417 struct fd_batch *other = NULL; 418 fd_batch_reference_locked(&other, dep); 419 batch->dependents_mask |= (1 << dep->idx); 420 DBG("%p: added dependency on %p", batch, dep); 421} 422 423static void 424flush_write_batch(struct fd_resource *rsc) assert_dt 425{ 426 struct fd_batch *b = NULL; 427 fd_batch_reference_locked(&b, rsc->track->write_batch); 428 429 fd_screen_unlock(b->ctx->screen); 430 fd_batch_flush(b); 431 fd_screen_lock(b->ctx->screen); 432 433 fd_batch_reference_locked(&b, NULL); 434} 435 436static void 437fd_batch_add_resource(struct fd_batch *batch, struct fd_resource *rsc) 438{ 439 440 if (likely(fd_batch_references_resource(batch, rsc))) { 441 assert(_mesa_set_search_pre_hashed(batch->resources, rsc->hash, rsc)); 442 return; 443 } 444 445 assert(!_mesa_set_search(batch->resources, rsc)); 446 447 _mesa_set_add_pre_hashed(batch->resources, rsc->hash, rsc); 448 rsc->track->batch_mask |= (1 << batch->idx); 449} 450 451void 452fd_batch_resource_write(struct fd_batch *batch, struct fd_resource *rsc) 453{ 454 fd_screen_assert_locked(batch->ctx->screen); 455 456 DBG("%p: write %p", batch, rsc); 457 458 /* Must do this before the early out, so we unset a previous resource 459 * invalidate (which may have left the write_batch state in place). 460 */ 461 rsc->valid = true; 462 463 if (rsc->track->write_batch == batch) 464 return; 465 466 fd_batch_write_prep(batch, rsc); 467 468 if (rsc->stencil) 469 fd_batch_resource_write(batch, rsc->stencil); 470 471 /* note, invalidate write batch, to avoid further writes to rsc 472 * resulting in a write-after-read hazard. 473 */ 474 /* if we are pending read or write by any other batch: */ 475 if (unlikely(rsc->track->batch_mask & ~(1 << batch->idx))) { 476 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 477 struct fd_batch *dep; 478 479 if (rsc->track->write_batch) 480 flush_write_batch(rsc); 481 482 foreach_batch (dep, cache, rsc->track->batch_mask) { 483 struct fd_batch *b = NULL; 484 if (dep == batch) 485 continue; 486 /* note that batch_add_dep could flush and unref dep, so 487 * we need to hold a reference to keep it live for the 488 * fd_bc_invalidate_batch() 489 */ 490 fd_batch_reference(&b, dep); 491 fd_batch_add_dep(batch, b); 492 fd_bc_invalidate_batch(b, false); 493 fd_batch_reference_locked(&b, NULL); 494 } 495 } 496 fd_batch_reference_locked(&rsc->track->write_batch, batch); 497 498 fd_batch_add_resource(batch, rsc); 499} 500 501void 502fd_batch_resource_read_slowpath(struct fd_batch *batch, struct fd_resource *rsc) 503{ 504 fd_screen_assert_locked(batch->ctx->screen); 505 506 if (rsc->stencil) 507 fd_batch_resource_read(batch, rsc->stencil); 508 509 DBG("%p: read %p", batch, rsc); 510 511 /* If reading a resource pending a write, go ahead and flush the 512 * writer. This avoids situations where we end up having to 513 * flush the current batch in _resource_used() 514 */ 515 if (unlikely(rsc->track->write_batch && rsc->track->write_batch != batch)) 516 flush_write_batch(rsc); 517 518 fd_batch_add_resource(batch, rsc); 519} 520 521void 522fd_batch_check_size(struct fd_batch *batch) 523{ 524 if (FD_DBG(FLUSH)) { 525 fd_batch_flush(batch); 526 return; 527 } 528 529 /* Place a reasonable upper bound on prim/draw stream buffer size: */ 530 const unsigned limit_bits = 8 * 8 * 1024 * 1024; 531 if ((batch->prim_strm_bits > limit_bits) || 532 (batch->draw_strm_bits > limit_bits)) { 533 fd_batch_flush(batch); 534 return; 535 } 536 537 if (!fd_ringbuffer_check_size(batch->draw)) 538 fd_batch_flush(batch); 539} 540 541/* emit a WAIT_FOR_IDLE only if needed, ie. if there has not already 542 * been one since last draw: 543 */ 544void 545fd_wfi(struct fd_batch *batch, struct fd_ringbuffer *ring) 546{ 547 if (batch->needs_wfi) { 548 if (batch->ctx->screen->gen >= 5) 549 OUT_WFI5(ring); 550 else 551 OUT_WFI(ring); 552 batch->needs_wfi = false; 553 } 554} 555