1/* 2 * Copyright (C) 2019-2020 Collabora, Ltd. 3 * © Copyright 2018 Alyssa Rosenzweig 4 * Copyright © 2014-2017 Broadcom 5 * Copyright (C) 2017 Intel Corporation 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 * SOFTWARE. 25 * 26 */ 27 28#include <sys/poll.h> 29#include <errno.h> 30 31#include "pan_bo.h" 32#include "pan_context.h" 33#include "pan_minmax_cache.h" 34 35#include "util/macros.h" 36#include "util/format/u_format.h" 37#include "util/u_inlines.h" 38#include "util/u_upload_mgr.h" 39#include "util/u_memory.h" 40#include "util/u_vbuf.h" 41#include "util/half_float.h" 42#include "util/u_helpers.h" 43#include "util/format/u_format.h" 44#include "util/u_prim.h" 45#include "util/u_prim_restart.h" 46#include "tgsi/tgsi_parse.h" 47#include "tgsi/tgsi_from_mesa.h" 48#include "nir/tgsi_to_nir.h" 49#include "util/u_math.h" 50 51#include "pan_screen.h" 52#include "pan_util.h" 53#include "decode.h" 54#include "util/pan_lower_framebuffer.h" 55 56static void 57panfrost_clear( 58 struct pipe_context *pipe, 59 unsigned buffers, 60 const struct pipe_scissor_state *scissor_state, 61 const union pipe_color_union *color, 62 double depth, unsigned stencil) 63{ 64 struct panfrost_context *ctx = pan_context(pipe); 65 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx); 66 67 if (!panfrost_render_condition_check(ctx)) 68 return; 69 70 /* At the start of the batch, we can clear for free */ 71 if (!batch->scoreboard.first_job) { 72 panfrost_batch_clear(batch, buffers, color, depth, stencil); 73 return; 74 } 75 76 /* Once there is content, clear with a fullscreen quad */ 77 panfrost_blitter_save(ctx, false /* render condition */); 78 79 util_blitter_clear(ctx->blitter, 80 ctx->pipe_framebuffer.width, 81 ctx->pipe_framebuffer.height, 82 util_framebuffer_get_num_layers(&ctx->pipe_framebuffer), 83 buffers, color, depth, stencil, 84 util_framebuffer_get_num_samples(&ctx->pipe_framebuffer) > 1); 85} 86 87bool 88panfrost_writes_point_size(struct panfrost_context *ctx) 89{ 90 assert(ctx->shader[PIPE_SHADER_VERTEX]); 91 struct panfrost_shader_state *vs = panfrost_get_shader_state(ctx, PIPE_SHADER_VERTEX); 92 93 return vs->info.vs.writes_point_size && ctx->active_prim == PIPE_PRIM_POINTS; 94} 95 96/* The entire frame is in memory -- send it off to the kernel! */ 97 98void 99panfrost_flush( 100 struct pipe_context *pipe, 101 struct pipe_fence_handle **fence, 102 unsigned flags) 103{ 104 struct panfrost_context *ctx = pan_context(pipe); 105 struct panfrost_device *dev = pan_device(pipe->screen); 106 107 108 /* Submit all pending jobs */ 109 panfrost_flush_all_batches(ctx, NULL); 110 111 if (fence) { 112 struct pipe_fence_handle *f = panfrost_fence_create(ctx); 113 pipe->screen->fence_reference(pipe->screen, fence, NULL); 114 *fence = f; 115 } 116 117 if (dev->debug & PAN_DBG_TRACE) 118 pandecode_next_frame(); 119} 120 121static void 122panfrost_texture_barrier(struct pipe_context *pipe, unsigned flags) 123{ 124 struct panfrost_context *ctx = pan_context(pipe); 125 panfrost_flush_all_batches(ctx, "Texture barrier"); 126} 127 128static void 129panfrost_set_frontend_noop(struct pipe_context *pipe, bool enable) 130{ 131 struct panfrost_context *ctx = pan_context(pipe); 132 panfrost_flush_all_batches(ctx, "Frontend no-op change"); 133 ctx->is_noop = enable; 134} 135 136 137static void 138panfrost_generic_cso_delete(struct pipe_context *pctx, void *hwcso) 139{ 140 free(hwcso); 141} 142 143static void 144panfrost_bind_blend_state(struct pipe_context *pipe, void *cso) 145{ 146 struct panfrost_context *ctx = pan_context(pipe); 147 ctx->blend = cso; 148 ctx->dirty |= PAN_DIRTY_BLEND; 149} 150 151static void 152panfrost_set_blend_color(struct pipe_context *pipe, 153 const struct pipe_blend_color *blend_color) 154{ 155 struct panfrost_context *ctx = pan_context(pipe); 156 ctx->dirty |= PAN_DIRTY_BLEND; 157 158 if (blend_color) 159 ctx->blend_color = *blend_color; 160} 161 162/* Create a final blend given the context */ 163 164mali_ptr 165panfrost_get_blend(struct panfrost_batch *batch, unsigned rti, struct panfrost_bo **bo, unsigned *shader_offset) 166{ 167 struct panfrost_context *ctx = batch->ctx; 168 struct panfrost_device *dev = pan_device(ctx->base.screen); 169 struct panfrost_blend_state *blend = ctx->blend; 170 struct pan_blend_info info = blend->info[rti]; 171 struct pipe_surface *surf = batch->key.cbufs[rti]; 172 enum pipe_format fmt = surf->format; 173 174 /* Use fixed-function if the equation permits, the format is blendable, 175 * and no more than one unique constant is accessed */ 176 if (info.fixed_function && panfrost_blendable_formats_v7[fmt].internal && 177 pan_blend_is_homogenous_constant(info.constant_mask, 178 ctx->blend_color.color)) { 179 return 0; 180 } 181 182 /* Otherwise, we need to grab a shader */ 183 struct pan_blend_state pan_blend = blend->pan; 184 unsigned nr_samples = surf->nr_samples ? : surf->texture->nr_samples; 185 186 pan_blend.rts[rti].format = fmt; 187 pan_blend.rts[rti].nr_samples = nr_samples; 188 memcpy(pan_blend.constants, ctx->blend_color.color, 189 sizeof(pan_blend.constants)); 190 191 /* Upload the shader, sharing a BO */ 192 if (!(*bo)) { 193 *bo = panfrost_batch_create_bo(batch, 4096, PAN_BO_EXECUTE, 194 PIPE_SHADER_FRAGMENT, "Blend shader"); 195 } 196 197 struct panfrost_shader_state *ss = panfrost_get_shader_state(ctx, PIPE_SHADER_FRAGMENT); 198 199 /* Default for Midgard */ 200 nir_alu_type col0_type = nir_type_float32; 201 nir_alu_type col1_type = nir_type_float32; 202 203 /* Bifrost has per-output types, respect them */ 204 if (dev->arch >= 6) { 205 col0_type = ss->info.bifrost.blend[rti].type; 206 col1_type = ss->info.bifrost.blend_src1_type; 207 } 208 209 pthread_mutex_lock(&dev->blend_shaders.lock); 210 struct pan_blend_shader_variant *shader = 211 pan_screen(ctx->base.screen)->vtbl.get_blend_shader(dev, 212 &pan_blend, 213 col0_type, 214 col1_type, 215 rti); 216 217 /* Size check and upload */ 218 unsigned offset = *shader_offset; 219 assert((offset + shader->binary.size) < 4096); 220 memcpy((*bo)->ptr.cpu + offset, shader->binary.data, shader->binary.size); 221 *shader_offset += shader->binary.size; 222 pthread_mutex_unlock(&dev->blend_shaders.lock); 223 224 return ((*bo)->ptr.gpu + offset) | shader->first_tag; 225} 226 227static void 228panfrost_bind_rasterizer_state( 229 struct pipe_context *pctx, 230 void *hwcso) 231{ 232 struct panfrost_context *ctx = pan_context(pctx); 233 ctx->rasterizer = hwcso; 234 235 /* We can assume rasterizer is always dirty, the dependencies are 236 * too intricate to bother tracking in detail. However we could 237 * probably diff the renderers for viewport dirty tracking, that 238 * just cares about the scissor enable and the depth clips. */ 239 ctx->dirty |= PAN_DIRTY_SCISSOR | PAN_DIRTY_RASTERIZER; 240} 241 242static void 243panfrost_set_shader_images( 244 struct pipe_context *pctx, 245 enum pipe_shader_type shader, 246 unsigned start_slot, unsigned count, unsigned unbind_num_trailing_slots, 247 const struct pipe_image_view *iviews) 248{ 249 struct panfrost_context *ctx = pan_context(pctx); 250 ctx->dirty_shader[PIPE_SHADER_FRAGMENT] |= PAN_DIRTY_STAGE_IMAGE; 251 252 /* Unbind start_slot...start_slot+count */ 253 if (!iviews) { 254 for (int i = start_slot; i < start_slot + count + unbind_num_trailing_slots; i++) { 255 pipe_resource_reference(&ctx->images[shader][i].resource, NULL); 256 } 257 258 ctx->image_mask[shader] &= ~(((1ull << count) - 1) << start_slot); 259 return; 260 } 261 262 /* Bind start_slot...start_slot+count */ 263 for (int i = 0; i < count; i++) { 264 const struct pipe_image_view *image = &iviews[i]; 265 SET_BIT(ctx->image_mask[shader], 1 << (start_slot + i), image->resource); 266 267 if (!image->resource) { 268 util_copy_image_view(&ctx->images[shader][start_slot+i], NULL); 269 continue; 270 } 271 272 struct panfrost_resource *rsrc = pan_resource(image->resource); 273 274 /* Images don't work with AFBC, since they require pixel-level granularity */ 275 if (drm_is_afbc(rsrc->image.layout.modifier)) { 276 pan_resource_modifier_convert(ctx, rsrc, 277 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, 278 "Shader image"); 279 } 280 281 util_copy_image_view(&ctx->images[shader][start_slot+i], image); 282 } 283 284 /* Unbind start_slot+count...start_slot+count+unbind_num_trailing_slots */ 285 for (int i = 0; i < unbind_num_trailing_slots; i++) { 286 SET_BIT(ctx->image_mask[shader], 1 << (start_slot + count + i), NULL); 287 util_copy_image_view(&ctx->images[shader][start_slot+count+i], NULL); 288 } 289} 290 291static void 292panfrost_bind_vertex_elements_state( 293 struct pipe_context *pctx, 294 void *hwcso) 295{ 296 struct panfrost_context *ctx = pan_context(pctx); 297 ctx->vertex = hwcso; 298 ctx->dirty |= PAN_DIRTY_VERTEX; 299} 300 301static void * 302panfrost_create_shader_state( 303 struct pipe_context *pctx, 304 const struct pipe_shader_state *cso) 305{ 306 struct panfrost_shader_variants *so = CALLOC_STRUCT(panfrost_shader_variants); 307 struct panfrost_device *dev = pan_device(pctx->screen); 308 309 simple_mtx_init(&so->lock, mtx_plain); 310 311 so->stream_output = cso->stream_output; 312 313 if (cso->type == PIPE_SHADER_IR_TGSI) 314 so->nir = tgsi_to_nir(cso->tokens, pctx->screen, false); 315 else 316 so->nir = cso->ir.nir; 317 318 /* Fix linkage early */ 319 if (so->nir->info.stage == MESA_SHADER_VERTEX) { 320 so->fixed_varying_mask = 321 (so->nir->info.outputs_written & BITFIELD_MASK(VARYING_SLOT_VAR0)) & 322 ~VARYING_BIT_POS & ~VARYING_BIT_PSIZ; 323 } 324 325 /* Precompile for shader-db if we need to */ 326 if (unlikely(dev->debug & PAN_DBG_PRECOMPILE)) { 327 struct panfrost_context *ctx = pan_context(pctx); 328 329 struct panfrost_shader_state state = { 0 }; 330 331 panfrost_shader_compile(pctx->screen, 332 &ctx->shaders, &ctx->descs, 333 so->nir, &state); 334 } 335 336 return so; 337} 338 339static void 340panfrost_delete_shader_state( 341 struct pipe_context *pctx, 342 void *so) 343{ 344 struct panfrost_shader_variants *cso = (struct panfrost_shader_variants *) so; 345 346 ralloc_free(cso->nir); 347 348 for (unsigned i = 0; i < cso->variant_count; ++i) { 349 struct panfrost_shader_state *shader_state = &cso->variants[i]; 350 panfrost_bo_unreference(shader_state->bin.bo); 351 panfrost_bo_unreference(shader_state->state.bo); 352 panfrost_bo_unreference(shader_state->linkage.bo); 353 354 if (shader_state->xfb) { 355 panfrost_bo_unreference(shader_state->xfb->bin.bo); 356 panfrost_bo_unreference(shader_state->xfb->state.bo); 357 panfrost_bo_unreference(shader_state->xfb->linkage.bo); 358 free(shader_state->xfb); 359 } 360 } 361 362 simple_mtx_destroy(&cso->lock); 363 364 free(cso->variants); 365 free(so); 366} 367 368static void 369panfrost_bind_sampler_states( 370 struct pipe_context *pctx, 371 enum pipe_shader_type shader, 372 unsigned start_slot, unsigned num_sampler, 373 void **sampler) 374{ 375 assert(start_slot == 0); 376 377 struct panfrost_context *ctx = pan_context(pctx); 378 ctx->dirty_shader[shader] |= PAN_DIRTY_STAGE_SAMPLER; 379 380 ctx->sampler_count[shader] = sampler ? num_sampler : 0; 381 if (sampler) 382 memcpy(ctx->samplers[shader], sampler, num_sampler * sizeof (void *)); 383} 384 385static void 386panfrost_build_key(struct panfrost_context *ctx, 387 struct panfrost_shader_key *key, 388 nir_shader *nir) 389{ 390 /* We don't currently have vertex shader variants */ 391 if (nir->info.stage != MESA_SHADER_FRAGMENT) 392 return; 393 394 struct panfrost_device *dev = pan_device(ctx->base.screen); 395 struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer; 396 struct pipe_rasterizer_state *rast = (void *) ctx->rasterizer; 397 struct panfrost_shader_variants *vs = ctx->shader[MESA_SHADER_VERTEX]; 398 399 key->fs.nr_cbufs = fb->nr_cbufs; 400 401 /* Point sprite lowering needed on Bifrost and newer */ 402 if (dev->arch >= 6 && rast && ctx->active_prim == PIPE_PRIM_POINTS) { 403 key->fs.sprite_coord_enable = rast->sprite_coord_enable; 404 } 405 406 /* User clip plane lowering needed everywhere */ 407 if (rast) { 408 key->fs.clip_plane_enable = rast->clip_plane_enable; 409 } 410 411 if (dev->arch <= 5) { 412 u_foreach_bit(i, (nir->info.outputs_read >> FRAG_RESULT_DATA0)) { 413 enum pipe_format fmt = PIPE_FORMAT_R8G8B8A8_UNORM; 414 415 if ((fb->nr_cbufs > i) && fb->cbufs[i]) 416 fmt = fb->cbufs[i]->format; 417 418 if (panfrost_blendable_formats_v6[fmt].internal) 419 fmt = PIPE_FORMAT_NONE; 420 421 key->fs.rt_formats[i] = fmt; 422 } 423 } 424 425 /* Funny desktop GL varying lowering on Valhall */ 426 if (dev->arch >= 9) { 427 assert(vs != NULL && "too early"); 428 key->fixed_varying_mask = vs->fixed_varying_mask; 429 } 430} 431 432/** 433 * Fix an uncompiled shader's stream output info, and produce a bitmask 434 * of which VARYING_SLOT_* are captured for stream output. 435 * 436 * Core Gallium stores output->register_index as a "slot" number, where 437 * slots are assigned consecutively to all outputs in info->outputs_written. 438 * This naive packing of outputs doesn't work for us - we too have slots, 439 * but the layout is defined by the VUE map, which we won't have until we 440 * compile a specific shader variant. So, we remap these and simply store 441 * VARYING_SLOT_* in our copy's output->register_index fields. 442 * 443 * We then produce a bitmask of outputs which are used for SO. 444 * 445 * Implementation from iris. 446 */ 447 448static uint64_t 449update_so_info(struct pipe_stream_output_info *so_info, 450 uint64_t outputs_written) 451{ 452 uint64_t so_outputs = 0; 453 uint8_t reverse_map[64] = {0}; 454 unsigned slot = 0; 455 456 while (outputs_written) 457 reverse_map[slot++] = u_bit_scan64(&outputs_written); 458 459 for (unsigned i = 0; i < so_info->num_outputs; i++) { 460 struct pipe_stream_output *output = &so_info->output[i]; 461 462 /* Map Gallium's condensed "slots" back to real VARYING_SLOT_* enums */ 463 output->register_index = reverse_map[output->register_index]; 464 465 so_outputs |= 1ull << output->register_index; 466 } 467 468 return so_outputs; 469} 470 471static unsigned 472panfrost_new_variant_locked( 473 struct panfrost_context *ctx, 474 struct panfrost_shader_variants *variants, 475 struct panfrost_shader_key *key) 476{ 477 unsigned variant = variants->variant_count++; 478 479 if (variants->variant_count > variants->variant_space) { 480 unsigned old_space = variants->variant_space; 481 482 variants->variant_space *= 2; 483 if (variants->variant_space == 0) 484 variants->variant_space = 1; 485 486 /* Arbitrary limit to stop runaway programs from 487 * creating an unbounded number of shader variants. */ 488 assert(variants->variant_space < 1024); 489 490 unsigned msize = sizeof(struct panfrost_shader_state); 491 variants->variants = realloc(variants->variants, 492 variants->variant_space * msize); 493 494 memset(&variants->variants[old_space], 0, 495 (variants->variant_space - old_space) * msize); 496 } 497 498 variants->variants[variant].key = *key; 499 500 struct panfrost_shader_state *shader_state = &variants->variants[variant]; 501 502 /* We finally have a variant, so compile it */ 503 panfrost_shader_compile(ctx->base.screen, 504 &ctx->shaders, &ctx->descs, 505 variants->nir, shader_state); 506 507 /* Fixup the stream out information */ 508 shader_state->stream_output = variants->stream_output; 509 shader_state->so_mask = 510 update_so_info(&shader_state->stream_output, 511 shader_state->info.outputs_written); 512 513 shader_state->earlyzs = pan_earlyzs_analyze(&shader_state->info); 514 515 return variant; 516} 517 518static void 519panfrost_bind_shader_state( 520 struct pipe_context *pctx, 521 void *hwcso, 522 enum pipe_shader_type type) 523{ 524 struct panfrost_context *ctx = pan_context(pctx); 525 ctx->shader[type] = hwcso; 526 527 ctx->dirty |= PAN_DIRTY_TLS_SIZE; 528 ctx->dirty_shader[type] |= PAN_DIRTY_STAGE_SHADER; 529 530 if (hwcso) 531 panfrost_update_shader_variant(ctx, type); 532} 533 534void 535panfrost_update_shader_variant(struct panfrost_context *ctx, 536 enum pipe_shader_type type) 537{ 538 /* No shader variants for compute */ 539 if (type == PIPE_SHADER_COMPUTE) 540 return; 541 542 /* We need linking information, defer this */ 543 if (type == PIPE_SHADER_FRAGMENT && !ctx->shader[PIPE_SHADER_VERTEX]) 544 return; 545 546 /* Also defer, happens with GALLIUM_HUD */ 547 if (!ctx->shader[type]) 548 return; 549 550 /* Match the appropriate variant */ 551 signed variant = -1; 552 struct panfrost_shader_variants *variants = ctx->shader[type]; 553 554 simple_mtx_lock(&variants->lock); 555 556 struct panfrost_shader_key key = { 557 .fixed_varying_mask = variants->fixed_varying_mask 558 }; 559 560 panfrost_build_key(ctx, &key, variants->nir); 561 562 for (unsigned i = 0; i < variants->variant_count; ++i) { 563 if (memcmp(&key, &variants->variants[i].key, sizeof(key)) == 0) { 564 variant = i; 565 break; 566 } 567 } 568 569 if (variant == -1) 570 variant = panfrost_new_variant_locked(ctx, variants, &key); 571 572 variants->active_variant = variant; 573 574 /* TODO: it would be more efficient to release the lock before 575 * compiling instead of after, but that can race if thread A compiles a 576 * variant while thread B searches for that same variant */ 577 simple_mtx_unlock(&variants->lock); 578} 579 580static void 581panfrost_bind_vs_state(struct pipe_context *pctx, void *hwcso) 582{ 583 panfrost_bind_shader_state(pctx, hwcso, PIPE_SHADER_VERTEX); 584 585 /* Fragment shaders are linked with vertex shaders */ 586 struct panfrost_context *ctx = pan_context(pctx); 587 panfrost_update_shader_variant(ctx, PIPE_SHADER_FRAGMENT); 588} 589 590static void 591panfrost_bind_fs_state(struct pipe_context *pctx, void *hwcso) 592{ 593 panfrost_bind_shader_state(pctx, hwcso, PIPE_SHADER_FRAGMENT); 594} 595 596static void 597panfrost_set_vertex_buffers( 598 struct pipe_context *pctx, 599 unsigned start_slot, 600 unsigned num_buffers, 601 unsigned unbind_num_trailing_slots, 602 bool take_ownership, 603 const struct pipe_vertex_buffer *buffers) 604{ 605 struct panfrost_context *ctx = pan_context(pctx); 606 607 util_set_vertex_buffers_mask(ctx->vertex_buffers, &ctx->vb_mask, buffers, 608 start_slot, num_buffers, unbind_num_trailing_slots, 609 take_ownership); 610 611 ctx->dirty |= PAN_DIRTY_VERTEX; 612} 613 614static void 615panfrost_set_constant_buffer( 616 struct pipe_context *pctx, 617 enum pipe_shader_type shader, uint index, bool take_ownership, 618 const struct pipe_constant_buffer *buf) 619{ 620 struct panfrost_context *ctx = pan_context(pctx); 621 struct panfrost_constant_buffer *pbuf = &ctx->constant_buffer[shader]; 622 623 util_copy_constant_buffer(&pbuf->cb[index], buf, take_ownership); 624 625 unsigned mask = (1 << index); 626 627 if (unlikely(!buf)) { 628 pbuf->enabled_mask &= ~mask; 629 return; 630 } 631 632 pbuf->enabled_mask |= mask; 633 ctx->dirty_shader[shader] |= PAN_DIRTY_STAGE_CONST; 634} 635 636static void 637panfrost_set_stencil_ref( 638 struct pipe_context *pctx, 639 const struct pipe_stencil_ref ref) 640{ 641 struct panfrost_context *ctx = pan_context(pctx); 642 ctx->stencil_ref = ref; 643 ctx->dirty |= PAN_DIRTY_ZS; 644} 645 646static void 647panfrost_set_sampler_views( 648 struct pipe_context *pctx, 649 enum pipe_shader_type shader, 650 unsigned start_slot, unsigned num_views, 651 unsigned unbind_num_trailing_slots, 652 bool take_ownership, 653 struct pipe_sampler_view **views) 654{ 655 struct panfrost_context *ctx = pan_context(pctx); 656 ctx->dirty_shader[shader] |= PAN_DIRTY_STAGE_TEXTURE; 657 658 unsigned new_nr = 0; 659 unsigned i; 660 661 for (i = 0; i < num_views; ++i) { 662 struct pipe_sampler_view *view = views ? views[i] : NULL; 663 unsigned p = i + start_slot; 664 665 if (view) 666 new_nr = p + 1; 667 668 if (take_ownership) { 669 pipe_sampler_view_reference((struct pipe_sampler_view **)&ctx->sampler_views[shader][p], 670 NULL); 671 ctx->sampler_views[shader][i] = (struct panfrost_sampler_view *)view; 672 } else { 673 pipe_sampler_view_reference((struct pipe_sampler_view **)&ctx->sampler_views[shader][p], 674 view); 675 } 676 } 677 678 for (; i < num_views + unbind_num_trailing_slots; i++) { 679 unsigned p = i + start_slot; 680 pipe_sampler_view_reference((struct pipe_sampler_view **)&ctx->sampler_views[shader][p], 681 NULL); 682 } 683 684 /* If the sampler view count is higher than the greatest sampler view 685 * we touch, it can't change */ 686 if (ctx->sampler_view_count[shader] > start_slot + num_views + unbind_num_trailing_slots) 687 return; 688 689 /* If we haven't set any sampler views here, search lower numbers for 690 * set sampler views */ 691 if (new_nr == 0) { 692 for (i = 0; i < start_slot; ++i) { 693 if (ctx->sampler_views[shader][i]) 694 new_nr = i + 1; 695 } 696 } 697 698 ctx->sampler_view_count[shader] = new_nr; 699} 700 701static void 702panfrost_set_shader_buffers( 703 struct pipe_context *pctx, 704 enum pipe_shader_type shader, 705 unsigned start, unsigned count, 706 const struct pipe_shader_buffer *buffers, 707 unsigned writable_bitmask) 708{ 709 struct panfrost_context *ctx = pan_context(pctx); 710 711 util_set_shader_buffers_mask(ctx->ssbo[shader], &ctx->ssbo_mask[shader], 712 buffers, start, count); 713 714 ctx->dirty_shader[shader] |= PAN_DIRTY_STAGE_SSBO; 715} 716 717static void 718panfrost_set_framebuffer_state(struct pipe_context *pctx, 719 const struct pipe_framebuffer_state *fb) 720{ 721 struct panfrost_context *ctx = pan_context(pctx); 722 723 util_copy_framebuffer_state(&ctx->pipe_framebuffer, fb); 724 ctx->batch = NULL; 725 726 /* Hot draw call path needs the mask of active render targets */ 727 ctx->fb_rt_mask = 0; 728 729 for (unsigned i = 0; i < ctx->pipe_framebuffer.nr_cbufs; ++i) { 730 if (ctx->pipe_framebuffer.cbufs[i]) 731 ctx->fb_rt_mask |= BITFIELD_BIT(i); 732 } 733} 734 735static void 736panfrost_bind_depth_stencil_state(struct pipe_context *pipe, 737 void *cso) 738{ 739 struct panfrost_context *ctx = pan_context(pipe); 740 ctx->depth_stencil = cso; 741 ctx->dirty |= PAN_DIRTY_ZS; 742} 743 744static void 745panfrost_set_sample_mask(struct pipe_context *pipe, 746 unsigned sample_mask) 747{ 748 struct panfrost_context *ctx = pan_context(pipe); 749 ctx->sample_mask = sample_mask; 750 ctx->dirty |= PAN_DIRTY_MSAA; 751} 752 753static void 754panfrost_set_min_samples(struct pipe_context *pipe, 755 unsigned min_samples) 756{ 757 struct panfrost_context *ctx = pan_context(pipe); 758 ctx->min_samples = min_samples; 759 ctx->dirty |= PAN_DIRTY_MSAA; 760} 761 762static void 763panfrost_set_clip_state(struct pipe_context *pipe, 764 const struct pipe_clip_state *clip) 765{ 766 //struct panfrost_context *panfrost = pan_context(pipe); 767} 768 769static void 770panfrost_set_viewport_states(struct pipe_context *pipe, 771 unsigned start_slot, 772 unsigned num_viewports, 773 const struct pipe_viewport_state *viewports) 774{ 775 struct panfrost_context *ctx = pan_context(pipe); 776 777 assert(start_slot == 0); 778 assert(num_viewports == 1); 779 780 ctx->pipe_viewport = *viewports; 781 ctx->dirty |= PAN_DIRTY_VIEWPORT; 782} 783 784static void 785panfrost_set_scissor_states(struct pipe_context *pipe, 786 unsigned start_slot, 787 unsigned num_scissors, 788 const struct pipe_scissor_state *scissors) 789{ 790 struct panfrost_context *ctx = pan_context(pipe); 791 792 assert(start_slot == 0); 793 assert(num_scissors == 1); 794 795 ctx->scissor = *scissors; 796 ctx->dirty |= PAN_DIRTY_SCISSOR; 797} 798 799static void 800panfrost_set_polygon_stipple(struct pipe_context *pipe, 801 const struct pipe_poly_stipple *stipple) 802{ 803 //struct panfrost_context *panfrost = pan_context(pipe); 804} 805 806static void 807panfrost_set_active_query_state(struct pipe_context *pipe, 808 bool enable) 809{ 810 struct panfrost_context *ctx = pan_context(pipe); 811 ctx->active_queries = enable; 812 ctx->dirty |= PAN_DIRTY_OQ; 813} 814 815static void 816panfrost_render_condition(struct pipe_context *pipe, 817 struct pipe_query *query, 818 bool condition, 819 enum pipe_render_cond_flag mode) 820{ 821 struct panfrost_context *ctx = pan_context(pipe); 822 823 ctx->cond_query = (struct panfrost_query *)query; 824 ctx->cond_cond = condition; 825 ctx->cond_mode = mode; 826} 827 828static void 829panfrost_destroy(struct pipe_context *pipe) 830{ 831 struct panfrost_context *panfrost = pan_context(pipe); 832 833 _mesa_hash_table_destroy(panfrost->writers, NULL); 834 835 if (panfrost->blitter) 836 util_blitter_destroy(panfrost->blitter); 837 838 util_unreference_framebuffer_state(&panfrost->pipe_framebuffer); 839 u_upload_destroy(pipe->stream_uploader); 840 841 panfrost_pool_cleanup(&panfrost->descs); 842 panfrost_pool_cleanup(&panfrost->shaders); 843 844 ralloc_free(pipe); 845} 846 847static struct pipe_query * 848panfrost_create_query(struct pipe_context *pipe, 849 unsigned type, 850 unsigned index) 851{ 852 struct panfrost_query *q = rzalloc(pipe, struct panfrost_query); 853 854 q->type = type; 855 q->index = index; 856 857 return (struct pipe_query *) q; 858} 859 860static void 861panfrost_destroy_query(struct pipe_context *pipe, struct pipe_query *q) 862{ 863 struct panfrost_query *query = (struct panfrost_query *) q; 864 865 if (query->rsrc) 866 pipe_resource_reference(&query->rsrc, NULL); 867 868 ralloc_free(q); 869} 870 871static bool 872panfrost_begin_query(struct pipe_context *pipe, struct pipe_query *q) 873{ 874 struct panfrost_context *ctx = pan_context(pipe); 875 struct panfrost_device *dev = pan_device(ctx->base.screen); 876 struct panfrost_query *query = (struct panfrost_query *) q; 877 878 switch (query->type) { 879 case PIPE_QUERY_OCCLUSION_COUNTER: 880 case PIPE_QUERY_OCCLUSION_PREDICATE: 881 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: { 882 unsigned size = sizeof(uint64_t) * dev->core_id_range; 883 884 /* Allocate a resource for the query results to be stored */ 885 if (!query->rsrc) { 886 query->rsrc = pipe_buffer_create(ctx->base.screen, 887 PIPE_BIND_QUERY_BUFFER, 0, size); 888 } 889 890 /* Default to 0 if nothing at all drawn. */ 891 uint8_t *zeroes = alloca(size); 892 memset(zeroes, 0, size); 893 pipe_buffer_write(pipe, query->rsrc, 0, size, zeroes); 894 895 query->msaa = (ctx->pipe_framebuffer.samples > 1); 896 ctx->occlusion_query = query; 897 ctx->dirty |= PAN_DIRTY_OQ; 898 break; 899 } 900 901 /* Geometry statistics are computed in the driver. XXX: geom/tess 902 * shaders.. */ 903 904 case PIPE_QUERY_PRIMITIVES_GENERATED: 905 query->start = ctx->prims_generated; 906 break; 907 case PIPE_QUERY_PRIMITIVES_EMITTED: 908 query->start = ctx->tf_prims_generated; 909 break; 910 911 default: 912 /* TODO: timestamp queries, etc? */ 913 break; 914 } 915 916 return true; 917} 918 919static bool 920panfrost_end_query(struct pipe_context *pipe, struct pipe_query *q) 921{ 922 struct panfrost_context *ctx = pan_context(pipe); 923 struct panfrost_query *query = (struct panfrost_query *) q; 924 925 switch (query->type) { 926 case PIPE_QUERY_OCCLUSION_COUNTER: 927 case PIPE_QUERY_OCCLUSION_PREDICATE: 928 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 929 ctx->occlusion_query = NULL; 930 ctx->dirty |= PAN_DIRTY_OQ; 931 break; 932 case PIPE_QUERY_PRIMITIVES_GENERATED: 933 query->end = ctx->prims_generated; 934 break; 935 case PIPE_QUERY_PRIMITIVES_EMITTED: 936 query->end = ctx->tf_prims_generated; 937 break; 938 } 939 940 return true; 941} 942 943static bool 944panfrost_get_query_result(struct pipe_context *pipe, 945 struct pipe_query *q, 946 bool wait, 947 union pipe_query_result *vresult) 948{ 949 struct panfrost_query *query = (struct panfrost_query *) q; 950 struct panfrost_context *ctx = pan_context(pipe); 951 struct panfrost_device *dev = pan_device(ctx->base.screen); 952 struct panfrost_resource *rsrc = pan_resource(query->rsrc); 953 954 switch (query->type) { 955 case PIPE_QUERY_OCCLUSION_COUNTER: 956 case PIPE_QUERY_OCCLUSION_PREDICATE: 957 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 958 panfrost_flush_writer(ctx, rsrc, "Occlusion query"); 959 panfrost_bo_wait(rsrc->image.data.bo, INT64_MAX, false); 960 961 /* Read back the query results */ 962 uint64_t *result = (uint64_t *) rsrc->image.data.bo->ptr.cpu; 963 964 if (query->type == PIPE_QUERY_OCCLUSION_COUNTER) { 965 uint64_t passed = 0; 966 for (int i = 0; i < dev->core_id_range; ++i) 967 passed += result[i]; 968 969 if (dev->arch <= 5 && !query->msaa) 970 passed /= 4; 971 972 vresult->u64 = passed; 973 } else { 974 vresult->b = !!result[0]; 975 } 976 977 break; 978 979 case PIPE_QUERY_PRIMITIVES_GENERATED: 980 case PIPE_QUERY_PRIMITIVES_EMITTED: 981 panfrost_flush_all_batches(ctx, "Primitive count query"); 982 vresult->u64 = query->end - query->start; 983 break; 984 985 default: 986 /* TODO: more queries */ 987 break; 988 } 989 990 return true; 991} 992 993bool 994panfrost_render_condition_check(struct panfrost_context *ctx) 995{ 996 if (!ctx->cond_query) 997 return true; 998 999 perf_debug_ctx(ctx, "Implementing conditional rendering on the CPU"); 1000 1001 union pipe_query_result res = { 0 }; 1002 bool wait = 1003 ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT && 1004 ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT; 1005 1006 struct pipe_query *pq = (struct pipe_query *)ctx->cond_query; 1007 1008 if (panfrost_get_query_result(&ctx->base, pq, wait, &res)) 1009 return res.u64 != ctx->cond_cond; 1010 1011 return true; 1012} 1013 1014static struct pipe_stream_output_target * 1015panfrost_create_stream_output_target(struct pipe_context *pctx, 1016 struct pipe_resource *prsc, 1017 unsigned buffer_offset, 1018 unsigned buffer_size) 1019{ 1020 struct pipe_stream_output_target *target; 1021 1022 target = &rzalloc(pctx, struct panfrost_streamout_target)->base; 1023 1024 if (!target) 1025 return NULL; 1026 1027 pipe_reference_init(&target->reference, 1); 1028 pipe_resource_reference(&target->buffer, prsc); 1029 1030 target->context = pctx; 1031 target->buffer_offset = buffer_offset; 1032 target->buffer_size = buffer_size; 1033 1034 return target; 1035} 1036 1037static void 1038panfrost_stream_output_target_destroy(struct pipe_context *pctx, 1039 struct pipe_stream_output_target *target) 1040{ 1041 pipe_resource_reference(&target->buffer, NULL); 1042 ralloc_free(target); 1043} 1044 1045static void 1046panfrost_set_stream_output_targets(struct pipe_context *pctx, 1047 unsigned num_targets, 1048 struct pipe_stream_output_target **targets, 1049 const unsigned *offsets) 1050{ 1051 struct panfrost_context *ctx = pan_context(pctx); 1052 struct panfrost_streamout *so = &ctx->streamout; 1053 1054 assert(num_targets <= ARRAY_SIZE(so->targets)); 1055 1056 for (unsigned i = 0; i < num_targets; i++) { 1057 if (targets[i] && offsets[i] != -1) 1058 pan_so_target(targets[i])->offset = offsets[i]; 1059 1060 pipe_so_target_reference(&so->targets[i], targets[i]); 1061 } 1062 1063 for (unsigned i = num_targets; i < so->num_targets; i++) 1064 pipe_so_target_reference(&so->targets[i], NULL); 1065 1066 so->num_targets = num_targets; 1067 ctx->dirty |= PAN_DIRTY_SO; 1068} 1069 1070struct pipe_context * 1071panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags) 1072{ 1073 struct panfrost_context *ctx = rzalloc(NULL, struct panfrost_context); 1074 struct pipe_context *gallium = (struct pipe_context *) ctx; 1075 struct panfrost_device *dev = pan_device(screen); 1076 1077 gallium->screen = screen; 1078 1079 gallium->destroy = panfrost_destroy; 1080 1081 gallium->set_framebuffer_state = panfrost_set_framebuffer_state; 1082 1083 gallium->flush = panfrost_flush; 1084 gallium->clear = panfrost_clear; 1085 gallium->texture_barrier = panfrost_texture_barrier; 1086 gallium->set_frontend_noop = panfrost_set_frontend_noop; 1087 1088 gallium->set_vertex_buffers = panfrost_set_vertex_buffers; 1089 gallium->set_constant_buffer = panfrost_set_constant_buffer; 1090 gallium->set_shader_buffers = panfrost_set_shader_buffers; 1091 gallium->set_shader_images = panfrost_set_shader_images; 1092 1093 gallium->set_stencil_ref = panfrost_set_stencil_ref; 1094 1095 gallium->set_sampler_views = panfrost_set_sampler_views; 1096 1097 gallium->bind_rasterizer_state = panfrost_bind_rasterizer_state; 1098 gallium->delete_rasterizer_state = panfrost_generic_cso_delete; 1099 1100 gallium->bind_vertex_elements_state = panfrost_bind_vertex_elements_state; 1101 gallium->delete_vertex_elements_state = panfrost_generic_cso_delete; 1102 1103 gallium->create_fs_state = panfrost_create_shader_state; 1104 gallium->delete_fs_state = panfrost_delete_shader_state; 1105 gallium->bind_fs_state = panfrost_bind_fs_state; 1106 1107 gallium->create_vs_state = panfrost_create_shader_state; 1108 gallium->delete_vs_state = panfrost_delete_shader_state; 1109 gallium->bind_vs_state = panfrost_bind_vs_state; 1110 1111 gallium->delete_sampler_state = panfrost_generic_cso_delete; 1112 gallium->bind_sampler_states = panfrost_bind_sampler_states; 1113 1114 gallium->bind_depth_stencil_alpha_state = panfrost_bind_depth_stencil_state; 1115 gallium->delete_depth_stencil_alpha_state = panfrost_generic_cso_delete; 1116 1117 gallium->set_sample_mask = panfrost_set_sample_mask; 1118 gallium->set_min_samples = panfrost_set_min_samples; 1119 1120 gallium->set_clip_state = panfrost_set_clip_state; 1121 gallium->set_viewport_states = panfrost_set_viewport_states; 1122 gallium->set_scissor_states = panfrost_set_scissor_states; 1123 gallium->set_polygon_stipple = panfrost_set_polygon_stipple; 1124 gallium->set_active_query_state = panfrost_set_active_query_state; 1125 gallium->render_condition = panfrost_render_condition; 1126 1127 gallium->create_query = panfrost_create_query; 1128 gallium->destroy_query = panfrost_destroy_query; 1129 gallium->begin_query = panfrost_begin_query; 1130 gallium->end_query = panfrost_end_query; 1131 gallium->get_query_result = panfrost_get_query_result; 1132 1133 gallium->create_stream_output_target = panfrost_create_stream_output_target; 1134 gallium->stream_output_target_destroy = panfrost_stream_output_target_destroy; 1135 gallium->set_stream_output_targets = panfrost_set_stream_output_targets; 1136 1137 gallium->bind_blend_state = panfrost_bind_blend_state; 1138 gallium->delete_blend_state = panfrost_generic_cso_delete; 1139 1140 gallium->set_blend_color = panfrost_set_blend_color; 1141 1142 pan_screen(screen)->vtbl.context_init(gallium); 1143 1144 panfrost_resource_context_init(gallium); 1145 panfrost_compute_context_init(gallium); 1146 1147 gallium->stream_uploader = u_upload_create_default(gallium); 1148 gallium->const_uploader = gallium->stream_uploader; 1149 1150 panfrost_pool_init(&ctx->descs, ctx, dev, 1151 0, 4096, "Descriptors", true, false); 1152 1153 panfrost_pool_init(&ctx->shaders, ctx, dev, 1154 PAN_BO_EXECUTE, 4096, "Shaders", true, false); 1155 1156 ctx->blitter = util_blitter_create(gallium); 1157 1158 ctx->writers = _mesa_hash_table_create(gallium, _mesa_hash_pointer, 1159 _mesa_key_pointer_equal); 1160 1161 assert(ctx->blitter); 1162 1163 /* Prepare for render! */ 1164 1165 /* By default mask everything on */ 1166 ctx->sample_mask = ~0; 1167 ctx->active_queries = true; 1168 1169 int ASSERTED ret; 1170 1171 /* Create a syncobj in a signaled state. Will be updated to point to the 1172 * last queued job out_sync every time we submit a new job. 1173 */ 1174 ret = drmSyncobjCreate(dev->fd, DRM_SYNCOBJ_CREATE_SIGNALED, &ctx->syncobj); 1175 assert(!ret && ctx->syncobj); 1176 1177 return gallium; 1178} 1179