1/************************************************************************** 2 * 3 * Copyright 2009 VMware, Inc. 4 * Copyright 2007 VMware, Inc. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 29/** 30 * @file 31 * Code generate the whole fragment pipeline. 32 * 33 * The fragment pipeline consists of the following stages: 34 * - early depth test 35 * - fragment shader 36 * - alpha test 37 * - depth/stencil test 38 * - blending 39 * 40 * This file has only the glue to assemble the fragment pipeline. The actual 41 * plumbing of converting Gallium state into LLVM IR is done elsewhere, in the 42 * lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we 43 * muster the LLVM JIT execution engine to create a function that follows an 44 * established binary interface and that can be called from C directly. 45 * 46 * A big source of complexity here is that we often want to run different 47 * stages with different precisions and data types and precisions. For example, 48 * the fragment shader needs typically to be done in floats, but the 49 * depth/stencil test and blending is better done in the type that most closely 50 * matches the depth/stencil and color buffer respectively. 51 * 52 * Since the width of a SIMD vector register stays the same regardless of the 53 * element type, different types imply different number of elements, so we must 54 * code generate more instances of the stages with larger types to be able to 55 * feed/consume the stages with smaller types. 56 * 57 * @author Jose Fonseca <jfonseca@vmware.com> 58 */ 59 60#include <limits.h> 61#include "pipe/p_defines.h" 62#include "util/u_inlines.h" 63#include "util/u_memory.h" 64#include "util/u_pointer.h" 65#include "util/format/u_format.h" 66#include "util/u_dump.h" 67#include "util/u_string.h" 68#include "util/u_dual_blend.h" 69#include "util/u_upload_mgr.h" 70#include "util/os_time.h" 71#include "pipe/p_shader_tokens.h" 72#include "draw/draw_context.h" 73#include "tgsi/tgsi_dump.h" 74#include "tgsi/tgsi_scan.h" 75#include "tgsi/tgsi_parse.h" 76#include "gallivm/lp_bld_type.h" 77#include "gallivm/lp_bld_const.h" 78#include "gallivm/lp_bld_conv.h" 79#include "gallivm/lp_bld_init.h" 80#include "gallivm/lp_bld_intr.h" 81#include "gallivm/lp_bld_logic.h" 82#include "gallivm/lp_bld_tgsi.h" 83#include "gallivm/lp_bld_nir.h" 84#include "gallivm/lp_bld_swizzle.h" 85#include "gallivm/lp_bld_flow.h" 86#include "gallivm/lp_bld_debug.h" 87#include "gallivm/lp_bld_arit.h" 88#include "gallivm/lp_bld_bitarit.h" 89#include "gallivm/lp_bld_pack.h" 90#include "gallivm/lp_bld_format.h" 91#include "gallivm/lp_bld_quad.h" 92#include "gallivm/lp_bld_gather.h" 93 94#include "lp_bld_alpha.h" 95#include "lp_bld_blend.h" 96#include "lp_bld_depth.h" 97#include "lp_bld_interp.h" 98#include "lp_context.h" 99#include "lp_debug.h" 100#include "lp_perf.h" 101#include "lp_setup.h" 102#include "lp_state.h" 103#include "lp_tex_sample.h" 104#include "lp_flush.h" 105#include "lp_state_fs.h" 106#include "lp_rast.h" 107#include "nir/nir_to_tgsi_info.h" 108 109#include "lp_screen.h" 110#include "compiler/nir/nir_serialize.h" 111#include "util/mesa-sha1.h" 112 113 114/** Fragment shader number (for debugging) */ 115static unsigned fs_no = 0; 116 117 118static void 119load_unswizzled_block(struct gallivm_state *gallivm, 120 LLVMValueRef base_ptr, 121 LLVMValueRef stride, 122 unsigned block_width, 123 unsigned block_height, 124 LLVMValueRef* dst, 125 struct lp_type dst_type, 126 unsigned dst_count, 127 unsigned dst_alignment); 128/** 129 * Checks if a format description is an arithmetic format 130 * 131 * A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5. 132 */ 133static inline boolean 134is_arithmetic_format(const struct util_format_description *format_desc) 135{ 136 boolean arith = false; 137 138 for (unsigned i = 0; i < format_desc->nr_channels; ++i) { 139 arith |= format_desc->channel[i].size != format_desc->channel[0].size; 140 arith |= (format_desc->channel[i].size % 8) != 0; 141 } 142 143 return arith; 144} 145 146 147/** 148 * Checks if this format requires special handling due to required expansion 149 * to floats for blending, and furthermore has "natural" packed AoS -> unpacked 150 * SoA conversion. 151 */ 152static inline boolean 153format_expands_to_float_soa(const struct util_format_description *format_desc) 154{ 155 if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT || 156 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) { 157 return true; 158 } 159 return false; 160} 161 162 163/** 164 * Retrieves the type representing the memory layout for a format 165 * 166 * e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte 167 */ 168static inline void 169lp_mem_type_from_format_desc(const struct util_format_description *format_desc, 170 struct lp_type* type) 171{ 172 unsigned i; 173 unsigned chan; 174 175 if (format_expands_to_float_soa(format_desc)) { 176 /* just make this a uint with width of block */ 177 type->floating = false; 178 type->fixed = false; 179 type->sign = false; 180 type->norm = false; 181 type->width = format_desc->block.bits; 182 type->length = 1; 183 return; 184 } 185 186 for (i = 0; i < 4; i++) { 187 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) 188 break; 189 } 190 chan = i; 191 192 memset(type, 0, sizeof(struct lp_type)); 193 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT; 194 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED; 195 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED; 196 type->norm = format_desc->channel[chan].normalized; 197 198 if (is_arithmetic_format(format_desc)) { 199 type->width = 0; 200 type->length = 1; 201 202 for (unsigned i = 0; i < format_desc->nr_channels; ++i) { 203 type->width += format_desc->channel[i].size; 204 } 205 } else { 206 type->width = format_desc->channel[chan].size; 207 type->length = format_desc->nr_channels; 208 } 209} 210 211/** 212 * Expand the relevant bits of mask_input to a n*4-dword mask for the 213 * n*four pixels in n 2x2 quads. This will set the n*four elements of the 214 * quad mask vector to 0 or ~0. 215 * Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid 216 * quad arguments with fs length 8. 217 * 218 * \param first_quad which quad(s) of the quad group to test, in [0,3] 219 * \param mask_input bitwise mask for the whole 4x4 stamp 220 */ 221static LLVMValueRef 222generate_quad_mask(struct gallivm_state *gallivm, 223 struct lp_type fs_type, 224 unsigned first_quad, 225 unsigned sample, 226 LLVMValueRef mask_input) /* int64 */ 227{ 228 LLVMBuilderRef builder = gallivm->builder; 229 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); 230 LLVMValueRef bits[16]; 231 LLVMValueRef mask, bits_vec; 232 233 /* 234 * XXX: We'll need a different path for 16 x u8 235 */ 236 assert(fs_type.width == 32); 237 assert(fs_type.length <= ARRAY_SIZE(bits)); 238 struct lp_type mask_type = lp_int_type(fs_type); 239 240 /* 241 * mask_input >>= (quad * 4) 242 */ 243 int shift; 244 switch (first_quad) { 245 case 0: 246 shift = 0; 247 break; 248 case 1: 249 assert(fs_type.length == 4); 250 shift = 2; 251 break; 252 case 2: 253 shift = 8; 254 break; 255 case 3: 256 assert(fs_type.length == 4); 257 shift = 10; 258 break; 259 default: 260 assert(0); 261 shift = 0; 262 } 263 264 mask_input = LLVMBuildLShr(builder, mask_input, 265 lp_build_const_int64(gallivm, 16 * sample), ""); 266 mask_input = LLVMBuildTrunc(builder, mask_input, i32t, ""); 267 mask_input = LLVMBuildAnd(builder, mask_input, 268 lp_build_const_int32(gallivm, 0xffff), ""); 269 mask_input = LLVMBuildLShr(builder, mask_input, 270 LLVMConstInt(i32t, shift, 0), ""); 271 272 /* 273 * mask = { mask_input & (1 << i), for i in [0,3] } 274 */ 275 mask = lp_build_broadcast(gallivm, 276 lp_build_vec_type(gallivm, mask_type), 277 mask_input); 278 279 for (int i = 0; i < fs_type.length / 4; i++) { 280 unsigned j = 2 * (i % 2) + (i / 2) * 8; 281 bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0); 282 bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0); 283 bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0); 284 bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0); 285 } 286 bits_vec = LLVMConstVector(bits, fs_type.length); 287 mask = LLVMBuildAnd(builder, mask, bits_vec, ""); 288 289 /* 290 * mask = mask == bits ? ~0 : 0 291 */ 292 mask = lp_build_compare(gallivm, 293 mask_type, PIPE_FUNC_EQUAL, 294 mask, bits_vec); 295 296 return mask; 297} 298 299 300#define EARLY_DEPTH_TEST 0x1 301#define LATE_DEPTH_TEST 0x2 302#define EARLY_DEPTH_WRITE 0x4 303#define LATE_DEPTH_WRITE 0x8 304#define EARLY_DEPTH_TEST_INFERRED 0x10 //only with EARLY_DEPTH_TEST 305 306 307static int 308find_output_by_semantic(const struct tgsi_shader_info *info, 309 enum tgsi_semantic semantic, 310 unsigned index) 311{ 312 for (int i = 0; i < info->num_outputs; i++) 313 if (info->output_semantic_name[i] == semantic && 314 info->output_semantic_index[i] == index) 315 return i; 316 317 return -1; 318} 319 320 321/** 322 * Fetch the specified lp_jit_viewport structure for a given viewport_index. 323 */ 324static LLVMValueRef 325lp_llvm_viewport(LLVMValueRef context_ptr, 326 struct gallivm_state *gallivm, 327 LLVMValueRef viewport_index) 328{ 329 LLVMBuilderRef builder = gallivm->builder; 330 LLVMValueRef ptr; 331 LLVMValueRef res; 332 struct lp_type viewport_type = 333 lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS); 334 335 ptr = lp_jit_context_viewports(gallivm, context_ptr); 336 ptr = LLVMBuildPointerCast(builder, ptr, 337 LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), ""); 338 339 res = lp_build_pointer_get(builder, ptr, viewport_index); 340 341 return res; 342} 343 344 345static LLVMValueRef 346lp_build_depth_clamp(struct gallivm_state *gallivm, 347 LLVMBuilderRef builder, 348 bool depth_clamp, 349 bool restrict_depth, 350 struct lp_type type, 351 LLVMValueRef context_ptr, 352 LLVMValueRef thread_data_ptr, 353 LLVMValueRef z) 354{ 355 LLVMValueRef viewport, min_depth, max_depth; 356 LLVMValueRef viewport_index; 357 struct lp_build_context f32_bld; 358 359 assert(type.floating); 360 lp_build_context_init(&f32_bld, gallivm, type); 361 362 if (restrict_depth) 363 z = lp_build_clamp(&f32_bld, z, f32_bld.zero, f32_bld.one); 364 365 if (!depth_clamp) 366 return z; 367 368 /* 369 * Assumes clamping of the viewport index will occur in setup/gs. Value 370 * is passed through the rasterization stage via lp_rast_shader_inputs. 371 * 372 * See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping 373 * semantics. 374 */ 375 viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm, 376 thread_data_ptr); 377 378 /* 379 * Load the min and max depth from the lp_jit_context.viewports 380 * array of lp_jit_viewport structures. 381 */ 382 viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index); 383 384 /* viewports[viewport_index].min_depth */ 385 min_depth = LLVMBuildExtractElement(builder, viewport, 386 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), ""); 387 min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth); 388 389 /* viewports[viewport_index].max_depth */ 390 max_depth = LLVMBuildExtractElement(builder, viewport, 391 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), ""); 392 max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth); 393 394 /* 395 * Clamp to the min and max depth values for the given viewport. 396 */ 397 return lp_build_clamp(&f32_bld, z, min_depth, max_depth); 398} 399 400 401static void 402lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm, 403 struct lp_type type, 404 unsigned coverage_samples, 405 LLVMValueRef num_loop, 406 LLVMValueRef loop_counter, 407 LLVMValueRef coverage_mask_store, 408 LLVMValueRef alpha) 409{ 410 struct lp_build_context bld; 411 LLVMBuilderRef builder = gallivm->builder; 412 float step = 1.0 / coverage_samples; 413 414 lp_build_context_init(&bld, gallivm, type); 415 for (unsigned s = 0; s < coverage_samples; s++) { 416 LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s); 417 LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value); 418 419 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, ""); 420 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, ""); 421 LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, ""); 422 LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 423 s_mask = LLVMBuildAnd(builder, s_mask, test, ""); 424 LLVMBuildStore(builder, s_mask, s_mask_ptr); 425 } 426}; 427 428 429struct lp_build_fs_llvm_iface { 430 struct lp_build_fs_iface base; 431 struct lp_build_interp_soa_context *interp; 432 struct lp_build_for_loop_state *loop_state; 433 LLVMValueRef mask_store; 434 LLVMValueRef sample_id; 435 LLVMValueRef color_ptr_ptr; 436 LLVMValueRef color_stride_ptr; 437 LLVMValueRef color_sample_stride_ptr; 438 LLVMValueRef zs_base_ptr; 439 LLVMValueRef zs_stride; 440 LLVMValueRef zs_sample_stride; 441 const struct lp_fragment_shader_variant_key *key; 442}; 443 444 445static LLVMValueRef 446fs_interp(const struct lp_build_fs_iface *iface, 447 struct lp_build_context *bld, 448 unsigned attrib, unsigned chan, 449 bool centroid, bool sample, 450 LLVMValueRef attrib_indir, 451 LLVMValueRef offsets[2]) 452{ 453 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface; 454 struct lp_build_interp_soa_context *interp = fs_iface->interp; 455 unsigned loc = TGSI_INTERPOLATE_LOC_CENTER; 456 if (centroid) 457 loc = TGSI_INTERPOLATE_LOC_CENTROID; 458 if (sample) 459 loc = TGSI_INTERPOLATE_LOC_SAMPLE; 460 461 return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter, 462 fs_iface->mask_store, 463 attrib, chan, loc, attrib_indir, offsets); 464} 465 466/* Convert depth-stencil format to a single component one, returning 467 * PIPE_FORMAT_NONE if it doesn't contain the required component. */ 468static enum pipe_format 469select_zs_component_format(enum pipe_format format, 470 bool fetch_stencil) 471{ 472 const struct util_format_description* desc = util_format_description(format); 473 if (fetch_stencil && !util_format_has_stencil(desc)) 474 return PIPE_FORMAT_NONE; 475 if (!fetch_stencil && !util_format_has_depth(desc)) 476 return PIPE_FORMAT_NONE; 477 478 switch (format) { 479 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 480 return fetch_stencil ? PIPE_FORMAT_X24S8_UINT : PIPE_FORMAT_Z24X8_UNORM; 481 case PIPE_FORMAT_S8_UINT_Z24_UNORM: 482 return fetch_stencil ? PIPE_FORMAT_S8X24_UINT : PIPE_FORMAT_X8Z24_UNORM; 483 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 484 return fetch_stencil ? PIPE_FORMAT_X32_S8X24_UINT : format; 485 default: 486 return format; 487 } 488} 489 490static void 491fs_fb_fetch(const struct lp_build_fs_iface *iface, 492 struct lp_build_context *bld, 493 int location, 494 LLVMValueRef result[4]) 495{ 496 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface; 497 struct gallivm_state *gallivm = bld->gallivm; 498 LLVMBuilderRef builder = gallivm->builder; 499 const struct lp_fragment_shader_variant_key *key = fs_iface->key; 500 501 LLVMValueRef buf_ptr; 502 LLVMValueRef stride; 503 enum pipe_format buf_format; 504 505 const bool fetch_stencil = location == FRAG_RESULT_STENCIL; 506 const bool fetch_zs = fetch_stencil || location == FRAG_RESULT_DEPTH; 507 if (fetch_zs) { 508 buf_ptr = fs_iface->zs_base_ptr; 509 stride = fs_iface->zs_stride; 510 buf_format = select_zs_component_format(key->zsbuf_format, fetch_stencil); 511 } 512 else { 513 assert(location >= FRAG_RESULT_DATA0 && location <= FRAG_RESULT_DATA7); 514 const int cbuf = location - FRAG_RESULT_DATA0; 515 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf); 516 517 buf_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_ptr_ptr, &index, 1, ""), ""); 518 stride = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_stride_ptr, &index, 1, ""), ""); 519 buf_format = key->cbuf_format[cbuf]; 520 } 521 522 const struct util_format_description* out_format_desc = util_format_description(buf_format); 523 if (out_format_desc->format == PIPE_FORMAT_NONE) { 524 result[0] = result[1] = result[2] = result[3] = bld->undef; 525 return; 526 } 527 528 unsigned block_size = bld->type.length; 529 unsigned block_height = key->resource_1d ? 1 : 2; 530 unsigned block_width = block_size / block_height; 531 532 if (key->multisample) { 533 LLVMValueRef sample_stride; 534 535 if (fetch_zs) { 536 sample_stride = fs_iface->zs_sample_stride; 537 } 538 else { 539 LLVMValueRef index = lp_build_const_int32(gallivm, location - FRAG_RESULT_DATA0); 540 sample_stride = LLVMBuildLoad(builder, 541 LLVMBuildGEP(builder, fs_iface->color_sample_stride_ptr, 542 &index, 1, ""), ""); 543 } 544 545 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, fs_iface->sample_id, ""); 546 buf_ptr = LLVMBuildGEP(builder, buf_ptr, &sample_offset, 1, ""); 547 } 548 549 /* fragment shader executes on 4x4 blocks. depending on vector width it can 550 * execute 2 or 4 iterations. only move to the next row once the top row 551 * has completed 8 wide 1 iteration, 4 wide 2 iterations */ 552 LLVMValueRef x_offset = NULL, y_offset = NULL; 553 if (!key->resource_1d) { 554 LLVMValueRef counter = fs_iface->loop_state->counter; 555 556 if (block_size == 4) { 557 x_offset = LLVMBuildShl(builder, 558 LLVMBuildAnd(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""), 559 lp_build_const_int32(gallivm, 1), ""); 560 counter = LLVMBuildLShr(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""); 561 } 562 y_offset = LLVMBuildMul(builder, counter, lp_build_const_int32(gallivm, 2), ""); 563 } 564 565 LLVMValueRef offsets[4 * 4]; 566 for (unsigned i = 0; i < block_size; i++) { 567 unsigned x = i % block_width; 568 unsigned y = i / block_width; 569 570 if (block_size == 8) { 571 /* remap the raw slots into the fragment shader execution mode. */ 572 /* this math took me way too long to work out, I'm sure it's overkill. */ 573 x = (i & 1) + ((i >> 2) << 1); 574 if (!key->resource_1d) 575 y = (i & 2) >> 1; 576 } 577 578 LLVMValueRef x_val; 579 if (x_offset) { 580 x_val = LLVMBuildAdd(builder, lp_build_const_int32(gallivm, x), x_offset, ""); 581 x_val = LLVMBuildMul(builder, x_val, lp_build_const_int32(gallivm, out_format_desc->block.bits / 8), ""); 582 } else { 583 x_val = lp_build_const_int32(gallivm, x * (out_format_desc->block.bits / 8)); 584 } 585 586 LLVMValueRef y_val = lp_build_const_int32(gallivm, y); 587 if (y_offset) 588 y_val = LLVMBuildAdd(builder, y_val, y_offset, ""); 589 y_val = LLVMBuildMul(builder, y_val, stride, ""); 590 591 offsets[i] = LLVMBuildAdd(builder, x_val, y_val, ""); 592 } 593 LLVMValueRef offset = lp_build_gather_values(gallivm, offsets, block_size); 594 595 struct lp_type texel_type = bld->type; 596 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB && 597 out_format_desc->channel[0].pure_integer) { 598 if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) { 599 texel_type = lp_type_int_vec(bld->type.width, bld->type.width * bld->type.length); 600 } 601 else if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) { 602 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length); 603 } 604 } else if (fetch_stencil) { 605 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length); 606 } 607 608 lp_build_fetch_rgba_soa(gallivm, out_format_desc, texel_type, 609 true, buf_ptr, offset, 610 NULL, NULL, NULL, result); 611} 612 613 614/** 615 * Generate the fragment shader, depth/stencil test, and alpha tests. 616 */ 617static void 618generate_fs_loop(struct gallivm_state *gallivm, 619 struct lp_fragment_shader *shader, 620 const struct lp_fragment_shader_variant_key *key, 621 LLVMBuilderRef builder, 622 struct lp_type type, 623 LLVMValueRef context_ptr, 624 LLVMValueRef sample_pos_array, 625 LLVMValueRef num_loop, 626 struct lp_build_interp_soa_context *interp, 627 const struct lp_build_sampler_soa *sampler, 628 const struct lp_build_image_soa *image, 629 LLVMValueRef mask_store, 630 LLVMValueRef (*out_color)[4], 631 LLVMValueRef depth_base_ptr, 632 LLVMValueRef depth_stride, 633 LLVMValueRef depth_sample_stride, 634 LLVMValueRef color_ptr_ptr, 635 LLVMValueRef color_stride_ptr, 636 LLVMValueRef color_sample_stride_ptr, 637 LLVMValueRef facing, 638 LLVMValueRef thread_data_ptr) 639{ 640 const struct tgsi_token *tokens = shader->base.tokens; 641 struct lp_type int_type = lp_int_type(type); 642 LLVMValueRef mask_ptr = NULL, mask_val = NULL; 643 LLVMValueRef z; 644 LLVMValueRef z_value, s_value; 645 LLVMValueRef z_fb, s_fb; 646 LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples); 647 LLVMValueRef z_out = NULL, s_out = NULL; 648 struct lp_build_for_loop_state loop_state, sample_loop_state = {0}; 649 struct lp_build_mask_context mask; 650 /* 651 * TODO: figure out if simple_shader optimization is really worthwile to 652 * keep. Disabled because it may hide some real bugs in the (depth/stencil) 653 * code since tests tend to take another codepath than real shaders. 654 */ 655 boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 && 656 shader->info.base.num_inputs < 3 && 657 shader->info.base.num_instructions < 8) && 0; 658 const boolean dual_source_blend = key->blend.rt[0].blend_enable && 659 util_blend_state_is_dual(&key->blend, 0); 660 const bool post_depth_coverage = shader->info.base.properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE]; 661 662 struct lp_bld_tgsi_system_values system_values; 663 664 memset(&system_values, 0, sizeof(system_values)); 665 666 /* truncate then sign extend. */ 667 system_values.front_facing = 668 LLVMBuildTrunc(gallivm->builder, facing, 669 LLVMInt1TypeInContext(gallivm->context), ""); 670 system_values.front_facing = 671 LLVMBuildSExt(gallivm->builder, system_values.front_facing, 672 LLVMInt32TypeInContext(gallivm->context), ""); 673 system_values.view_index = 674 lp_jit_thread_data_raster_state_view_index(gallivm, thread_data_ptr); 675 676 unsigned depth_mode; 677 const struct util_format_description *zs_format_desc = NULL; 678 if (key->depth.enabled || 679 key->stencil[0].enabled) { 680 zs_format_desc = util_format_description(key->zsbuf_format); 681 682 if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL]) 683 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE; 684 else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil && 685 !shader->info.base.uses_fbfetch && !shader->info.base.writes_memory) { 686 if (key->alpha.enabled || 687 key->blend.alpha_to_coverage || 688 shader->info.base.uses_kill || 689 shader->info.base.writes_samplemask) { 690 /* With alpha test and kill, can do the depth test early 691 * and hopefully eliminate some quads. But need to do a 692 * special deferred depth write once the final mask value 693 * is known. This only works though if there's either no 694 * stencil test or the stencil value isn't written. 695 */ 696 if (key->stencil[0].enabled && (key->stencil[0].writemask || 697 (key->stencil[1].enabled && 698 key->stencil[1].writemask))) 699 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE; 700 else 701 depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED; 702 } 703 else 704 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE | EARLY_DEPTH_TEST_INFERRED; 705 } 706 else { 707 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE; 708 } 709 710 if (!(key->depth.enabled && key->depth.writemask) && 711 !(key->stencil[0].enabled && (key->stencil[0].writemask || 712 (key->stencil[1].enabled && 713 key->stencil[1].writemask)))) 714 depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE); 715 } 716 else { 717 depth_mode = 0; 718 } 719 720 LLVMTypeRef vec_type = lp_build_vec_type(gallivm, type); 721 LLVMTypeRef int_vec_type = lp_build_vec_type(gallivm, int_type); 722 723 LLVMValueRef stencil_refs[2]; 724 stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr); 725 stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr); 726 /* convert scalar stencil refs into vectors */ 727 stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]); 728 stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]); 729 730 LLVMValueRef consts_ptr = lp_jit_context_constants(gallivm, context_ptr); 731 LLVMValueRef num_consts_ptr = lp_jit_context_num_constants(gallivm, 732 context_ptr); 733 734 LLVMValueRef ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr); 735 LLVMValueRef num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr); 736 737 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 738 memset(outputs, 0, sizeof outputs); 739 740 /* Allocate color storage for each fragment sample */ 741 LLVMValueRef color_store_size = num_loop; 742 if (key->min_samples > 1) 743 color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), ""); 744 745 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 746 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 747 out_color[cbuf][chan] = lp_build_array_alloca(gallivm, 748 lp_build_vec_type(gallivm, 749 type), 750 color_store_size, "color"); 751 } 752 } 753 if (dual_source_blend) { 754 assert(key->nr_cbufs <= 1); 755 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 756 out_color[1][chan] = lp_build_array_alloca(gallivm, 757 lp_build_vec_type(gallivm, 758 type), 759 color_store_size, "color1"); 760 } 761 } 762 if (shader->info.base.writes_z) { 763 z_out = lp_build_array_alloca(gallivm, 764 lp_build_vec_type(gallivm, type), 765 color_store_size, "depth"); 766 } 767 768 if (shader->info.base.writes_stencil) { 769 s_out = lp_build_array_alloca(gallivm, 770 lp_build_vec_type(gallivm, type), 771 color_store_size, "depth"); 772 } 773 774 lp_build_for_loop_begin(&loop_state, gallivm, 775 lp_build_const_int32(gallivm, 0), 776 LLVMIntULT, 777 num_loop, 778 lp_build_const_int32(gallivm, 1)); 779 780 LLVMValueRef sample_mask_in; 781 if (key->multisample) { 782 sample_mask_in = lp_build_const_int_vec(gallivm, type, 0); 783 /* create shader execution mask by combining all sample masks. */ 784 for (unsigned s = 0; s < key->coverage_samples; s++) { 785 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), ""); 786 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 787 LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx); 788 if (s == 0) 789 mask_val = s_mask; 790 else 791 mask_val = LLVMBuildOr(builder, s_mask, mask_val, ""); 792 793 LLVMValueRef mask_in = LLVMBuildAnd(builder, s_mask, lp_build_const_int_vec(gallivm, type, (1ll << s)), ""); 794 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, ""); 795 } 796 } else { 797 sample_mask_in = lp_build_const_int_vec(gallivm, type, 1); 798 mask_ptr = LLVMBuildGEP(builder, mask_store, 799 &loop_state.counter, 1, "mask_ptr"); 800 mask_val = LLVMBuildLoad(builder, mask_ptr, ""); 801 802 LLVMValueRef mask_in = LLVMBuildAnd(builder, mask_val, lp_build_const_int_vec(gallivm, type, 1), ""); 803 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, ""); 804 } 805 806 /* 'mask' will control execution based on quad's pixel alive/killed state */ 807 lp_build_mask_begin(&mask, gallivm, type, mask_val); 808 809 if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader) 810 lp_build_mask_check(&mask); 811 812 /* Create storage for recombining sample masks after early Z pass. */ 813 LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth"); 814 LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or); 815 816 /* Create storage for post depth sample mask */ 817 LLVMValueRef post_depth_sample_mask_in = NULL; 818 if (post_depth_coverage) 819 post_depth_sample_mask_in = lp_build_alloca(gallivm, int_vec_type, "post_depth_sample_mask_in"); 820 821 LLVMValueRef s_mask = NULL, s_mask_ptr = NULL; 822 LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL; 823 LLVMValueRef z_fb_store = NULL, s_fb_store = NULL; 824 LLVMTypeRef z_type = NULL, z_fb_type = NULL; 825 826 /* Run early depth once per sample */ 827 if (key->multisample) { 828 829 if (zs_format_desc) { 830 struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length); 831 struct lp_type z_type = zs_type; 832 struct lp_type s_type = zs_type; 833 if (zs_format_desc->block.bits < type.width) 834 z_type.width = type.width; 835 if (zs_format_desc->block.bits == 8) 836 s_type.width = type.width; 837 838 else if (zs_format_desc->block.bits > 32) { 839 z_type.width = z_type.width / 2; 840 s_type.width = s_type.width / 2; 841 s_type.floating = 0; 842 } 843 z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type), 844 zs_samples, "z_sample_store"); 845 s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type), 846 zs_samples, "s_sample_store"); 847 z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type), 848 zs_samples, "z_fb_store"); 849 s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type), 850 zs_samples, "s_fb_store"); 851 } 852 lp_build_for_loop_begin(&sample_loop_state, gallivm, 853 lp_build_const_int32(gallivm, 0), 854 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples), 855 lp_build_const_int32(gallivm, 1)); 856 857 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 858 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 859 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 860 861 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 862 s_mask = LLVMBuildAnd(builder, s_mask, mask_val, ""); 863 } 864 865 866 /* for multisample Z needs to be interpolated at sample points for testing. */ 867 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, 868 key->multisample 869 ? sample_loop_state.counter : NULL); 870 z = interp->pos[2]; 871 872 LLVMValueRef depth_ptr = depth_base_ptr; 873 if (key->multisample) { 874 LLVMValueRef sample_offset = 875 LLVMBuildMul(builder, sample_loop_state.counter, 876 depth_sample_stride, ""); 877 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, ""); 878 } 879 880 if (depth_mode & EARLY_DEPTH_TEST) { 881 z = lp_build_depth_clamp(gallivm, builder, key->depth_clamp, 882 key->restrict_depth_values, type, context_ptr, 883 thread_data_ptr, z); 884 885 lp_build_depth_stencil_load_swizzled(gallivm, type, 886 zs_format_desc, key->resource_1d, 887 depth_ptr, depth_stride, 888 &z_fb, &s_fb, loop_state.counter); 889 lp_build_depth_stencil_test(gallivm, 890 &key->depth, 891 key->stencil, 892 type, 893 zs_format_desc, 894 key->multisample ? NULL : &mask, 895 &s_mask, 896 stencil_refs, 897 z, z_fb, s_fb, 898 facing, 899 &z_value, &s_value, 900 !simple_shader && !key->multisample, 901 key->restrict_depth_values); 902 903 if (depth_mode & EARLY_DEPTH_WRITE) { 904 lp_build_depth_stencil_write_swizzled(gallivm, type, 905 zs_format_desc, key->resource_1d, 906 NULL, NULL, NULL, loop_state.counter, 907 depth_ptr, depth_stride, 908 z_value, s_value); 909 } 910 /* 911 * Note mask check if stencil is enabled must be after ds write not after 912 * stencil test otherwise new stencil values may not get written if all 913 * fragments got killed by depth/stencil test. 914 */ 915 if (!simple_shader && key->stencil[0].enabled && !key->multisample) 916 lp_build_mask_check(&mask); 917 918 if (key->multisample) { 919 z_fb_type = LLVMTypeOf(z_fb); 920 z_type = LLVMTypeOf(z_value); 921 lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), "")); 922 lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), "")); 923 lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb); 924 lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb); 925 } 926 } 927 928 if (key->multisample) { 929 /* 930 * Store the post-early Z coverage mask. 931 * Recombine the resulting coverage masks post early Z into the fragment 932 * shader execution mask. 933 */ 934 LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, ""); 935 tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, ""); 936 LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or); 937 938 if (post_depth_coverage) { 939 LLVMValueRef mask_bit_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 940 LLVMValueRef post_depth_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, ""); 941 mask_bit_idx = LLVMBuildAnd(builder, s_mask, lp_build_broadcast(gallivm, int_vec_type, mask_bit_idx), ""); 942 post_depth_mask_in = LLVMBuildOr(builder, post_depth_mask_in, mask_bit_idx, ""); 943 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in); 944 } 945 946 LLVMBuildStore(builder, s_mask, s_mask_ptr); 947 948 lp_build_for_loop_end(&sample_loop_state); 949 950 /* recombined all the coverage masks in the shader exec mask. */ 951 tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, ""); 952 lp_build_mask_update(&mask, tmp_s_mask_or); 953 954 if (key->min_samples == 1) { 955 /* for multisample Z needs to be re interpolated at pixel center */ 956 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL); 957 z = interp->pos[2]; 958 lp_build_mask_update(&mask, tmp_s_mask_or); 959 } 960 } else { 961 if (post_depth_coverage) { 962 LLVMValueRef post_depth_mask_in = LLVMBuildAnd(builder, lp_build_mask_value(&mask), lp_build_const_int_vec(gallivm, type, 1), ""); 963 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in); 964 } 965 } 966 967 LLVMValueRef out_sample_mask_storage = NULL; 968 if (shader->info.base.writes_samplemask) { 969 out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask"); 970 if (key->min_samples > 1) 971 LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage); 972 } 973 974 if (post_depth_coverage) { 975 system_values.sample_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, ""); 976 } 977 else 978 system_values.sample_mask_in = sample_mask_in; 979 if (key->multisample && key->min_samples > 1) { 980 lp_build_for_loop_begin(&sample_loop_state, gallivm, 981 lp_build_const_int32(gallivm, 0), 982 LLVMIntULT, 983 lp_build_const_int32(gallivm, key->min_samples), 984 lp_build_const_int32(gallivm, 1)); 985 986 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 987 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 988 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 989 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 990 lp_build_mask_force(&mask, s_mask); 991 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter); 992 system_values.sample_id = sample_loop_state.counter; 993 system_values.sample_mask_in = LLVMBuildAnd(builder, system_values.sample_mask_in, 994 lp_build_broadcast(gallivm, int_vec_type, 995 LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "")), ""); 996 } else { 997 system_values.sample_id = lp_build_const_int32(gallivm, 0); 998 999 } 1000 system_values.sample_pos = sample_pos_array; 1001 1002 lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter); 1003 1004 struct lp_build_fs_llvm_iface fs_iface = { 1005 .base.interp_fn = fs_interp, 1006 .base.fb_fetch = fs_fb_fetch, 1007 .interp = interp, 1008 .loop_state = &loop_state, 1009 .sample_id = system_values.sample_id, 1010 .mask_store = mask_store, 1011 .color_ptr_ptr = color_ptr_ptr, 1012 .color_stride_ptr = color_stride_ptr, 1013 .color_sample_stride_ptr = color_sample_stride_ptr, 1014 .zs_base_ptr = depth_base_ptr, 1015 .zs_stride = depth_stride, 1016 .zs_sample_stride = depth_sample_stride, 1017 .key = key, 1018 }; 1019 1020 struct lp_build_tgsi_params params; 1021 memset(¶ms, 0, sizeof(params)); 1022 1023 params.type = type; 1024 params.mask = &mask; 1025 params.fs_iface = &fs_iface.base; 1026 params.consts_ptr = consts_ptr; 1027 params.const_sizes_ptr = num_consts_ptr; 1028 params.system_values = &system_values; 1029 params.inputs = interp->inputs; 1030 params.context_ptr = context_ptr; 1031 params.thread_data_ptr = thread_data_ptr; 1032 params.sampler = sampler; 1033 params.info = &shader->info.base; 1034 params.ssbo_ptr = ssbo_ptr; 1035 params.ssbo_sizes_ptr = num_ssbo_ptr; 1036 params.image = image; 1037 params.aniso_filter_table = lp_jit_context_aniso_filter_table(gallivm, context_ptr); 1038 1039 /* Build the actual shader */ 1040 if (shader->base.type == PIPE_SHADER_IR_TGSI) 1041 lp_build_tgsi_soa(gallivm, tokens, ¶ms, 1042 outputs); 1043 else 1044 lp_build_nir_soa(gallivm, shader->base.ir.nir, ¶ms, 1045 outputs); 1046 1047 /* Alpha test */ 1048 if (key->alpha.enabled) { 1049 int color0 = find_output_by_semantic(&shader->info.base, 1050 TGSI_SEMANTIC_COLOR, 1051 0); 1052 1053 if (color0 != -1 && outputs[color0][3]) { 1054 const struct util_format_description *cbuf_format_desc; 1055 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha"); 1056 LLVMValueRef alpha_ref_value; 1057 1058 alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr); 1059 alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value); 1060 1061 cbuf_format_desc = util_format_description(key->cbuf_format[0]); 1062 1063 lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc, 1064 &mask, alpha, alpha_ref_value, 1065 ((depth_mode & LATE_DEPTH_TEST) != 0) && !key->multisample); 1066 } 1067 } 1068 1069 /* Emulate Alpha to Coverage with Alpha test */ 1070 if (key->blend.alpha_to_coverage) { 1071 int color0 = find_output_by_semantic(&shader->info.base, 1072 TGSI_SEMANTIC_COLOR, 1073 0); 1074 1075 if (color0 != -1 && outputs[color0][3]) { 1076 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha"); 1077 1078 if (!key->multisample) { 1079 lp_build_alpha_to_coverage(gallivm, type, 1080 &mask, alpha, 1081 (depth_mode & LATE_DEPTH_TEST) != 0); 1082 } else { 1083 lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop, 1084 loop_state.counter, 1085 mask_store, alpha); 1086 } 1087 } 1088 } 1089 1090 if (key->blend.alpha_to_one) { 1091 for (unsigned attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) { 1092 unsigned cbuf = shader->info.base.output_semantic_index[attrib]; 1093 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) && 1094 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend))) 1095 if (outputs[cbuf][3]) { 1096 LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0), 1097 outputs[cbuf][3]); 1098 } 1099 } 1100 } 1101 1102 if (shader->info.base.writes_samplemask) { 1103 LLVMValueRef output_smask = NULL; 1104 int smaski = find_output_by_semantic(&shader->info.base, 1105 TGSI_SEMANTIC_SAMPLEMASK, 1106 0); 1107 struct lp_build_context smask_bld; 1108 lp_build_context_init(&smask_bld, gallivm, int_type); 1109 1110 assert(smaski >= 0); 1111 output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask"); 1112 output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, ""); 1113 if (!key->multisample && key->no_ms_sample_mask_out) { 1114 output_smask = lp_build_and(&smask_bld, output_smask, smask_bld.one); 1115 output_smask = lp_build_cmp(&smask_bld, PIPE_FUNC_NOTEQUAL, output_smask, smask_bld.zero); 1116 lp_build_mask_update(&mask, output_smask); 1117 } 1118 1119 if (key->min_samples > 1) { 1120 /* only the bit corresponding to this sample is to be used. */ 1121 LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask"); 1122 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 1123 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), ""); 1124 output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, ""); 1125 } 1126 1127 LLVMBuildStore(builder, output_smask, out_sample_mask_storage); 1128 } 1129 1130 if (shader->info.base.writes_z) { 1131 int pos0 = find_output_by_semantic(&shader->info.base, 1132 TGSI_SEMANTIC_POSITION, 1133 0); 1134 LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos0][2], ""); 1135 LLVMValueRef idx = loop_state.counter; 1136 if (key->min_samples > 1) 1137 idx = LLVMBuildAdd(builder, idx, 1138 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1139 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, ""); 1140 LLVMBuildStore(builder, out, ptr); 1141 } 1142 1143 if (shader->info.base.writes_stencil) { 1144 int sten_out = find_output_by_semantic(&shader->info.base, 1145 TGSI_SEMANTIC_STENCIL, 1146 0); 1147 LLVMValueRef out = LLVMBuildLoad(builder, outputs[sten_out][1], "output.s"); 1148 LLVMValueRef idx = loop_state.counter; 1149 if (key->min_samples > 1) 1150 idx = LLVMBuildAdd(builder, idx, 1151 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1152 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, ""); 1153 LLVMBuildStore(builder, out, ptr); 1154 } 1155 1156 bool has_cbuf0_write = false; 1157 /* Color write - per fragment sample */ 1158 for (unsigned attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) { 1159 unsigned cbuf = shader->info.base.output_semantic_index[attrib]; 1160 if ((shader->info.base.output_semantic_name[attrib] 1161 == TGSI_SEMANTIC_COLOR) && 1162 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend))) { 1163 if (cbuf == 0 && 1164 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]) { 1165 /* XXX: there is an edge case with FB fetch where gl_FragColor and 1166 * gl_LastFragData[0] are used together. This creates both 1167 * FRAG_RESULT_COLOR and FRAG_RESULT_DATA* output variables. This 1168 * loop then writes to cbuf 0 twice, owerwriting the correct value 1169 * from gl_FragColor with some garbage. This case is excercised in 1170 * one of deqp tests. A similar bug can happen if 1171 * gl_SecondaryFragColorEXT and gl_LastFragData[1] are mixed in 1172 * the same fashion... This workaround will break if 1173 * gl_LastFragData[0] goes in outputs list before 1174 * gl_FragColor. This doesn't seem to happen though. 1175 */ 1176 if (has_cbuf0_write) 1177 continue; 1178 has_cbuf0_write = true; 1179 } 1180 1181 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 1182 if (outputs[attrib][chan]) { 1183 /* XXX: just initialize outputs to point at colors[] and 1184 * skip this. 1185 */ 1186 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 1187 LLVMValueRef color_ptr; 1188 LLVMValueRef color_idx = loop_state.counter; 1189 if (key->min_samples > 1) 1190 color_idx = LLVMBuildAdd(builder, color_idx, 1191 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1192 color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan], 1193 &color_idx, 1, ""); 1194 lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]); 1195 LLVMBuildStore(builder, out, color_ptr); 1196 } 1197 } 1198 } 1199 } 1200 1201 if (key->multisample && key->min_samples > 1) { 1202 LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr); 1203 lp_build_for_loop_end(&sample_loop_state); 1204 } 1205 1206 if (key->multisample) { 1207 /* execute depth test for each sample */ 1208 lp_build_for_loop_begin(&sample_loop_state, gallivm, 1209 lp_build_const_int32(gallivm, 0), 1210 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples), 1211 lp_build_const_int32(gallivm, 1)); 1212 1213 /* load the per-sample coverage mask */ 1214 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 1215 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 1216 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 1217 1218 /* combine the execution mask post fragment shader with the coverage mask. */ 1219 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 1220 if (key->min_samples == 1) 1221 s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), ""); 1222 1223 /* if the shader writes sample mask use that, 1224 * but only if this isn't genuine early-depth to avoid breaking occlusion query */ 1225 if (shader->info.base.writes_samplemask && 1226 (!(depth_mode & EARLY_DEPTH_TEST) || (depth_mode & (EARLY_DEPTH_TEST_INFERRED)))) { 1227 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 1228 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx); 1229 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, ""); 1230 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, ""); 1231 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), ""); 1232 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, ""); 1233 1234 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, ""); 1235 } 1236 } 1237 1238 depth_ptr = depth_base_ptr; 1239 if (key->multisample) { 1240 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, ""); 1241 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, ""); 1242 } 1243 1244 /* Late Z test */ 1245 if (depth_mode & LATE_DEPTH_TEST) { 1246 if (shader->info.base.writes_z) { 1247 LLVMValueRef idx = loop_state.counter; 1248 if (key->min_samples > 1) 1249 idx = LLVMBuildAdd(builder, idx, 1250 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1251 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, ""); 1252 z = LLVMBuildLoad(builder, ptr, "output.z"); 1253 } else { 1254 if (key->multisample) { 1255 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL); 1256 z = interp->pos[2]; 1257 } 1258 } 1259 1260 /* 1261 * Clamp according to ARB_depth_clamp semantics. 1262 */ 1263 z = lp_build_depth_clamp(gallivm, builder, key->depth_clamp, 1264 key->restrict_depth_values, type, context_ptr, 1265 thread_data_ptr, z); 1266 1267 if (shader->info.base.writes_stencil) { 1268 LLVMValueRef idx = loop_state.counter; 1269 if (key->min_samples > 1) 1270 idx = LLVMBuildAdd(builder, idx, 1271 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1272 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, ""); 1273 stencil_refs[0] = LLVMBuildLoad(builder, ptr, "output.s"); 1274 /* there's only one value, and spec says to discard additional bits */ 1275 LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255); 1276 stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, ""); 1277 stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, ""); 1278 stencil_refs[1] = stencil_refs[0]; 1279 } 1280 1281 lp_build_depth_stencil_load_swizzled(gallivm, type, 1282 zs_format_desc, key->resource_1d, 1283 depth_ptr, depth_stride, 1284 &z_fb, &s_fb, loop_state.counter); 1285 1286 lp_build_depth_stencil_test(gallivm, 1287 &key->depth, 1288 key->stencil, 1289 type, 1290 zs_format_desc, 1291 key->multisample ? NULL : &mask, 1292 &s_mask, 1293 stencil_refs, 1294 z, z_fb, s_fb, 1295 facing, 1296 &z_value, &s_value, 1297 !simple_shader, 1298 key->restrict_depth_values); 1299 /* Late Z write */ 1300 if (depth_mode & LATE_DEPTH_WRITE) { 1301 lp_build_depth_stencil_write_swizzled(gallivm, type, 1302 zs_format_desc, key->resource_1d, 1303 NULL, NULL, NULL, loop_state.counter, 1304 depth_ptr, depth_stride, 1305 z_value, s_value); 1306 } 1307 } 1308 else if ((depth_mode & EARLY_DEPTH_TEST) && 1309 (depth_mode & LATE_DEPTH_WRITE)) 1310 { 1311 /* Need to apply a reduced mask to the depth write. Reload the 1312 * depth value, update from zs_value with the new mask value and 1313 * write that out. 1314 */ 1315 if (key->multisample) { 1316 z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");; 1317 s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter); 1318 z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, ""); 1319 s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter); 1320 } 1321 lp_build_depth_stencil_write_swizzled(gallivm, type, 1322 zs_format_desc, key->resource_1d, 1323 key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter, 1324 depth_ptr, depth_stride, 1325 z_value, s_value); 1326 } 1327 1328 if (key->occlusion_count) { 1329 LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr); 1330 lp_build_name(counter, "counter"); 1331 1332 lp_build_occlusion_count(gallivm, type, 1333 key->multisample ? s_mask : lp_build_mask_value(&mask), counter); 1334 } 1335 1336 /* if this is genuine early-depth in the shader, write samplemask now 1337 * after occlusion count has been updated 1338 */ 1339 if (key->multisample && shader->info.base.writes_samplemask && 1340 (depth_mode & (EARLY_DEPTH_TEST_INFERRED | EARLY_DEPTH_TEST)) == EARLY_DEPTH_TEST) { 1341 /* if the shader writes sample mask use that */ 1342 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 1343 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx); 1344 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, ""); 1345 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, ""); 1346 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), ""); 1347 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, ""); 1348 1349 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, ""); 1350 } 1351 1352 1353 if (key->multisample) { 1354 /* store the sample mask for this loop */ 1355 LLVMBuildStore(builder, s_mask, s_mask_ptr); 1356 lp_build_for_loop_end(&sample_loop_state); 1357 } 1358 1359 mask_val = lp_build_mask_end(&mask); 1360 if (!key->multisample) 1361 LLVMBuildStore(builder, mask_val, mask_ptr); 1362 lp_build_for_loop_end(&loop_state); 1363} 1364 1365 1366/** 1367 * This function will reorder pixels from the fragment shader SoA to memory layout AoS 1368 * 1369 * Fragment Shader outputs pixels in small 2x2 blocks 1370 * e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ... 1371 * 1372 * However in memory pixels are stored in rows 1373 * e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ... 1374 * 1375 * @param type fragment shader type (4x or 8x float) 1376 * @param num_fs number of fs_src 1377 * @param is_1d whether we're outputting to a 1d resource 1378 * @param dst_channels number of output channels 1379 * @param fs_src output from fragment shader 1380 * @param dst pointer to store result 1381 * @param pad_inline is channel padding inline or at end of row 1382 * @return the number of dsts 1383 */ 1384static int 1385generate_fs_twiddle(struct gallivm_state *gallivm, 1386 struct lp_type type, 1387 unsigned num_fs, 1388 unsigned dst_channels, 1389 LLVMValueRef fs_src[][4], 1390 LLVMValueRef* dst, 1391 bool pad_inline) 1392{ 1393 LLVMValueRef src[16]; 1394 1395 bool swizzle_pad; 1396 bool twiddle; 1397 bool split; 1398 1399 unsigned pixels = type.length / 4; 1400 unsigned reorder_group; 1401 unsigned src_channels; 1402 unsigned src_count; 1403 unsigned i; 1404 1405 src_channels = dst_channels < 3 ? dst_channels : 4; 1406 src_count = num_fs * src_channels; 1407 1408 assert(pixels == 2 || pixels == 1); 1409 assert(num_fs * src_channels <= ARRAY_SIZE(src)); 1410 1411 /* 1412 * Transpose from SoA -> AoS 1413 */ 1414 for (i = 0; i < num_fs; ++i) { 1415 lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]); 1416 } 1417 1418 /* 1419 * Pick transformation options 1420 */ 1421 swizzle_pad = false; 1422 twiddle = false; 1423 split = false; 1424 reorder_group = 0; 1425 1426 if (dst_channels == 1) { 1427 twiddle = true; 1428 1429 if (pixels == 2) { 1430 split = true; 1431 } 1432 } else if (dst_channels == 2) { 1433 if (pixels == 1) { 1434 reorder_group = 1; 1435 } 1436 } else if (dst_channels > 2) { 1437 if (pixels == 1) { 1438 reorder_group = 2; 1439 } else { 1440 twiddle = true; 1441 } 1442 1443 if (!pad_inline && dst_channels == 3 && pixels > 1) { 1444 swizzle_pad = true; 1445 } 1446 } 1447 1448 /* 1449 * Split the src in half 1450 */ 1451 if (split) { 1452 for (i = num_fs; i > 0; --i) { 1453 src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4); 1454 src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4); 1455 } 1456 1457 src_count *= 2; 1458 type.length = 4; 1459 } 1460 1461 /* 1462 * Ensure pixels are in memory order 1463 */ 1464 if (reorder_group) { 1465 /* Twiddle pixels by reordering the array, e.g.: 1466 * 1467 * src_count = 8 -> 0 2 1 3 4 6 5 7 1468 * src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15 1469 */ 1470 const unsigned reorder_sw[] = { 0, 2, 1, 3 }; 1471 1472 for (i = 0; i < src_count; ++i) { 1473 unsigned group = i / reorder_group; 1474 unsigned block = (group / 4) * 4 * reorder_group; 1475 unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group); 1476 dst[i] = src[j]; 1477 } 1478 } else if (twiddle) { 1479 /* Twiddle pixels across elements of array */ 1480 /* 1481 * XXX: we should avoid this in some cases, but would need to tell 1482 * lp_build_conv to reorder (or deal with it ourselves). 1483 */ 1484 lp_bld_quad_twiddle(gallivm, type, src, src_count, dst); 1485 } else { 1486 /* Do nothing */ 1487 memcpy(dst, src, sizeof(LLVMValueRef) * src_count); 1488 } 1489 1490 /* 1491 * Moves any padding between pixels to the end 1492 * e.g. RGBXRGBX -> RGBRGBXX 1493 */ 1494 if (swizzle_pad) { 1495 unsigned char swizzles[16]; 1496 unsigned elems = pixels * dst_channels; 1497 1498 for (i = 0; i < type.length; ++i) { 1499 if (i < elems) 1500 swizzles[i] = i % dst_channels + (i / dst_channels) * 4; 1501 else 1502 swizzles[i] = LP_BLD_SWIZZLE_DONTCARE; 1503 } 1504 1505 for (i = 0; i < src_count; ++i) { 1506 dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length); 1507 } 1508 } 1509 1510 return src_count; 1511} 1512 1513 1514/* 1515 * Untwiddle and transpose, much like the above. 1516 * However, this is after conversion, so we get packed vectors. 1517 * At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data, 1518 * the vectors will look like: 1519 * r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may 1520 * be swizzled here). Extending to 16bit should be trivial. 1521 * Should also be extended to handle twice wide vectors with AVX2... 1522 */ 1523static void 1524fs_twiddle_transpose(struct gallivm_state *gallivm, 1525 struct lp_type type, 1526 LLVMValueRef *src, 1527 unsigned src_count, 1528 LLVMValueRef *dst) 1529{ 1530 struct lp_type type64, type16, type32; 1531 LLVMTypeRef type64_t, type8_t, type16_t, type32_t; 1532 LLVMBuilderRef builder = gallivm->builder; 1533 LLVMValueRef tmp[4], shuf[8]; 1534 for (unsigned j = 0; j < 2; j++) { 1535 shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0); 1536 shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2); 1537 shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1); 1538 shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3); 1539 } 1540 1541 assert(src_count == 4 || src_count == 2 || src_count == 1); 1542 assert(type.width == 8); 1543 assert(type.length == 16); 1544 1545 type8_t = lp_build_vec_type(gallivm, type); 1546 1547 type64 = type; 1548 type64.length /= 8; 1549 type64.width *= 8; 1550 type64_t = lp_build_vec_type(gallivm, type64); 1551 1552 type16 = type; 1553 type16.length /= 2; 1554 type16.width *= 2; 1555 type16_t = lp_build_vec_type(gallivm, type16); 1556 1557 type32 = type; 1558 type32.length /= 4; 1559 type32.width *= 4; 1560 type32_t = lp_build_vec_type(gallivm, type32); 1561 1562 lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp); 1563 1564 if (src_count == 1) { 1565 /* transpose was no-op, just untwiddle */ 1566 LLVMValueRef shuf_vec; 1567 shuf_vec = LLVMConstVector(shuf, 8); 1568 tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, ""); 1569 tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, ""); 1570 dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, ""); 1571 } else if (src_count == 2) { 1572 LLVMValueRef shuf_vec; 1573 shuf_vec = LLVMConstVector(shuf, 4); 1574 1575 for (unsigned i = 0; i < 2; i++) { 1576 tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, ""); 1577 tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, ""); 1578 dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, ""); 1579 } 1580 } else { 1581 for (unsigned j = 0; j < 2; j++) { 1582 LLVMValueRef lo, hi, lo2, hi2; 1583 /* 1584 * Note that if we only really have 3 valid channels (rgb) 1585 * and we don't need alpha we could substitute a undef here 1586 * for the respective channel (causing llvm to drop conversion 1587 * for alpha). 1588 */ 1589 /* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */ 1590 lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, ""); 1591 hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, ""); 1592 lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0); 1593 hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1); 1594 dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, ""); 1595 dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, ""); 1596 } 1597 } 1598} 1599 1600 1601/** 1602 * Load an unswizzled block of pixels from memory 1603 */ 1604static void 1605load_unswizzled_block(struct gallivm_state *gallivm, 1606 LLVMValueRef base_ptr, 1607 LLVMValueRef stride, 1608 unsigned block_width, 1609 unsigned block_height, 1610 LLVMValueRef* dst, 1611 struct lp_type dst_type, 1612 unsigned dst_count, 1613 unsigned dst_alignment) 1614{ 1615 LLVMBuilderRef builder = gallivm->builder; 1616 const unsigned row_size = dst_count / block_height; 1617 1618 /* Ensure block exactly fits into dst */ 1619 assert((block_width * block_height) % dst_count == 0); 1620 1621 for (unsigned i = 0; i < dst_count; ++i) { 1622 unsigned x = i % row_size; 1623 unsigned y = i / row_size; 1624 1625 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length); 1626 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, ""); 1627 1628 LLVMValueRef gep[2]; 1629 LLVMValueRef dst_ptr; 1630 1631 gep[0] = lp_build_const_int32(gallivm, 0); 1632 gep[1] = LLVMBuildAdd(builder, bx, by, ""); 1633 1634 dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, ""); 1635 dst_ptr = LLVMBuildBitCast(builder, dst_ptr, 1636 LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), ""); 1637 1638 dst[i] = LLVMBuildLoad(builder, dst_ptr, ""); 1639 1640 LLVMSetAlignment(dst[i], dst_alignment); 1641 } 1642} 1643 1644 1645/** 1646 * Store an unswizzled block of pixels to memory 1647 */ 1648static void 1649store_unswizzled_block(struct gallivm_state *gallivm, 1650 LLVMValueRef base_ptr, 1651 LLVMValueRef stride, 1652 unsigned block_width, 1653 unsigned block_height, 1654 LLVMValueRef* src, 1655 struct lp_type src_type, 1656 unsigned src_count, 1657 unsigned src_alignment) 1658{ 1659 LLVMBuilderRef builder = gallivm->builder; 1660 const unsigned row_size = src_count / block_height; 1661 1662 /* Ensure src exactly fits into block */ 1663 assert((block_width * block_height) % src_count == 0); 1664 1665 for (unsigned i = 0; i < src_count; ++i) { 1666 unsigned x = i % row_size; 1667 unsigned y = i / row_size; 1668 1669 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length); 1670 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, ""); 1671 1672 LLVMValueRef gep[2]; 1673 LLVMValueRef src_ptr; 1674 1675 gep[0] = lp_build_const_int32(gallivm, 0); 1676 gep[1] = LLVMBuildAdd(builder, bx, by, ""); 1677 1678 src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, ""); 1679 src_ptr = LLVMBuildBitCast(builder, src_ptr, 1680 LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), ""); 1681 1682 src_ptr = LLVMBuildStore(builder, src[i], src_ptr); 1683 1684 LLVMSetAlignment(src_ptr, src_alignment); 1685 } 1686} 1687 1688 1689 1690/** 1691 * Retrieves the type for a format which is usable in the blending code. 1692 * 1693 * e.g. RGBA16F = 4x float, R3G3B2 = 3x byte 1694 */ 1695static inline void 1696lp_blend_type_from_format_desc(const struct util_format_description *format_desc, 1697 struct lp_type* type) 1698{ 1699 if (format_expands_to_float_soa(format_desc)) { 1700 /* always use ordinary floats for blending */ 1701 type->floating = true; 1702 type->fixed = false; 1703 type->sign = true; 1704 type->norm = false; 1705 type->width = 32; 1706 type->length = 4; 1707 return; 1708 } 1709 1710 unsigned i; 1711 for (i = 0; i < 4; i++) 1712 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) 1713 break; 1714 const unsigned chan = i; 1715 1716 memset(type, 0, sizeof(struct lp_type)); 1717 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT; 1718 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED; 1719 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED; 1720 type->norm = format_desc->channel[chan].normalized; 1721 type->width = format_desc->channel[chan].size; 1722 type->length = format_desc->nr_channels; 1723 1724 for (unsigned i = 1; i < format_desc->nr_channels; ++i) { 1725 if (format_desc->channel[i].size > type->width) 1726 type->width = format_desc->channel[i].size; 1727 } 1728 1729 if (type->floating) { 1730 type->width = 32; 1731 } else { 1732 if (type->width <= 8) { 1733 type->width = 8; 1734 } else if (type->width <= 16) { 1735 type->width = 16; 1736 } else { 1737 type->width = 32; 1738 } 1739 } 1740 1741 if (is_arithmetic_format(format_desc) && type->length == 3) { 1742 type->length = 4; 1743 } 1744} 1745 1746 1747/** 1748 * Scale a normalized value from src_bits to dst_bits. 1749 * 1750 * The exact calculation is 1751 * 1752 * dst = iround(src * dst_mask / src_mask) 1753 * 1754 * or with integer rounding 1755 * 1756 * dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask) 1757 * 1758 * where 1759 * 1760 * src_mask = (1 << src_bits) - 1 1761 * dst_mask = (1 << dst_bits) - 1 1762 * 1763 * but we try to avoid division and multiplication through shifts. 1764 */ 1765static inline LLVMValueRef 1766scale_bits(struct gallivm_state *gallivm, 1767 int src_bits, 1768 int dst_bits, 1769 LLVMValueRef src, 1770 struct lp_type src_type) 1771{ 1772 LLVMBuilderRef builder = gallivm->builder; 1773 LLVMValueRef result = src; 1774 1775 if (dst_bits < src_bits) { 1776 int delta_bits = src_bits - dst_bits; 1777 1778 if (delta_bits <= dst_bits) { 1779 1780 if (dst_bits == 4) { 1781 struct lp_type flt_type = lp_type_float_vec(32, src_type.length * 32); 1782 1783 result = lp_build_unsigned_norm_to_float(gallivm, src_bits, flt_type, src); 1784 result = lp_build_clamped_float_to_unsigned_norm(gallivm, flt_type, dst_bits, result); 1785 result = LLVMBuildTrunc(gallivm->builder, result, lp_build_int_vec_type(gallivm, src_type), ""); 1786 return result; 1787 } 1788 1789 /* 1790 * Approximate the rescaling with a single shift. 1791 * 1792 * This gives the wrong rounding. 1793 */ 1794 1795 result = LLVMBuildLShr(builder, 1796 src, 1797 lp_build_const_int_vec(gallivm, src_type, delta_bits), 1798 ""); 1799 1800 } else { 1801 /* 1802 * Try more accurate rescaling. 1803 */ 1804 1805 /* 1806 * Drop the least significant bits to make space for the multiplication. 1807 * 1808 * XXX: A better approach would be to use a wider integer type as intermediate. But 1809 * this is enough to convert alpha from 16bits -> 2 when rendering to 1810 * PIPE_FORMAT_R10G10B10A2_UNORM. 1811 */ 1812 result = LLVMBuildLShr(builder, 1813 src, 1814 lp_build_const_int_vec(gallivm, src_type, dst_bits), 1815 ""); 1816 1817 1818 result = LLVMBuildMul(builder, 1819 result, 1820 lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1), 1821 ""); 1822 1823 /* 1824 * Add a rounding term before the division. 1825 * 1826 * TODO: Handle signed integers too. 1827 */ 1828 if (!src_type.sign) { 1829 result = LLVMBuildAdd(builder, 1830 result, 1831 lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))), 1832 ""); 1833 } 1834 1835 /* 1836 * Approximate the division by src_mask with a src_bits shift. 1837 * 1838 * Given the src has already been shifted by dst_bits, all we need 1839 * to do is to shift by the difference. 1840 */ 1841 1842 result = LLVMBuildLShr(builder, 1843 result, 1844 lp_build_const_int_vec(gallivm, src_type, delta_bits), 1845 ""); 1846 } 1847 1848 } else if (dst_bits > src_bits) { 1849 /* Scale up bits */ 1850 int db = dst_bits - src_bits; 1851 1852 /* Shift left by difference in bits */ 1853 result = LLVMBuildShl(builder, 1854 src, 1855 lp_build_const_int_vec(gallivm, src_type, db), 1856 ""); 1857 1858 if (db <= src_bits) { 1859 /* Enough bits in src to fill the remainder */ 1860 LLVMValueRef lower = LLVMBuildLShr(builder, 1861 src, 1862 lp_build_const_int_vec(gallivm, src_type, src_bits - db), 1863 ""); 1864 1865 result = LLVMBuildOr(builder, result, lower, ""); 1866 } else if (db > src_bits) { 1867 /* Need to repeatedly copy src bits to fill remainder in dst */ 1868 unsigned n; 1869 1870 for (n = src_bits; n < dst_bits; n *= 2) { 1871 LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n); 1872 1873 result = LLVMBuildOr(builder, 1874 result, 1875 LLVMBuildLShr(builder, result, shuv, ""), 1876 ""); 1877 } 1878 } 1879 } 1880 1881 return result; 1882} 1883 1884/** 1885 * If RT is a smallfloat (needing denorms) format 1886 */ 1887static inline int 1888have_smallfloat_format(struct lp_type dst_type, 1889 enum pipe_format format) 1890{ 1891 return ((dst_type.floating && dst_type.width != 32) || 1892 /* due to format handling hacks this format doesn't have floating set 1893 * here (and actually has width set to 32 too) so special case this. */ 1894 (format == PIPE_FORMAT_R11G11B10_FLOAT)); 1895} 1896 1897 1898/** 1899 * Convert from memory format to blending format 1900 * 1901 * e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending 1902 */ 1903static void 1904convert_to_blend_type(struct gallivm_state *gallivm, 1905 unsigned block_size, 1906 const struct util_format_description *src_fmt, 1907 struct lp_type src_type, 1908 struct lp_type dst_type, 1909 LLVMValueRef* src, // and dst 1910 unsigned num_srcs) 1911{ 1912 LLVMValueRef *dst = src; 1913 LLVMBuilderRef builder = gallivm->builder; 1914 struct lp_type blend_type; 1915 struct lp_type mem_type; 1916 unsigned i, j; 1917 unsigned pixels = block_size / num_srcs; 1918 bool is_arith; 1919 1920 /* 1921 * full custom path for packed floats and srgb formats - none of the later 1922 * functions would do anything useful, and given the lp_type representation they 1923 * can't be fixed. Should really have some SoA blend path for these kind of 1924 * formats rather than hacking them in here. 1925 */ 1926 if (format_expands_to_float_soa(src_fmt)) { 1927 LLVMValueRef tmpsrc[4]; 1928 /* 1929 * This is pretty suboptimal for this case blending in SoA would be much 1930 * better, since conversion gets us SoA values so need to convert back. 1931 */ 1932 assert(src_type.width == 32 || src_type.width == 16); 1933 assert(dst_type.floating); 1934 assert(dst_type.width == 32); 1935 assert(dst_type.length % 4 == 0); 1936 assert(num_srcs % 4 == 0); 1937 1938 if (src_type.width == 16) { 1939 /* expand 4x16bit values to 4x32bit */ 1940 struct lp_type type32x4 = src_type; 1941 LLVMTypeRef ltype32x4; 1942 unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4; 1943 type32x4.width = 32; 1944 ltype32x4 = lp_build_vec_type(gallivm, type32x4); 1945 for (i = 0; i < num_fetch; i++) { 1946 src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, ""); 1947 } 1948 src_type.width = 32; 1949 } 1950 for (i = 0; i < 4; i++) { 1951 tmpsrc[i] = src[i]; 1952 } 1953 for (i = 0; i < num_srcs / 4; i++) { 1954 LLVMValueRef tmpsoa[4]; 1955 LLVMValueRef tmps = tmpsrc[i]; 1956 if (dst_type.length == 8) { 1957 LLVMValueRef shuffles[8]; 1958 unsigned j; 1959 /* fetch was 4 values but need 8-wide output values */ 1960 tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2); 1961 /* 1962 * for 8-wide aos transpose would give us wrong order not matching 1963 * incoming converted fs values and mask. ARGH. 1964 */ 1965 for (j = 0; j < 4; j++) { 1966 shuffles[j] = lp_build_const_int32(gallivm, j * 2); 1967 shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1); 1968 } 1969 tmps = LLVMBuildShuffleVector(builder, tmps, tmps, 1970 LLVMConstVector(shuffles, 8), ""); 1971 } 1972 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) { 1973 lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa); 1974 } 1975 else { 1976 lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa); 1977 } 1978 lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]); 1979 } 1980 return; 1981 } 1982 1983 lp_mem_type_from_format_desc(src_fmt, &mem_type); 1984 lp_blend_type_from_format_desc(src_fmt, &blend_type); 1985 1986 /* Is the format arithmetic */ 1987 is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length; 1988 is_arith &= !(mem_type.width == 16 && mem_type.floating); 1989 1990 /* Pad if necessary */ 1991 if (!is_arith && src_type.length < dst_type.length) { 1992 for (i = 0; i < num_srcs; ++i) { 1993 dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length); 1994 } 1995 1996 src_type.length = dst_type.length; 1997 } 1998 1999 /* Special case for half-floats */ 2000 if (mem_type.width == 16 && mem_type.floating) { 2001 assert(blend_type.width == 32 && blend_type.floating); 2002 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst); 2003 is_arith = false; 2004 } 2005 2006 if (!is_arith) { 2007 return; 2008 } 2009 2010 src_type.width = blend_type.width * blend_type.length; 2011 blend_type.length *= pixels; 2012 src_type.length *= pixels / (src_type.length / mem_type.length); 2013 2014 for (i = 0; i < num_srcs; ++i) { 2015 LLVMValueRef chans; 2016 LLVMValueRef res = NULL; 2017 2018 dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), ""); 2019 2020 for (j = 0; j < src_fmt->nr_channels; ++j) { 2021 unsigned mask = 0; 2022 unsigned sa = src_fmt->channel[j].shift; 2023#if UTIL_ARCH_LITTLE_ENDIAN 2024 unsigned from_lsb = j; 2025#else 2026 unsigned from_lsb = (blend_type.length / pixels) - j - 1; 2027#endif 2028 2029 mask = (1 << src_fmt->channel[j].size) - 1; 2030 2031 /* Extract bits from source */ 2032 chans = LLVMBuildLShr(builder, 2033 dst[i], 2034 lp_build_const_int_vec(gallivm, src_type, sa), 2035 ""); 2036 2037 chans = LLVMBuildAnd(builder, 2038 chans, 2039 lp_build_const_int_vec(gallivm, src_type, mask), 2040 ""); 2041 2042 /* Scale bits */ 2043 if (src_type.norm) { 2044 chans = scale_bits(gallivm, src_fmt->channel[j].size, 2045 blend_type.width, chans, src_type); 2046 } 2047 2048 /* Insert bits into correct position */ 2049 chans = LLVMBuildShl(builder, 2050 chans, 2051 lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width), 2052 ""); 2053 2054 if (j == 0) { 2055 res = chans; 2056 } else { 2057 res = LLVMBuildOr(builder, res, chans, ""); 2058 } 2059 } 2060 2061 dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), ""); 2062 } 2063} 2064 2065 2066/** 2067 * Convert from blending format to memory format 2068 * 2069 * e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory 2070 */ 2071static void 2072convert_from_blend_type(struct gallivm_state *gallivm, 2073 unsigned block_size, 2074 const struct util_format_description *src_fmt, 2075 struct lp_type src_type, 2076 struct lp_type dst_type, 2077 LLVMValueRef* src, // and dst 2078 unsigned num_srcs) 2079{ 2080 LLVMValueRef* dst = src; 2081 unsigned i, j, k; 2082 struct lp_type mem_type; 2083 struct lp_type blend_type; 2084 LLVMBuilderRef builder = gallivm->builder; 2085 unsigned pixels = block_size / num_srcs; 2086 bool is_arith; 2087 2088 /* 2089 * full custom path for packed floats and srgb formats - none of the later 2090 * functions would do anything useful, and given the lp_type representation they 2091 * can't be fixed. Should really have some SoA blend path for these kind of 2092 * formats rather than hacking them in here. 2093 */ 2094 if (format_expands_to_float_soa(src_fmt)) { 2095 /* 2096 * This is pretty suboptimal for this case blending in SoA would be much 2097 * better - we need to transpose the AoS values back to SoA values for 2098 * conversion/packing. 2099 */ 2100 assert(src_type.floating); 2101 assert(src_type.width == 32); 2102 assert(src_type.length % 4 == 0); 2103 assert(dst_type.width == 32 || dst_type.width == 16); 2104 2105 for (i = 0; i < num_srcs / 4; i++) { 2106 LLVMValueRef tmpsoa[4], tmpdst; 2107 lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa); 2108 /* really really need SoA here */ 2109 2110 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) { 2111 tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa); 2112 } 2113 else { 2114 tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt, 2115 src_type, tmpsoa); 2116 } 2117 2118 if (src_type.length == 8) { 2119 LLVMValueRef tmpaos, shuffles[8]; 2120 unsigned j; 2121 /* 2122 * for 8-wide aos transpose has given us wrong order not matching 2123 * output order. HMPF. Also need to split the output values manually. 2124 */ 2125 for (j = 0; j < 4; j++) { 2126 shuffles[j * 2] = lp_build_const_int32(gallivm, j); 2127 shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4); 2128 } 2129 tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst, 2130 LLVMConstVector(shuffles, 8), ""); 2131 src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4); 2132 src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4); 2133 } 2134 else { 2135 src[i] = tmpdst; 2136 } 2137 } 2138 if (dst_type.width == 16) { 2139 struct lp_type type16x8 = dst_type; 2140 struct lp_type type32x4 = dst_type; 2141 LLVMTypeRef ltype16x4, ltypei64, ltypei128; 2142 unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4; 2143 type16x8.length = 8; 2144 type32x4.width = 32; 2145 ltypei128 = LLVMIntTypeInContext(gallivm->context, 128); 2146 ltypei64 = LLVMIntTypeInContext(gallivm->context, 64); 2147 ltype16x4 = lp_build_vec_type(gallivm, dst_type); 2148 /* We could do vector truncation but it doesn't generate very good code */ 2149 for (i = 0; i < num_fetch; i++) { 2150 src[i] = lp_build_pack2(gallivm, type32x4, type16x8, 2151 src[i], lp_build_zero(gallivm, type32x4)); 2152 src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, ""); 2153 src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, ""); 2154 src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, ""); 2155 } 2156 } 2157 return; 2158 } 2159 2160 lp_mem_type_from_format_desc(src_fmt, &mem_type); 2161 lp_blend_type_from_format_desc(src_fmt, &blend_type); 2162 2163 is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length); 2164 2165 /* Special case for half-floats */ 2166 if (mem_type.width == 16 && mem_type.floating) { 2167 int length = dst_type.length; 2168 assert(blend_type.width == 32 && blend_type.floating); 2169 2170 dst_type.length = src_type.length; 2171 2172 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst); 2173 2174 dst_type.length = length; 2175 is_arith = false; 2176 } 2177 2178 /* Remove any padding */ 2179 if (!is_arith && (src_type.length % mem_type.length)) { 2180 src_type.length -= (src_type.length % mem_type.length); 2181 2182 for (i = 0; i < num_srcs; ++i) { 2183 dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length); 2184 } 2185 } 2186 2187 /* No bit arithmetic to do */ 2188 if (!is_arith) { 2189 return; 2190 } 2191 2192 src_type.length = pixels; 2193 src_type.width = blend_type.length * blend_type.width; 2194 dst_type.length = pixels; 2195 2196 for (i = 0; i < num_srcs; ++i) { 2197 LLVMValueRef chans; 2198 LLVMValueRef res = NULL; 2199 2200 dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), ""); 2201 2202 for (j = 0; j < src_fmt->nr_channels; ++j) { 2203 unsigned mask = 0; 2204 unsigned sa = src_fmt->channel[j].shift; 2205 unsigned sz_a = src_fmt->channel[j].size; 2206#if UTIL_ARCH_LITTLE_ENDIAN 2207 unsigned from_lsb = j; 2208#else 2209 unsigned from_lsb = blend_type.length - j - 1; 2210#endif 2211 2212 assert(blend_type.width > src_fmt->channel[j].size); 2213 2214 for (k = 0; k < blend_type.width; ++k) { 2215 mask |= 1 << k; 2216 } 2217 2218 /* Extract bits */ 2219 chans = LLVMBuildLShr(builder, 2220 dst[i], 2221 lp_build_const_int_vec(gallivm, src_type, 2222 from_lsb * blend_type.width), 2223 ""); 2224 2225 chans = LLVMBuildAnd(builder, 2226 chans, 2227 lp_build_const_int_vec(gallivm, src_type, mask), 2228 ""); 2229 2230 /* Scale down bits */ 2231 if (src_type.norm) { 2232 chans = scale_bits(gallivm, blend_type.width, 2233 src_fmt->channel[j].size, chans, src_type); 2234 } else if (!src_type.floating && sz_a < blend_type.width) { 2235 LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1); 2236 LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans, mask_val, ""); 2237 chans = LLVMBuildSelect(builder, mask, mask_val, chans, ""); 2238 } 2239 2240 /* Insert bits */ 2241 chans = LLVMBuildShl(builder, 2242 chans, 2243 lp_build_const_int_vec(gallivm, src_type, sa), 2244 ""); 2245 2246 sa += src_fmt->channel[j].size; 2247 2248 if (j == 0) { 2249 res = chans; 2250 } else { 2251 res = LLVMBuildOr(builder, res, chans, ""); 2252 } 2253 } 2254 2255 assert (dst_type.width != 24); 2256 2257 dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), ""); 2258 } 2259} 2260 2261 2262/** 2263 * Convert alpha to same blend type as src 2264 */ 2265static void 2266convert_alpha(struct gallivm_state *gallivm, 2267 struct lp_type row_type, 2268 struct lp_type alpha_type, 2269 const unsigned block_size, 2270 const unsigned block_height, 2271 const unsigned src_count, 2272 const unsigned dst_channels, 2273 const bool pad_inline, 2274 LLVMValueRef* src_alpha) 2275{ 2276 LLVMBuilderRef builder = gallivm->builder; 2277 unsigned i, j; 2278 unsigned length = row_type.length; 2279 row_type.length = alpha_type.length; 2280 2281 /* Twiddle the alpha to match pixels */ 2282 lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha); 2283 2284 /* 2285 * TODO this should use single lp_build_conv call for 2286 * src_count == 1 && dst_channels == 1 case (dropping the concat below) 2287 */ 2288 for (i = 0; i < block_height; ++i) { 2289 lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1); 2290 } 2291 2292 alpha_type = row_type; 2293 row_type.length = length; 2294 2295 /* If only one channel we can only need the single alpha value per pixel */ 2296 if (src_count == 1 && dst_channels == 1) { 2297 2298 lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count); 2299 } else { 2300 /* If there are more srcs than rows then we need to split alpha up */ 2301 if (src_count > block_height) { 2302 for (i = src_count; i > 0; --i) { 2303 unsigned pixels = block_size / src_count; 2304 unsigned idx = i - 1; 2305 2306 src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4], 2307 (idx * pixels) % 4, pixels); 2308 } 2309 } 2310 2311 /* If there is a src for each pixel broadcast the alpha across whole row */ 2312 if (src_count == block_size) { 2313 for (i = 0; i < src_count; ++i) { 2314 src_alpha[i] = lp_build_broadcast(gallivm, 2315 lp_build_vec_type(gallivm, row_type), src_alpha[i]); 2316 } 2317 } else { 2318 unsigned pixels = block_size / src_count; 2319 unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels; 2320 unsigned alpha_span = 1; 2321 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; 2322 2323 /* Check if we need 2 src_alphas for our shuffles */ 2324 if (pixels > alpha_type.length) { 2325 alpha_span = 2; 2326 } 2327 2328 /* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */ 2329 for (j = 0; j < row_type.length; ++j) { 2330 if (j < pixels * channels) { 2331 shuffles[j] = lp_build_const_int32(gallivm, j / channels); 2332 } else { 2333 shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context)); 2334 } 2335 } 2336 2337 for (i = 0; i < src_count; ++i) { 2338 unsigned idx1 = i, idx2 = i; 2339 2340 if (alpha_span > 1){ 2341 idx1 *= alpha_span; 2342 idx2 = idx1 + 1; 2343 } 2344 2345 src_alpha[i] = LLVMBuildShuffleVector(builder, 2346 src_alpha[idx1], 2347 src_alpha[idx2], 2348 LLVMConstVector(shuffles, row_type.length), 2349 ""); 2350 } 2351 } 2352 } 2353} 2354 2355 2356/** 2357 * Generates the blend function for unswizzled colour buffers 2358 * Also generates the read & write from colour buffer 2359 */ 2360static void 2361generate_unswizzled_blend(struct gallivm_state *gallivm, 2362 unsigned rt, 2363 struct lp_fragment_shader_variant *variant, 2364 enum pipe_format out_format, 2365 unsigned int num_fs, 2366 struct lp_type fs_type, 2367 LLVMValueRef* fs_mask, 2368 LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4], 2369 LLVMValueRef context_ptr, 2370 LLVMValueRef color_ptr, 2371 LLVMValueRef stride, 2372 unsigned partial_mask, 2373 boolean do_branch) 2374{ 2375 const unsigned alpha_channel = 3; 2376 const unsigned block_width = LP_RASTER_BLOCK_SIZE; 2377 const unsigned block_height = LP_RASTER_BLOCK_SIZE; 2378 const unsigned block_size = block_width * block_height; 2379 const unsigned lp_integer_vector_width = 128; 2380 2381 LLVMBuilderRef builder = gallivm->builder; 2382 LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS]; 2383 LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS]; 2384 LLVMValueRef src_alpha[4 * 4]; 2385 LLVMValueRef src1_alpha[4 * 4] = { NULL }; 2386 LLVMValueRef src_mask[4 * 4]; 2387 LLVMValueRef src[4 * 4]; 2388 LLVMValueRef src1[4 * 4]; 2389 LLVMValueRef dst[4 * 4]; 2390 LLVMValueRef blend_color; 2391 LLVMValueRef blend_alpha; 2392 LLVMValueRef i32_zero; 2393 LLVMValueRef check_mask; 2394 LLVMValueRef undef_src_val; 2395 2396 struct lp_build_mask_context mask_ctx; 2397 struct lp_type mask_type; 2398 struct lp_type blend_type; 2399 struct lp_type row_type; 2400 struct lp_type dst_type; 2401 struct lp_type ls_type; 2402 2403 unsigned char swizzle[TGSI_NUM_CHANNELS]; 2404 unsigned vector_width; 2405 unsigned src_channels = TGSI_NUM_CHANNELS; 2406 unsigned dst_channels; 2407 unsigned dst_count; 2408 unsigned src_count; 2409 2410 const struct util_format_description* out_format_desc = util_format_description(out_format); 2411 2412 unsigned dst_alignment; 2413 2414 bool pad_inline = is_arithmetic_format(out_format_desc); 2415 bool has_alpha = false; 2416 const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable && 2417 util_blend_state_is_dual(&variant->key.blend, 0); 2418 2419 const boolean is_1d = variant->key.resource_1d; 2420 boolean twiddle_after_convert = FALSE; 2421 unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs; 2422 LLVMValueRef fpstate = 0; 2423 2424 /* Get type from output format */ 2425 lp_blend_type_from_format_desc(out_format_desc, &row_type); 2426 lp_mem_type_from_format_desc(out_format_desc, &dst_type); 2427 2428 /* 2429 * Technically this code should go into lp_build_smallfloat_to_float 2430 * and lp_build_float_to_smallfloat but due to the 2431 * http://llvm.org/bugs/show_bug.cgi?id=6393 2432 * llvm reorders the mxcsr intrinsics in a way that breaks the code. 2433 * So the ordering is important here and there shouldn't be any 2434 * llvm ir instrunctions in this function before 2435 * this, otherwise half-float format conversions won't work 2436 * (again due to llvm bug #6393). 2437 */ 2438 if (have_smallfloat_format(dst_type, out_format)) { 2439 /* We need to make sure that denorms are ok for half float 2440 conversions */ 2441 fpstate = lp_build_fpstate_get(gallivm); 2442 lp_build_fpstate_set_denorms_zero(gallivm, FALSE); 2443 } 2444 2445 mask_type = lp_int32_vec4_type(); 2446 mask_type.length = fs_type.length; 2447 2448 for (unsigned i = num_fs; i < num_fullblock_fs; i++) { 2449 fs_mask[i] = lp_build_zero(gallivm, mask_type); 2450 } 2451 2452 /* Do not bother executing code when mask is empty.. */ 2453 if (do_branch) { 2454 check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type)); 2455 2456 for (unsigned i = 0; i < num_fullblock_fs; ++i) { 2457 check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], ""); 2458 } 2459 2460 lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask); 2461 lp_build_mask_check(&mask_ctx); 2462 } 2463 2464 partial_mask |= !variant->opaque; 2465 i32_zero = lp_build_const_int32(gallivm, 0); 2466 2467 undef_src_val = lp_build_undef(gallivm, fs_type); 2468 2469 row_type.length = fs_type.length; 2470 vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width; 2471 2472 /* Compute correct swizzle and count channels */ 2473 memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS); 2474 dst_channels = 0; 2475 2476 for (unsigned i = 0; i < TGSI_NUM_CHANNELS; ++i) { 2477 /* Ensure channel is used */ 2478 if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) { 2479 continue; 2480 } 2481 2482 /* Ensure not already written to (happens in case with GL_ALPHA) */ 2483 if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) { 2484 continue; 2485 } 2486 2487 /* Ensure we haven't already found all channels */ 2488 if (dst_channels >= out_format_desc->nr_channels) { 2489 continue; 2490 } 2491 2492 swizzle[out_format_desc->swizzle[i]] = i; 2493 ++dst_channels; 2494 2495 if (i == alpha_channel) { 2496 has_alpha = true; 2497 } 2498 } 2499 2500 if (format_expands_to_float_soa(out_format_desc)) { 2501 /* 2502 * the code above can't work for layout_other 2503 * for srgb it would sort of work but we short-circuit swizzles, etc. 2504 * as that is done as part of unpack / pack. 2505 */ 2506 dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */ 2507 has_alpha = true; 2508 swizzle[0] = 0; 2509 swizzle[1] = 1; 2510 swizzle[2] = 2; 2511 swizzle[3] = 3; 2512 pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */ 2513 } 2514 2515 /* If 3 channels then pad to include alpha for 4 element transpose */ 2516 if (dst_channels == 3) { 2517 assert (!has_alpha); 2518 for (unsigned i = 0; i < TGSI_NUM_CHANNELS; i++) { 2519 if (swizzle[i] > TGSI_NUM_CHANNELS) 2520 swizzle[i] = 3; 2521 } 2522 if (out_format_desc->nr_channels == 4) { 2523 dst_channels = 4; 2524 /* 2525 * We use alpha from the color conversion, not separate one. 2526 * We had to include it for transpose, hence it will get converted 2527 * too (albeit when doing transpose after conversion, that would 2528 * no longer be the case necessarily). 2529 * (It works only with 4 channel dsts, e.g. rgbx formats, because 2530 * otherwise we really have padding, not alpha, included.) 2531 */ 2532 has_alpha = true; 2533 } 2534 } 2535 2536 /* 2537 * Load shader output 2538 */ 2539 for (unsigned i = 0; i < num_fullblock_fs; ++i) { 2540 /* Always load alpha for use in blending */ 2541 LLVMValueRef alpha; 2542 if (i < num_fs) { 2543 alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], ""); 2544 } 2545 else { 2546 alpha = undef_src_val; 2547 } 2548 2549 /* Load each channel */ 2550 for (unsigned j = 0; j < dst_channels; ++j) { 2551 assert(swizzle[j] < 4); 2552 if (i < num_fs) { 2553 fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], ""); 2554 } 2555 else { 2556 fs_src[i][j] = undef_src_val; 2557 } 2558 } 2559 2560 /* If 3 channels then pad to include alpha for 4 element transpose */ 2561 /* 2562 * XXX If we include that here maybe could actually use it instead of 2563 * separate alpha for blending? 2564 * (Difficult though we actually convert pad channels, not alpha.) 2565 */ 2566 if (dst_channels == 3 && !has_alpha) { 2567 fs_src[i][3] = alpha; 2568 } 2569 2570 /* We split the row_mask and row_alpha as we want 128bit interleave */ 2571 if (fs_type.length == 8) { 2572 src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i], 2573 0, src_channels); 2574 src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i], 2575 src_channels, src_channels); 2576 2577 src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels); 2578 src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha, 2579 src_channels, src_channels); 2580 } else { 2581 src_mask[i] = fs_mask[i]; 2582 src_alpha[i] = alpha; 2583 } 2584 } 2585 if (dual_source_blend) { 2586 /* same as above except different src/dst, skip masks and comments... */ 2587 for (unsigned i = 0; i < num_fullblock_fs; ++i) { 2588 LLVMValueRef alpha; 2589 if (i < num_fs) { 2590 alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], ""); 2591 } 2592 else { 2593 alpha = undef_src_val; 2594 } 2595 2596 for (unsigned j = 0; j < dst_channels; ++j) { 2597 assert(swizzle[j] < 4); 2598 if (i < num_fs) { 2599 fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], ""); 2600 } 2601 else { 2602 fs_src1[i][j] = undef_src_val; 2603 } 2604 } 2605 if (dst_channels == 3 && !has_alpha) { 2606 fs_src1[i][3] = alpha; 2607 } 2608 if (fs_type.length == 8) { 2609 src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels); 2610 src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha, 2611 src_channels, src_channels); 2612 } else { 2613 src1_alpha[i] = alpha; 2614 } 2615 } 2616 } 2617 2618 if (util_format_is_pure_integer(out_format)) { 2619 /* 2620 * In this case fs_type was really ints or uints disguised as floats, 2621 * fix that up now. 2622 */ 2623 fs_type.floating = 0; 2624 fs_type.sign = dst_type.sign; 2625 for (unsigned i = 0; i < num_fullblock_fs; ++i) { 2626 for (unsigned j = 0; j < dst_channels; ++j) { 2627 fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j], 2628 lp_build_vec_type(gallivm, fs_type), ""); 2629 } 2630 if (dst_channels == 3 && !has_alpha) { 2631 fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3], 2632 lp_build_vec_type(gallivm, fs_type), ""); 2633 } 2634 } 2635 } 2636 2637 /* 2638 * We actually should generally do conversion first (for non-1d cases) 2639 * when the blend format is 8 or 16 bits. The reason is obvious, 2640 * there's 2 or 4 times less vectors to deal with for the interleave... 2641 * Albeit for the AVX (not AVX2) case there's no benefit with 16 bit 2642 * vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit 2643 * unpack only with 128bit vectors). 2644 * Note: for 16bit sizes really need matching pack conversion code 2645 */ 2646 if (!is_1d && dst_channels != 3 && dst_type.width == 8) { 2647 twiddle_after_convert = TRUE; 2648 } 2649 2650 /* 2651 * Pixel twiddle from fragment shader order to memory order 2652 */ 2653 if (!twiddle_after_convert) { 2654 src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, 2655 dst_channels, fs_src, src, pad_inline); 2656 if (dual_source_blend) { 2657 generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels, 2658 fs_src1, src1, pad_inline); 2659 } 2660 } else { 2661 src_count = num_fullblock_fs * dst_channels; 2662 /* 2663 * We reorder things a bit here, so the cases for 4-wide and 8-wide 2664 * (AVX) turn out the same later when untwiddling/transpose (albeit 2665 * for true AVX2 path untwiddle needs to be different). 2666 * For now just order by colors first (so we can use unpack later). 2667 */ 2668 for (unsigned j = 0; j < num_fullblock_fs; j++) { 2669 for (unsigned i = 0; i < dst_channels; i++) { 2670 src[i*num_fullblock_fs + j] = fs_src[j][i]; 2671 if (dual_source_blend) { 2672 src1[i*num_fullblock_fs + j] = fs_src1[j][i]; 2673 } 2674 } 2675 } 2676 } 2677 2678 src_channels = dst_channels < 3 ? dst_channels : 4; 2679 if (src_count != num_fullblock_fs * src_channels) { 2680 unsigned ds = src_count / (num_fullblock_fs * src_channels); 2681 row_type.length /= ds; 2682 fs_type.length = row_type.length; 2683 } 2684 2685 blend_type = row_type; 2686 mask_type.length = 4; 2687 2688 /* Convert src to row_type */ 2689 if (dual_source_blend) { 2690 struct lp_type old_row_type = row_type; 2691 lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src); 2692 src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1); 2693 } 2694 else { 2695 src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src); 2696 } 2697 2698 /* If the rows are not an SSE vector, combine them to become SSE size! */ 2699 if ((row_type.width * row_type.length) % 128) { 2700 unsigned bits = row_type.width * row_type.length; 2701 unsigned combined; 2702 2703 assert(src_count >= (vector_width / bits)); 2704 2705 dst_count = src_count / (vector_width / bits); 2706 2707 combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count); 2708 if (dual_source_blend) { 2709 lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count); 2710 } 2711 2712 row_type.length *= combined; 2713 src_count /= combined; 2714 2715 bits = row_type.width * row_type.length; 2716 assert(bits == 128 || bits == 256); 2717 } 2718 2719 if (twiddle_after_convert) { 2720 fs_twiddle_transpose(gallivm, row_type, src, src_count, src); 2721 if (dual_source_blend) { 2722 fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1); 2723 } 2724 } 2725 2726 /* 2727 * Blend Colour conversion 2728 */ 2729 blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr); 2730 blend_color = LLVMBuildPointerCast(builder, blend_color, 2731 LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), ""); 2732 blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color, 2733 &i32_zero, 1, ""), ""); 2734 2735 /* Convert */ 2736 lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1); 2737 2738 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) { 2739 /* 2740 * since blending is done with floats, there was no conversion. 2741 * However, the rules according to fixed point renderbuffers still 2742 * apply, that is we must clamp inputs to 0.0/1.0. 2743 * (This would apply to separate alpha conversion too but we currently 2744 * force has_alpha to be true.) 2745 * TODO: should skip this with "fake" blend, since post-blend conversion 2746 * will clamp anyway. 2747 * TODO: could also skip this if fragment color clamping is enabled. 2748 * We don't support it natively so it gets baked into the shader 2749 * however, so can't really tell here. 2750 */ 2751 struct lp_build_context f32_bld; 2752 assert(row_type.floating); 2753 lp_build_context_init(&f32_bld, gallivm, row_type); 2754 for (unsigned i = 0; i < src_count; i++) { 2755 src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]); 2756 } 2757 if (dual_source_blend) { 2758 for (unsigned i = 0; i < src_count; i++) { 2759 src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]); 2760 } 2761 } 2762 /* probably can't be different than row_type but better safe than sorry... */ 2763 lp_build_context_init(&f32_bld, gallivm, blend_type); 2764 blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one); 2765 } 2766 2767 /* Extract alpha */ 2768 blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3)); 2769 2770 /* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */ 2771 pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width; 2772 if (pad_inline) { 2773 /* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */ 2774 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length); 2775 } else { 2776 /* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */ 2777 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length); 2778 } 2779 2780 /* 2781 * Mask conversion 2782 */ 2783 lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]); 2784 2785 if (src_count < block_height) { 2786 lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count); 2787 } else if (src_count > block_height) { 2788 for (unsigned i = src_count; i > 0; --i) { 2789 unsigned pixels = block_size / src_count; 2790 unsigned idx = i - 1; 2791 2792 src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4], 2793 (idx * pixels) % 4, pixels); 2794 } 2795 } 2796 2797 assert(mask_type.width == 32); 2798 2799 for (unsigned i = 0; i < src_count; ++i) { 2800 unsigned pixels = block_size / src_count; 2801 unsigned pixel_width = row_type.width * dst_channels; 2802 2803 if (pixel_width == 24) { 2804 mask_type.width = 8; 2805 mask_type.length = vector_width / mask_type.width; 2806 } else { 2807 mask_type.length = pixels; 2808 mask_type.width = row_type.width * dst_channels; 2809 2810 /* 2811 * If mask_type width is smaller than 32bit, this doesn't quite 2812 * generate the most efficient code (could use some pack). 2813 */ 2814 src_mask[i] = LLVMBuildIntCast(builder, src_mask[i], 2815 lp_build_int_vec_type(gallivm, mask_type), ""); 2816 2817 mask_type.length *= dst_channels; 2818 mask_type.width /= dst_channels; 2819 } 2820 2821 src_mask[i] = LLVMBuildBitCast(builder, src_mask[i], 2822 lp_build_int_vec_type(gallivm, mask_type), ""); 2823 src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length); 2824 } 2825 2826 /* 2827 * Alpha conversion 2828 */ 2829 if (!has_alpha) { 2830 struct lp_type alpha_type = fs_type; 2831 alpha_type.length = 4; 2832 convert_alpha(gallivm, row_type, alpha_type, 2833 block_size, block_height, 2834 src_count, dst_channels, 2835 pad_inline, src_alpha); 2836 if (dual_source_blend) { 2837 convert_alpha(gallivm, row_type, alpha_type, 2838 block_size, block_height, 2839 src_count, dst_channels, 2840 pad_inline, src1_alpha); 2841 } 2842 } 2843 2844 2845 /* 2846 * Load dst from memory 2847 */ 2848 if (src_count < block_height) { 2849 dst_count = block_height; 2850 } else { 2851 dst_count = src_count; 2852 } 2853 2854 dst_type.length *= block_size / dst_count; 2855 2856 if (format_expands_to_float_soa(out_format_desc)) { 2857 /* 2858 * we need multiple values at once for the conversion, so can as well 2859 * load them vectorized here too instead of concatenating later. 2860 * (Still need concatenation later for 8-wide vectors). 2861 */ 2862 dst_count = block_height; 2863 dst_type.length = block_width; 2864 } 2865 2866 /* 2867 * Compute the alignment of the destination pointer in bytes 2868 * We fetch 1-4 pixels, if the format has pot alignment then those fetches 2869 * are always aligned by MIN2(16, fetch_width) except for buffers (not 2870 * 1d tex but can't distinguish here) so need to stick with per-pixel 2871 * alignment in this case. 2872 */ 2873 if (is_1d) { 2874 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8); 2875 } 2876 else { 2877 dst_alignment = dst_type.length * dst_type.width / 8; 2878 } 2879 /* Force power-of-two alignment by extracting only the least-significant-bit */ 2880 dst_alignment = 1 << (ffs(dst_alignment) - 1); 2881 /* 2882 * Resource base and stride pointers are aligned to 16 bytes, so that's 2883 * the maximum alignment we can guarantee 2884 */ 2885 dst_alignment = MIN2(16, dst_alignment); 2886 2887 ls_type = dst_type; 2888 2889 if (dst_count > src_count) { 2890 if ((dst_type.width == 8 || dst_type.width == 16) && 2891 util_is_power_of_two_or_zero(dst_type.length) && 2892 dst_type.length * dst_type.width < 128) { 2893 /* 2894 * Never try to load values as 4xi8 which we will then 2895 * concatenate to larger vectors. This gives llvm a real 2896 * headache (the problem is the type legalizer (?) will 2897 * try to load that as 4xi8 zext to 4xi32 to fill the vector, 2898 * then the shuffles to concatenate are more or less impossible 2899 * - llvm is easily capable of generating a sequence of 32 2900 * pextrb/pinsrb instructions for that. Albeit it appears to 2901 * be fixed in llvm 4.0. So, load and concatenate with 32bit 2902 * width to avoid the trouble (16bit seems not as bad, llvm 2903 * probably recognizes the load+shuffle as only one shuffle 2904 * is necessary, but we can do just the same anyway). 2905 */ 2906 ls_type.length = dst_type.length * dst_type.width / 32; 2907 ls_type.width = 32; 2908 } 2909 } 2910 2911 if (is_1d) { 2912 load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1, 2913 dst, ls_type, dst_count / 4, dst_alignment); 2914 for (unsigned i = dst_count / 4; i < dst_count; i++) { 2915 dst[i] = lp_build_undef(gallivm, ls_type); 2916 } 2917 2918 } 2919 else { 2920 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height, 2921 dst, ls_type, dst_count, dst_alignment); 2922 } 2923 2924 2925 /* 2926 * Convert from dst/output format to src/blending format. 2927 * 2928 * This is necessary as we can only read 1 row from memory at a time, 2929 * so the minimum dst_count will ever be at this point is 4. 2930 * 2931 * With, for example, R8 format you can have all 16 pixels in a 128 bit 2932 * vector, this will take the 4 dsts and combine them into 1 src so we can 2933 * perform blending on all 16 pixels in that single vector at once. 2934 */ 2935 if (dst_count > src_count) { 2936 if (ls_type.length != dst_type.length && ls_type.length == 1) { 2937 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type); 2938 LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1); 2939 for (unsigned i = 0; i < dst_count; i++) { 2940 dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, ""); 2941 } 2942 } 2943 2944 lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count); 2945 2946 if (ls_type.length != dst_type.length) { 2947 struct lp_type tmp_type = dst_type; 2948 tmp_type.length = dst_type.length * 4 / src_count; 2949 for (unsigned i = 0; i < src_count; i++) { 2950 dst[i] = LLVMBuildBitCast(builder, dst[i], 2951 lp_build_vec_type(gallivm, tmp_type), ""); 2952 } 2953 } 2954 } 2955 2956 /* 2957 * Blending 2958 */ 2959 /* XXX this is broken for RGB8 formats - 2960 * they get expanded from 12 to 16 elements (to include alpha) 2961 * by convert_to_blend_type then reduced to 15 instead of 12 2962 * by convert_from_blend_type (a simple fix though breaks A8...). 2963 * R16G16B16 also crashes differently however something going wrong 2964 * inside llvm handling npot vector sizes seemingly. 2965 * It seems some cleanup could be done here (like skipping conversion/blend 2966 * when not needed). 2967 */ 2968 convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type, 2969 row_type, dst, src_count); 2970 2971 /* 2972 * FIXME: Really should get logic ops / masks out of generic blend / row 2973 * format. Logic ops will definitely not work on the blend float format 2974 * used for SRGB here and I think OpenGL expects this to work as expected 2975 * (that is incoming values converted to srgb then logic op applied). 2976 */ 2977 for (unsigned i = 0; i < src_count; ++i) { 2978 dst[i] = lp_build_blend_aos(gallivm, 2979 &variant->key.blend, 2980 out_format, 2981 row_type, 2982 rt, 2983 src[i], 2984 has_alpha ? NULL : src_alpha[i], 2985 src1[i], 2986 has_alpha ? NULL : src1_alpha[i], 2987 dst[i], 2988 partial_mask ? src_mask[i] : NULL, 2989 blend_color, 2990 has_alpha ? NULL : blend_alpha, 2991 swizzle, 2992 pad_inline ? 4 : dst_channels); 2993 } 2994 2995 convert_from_blend_type(gallivm, block_size, out_format_desc, 2996 row_type, dst_type, dst, src_count); 2997 2998 /* Split the blend rows back to memory rows */ 2999 if (dst_count > src_count) { 3000 row_type.length = dst_type.length * (dst_count / src_count); 3001 3002 if (src_count == 1) { 3003 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2); 3004 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2); 3005 3006 row_type.length /= 2; 3007 src_count *= 2; 3008 } 3009 3010 dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2); 3011 dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2); 3012 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2); 3013 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2); 3014 3015 row_type.length /= 2; 3016 src_count *= 2; 3017 } 3018 3019 /* 3020 * Store blend result to memory 3021 */ 3022 if (is_1d) { 3023 store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1, 3024 dst, dst_type, dst_count / 4, dst_alignment); 3025 } 3026 else { 3027 store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height, 3028 dst, dst_type, dst_count, dst_alignment); 3029 } 3030 3031 if (have_smallfloat_format(dst_type, out_format)) { 3032 lp_build_fpstate_set(gallivm, fpstate); 3033 } 3034 3035 if (do_branch) { 3036 lp_build_mask_end(&mask_ctx); 3037 } 3038} 3039 3040 3041/** 3042 * Generate the runtime callable function for the whole fragment pipeline. 3043 * Note that the function which we generate operates on a block of 16 3044 * pixels at at time. The block contains 2x2 quads. Each quad contains 3045 * 2x2 pixels. 3046 */ 3047static void 3048generate_fragment(struct llvmpipe_context *lp, 3049 struct lp_fragment_shader *shader, 3050 struct lp_fragment_shader_variant *variant, 3051 unsigned partial_mask) 3052{ 3053 assert(partial_mask == RAST_WHOLE || 3054 partial_mask == RAST_EDGE_TEST); 3055 3056 struct gallivm_state *gallivm = variant->gallivm; 3057 struct lp_fragment_shader_variant_key *key = &variant->key; 3058 struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS]; 3059 LLVMTypeRef fs_elem_type; 3060 LLVMTypeRef blend_vec_type; 3061 LLVMTypeRef arg_types[15]; 3062 LLVMTypeRef func_type; 3063 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 3064 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context); 3065 LLVMValueRef context_ptr; 3066 LLVMValueRef x; 3067 LLVMValueRef y; 3068 LLVMValueRef a0_ptr; 3069 LLVMValueRef dadx_ptr; 3070 LLVMValueRef dady_ptr; 3071 LLVMValueRef color_ptr_ptr; 3072 LLVMValueRef stride_ptr; 3073 LLVMValueRef color_sample_stride_ptr; 3074 LLVMValueRef depth_ptr; 3075 LLVMValueRef depth_stride; 3076 LLVMValueRef depth_sample_stride; 3077 LLVMValueRef mask_input; 3078 LLVMValueRef thread_data_ptr; 3079 LLVMBasicBlockRef block; 3080 LLVMBuilderRef builder; 3081 struct lp_build_interp_soa_context interp; 3082 LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES]; 3083 LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4]; 3084 LLVMValueRef function; 3085 LLVMValueRef facing; 3086 boolean cbuf0_write_all; 3087 const boolean dual_source_blend = key->blend.rt[0].blend_enable && 3088 util_blend_state_is_dual(&key->blend, 0); 3089 3090 assert(lp_native_vector_width / 32 >= 4); 3091 3092 /* Adjust color input interpolation according to flatshade state: 3093 */ 3094 memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]); 3095 for (unsigned i = 0; i < shader->info.base.num_inputs; i++) { 3096 if (inputs[i].interp == LP_INTERP_COLOR) { 3097 if (key->flatshade) 3098 inputs[i].interp = LP_INTERP_CONSTANT; 3099 else 3100 inputs[i].interp = LP_INTERP_PERSPECTIVE; 3101 } 3102 } 3103 3104 /* check if writes to cbuf[0] are to be copied to all cbufs */ 3105 cbuf0_write_all = 3106 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]; 3107 3108 /* TODO: actually pick these based on the fs and color buffer 3109 * characteristics. */ 3110 3111 struct lp_type fs_type; 3112 memset(&fs_type, 0, sizeof fs_type); 3113 fs_type.floating = TRUE; /* floating point values */ 3114 fs_type.sign = TRUE; /* values are signed */ 3115 fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 3116 fs_type.width = 32; /* 32-bit float */ 3117 fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */ 3118 3119 struct lp_type blend_type; 3120 memset(&blend_type, 0, sizeof blend_type); 3121 blend_type.floating = FALSE; /* values are integers */ 3122 blend_type.sign = FALSE; /* values are unsigned */ 3123 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */ 3124 blend_type.width = 8; /* 8-bit ubyte values */ 3125 blend_type.length = 16; /* 16 elements per vector */ 3126 3127 /* 3128 * Generate the function prototype. Any change here must be reflected in 3129 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa. 3130 */ 3131 3132 fs_elem_type = lp_build_elem_type(gallivm, fs_type); 3133 3134 blend_vec_type = lp_build_vec_type(gallivm, blend_type); 3135 3136 char func_name[64]; 3137 snprintf(func_name, sizeof(func_name), "fs_variant_%s", 3138 partial_mask ? "partial" : "whole"); 3139 3140 arg_types[0] = variant->jit_context_ptr_type; /* context */ 3141 arg_types[1] = int32_type; /* x */ 3142 arg_types[2] = int32_type; /* y */ 3143 arg_types[3] = int32_type; /* facing */ 3144 arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */ 3145 arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */ 3146 arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */ 3147 arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */ 3148 arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */ 3149 arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */ 3150 arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */ 3151 arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */ 3152 arg_types[12] = int32_type; /* depth_stride */ 3153 arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */ 3154 arg_types[14] = int32_type; /* depth sample stride */ 3155 3156 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), 3157 arg_types, ARRAY_SIZE(arg_types), 0); 3158 3159 function = LLVMAddFunction(gallivm->module, func_name, func_type); 3160 LLVMSetFunctionCallConv(function, LLVMCCallConv); 3161 3162 variant->function[partial_mask] = function; 3163 3164 /* XXX: need to propagate noalias down into color param now we are 3165 * passing a pointer-to-pointer? 3166 */ 3167 for (unsigned i = 0; i < ARRAY_SIZE(arg_types); ++i) 3168 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 3169 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS); 3170 3171 if (variant->gallivm->cache->data_size) 3172 return; 3173 3174 context_ptr = LLVMGetParam(function, 0); 3175 x = LLVMGetParam(function, 1); 3176 y = LLVMGetParam(function, 2); 3177 facing = LLVMGetParam(function, 3); 3178 a0_ptr = LLVMGetParam(function, 4); 3179 dadx_ptr = LLVMGetParam(function, 5); 3180 dady_ptr = LLVMGetParam(function, 6); 3181 color_ptr_ptr = LLVMGetParam(function, 7); 3182 depth_ptr = LLVMGetParam(function, 8); 3183 mask_input = LLVMGetParam(function, 9); 3184 thread_data_ptr = LLVMGetParam(function, 10); 3185 stride_ptr = LLVMGetParam(function, 11); 3186 depth_stride = LLVMGetParam(function, 12); 3187 color_sample_stride_ptr = LLVMGetParam(function, 13); 3188 depth_sample_stride = LLVMGetParam(function, 14); 3189 3190 lp_build_name(context_ptr, "context"); 3191 lp_build_name(x, "x"); 3192 lp_build_name(y, "y"); 3193 lp_build_name(a0_ptr, "a0"); 3194 lp_build_name(dadx_ptr, "dadx"); 3195 lp_build_name(dady_ptr, "dady"); 3196 lp_build_name(color_ptr_ptr, "color_ptr_ptr"); 3197 lp_build_name(depth_ptr, "depth"); 3198 lp_build_name(mask_input, "mask_input"); 3199 lp_build_name(thread_data_ptr, "thread_data"); 3200 lp_build_name(stride_ptr, "stride_ptr"); 3201 lp_build_name(depth_stride, "depth_stride"); 3202 lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr"); 3203 lp_build_name(depth_sample_stride, "depth_sample_stride"); 3204 3205 /* 3206 * Function body 3207 */ 3208 3209 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry"); 3210 builder = gallivm->builder; 3211 assert(builder); 3212 LLVMPositionBuilderAtEnd(builder, block); 3213 3214 /* 3215 * Must not count ps invocations if there's a null shader. 3216 * (It would be ok to count with null shader if there's d/s tests, 3217 * but only if there's d/s buffers too, which is different 3218 * to implicit rasterization disable which must not depend 3219 * on the d/s buffers.) 3220 * Could use popcount on mask, but pixel accuracy is not required. 3221 * Could disable if there's no stats query, but maybe not worth it. 3222 */ 3223 if (shader->info.base.num_instructions > 1) { 3224 LLVMValueRef invocs, val; 3225 invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr); 3226 val = LLVMBuildLoad(builder, invocs, ""); 3227 val = LLVMBuildAdd(builder, val, 3228 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0), 3229 "invoc_count"); 3230 LLVMBuildStore(builder, val, invocs); 3231 } 3232 3233 /* code generated texture sampling */ 3234 struct lp_build_sampler_soa *sampler = 3235 lp_llvm_sampler_soa_create(lp_fs_variant_key_samplers(key), 3236 MAX2(key->nr_samplers, 3237 key->nr_sampler_views)); 3238 struct lp_build_image_soa *image = 3239 lp_llvm_image_soa_create(lp_fs_variant_key_images(key), key->nr_images); 3240 3241 unsigned num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */ 3242 /* for 1d resources only run "upper half" of stamp */ 3243 if (key->resource_1d) 3244 num_fs /= 2; 3245 3246 { 3247 LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs); 3248 LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type); 3249 LLVMValueRef num_loop_samp = 3250 lp_build_const_int32(gallivm, num_fs * key->coverage_samples); 3251 LLVMValueRef mask_store = 3252 lp_build_array_alloca(gallivm, mask_type, 3253 num_loop_samp, "mask_store"); 3254 LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context); 3255 LLVMValueRef glob_sample_pos = 3256 LLVMAddGlobal(gallivm->module, 3257 LLVMArrayType(flt_type, key->coverage_samples * 2), ""); 3258 LLVMValueRef sample_pos_array; 3259 3260 if (key->multisample && key->coverage_samples == 4) { 3261 LLVMValueRef sample_pos_arr[8]; 3262 for (unsigned i = 0; i < 4; i++) { 3263 sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]); 3264 sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]); 3265 } 3266 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8); 3267 } else { 3268 LLVMValueRef sample_pos_arr[2]; 3269 sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5); 3270 sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5); 3271 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2); 3272 } 3273 LLVMSetInitializer(glob_sample_pos, sample_pos_array); 3274 3275 LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS]; 3276 boolean pixel_center_integer = 3277 shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER]; 3278 3279 /* 3280 * The shader input interpolation info is not explicitely baked in the 3281 * shader key, but everything it derives from (TGSI, and flatshade) is 3282 * already included in the shader key. 3283 */ 3284 lp_build_interp_soa_init(&interp, 3285 gallivm, 3286 shader->info.base.num_inputs, 3287 inputs, 3288 pixel_center_integer, 3289 key->coverage_samples, glob_sample_pos, 3290 num_loop, 3291 builder, fs_type, 3292 a0_ptr, dadx_ptr, dady_ptr, 3293 x, y); 3294 3295 for (unsigned i = 0; i < num_fs; i++) { 3296 if (key->multisample) { 3297 LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), ""); 3298 3299 /* 3300 * For multisampling, extract the per-sample mask from the 3301 * incoming 64-bit mask, store to the per sample mask storage. Or 3302 * all of them together to generate the fragment shader 3303 * mask. (sample shading TODO). Take the incoming state coverage 3304 * mask into account. 3305 */ 3306 for (unsigned s = 0; s < key->coverage_samples; s++) { 3307 LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs)); 3308 LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store, 3309 &sindexi, 1, "sample_mask_ptr"); 3310 LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type, 3311 i*fs_type.length/4, s, mask_input); 3312 3313 LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), ""); 3314 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), ""); 3315 smask_bit = LLVMBuildSExt(builder, cmp, int32_type, ""); 3316 smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit); 3317 3318 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, ""); 3319 LLVMBuildStore(builder, s_mask, sample_mask_ptr); 3320 } 3321 } else { 3322 LLVMValueRef mask; 3323 LLVMValueRef indexi = lp_build_const_int32(gallivm, i); 3324 LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store, 3325 &indexi, 1, "mask_ptr"); 3326 3327 if (partial_mask) { 3328 mask = generate_quad_mask(gallivm, fs_type, 3329 i*fs_type.length/4, 0, mask_input); 3330 } 3331 else { 3332 mask = lp_build_const_int_vec(gallivm, fs_type, ~0); 3333 } 3334 LLVMBuildStore(builder, mask, mask_ptr); 3335 } 3336 } 3337 3338 generate_fs_loop(gallivm, 3339 shader, key, 3340 builder, 3341 fs_type, 3342 context_ptr, 3343 glob_sample_pos, 3344 num_loop, 3345 &interp, 3346 sampler, 3347 image, 3348 mask_store, /* output */ 3349 color_store, 3350 depth_ptr, 3351 depth_stride, 3352 depth_sample_stride, 3353 color_ptr_ptr, 3354 stride_ptr, 3355 color_sample_stride_ptr, 3356 facing, 3357 thread_data_ptr); 3358 3359 for (unsigned i = 0; i < num_fs; i++) { 3360 LLVMValueRef ptr; 3361 for (unsigned s = 0; s < key->coverage_samples; s++) { 3362 int idx = (i + (s * num_fs)); 3363 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx); 3364 ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, ""); 3365 3366 fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask"); 3367 } 3368 3369 for (unsigned s = 0; s < key->min_samples; s++) { 3370 /* This is fucked up need to reorganize things */ 3371 int idx = s * num_fs + i; 3372 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx); 3373 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 3374 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 3375 ptr = LLVMBuildGEP(builder, 3376 color_store[cbuf * !cbuf0_write_all][chan], 3377 &sindexi, 1, ""); 3378 fs_out_color[s][cbuf][chan][i] = ptr; 3379 } 3380 } 3381 if (dual_source_blend) { 3382 /* only support one dual source blend target hence always use output 1 */ 3383 for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 3384 ptr = LLVMBuildGEP(builder, 3385 color_store[1][chan], 3386 &sindexi, 1, ""); 3387 fs_out_color[s][1][chan][i] = ptr; 3388 } 3389 } 3390 } 3391 } 3392 } 3393 3394 sampler->destroy(sampler); 3395 image->destroy(image); 3396 3397 /* Loop over color outputs / color buffers to do blending */ 3398 for (unsigned cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 3399 if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) { 3400 LLVMValueRef color_ptr; 3401 LLVMValueRef stride; 3402 LLVMValueRef sample_stride = NULL; 3403 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf); 3404 3405 boolean do_branch = ((key->depth.enabled 3406 || key->stencil[0].enabled 3407 || key->alpha.enabled) 3408 && !shader->info.base.uses_kill); 3409 3410 color_ptr = LLVMBuildLoad(builder, 3411 LLVMBuildGEP(builder, color_ptr_ptr, 3412 &index, 1, ""), 3413 ""); 3414 3415 stride = LLVMBuildLoad(builder, 3416 LLVMBuildGEP(builder, stride_ptr, 3417 &index, 1, ""), 3418 ""); 3419 3420 if (key->cbuf_nr_samples[cbuf] > 1) 3421 sample_stride = LLVMBuildLoad(builder, 3422 LLVMBuildGEP(builder, 3423 color_sample_stride_ptr, 3424 &index, 1, ""), ""); 3425 3426 for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) { 3427 unsigned mask_idx = num_fs * (key->multisample ? s : 0); 3428 unsigned out_idx = key->min_samples == 1 ? 0 : s; 3429 LLVMValueRef out_ptr = color_ptr;; 3430 3431 if (sample_stride) { 3432 LLVMValueRef sample_offset = 3433 LLVMBuildMul(builder, sample_stride, 3434 lp_build_const_int32(gallivm, s), ""); 3435 out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, ""); 3436 } 3437 out_ptr = LLVMBuildBitCast(builder, out_ptr, 3438 LLVMPointerType(blend_vec_type, 0), ""); 3439 3440 lp_build_name(out_ptr, "color_ptr%d", cbuf); 3441 3442 generate_unswizzled_blend(gallivm, cbuf, variant, 3443 key->cbuf_format[cbuf], 3444 num_fs, fs_type, &fs_mask[mask_idx], 3445 fs_out_color[out_idx], 3446 context_ptr, out_ptr, stride, 3447 partial_mask, do_branch); 3448 } 3449 } 3450 } 3451 3452 LLVMBuildRetVoid(builder); 3453 3454 gallivm_verify_function(gallivm, function); 3455} 3456 3457 3458static void 3459dump_fs_variant_key(struct lp_fragment_shader_variant_key *key) 3460{ 3461 debug_printf("fs variant %p:\n", (void *) key); 3462 3463 if (key->flatshade) { 3464 debug_printf("flatshade = 1\n"); 3465 } 3466 if (key->depth_clamp) 3467 debug_printf("depth_clamp = 1\n"); 3468 3469 if (key->restrict_depth_values) 3470 debug_printf("restrict_depth_values = 1\n"); 3471 3472 if (key->multisample) { 3473 debug_printf("multisample = 1\n"); 3474 debug_printf("coverage samples = %d\n", key->coverage_samples); 3475 debug_printf("min samples = %d\n", key->min_samples); 3476 } 3477 for (unsigned i = 0; i < key->nr_cbufs; ++i) { 3478 debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i])); 3479 debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]); 3480 } 3481 if (key->depth.enabled || key->stencil[0].enabled) { 3482 debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format)); 3483 debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples); 3484 } 3485 if (key->depth.enabled) { 3486 debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE)); 3487 debug_printf("depth.writemask = %u\n", key->depth.writemask); 3488 } 3489 3490 for (unsigned i = 0; i < 2; ++i) { 3491 if (key->stencil[i].enabled) { 3492 debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE)); 3493 debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE)); 3494 debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE)); 3495 debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE)); 3496 debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask); 3497 debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask); 3498 } 3499 } 3500 3501 if (key->alpha.enabled) { 3502 debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE)); 3503 } 3504 3505 if (key->occlusion_count) { 3506 debug_printf("occlusion_count = 1\n"); 3507 } 3508 3509 if (key->blend.logicop_enable) { 3510 debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE)); 3511 } 3512 else if (key->blend.rt[0].blend_enable) { 3513 debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE)); 3514 debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE)); 3515 debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE)); 3516 debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE)); 3517 debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE)); 3518 debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE)); 3519 } 3520 debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask); 3521 if (key->blend.alpha_to_coverage) { 3522 debug_printf("blend.alpha_to_coverage is enabled\n"); 3523 } 3524 for (unsigned i = 0; i < key->nr_samplers; ++i) { 3525 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key); 3526 const struct lp_static_sampler_state *sampler = &samplers[i].sampler_state; 3527 debug_printf("sampler[%u] = \n", i); 3528 debug_printf(" .wrap = %s %s %s\n", 3529 util_str_tex_wrap(sampler->wrap_s, TRUE), 3530 util_str_tex_wrap(sampler->wrap_t, TRUE), 3531 util_str_tex_wrap(sampler->wrap_r, TRUE)); 3532 debug_printf(" .min_img_filter = %s\n", 3533 util_str_tex_filter(sampler->min_img_filter, TRUE)); 3534 debug_printf(" .min_mip_filter = %s\n", 3535 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE)); 3536 debug_printf(" .mag_img_filter = %s\n", 3537 util_str_tex_filter(sampler->mag_img_filter, TRUE)); 3538 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE) 3539 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE)); 3540 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords); 3541 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal); 3542 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero); 3543 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod); 3544 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod); 3545 debug_printf(" .reduction_mode = %u\n", sampler->reduction_mode); 3546 debug_printf(" .aniso = %u\n", sampler->aniso); 3547 } 3548 for (unsigned i = 0; i < key->nr_sampler_views; ++i) { 3549 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key); 3550 const struct lp_static_texture_state *texture = &samplers[i].texture_state; 3551 debug_printf("texture[%u] = \n", i); 3552 debug_printf(" .format = %s\n", 3553 util_format_name(texture->format)); 3554 debug_printf(" .target = %s\n", 3555 util_str_tex_target(texture->target, TRUE)); 3556 debug_printf(" .level_zero_only = %u\n", 3557 texture->level_zero_only); 3558 debug_printf(" .pot = %u %u %u\n", 3559 texture->pot_width, 3560 texture->pot_height, 3561 texture->pot_depth); 3562 } 3563 struct lp_image_static_state *images = lp_fs_variant_key_images(key); 3564 for (unsigned i = 0; i < key->nr_images; ++i) { 3565 const struct lp_static_texture_state *image = &images[i].image_state; 3566 debug_printf("image[%u] = \n", i); 3567 debug_printf(" .format = %s\n", 3568 util_format_name(image->format)); 3569 debug_printf(" .target = %s\n", 3570 util_str_tex_target(image->target, TRUE)); 3571 debug_printf(" .level_zero_only = %u\n", 3572 image->level_zero_only); 3573 debug_printf(" .pot = %u %u %u\n", 3574 image->pot_width, 3575 image->pot_height, 3576 image->pot_depth); 3577 } 3578} 3579 3580 3581const char * 3582lp_debug_fs_kind(enum lp_fs_kind kind) 3583{ 3584 switch (kind) { 3585 case LP_FS_KIND_GENERAL: 3586 return "GENERAL"; 3587 case LP_FS_KIND_BLIT_RGBA: 3588 return "BLIT_RGBA"; 3589 case LP_FS_KIND_BLIT_RGB1: 3590 return "BLIT_RGB1"; 3591 case LP_FS_KIND_AERO_MINIFICATION: 3592 return "AERO_MINIFICATION"; 3593 case LP_FS_KIND_LLVM_LINEAR: 3594 return "LLVM_LINEAR"; 3595 default: 3596 return "unknown"; 3597 } 3598} 3599 3600 3601void 3602lp_debug_fs_variant(struct lp_fragment_shader_variant *variant) 3603{ 3604 debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n", 3605 variant->shader->no, variant->no); 3606 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI) 3607 tgsi_dump(variant->shader->base.tokens, 0); 3608 else 3609 nir_print_shader(variant->shader->base.ir.nir, stderr); 3610 dump_fs_variant_key(&variant->key); 3611 debug_printf("variant->opaque = %u\n", variant->opaque); 3612 debug_printf("variant->potentially_opaque = %u\n", variant->potentially_opaque); 3613 debug_printf("variant->blit = %u\n", variant->blit); 3614 debug_printf("shader->kind = %s\n", lp_debug_fs_kind(variant->shader->kind)); 3615 debug_printf("\n"); 3616} 3617 3618 3619static void 3620lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant *variant, 3621 unsigned char ir_sha1_cache_key[20]) 3622{ 3623 struct blob blob = { 0 }; 3624 unsigned ir_size; 3625 void *ir_binary; 3626 3627 blob_init(&blob); 3628 nir_serialize(&blob, variant->shader->base.ir.nir, true); 3629 ir_binary = blob.data; 3630 ir_size = blob.size; 3631 3632 struct mesa_sha1 ctx; 3633 _mesa_sha1_init(&ctx); 3634 _mesa_sha1_update(&ctx, &variant->key, variant->shader->variant_key_size); 3635 _mesa_sha1_update(&ctx, ir_binary, ir_size); 3636 _mesa_sha1_final(&ctx, ir_sha1_cache_key); 3637 3638 blob_finish(&blob); 3639} 3640 3641 3642/** 3643 * Generate a new fragment shader variant from the shader code and 3644 * other state indicated by the key. 3645 */ 3646static struct lp_fragment_shader_variant * 3647generate_variant(struct llvmpipe_context *lp, 3648 struct lp_fragment_shader *shader, 3649 const struct lp_fragment_shader_variant_key *key) 3650{ 3651 struct lp_fragment_shader_variant *variant = 3652 MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key); 3653 if (!variant) 3654 return NULL; 3655 3656 memset(variant, 0, sizeof(*variant)); 3657 3658 pipe_reference_init(&variant->reference, 1); 3659 lp_fs_reference(lp, &variant->shader, shader); 3660 3661 memcpy(&variant->key, key, shader->variant_key_size); 3662 3663 struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen); 3664 struct lp_cached_code cached = { 0 }; 3665 unsigned char ir_sha1_cache_key[20]; 3666 bool needs_caching = false; 3667 if (shader->base.ir.nir) { 3668 lp_fs_get_ir_cache_key(variant, ir_sha1_cache_key); 3669 3670 lp_disk_cache_find_shader(screen, &cached, ir_sha1_cache_key); 3671 if (!cached.data_size) 3672 needs_caching = true; 3673 } 3674 3675 char module_name[64]; 3676 snprintf(module_name, sizeof(module_name), "fs%u_variant%u", 3677 shader->no, shader->variants_created); 3678 variant->gallivm = gallivm_create(module_name, lp->context, &cached); 3679 if (!variant->gallivm) { 3680 FREE(variant); 3681 return NULL; 3682 } 3683 3684 variant->list_item_global.base = variant; 3685 variant->list_item_local.base = variant; 3686 variant->no = shader->variants_created++; 3687 3688 /* 3689 * Determine whether we are touching all channels in the color buffer. 3690 */ 3691 const struct util_format_description *cbuf0_format_desc = NULL; 3692 boolean fullcolormask = FALSE; 3693 if (key->nr_cbufs == 1) { 3694 cbuf0_format_desc = util_format_description(key->cbuf_format[0]); 3695 fullcolormask = util_format_colormask_full(cbuf0_format_desc, 3696 key->blend.rt[0].colormask); 3697 } 3698 3699 /* The scissor is ignored here as only tiles inside the scissoring 3700 * rectangle will refer to this */ 3701 const boolean no_kill = 3702 fullcolormask && 3703 !key->stencil[0].enabled && 3704 !key->alpha.enabled && 3705 !key->multisample && 3706 !key->blend.alpha_to_coverage && 3707 !key->depth.enabled && 3708 !shader->info.base.uses_kill && 3709 !shader->info.base.writes_samplemask && 3710 !shader->info.base.uses_fbfetch; 3711 3712 variant->opaque = 3713 no_kill && 3714 !key->blend.logicop_enable && 3715 !key->blend.rt[0].blend_enable 3716 ? TRUE : FALSE; 3717 3718 variant->potentially_opaque = 3719 no_kill && 3720 !key->blend.logicop_enable && 3721 key->blend.rt[0].blend_enable && 3722 key->blend.rt[0].rgb_func == PIPE_BLEND_ADD && 3723 key->blend.rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_INV_SRC_ALPHA && 3724 key->blend.rt[0].alpha_func == key->blend.rt[0].rgb_func && 3725 key->blend.rt[0].alpha_dst_factor == key->blend.rt[0].rgb_dst_factor && 3726 shader->base.type == PIPE_SHADER_IR_TGSI && 3727 /* 3728 * FIXME: for NIR, all of the fields of info.xxx (except info.base) 3729 * are zeros, hence shader analysis (here and elsewhere) using these 3730 * bits cannot work and will silently fail (cbuf is the only pointer 3731 * field, hence causing a crash). 3732 */ 3733 shader->info.cbuf[0][3].file != TGSI_FILE_NULL 3734 ? TRUE : FALSE; 3735 3736 /* We only care about opaque blits for now */ 3737 if (variant->opaque && 3738 (shader->kind == LP_FS_KIND_BLIT_RGBA || 3739 shader->kind == LP_FS_KIND_BLIT_RGB1)) { 3740 const struct lp_sampler_static_state *samp0 = 3741 lp_fs_variant_key_sampler_idx(key, 0); 3742 assert(samp0); 3743 3744 const enum pipe_format texture_format = samp0->texture_state.format; 3745 const enum pipe_texture_target target = samp0->texture_state.target; 3746 const unsigned min_img_filter = samp0->sampler_state.min_img_filter; 3747 const unsigned mag_img_filter = samp0->sampler_state.mag_img_filter; 3748 3749 unsigned min_mip_filter; 3750 if (samp0->texture_state.level_zero_only) { 3751 min_mip_filter = PIPE_TEX_MIPFILTER_NONE; 3752 } else { 3753 min_mip_filter = samp0->sampler_state.min_mip_filter; 3754 } 3755 3756 if (target == PIPE_TEXTURE_2D && 3757 min_img_filter == PIPE_TEX_FILTER_NEAREST && 3758 mag_img_filter == PIPE_TEX_FILTER_NEAREST && 3759 min_mip_filter == PIPE_TEX_MIPFILTER_NONE && 3760 ((texture_format && 3761 util_is_format_compatible(util_format_description(texture_format), 3762 cbuf0_format_desc)) || 3763 (shader->kind == LP_FS_KIND_BLIT_RGB1 && 3764 (texture_format == PIPE_FORMAT_B8G8R8A8_UNORM || 3765 texture_format == PIPE_FORMAT_B8G8R8X8_UNORM) && 3766 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM || 3767 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM)))) { 3768 variant->blit = 1; 3769 } 3770 } 3771 3772 /* Determine whether this shader + pipeline state is a candidate for 3773 * the linear path. 3774 */ 3775 const boolean linear_pipeline = 3776 !key->stencil[0].enabled && 3777 !key->depth.enabled && 3778 !shader->info.base.uses_kill && 3779 !key->blend.logicop_enable && 3780 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM || 3781 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM); 3782 3783 memcpy(&variant->key, key, sizeof *key); 3784 3785 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) { 3786 lp_debug_fs_variant(variant); 3787 } 3788 3789 llvmpipe_fs_variant_fastpath(variant); 3790 3791 lp_jit_init_types(variant); 3792 3793 if (variant->jit_function[RAST_EDGE_TEST] == NULL) 3794 generate_fragment(lp, shader, variant, RAST_EDGE_TEST); 3795 3796 if (variant->jit_function[RAST_WHOLE] == NULL) { 3797 if (variant->opaque) { 3798 /* Specialized shader, which doesn't need to read the color buffer. */ 3799 generate_fragment(lp, shader, variant, RAST_WHOLE); 3800 } 3801 } 3802 3803 if (linear_pipeline) { 3804 /* Currently keeping both the old fastpaths and new linear path 3805 * active. The older code is still somewhat faster for the cases 3806 * it covers. 3807 * 3808 * XXX: consider restricting this to aero-mode only. 3809 */ 3810 if (fullcolormask && 3811 !key->alpha.enabled && 3812 !key->blend.alpha_to_coverage) { 3813 llvmpipe_fs_variant_linear_fastpath(variant); 3814 } 3815 3816 /* If the original fastpath doesn't cover this variant, try the new 3817 * code: 3818 */ 3819 if (variant->jit_linear == NULL) { 3820 if (shader->kind == LP_FS_KIND_BLIT_RGBA || 3821 shader->kind == LP_FS_KIND_BLIT_RGB1 || 3822 shader->kind == LP_FS_KIND_LLVM_LINEAR) { 3823 llvmpipe_fs_variant_linear_llvm(lp, shader, variant); 3824 } 3825 } 3826 } else { 3827 if (LP_DEBUG & DEBUG_LINEAR) { 3828 lp_debug_fs_variant(variant); 3829 debug_printf(" ----> no linear path for this variant\n"); 3830 } 3831 } 3832 3833 /* 3834 * Compile everything 3835 */ 3836 3837 gallivm_compile_module(variant->gallivm); 3838 3839 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); 3840 3841 if (variant->function[RAST_EDGE_TEST]) { 3842 variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func) 3843 gallivm_jit_function(variant->gallivm, 3844 variant->function[RAST_EDGE_TEST]); 3845 } 3846 3847 if (variant->function[RAST_WHOLE]) { 3848 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) 3849 gallivm_jit_function(variant->gallivm, 3850 variant->function[RAST_WHOLE]); 3851 } else if (!variant->jit_function[RAST_WHOLE]) { 3852 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) 3853 variant->jit_function[RAST_EDGE_TEST]; 3854 } 3855 3856 if (linear_pipeline) { 3857 if (variant->linear_function) { 3858 variant->jit_linear_llvm = (lp_jit_linear_llvm_func) 3859 gallivm_jit_function(variant->gallivm, variant->linear_function); 3860 } 3861 3862 /* 3863 * This must be done after LLVM compilation, as it will call the JIT'ed 3864 * code to determine active inputs. 3865 */ 3866 lp_linear_check_variant(variant); 3867 } 3868 3869 if (needs_caching) { 3870 lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key); 3871 } 3872 3873 gallivm_free_ir(variant->gallivm); 3874 3875 return variant; 3876} 3877 3878 3879static void * 3880llvmpipe_create_fs_state(struct pipe_context *pipe, 3881 const struct pipe_shader_state *templ) 3882{ 3883 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 3884 3885 struct lp_fragment_shader *shader = CALLOC_STRUCT(lp_fragment_shader); 3886 if (!shader) 3887 return NULL; 3888 3889 pipe_reference_init(&shader->reference, 1); 3890 shader->no = fs_no++; 3891 list_inithead(&shader->variants.list); 3892 3893 shader->base.type = templ->type; 3894 if (templ->type == PIPE_SHADER_IR_TGSI) { 3895 /* get/save the summary info for this shader */ 3896 lp_build_tgsi_info(templ->tokens, &shader->info); 3897 3898 /* we need to keep a local copy of the tokens */ 3899 shader->base.tokens = tgsi_dup_tokens(templ->tokens); 3900 } else { 3901 shader->base.ir.nir = templ->ir.nir; 3902 nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true); 3903 } 3904 3905 shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ); 3906 if (shader->draw_data == NULL) { 3907 FREE((void *) shader->base.tokens); 3908 FREE(shader); 3909 return NULL; 3910 } 3911 3912 const int nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1; 3913 const int nr_sampler_views = 3914 shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 3915 const int nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1; 3916 3917 shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers, 3918 nr_sampler_views), 3919 nr_images); 3920 3921 for (int i = 0; i < shader->info.base.num_inputs; i++) { 3922 shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i]; 3923 shader->inputs[i].location = shader->info.base.input_interpolate_loc[i]; 3924 3925 switch (shader->info.base.input_interpolate[i]) { 3926 case TGSI_INTERPOLATE_CONSTANT: 3927 shader->inputs[i].interp = LP_INTERP_CONSTANT; 3928 break; 3929 case TGSI_INTERPOLATE_LINEAR: 3930 shader->inputs[i].interp = LP_INTERP_LINEAR; 3931 break; 3932 case TGSI_INTERPOLATE_PERSPECTIVE: 3933 shader->inputs[i].interp = LP_INTERP_PERSPECTIVE; 3934 break; 3935 case TGSI_INTERPOLATE_COLOR: 3936 shader->inputs[i].interp = LP_INTERP_COLOR; 3937 break; 3938 default: 3939 assert(0); 3940 break; 3941 } 3942 3943 switch (shader->info.base.input_semantic_name[i]) { 3944 case TGSI_SEMANTIC_FACE: 3945 shader->inputs[i].interp = LP_INTERP_FACING; 3946 break; 3947 case TGSI_SEMANTIC_POSITION: 3948 /* Position was already emitted above 3949 */ 3950 shader->inputs[i].interp = LP_INTERP_POSITION; 3951 shader->inputs[i].src_index = 0; 3952 continue; 3953 } 3954 3955 /* XXX this is a completely pointless index map... */ 3956 shader->inputs[i].src_index = i+1; 3957 } 3958 3959 if (LP_DEBUG & DEBUG_TGSI && templ->type == PIPE_SHADER_IR_TGSI) { 3960 debug_printf("llvmpipe: Create fragment shader #%u %p:\n", 3961 shader->no, (void *) shader); 3962 tgsi_dump(templ->tokens, 0); 3963 debug_printf("usage masks:\n"); 3964 for (unsigned attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) { 3965 unsigned usage_mask = shader->info.base.input_usage_mask[attrib]; 3966 debug_printf(" IN[%u].%s%s%s%s\n", 3967 attrib, 3968 usage_mask & TGSI_WRITEMASK_X ? "x" : "", 3969 usage_mask & TGSI_WRITEMASK_Y ? "y" : "", 3970 usage_mask & TGSI_WRITEMASK_Z ? "z" : "", 3971 usage_mask & TGSI_WRITEMASK_W ? "w" : ""); 3972 } 3973 debug_printf("\n"); 3974 } 3975 3976 /* This will put a derived copy of the tokens into shader->base.tokens */ 3977 if (templ->type == PIPE_SHADER_IR_TGSI) 3978 llvmpipe_fs_analyse(shader, templ->tokens); 3979 else 3980 llvmpipe_fs_analyse_nir(shader); 3981 3982 return shader; 3983} 3984 3985 3986static void 3987llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs) 3988{ 3989 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 3990 struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs; 3991 if (llvmpipe->fs == lp_fs) 3992 return; 3993 3994 draw_bind_fragment_shader(llvmpipe->draw, 3995 (lp_fs ? lp_fs->draw_data : NULL)); 3996 3997 lp_fs_reference(llvmpipe, &llvmpipe->fs, lp_fs); 3998 3999 /* invalidate the setup link, NEW_FS will make it update */ 4000 lp_setup_set_fs_variant(llvmpipe->setup, NULL); 4001 llvmpipe->dirty |= LP_NEW_FS; 4002} 4003 4004 4005/** 4006 * Remove shader variant from two lists: the shader's variant list 4007 * and the context's variant list. 4008 */ 4009static void 4010llvmpipe_remove_shader_variant(struct llvmpipe_context *lp, 4011 struct lp_fragment_shader_variant *variant) 4012{ 4013 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) { 4014 debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u " 4015 "v total cached %u inst %u total inst %u\n", 4016 variant->shader->no, variant->no, 4017 variant->shader->variants_created, 4018 variant->shader->variants_cached, 4019 lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs); 4020 } 4021 4022 /* remove from shader's list */ 4023 list_del(&variant->list_item_local.list); 4024 variant->shader->variants_cached--; 4025 4026 /* remove from context's list */ 4027 list_del(&variant->list_item_global.list); 4028 lp->nr_fs_variants--; 4029 lp->nr_fs_instrs -= variant->nr_instrs; 4030} 4031 4032 4033void 4034llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp, 4035 struct lp_fragment_shader_variant *variant) 4036{ 4037 gallivm_destroy(variant->gallivm); 4038 lp_fs_reference(lp, &variant->shader, NULL); 4039 FREE(variant); 4040} 4041 4042 4043void 4044llvmpipe_destroy_fs(struct llvmpipe_context *llvmpipe, 4045 struct lp_fragment_shader *shader) 4046{ 4047 /* Delete draw module's data */ 4048 draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data); 4049 4050 if (shader->base.ir.nir) 4051 ralloc_free(shader->base.ir.nir); 4052 assert(shader->variants_cached == 0); 4053 FREE((void *) shader->base.tokens); 4054 FREE(shader); 4055} 4056 4057 4058static void 4059llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs) 4060{ 4061 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4062 struct lp_fragment_shader *shader = fs; 4063 struct lp_fs_variant_list_item *li, *next; 4064 4065 /* Delete all the variants */ 4066 LIST_FOR_EACH_ENTRY_SAFE(li, next, &shader->variants.list, list) { 4067 struct lp_fragment_shader_variant *variant; 4068 variant = li->base; 4069 llvmpipe_remove_shader_variant(llvmpipe, li->base); 4070 lp_fs_variant_reference(llvmpipe, &variant, NULL); 4071 } 4072 4073 lp_fs_reference(llvmpipe, &shader, NULL); 4074} 4075 4076 4077static void 4078llvmpipe_set_constant_buffer(struct pipe_context *pipe, 4079 enum pipe_shader_type shader, uint index, 4080 bool take_ownership, 4081 const struct pipe_constant_buffer *cb) 4082{ 4083 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4084 struct pipe_constant_buffer *constants = &llvmpipe->constants[shader][index]; 4085 4086 assert(shader < PIPE_SHADER_TYPES); 4087 assert(index < ARRAY_SIZE(llvmpipe->constants[shader])); 4088 4089 /* note: reference counting */ 4090 util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb, 4091 take_ownership); 4092 4093 /* user_buffer is only valid until the next set_constant_buffer (at most, 4094 * possibly until shader deletion), so we need to upload it now to make sure 4095 * it doesn't get updated/freed out from under us. 4096 */ 4097 if (constants->user_buffer) { 4098 u_upload_data(llvmpipe->pipe.const_uploader, 0, constants->buffer_size, 4099 16, constants->user_buffer, &constants->buffer_offset, 4100 &constants->buffer); 4101 } 4102 if (constants->buffer) { 4103 if (!(constants->buffer->bind & PIPE_BIND_CONSTANT_BUFFER)) { 4104 debug_printf("Illegal set constant without bind flag\n"); 4105 constants->buffer->bind |= PIPE_BIND_CONSTANT_BUFFER; 4106 } 4107 } 4108 4109 if (shader == PIPE_SHADER_VERTEX || 4110 shader == PIPE_SHADER_GEOMETRY || 4111 shader == PIPE_SHADER_TESS_CTRL || 4112 shader == PIPE_SHADER_TESS_EVAL) { 4113 /* Pass the constants to the 'draw' module */ 4114 const unsigned size = cb ? cb->buffer_size : 0; 4115 4116 const ubyte *data = NULL; 4117 if (constants->buffer) { 4118 data = (ubyte *) llvmpipe_resource_data(constants->buffer) 4119 + constants->buffer_offset; 4120 } 4121 4122 draw_set_mapped_constant_buffer(llvmpipe->draw, shader, 4123 index, data, size); 4124 } else if (shader == PIPE_SHADER_COMPUTE) { 4125 llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS; 4126 } else { 4127 llvmpipe->dirty |= LP_NEW_FS_CONSTANTS; 4128 } 4129} 4130 4131 4132static void 4133llvmpipe_set_shader_buffers(struct pipe_context *pipe, 4134 enum pipe_shader_type shader, unsigned start_slot, 4135 unsigned count, 4136 const struct pipe_shader_buffer *buffers, 4137 unsigned writable_bitmask) 4138{ 4139 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4140 4141 unsigned i, idx; 4142 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) { 4143 const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL; 4144 4145 util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer); 4146 4147 if (buffer && buffer->buffer) { 4148 boolean read_only = !(writable_bitmask & (1 << idx)); 4149 llvmpipe_flush_resource(pipe, buffer->buffer, 0, read_only, false, 4150 false, "buffer"); 4151 } 4152 4153 if (shader == PIPE_SHADER_VERTEX || 4154 shader == PIPE_SHADER_GEOMETRY || 4155 shader == PIPE_SHADER_TESS_CTRL || 4156 shader == PIPE_SHADER_TESS_EVAL) { 4157 const unsigned size = buffer ? buffer->buffer_size : 0; 4158 const ubyte *data = NULL; 4159 if (buffer && buffer->buffer) 4160 data = (ubyte *) llvmpipe_resource_data(buffer->buffer); 4161 if (data) 4162 data += buffer->buffer_offset; 4163 draw_set_mapped_shader_buffer(llvmpipe->draw, shader, 4164 i, data, size); 4165 } else if (shader == PIPE_SHADER_COMPUTE) { 4166 llvmpipe->cs_dirty |= LP_CSNEW_SSBOS; 4167 } else if (shader == PIPE_SHADER_FRAGMENT) { 4168 llvmpipe->fs_ssbo_write_mask &= ~(((1 << count) - 1) << start_slot); 4169 llvmpipe->fs_ssbo_write_mask |= writable_bitmask << start_slot; 4170 llvmpipe->dirty |= LP_NEW_FS_SSBOS; 4171 } 4172 } 4173} 4174 4175 4176static void 4177llvmpipe_set_shader_images(struct pipe_context *pipe, 4178 enum pipe_shader_type shader, unsigned start_slot, 4179 unsigned count, unsigned unbind_num_trailing_slots, 4180 const struct pipe_image_view *images) 4181{ 4182 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4183 unsigned i, idx; 4184 4185 draw_flush(llvmpipe->draw); 4186 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) { 4187 const struct pipe_image_view *image = images ? &images[idx] : NULL; 4188 4189 util_copy_image_view(&llvmpipe->images[shader][i], image); 4190 4191 if (image && image->resource) { 4192 bool read_only = !(image->access & PIPE_IMAGE_ACCESS_WRITE); 4193 llvmpipe_flush_resource(pipe, image->resource, 0, read_only, false, 4194 false, "image"); 4195 } 4196 } 4197 4198 llvmpipe->num_images[shader] = start_slot + count; 4199 if (shader == PIPE_SHADER_VERTEX || 4200 shader == PIPE_SHADER_GEOMETRY || 4201 shader == PIPE_SHADER_TESS_CTRL || 4202 shader == PIPE_SHADER_TESS_EVAL) { 4203 draw_set_images(llvmpipe->draw, 4204 shader, 4205 llvmpipe->images[shader], 4206 start_slot + count); 4207 } else if (shader == PIPE_SHADER_COMPUTE) { 4208 llvmpipe->cs_dirty |= LP_CSNEW_IMAGES; 4209 } else { 4210 llvmpipe->dirty |= LP_NEW_FS_IMAGES; 4211 } 4212 4213 if (unbind_num_trailing_slots) { 4214 llvmpipe_set_shader_images(pipe, shader, start_slot + count, 4215 unbind_num_trailing_slots, 0, NULL); 4216 } 4217} 4218 4219 4220/** 4221 * Return the blend factor equivalent to a destination alpha of one. 4222 */ 4223static inline enum pipe_blendfactor 4224force_dst_alpha_one(enum pipe_blendfactor factor, boolean clamped_zero) 4225{ 4226 switch (factor) { 4227 case PIPE_BLENDFACTOR_DST_ALPHA: 4228 return PIPE_BLENDFACTOR_ONE; 4229 case PIPE_BLENDFACTOR_INV_DST_ALPHA: 4230 return PIPE_BLENDFACTOR_ZERO; 4231 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE: 4232 if (clamped_zero) 4233 return PIPE_BLENDFACTOR_ZERO; 4234 else 4235 return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE; 4236 default: 4237 return factor; 4238 } 4239} 4240 4241 4242/** 4243 * We need to generate several variants of the fragment pipeline to match 4244 * all the combinations of the contributing state atoms. 4245 * 4246 * TODO: there is actually no reason to tie this to context state -- the 4247 * generated code could be cached globally in the screen. 4248 */ 4249static struct lp_fragment_shader_variant_key * 4250make_variant_key(struct llvmpipe_context *lp, 4251 struct lp_fragment_shader *shader, 4252 char *store) 4253{ 4254 struct lp_fragment_shader_variant_key *key = 4255 (struct lp_fragment_shader_variant_key *)store; 4256 4257 memset(key, 0, sizeof(*key)); 4258 4259 if (lp->framebuffer.zsbuf) { 4260 const enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format; 4261 const struct util_format_description *zsbuf_desc = 4262 util_format_description(zsbuf_format); 4263 4264 if (lp->depth_stencil->depth_enabled && 4265 util_format_has_depth(zsbuf_desc)) { 4266 key->zsbuf_format = zsbuf_format; 4267 key->depth.enabled = lp->depth_stencil->depth_enabled; 4268 key->depth.writemask = lp->depth_stencil->depth_writemask; 4269 key->depth.func = lp->depth_stencil->depth_func; 4270 } 4271 if (lp->depth_stencil->stencil[0].enabled && 4272 util_format_has_stencil(zsbuf_desc)) { 4273 key->zsbuf_format = zsbuf_format; 4274 memcpy(&key->stencil, &lp->depth_stencil->stencil, 4275 sizeof key->stencil); 4276 } 4277 if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) { 4278 key->resource_1d = TRUE; 4279 } 4280 key->zsbuf_nr_samples = 4281 util_res_sample_count(lp->framebuffer.zsbuf->texture); 4282 4283 /* 4284 * Restrict depth values if the API is clamped (GL, VK with ext) 4285 * for non float Z buffer 4286 */ 4287 key->restrict_depth_values = 4288 !(lp->rasterizer->unclamped_fragment_depth_values && 4289 util_format_get_depth_only(zsbuf_format) == PIPE_FORMAT_Z32_FLOAT); 4290 } 4291 4292 /* 4293 * Propagate the depth clamp setting from the rasterizer state. 4294 */ 4295 key->depth_clamp = lp->rasterizer->depth_clamp; 4296 4297 /* alpha test only applies if render buffer 0 is non-integer 4298 * (or does not exist) 4299 */ 4300 if (!lp->framebuffer.nr_cbufs || 4301 !lp->framebuffer.cbufs[0] || 4302 !util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) { 4303 key->alpha.enabled = lp->depth_stencil->alpha_enabled; 4304 } 4305 if (key->alpha.enabled) { 4306 key->alpha.func = lp->depth_stencil->alpha_func; 4307 /* alpha.ref_value is passed in jit_context */ 4308 } 4309 4310 key->flatshade = lp->rasterizer->flatshade; 4311 key->multisample = lp->rasterizer->multisample; 4312 key->no_ms_sample_mask_out = lp->rasterizer->no_ms_sample_mask_out; 4313 if (lp->active_occlusion_queries && !lp->queries_disabled) { 4314 key->occlusion_count = TRUE; 4315 } 4316 4317 memcpy(&key->blend, lp->blend, sizeof key->blend); 4318 4319 key->coverage_samples = 1; 4320 key->min_samples = 1; 4321 if (key->multisample) { 4322 key->coverage_samples = 4323 util_framebuffer_get_num_samples(&lp->framebuffer); 4324 /* Per EXT_shader_framebuffer_fetch spec: 4325 * 4326 * "1. How is framebuffer data treated during multisample rendering? 4327 * 4328 * RESOLVED: Reading the value of gl_LastFragData produces a different 4329 * result for each sample. This implies that all or part of the shader be 4330 * run once for each sample, but has no additional implications on fragment 4331 * shader input variables which may still be interpolated per pixel by the 4332 * implementation." 4333 * 4334 * ARM_shader_framebuffer_fetch_depth_stencil spec further says: 4335 * 4336 * "(1) When multisampling is enabled, does the shader run per sample? 4337 * 4338 * RESOLVED. 4339 * 4340 * This behavior is inherited from either EXT_shader_framebuffer_fetch or 4341 * ARM_shader_framebuffer_fetch as described in the interactions section. 4342 * If neither extension is supported, the shader runs once per fragment." 4343 * 4344 * Therefore we should always enable per-sample shading when FB fetch is used. 4345 */ 4346 if (lp->min_samples > 1 || shader->info.base.uses_fbfetch) 4347 key->min_samples = key->coverage_samples; 4348 } 4349 key->nr_cbufs = lp->framebuffer.nr_cbufs; 4350 4351 if (!key->blend.independent_blend_enable) { 4352 // we always need independent blend otherwise the fixups below won't work 4353 for (unsigned i = 1; i < key->nr_cbufs; i++) { 4354 memcpy(&key->blend.rt[i], &key->blend.rt[0], 4355 sizeof(key->blend.rt[0])); 4356 } 4357 key->blend.independent_blend_enable = 1; 4358 } 4359 4360 for (unsigned i = 0; i < lp->framebuffer.nr_cbufs; i++) { 4361 struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i]; 4362 4363 if (lp->framebuffer.cbufs[i]) { 4364 const enum pipe_format format = lp->framebuffer.cbufs[i]->format; 4365 4366 key->cbuf_format[i] = format; 4367 key->cbuf_nr_samples[i] = 4368 util_res_sample_count(lp->framebuffer.cbufs[i]->texture); 4369 4370 /* 4371 * Figure out if this is a 1d resource. Note that OpenGL allows crazy 4372 * mixing of 2d textures with height 1 and 1d textures, so make sure 4373 * we pick 1d if any cbuf or zsbuf is 1d. 4374 */ 4375 if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) { 4376 key->resource_1d = TRUE; 4377 } 4378 4379 const struct util_format_description *format_desc = 4380 util_format_description(format); 4381 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB || 4382 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB); 4383 4384 /* 4385 * Mask out color channels not present in the color buffer. 4386 */ 4387 blend_rt->colormask &= util_format_colormask(format_desc); 4388 4389 /* 4390 * Disable blend for integer formats. 4391 */ 4392 if (util_format_is_pure_integer(format)) { 4393 blend_rt->blend_enable = 0; 4394 } 4395 4396 /* 4397 * Our swizzled render tiles always have an alpha channel, but the 4398 * linear render target format often does not, so force here the dst 4399 * alpha to be one. 4400 * 4401 * This is not a mere optimization. Wrong results will be produced if 4402 * the dst alpha is used, the dst format does not have alpha, and the 4403 * previous rendering was not flushed from the swizzled to linear 4404 * buffer. For example, NonPowTwo DCT. 4405 * 4406 * TODO: This should be generalized to all channels for better 4407 * performance, but only alpha causes correctness issues. 4408 * 4409 * Also, force rgb/alpha func/factors match, to make AoS blending 4410 * easier. 4411 */ 4412 if (format_desc->swizzle[3] > PIPE_SWIZZLE_W || 4413 format_desc->swizzle[3] == format_desc->swizzle[0]) { 4414 // Doesn't cover mixed snorm/unorm but can't render to them anyway 4415 boolean clamped_zero = !util_format_is_float(format) && 4416 !util_format_is_snorm(format); 4417 blend_rt->rgb_src_factor = 4418 force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero); 4419 blend_rt->rgb_dst_factor = 4420 force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero); 4421 blend_rt->alpha_func = blend_rt->rgb_func; 4422 blend_rt->alpha_src_factor = blend_rt->rgb_src_factor; 4423 blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor; 4424 } 4425 } 4426 else { 4427 /* no color buffer for this fragment output */ 4428 key->cbuf_format[i] = PIPE_FORMAT_NONE; 4429 key->cbuf_nr_samples[i] = 0; 4430 blend_rt->colormask = 0x0; 4431 blend_rt->blend_enable = 0; 4432 } 4433 } 4434 4435 /* This value will be the same for all the variants of a given shader: 4436 */ 4437 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1; 4438 4439 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 4440 key->nr_sampler_views = 4441 shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 4442 } 4443 4444 struct lp_sampler_static_state *fs_sampler = 4445 lp_fs_variant_key_samplers(key); 4446 4447 memset(fs_sampler, 0, 4448 MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler); 4449 4450 for (unsigned i = 0; i < key->nr_samplers; ++i) { 4451 if (shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) { 4452 lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state, 4453 lp->samplers[PIPE_SHADER_FRAGMENT][i]); 4454 } 4455 } 4456 4457 /* 4458 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes 4459 * are dx10-style? Can't really have mixed opcodes, at least not 4460 * if we want to skip the holes here (without rescanning tgsi). 4461 */ 4462 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 4463 for (unsigned i = 0; i < key->nr_sampler_views; ++i) { 4464 /* 4465 * Note sview may exceed what's representable by file_mask. 4466 * This will still work, the only downside is that not actually 4467 * used views may be included in the shader key. 4468 */ 4469 if ((shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] 4470 & (1u << (i & 31))) || i > 31) { 4471 lp_sampler_static_texture_state(&fs_sampler[i].texture_state, 4472 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]); 4473 } 4474 } 4475 } 4476 else { 4477 key->nr_sampler_views = key->nr_samplers; 4478 for (unsigned i = 0; i < key->nr_sampler_views; ++i) { 4479 if ((shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) || i > 31) { 4480 lp_sampler_static_texture_state(&fs_sampler[i].texture_state, 4481 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]); 4482 } 4483 } 4484 } 4485 4486 struct lp_image_static_state *lp_image = lp_fs_variant_key_images(key); 4487 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1; 4488 4489 if (key->nr_images) 4490 memset(lp_image, 0, 4491 key->nr_images * sizeof *lp_image); 4492 for (unsigned i = 0; i < key->nr_images; ++i) { 4493 if ((shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) || i > 31) { 4494 lp_sampler_static_texture_state_image(&lp_image[i].image_state, 4495 &lp->images[PIPE_SHADER_FRAGMENT][i]); 4496 } 4497 } 4498 4499 if (shader->kind == LP_FS_KIND_AERO_MINIFICATION) { 4500 struct lp_sampler_static_state *samp0 = 4501 lp_fs_variant_key_sampler_idx(key, 0); 4502 assert(samp0); 4503 samp0->sampler_state.min_img_filter = PIPE_TEX_FILTER_NEAREST; 4504 samp0->sampler_state.mag_img_filter = PIPE_TEX_FILTER_NEAREST; 4505 } 4506 4507 return key; 4508} 4509 4510 4511/** 4512 * Update fragment shader state. This is called just prior to drawing 4513 * something when some fragment-related state has changed. 4514 */ 4515void 4516llvmpipe_update_fs(struct llvmpipe_context *lp) 4517{ 4518 struct lp_fragment_shader *shader = lp->fs; 4519 4520 char store[LP_FS_MAX_VARIANT_KEY_SIZE]; 4521 const struct lp_fragment_shader_variant_key *key = 4522 make_variant_key(lp, shader, store); 4523 4524 struct lp_fragment_shader_variant *variant = NULL; 4525 struct lp_fs_variant_list_item *li; 4526 /* Search the variants for one which matches the key */ 4527 LIST_FOR_EACH_ENTRY(li, &shader->variants.list, list) { 4528 if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) { 4529 variant = li->base; 4530 break; 4531 } 4532 } 4533 4534 if (variant) { 4535 /* Move this variant to the head of the list to implement LRU 4536 * deletion of shader's when we have too many. 4537 */ 4538 list_move_to(&variant->list_item_global.list, &lp->fs_variants_list.list); 4539 } 4540 else { 4541 /* variant not found, create it now */ 4542 4543 if (LP_DEBUG & DEBUG_FS) { 4544 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n", 4545 lp->nr_fs_variants, 4546 lp->nr_fs_instrs, 4547 lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0); 4548 } 4549 4550 /* First, check if we've exceeded the max number of shader variants. 4551 * If so, free 6.25% of them (the least recently used ones). 4552 */ 4553 const unsigned variants_to_cull = 4554 lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS 4555 ? LP_MAX_SHADER_VARIANTS / 16 : 0; 4556 4557 if (variants_to_cull || 4558 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) { 4559 if (gallivm_debug & GALLIVM_DEBUG_PERF) { 4560 debug_printf("Evicting FS: %u fs variants,\t%u total variants," 4561 "\t%u instrs,\t%u instrs/variant\n", 4562 shader->variants_cached, 4563 lp->nr_fs_variants, lp->nr_fs_instrs, 4564 lp->nr_fs_instrs / lp->nr_fs_variants); 4565 } 4566 4567 /* 4568 * We need to re-check lp->nr_fs_variants because an arbitrarliy large 4569 * number of shader variants (potentially all of them) could be 4570 * pending for destruction on flush. 4571 */ 4572 4573 for (unsigned i = 0; 4574 i < variants_to_cull || 4575 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; 4576 i++) { 4577 struct lp_fs_variant_list_item *item; 4578 if (list_is_empty(&lp->fs_variants_list.list)) { 4579 break; 4580 } 4581 item = list_last_entry(&lp->fs_variants_list.list, 4582 struct lp_fs_variant_list_item, list); 4583 assert(item); 4584 assert(item->base); 4585 llvmpipe_remove_shader_variant(lp, item->base); 4586 struct lp_fragment_shader_variant *variant = item->base; 4587 lp_fs_variant_reference(lp, &variant, NULL); 4588 } 4589 } 4590 4591 /* 4592 * Generate the new variant. 4593 */ 4594 int64_t t0 = os_time_get(); 4595 variant = generate_variant(lp, shader, key); 4596 int64_t t1 = os_time_get(); 4597 int64_t dt = t1 - t0; 4598 LP_COUNT_ADD(llvm_compile_time, dt); 4599 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */ 4600 4601 /* Put the new variant into the list */ 4602 if (variant) { 4603 list_add(&variant->list_item_local.list, &shader->variants.list); 4604 list_add(&variant->list_item_global.list, &lp->fs_variants_list.list); 4605 lp->nr_fs_variants++; 4606 lp->nr_fs_instrs += variant->nr_instrs; 4607 shader->variants_cached++; 4608 } 4609 } 4610 4611 /* Bind this variant */ 4612 lp_setup_set_fs_variant(lp->setup, variant); 4613} 4614 4615 4616void 4617llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe) 4618{ 4619 llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state; 4620 llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state; 4621 llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state; 4622 llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer; 4623 llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers; 4624 llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images; 4625} 4626