1/************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28#include "draw_llvm.h" 29 30#include "draw_context.h" 31#include "draw_vs.h" 32#include "draw_gs.h" 33 34#include "gallivm/lp_bld_arit.h" 35#include "gallivm/lp_bld_arit_overflow.h" 36#include "gallivm/lp_bld_bitarit.h" 37#include "gallivm/lp_bld_gather.h" 38#include "gallivm/lp_bld_logic.h" 39#include "gallivm/lp_bld_const.h" 40#include "gallivm/lp_bld_coro.h" 41#include "gallivm/lp_bld_swizzle.h" 42#include "gallivm/lp_bld_struct.h" 43#include "gallivm/lp_bld_type.h" 44#include "gallivm/lp_bld_flow.h" 45#include "gallivm/lp_bld_debug.h" 46#include "gallivm/lp_bld_tgsi.h" 47#include "gallivm/lp_bld_nir.h" 48#include "gallivm/lp_bld_printf.h" 49#include "gallivm/lp_bld_intr.h" 50#include "gallivm/lp_bld_init.h" 51#include "gallivm/lp_bld_type.h" 52#include "gallivm/lp_bld_pack.h" 53#include "gallivm/lp_bld_format.h" 54#include "gallivm/lp_bld_misc.h" 55#include "tgsi/tgsi_exec.h" 56#include "tgsi/tgsi_dump.h" 57 58#include "util/u_math.h" 59#include "util/u_pointer.h" 60#include "util/u_string.h" 61#include "nir_serialize.h" 62#include "util/mesa-sha1.h" 63#define DEBUG_STORE 0 64 65 66static void 67draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var); 68 69 70struct draw_gs_llvm_iface { 71 struct lp_build_gs_iface base; 72 73 struct draw_gs_llvm_variant *variant; 74 LLVMValueRef input; 75}; 76 77static inline const struct draw_gs_llvm_iface * 78draw_gs_llvm_iface(const struct lp_build_gs_iface *iface) 79{ 80 return (const struct draw_gs_llvm_iface *)iface; 81} 82 83struct draw_tcs_llvm_iface { 84 struct lp_build_tcs_iface base; 85 86 struct draw_tcs_llvm_variant *variant; 87 LLVMValueRef input; 88 LLVMValueRef output; 89}; 90 91static inline const struct draw_tcs_llvm_iface * 92draw_tcs_llvm_iface(const struct lp_build_tcs_iface *iface) 93{ 94 return (const struct draw_tcs_llvm_iface *)iface; 95} 96 97struct draw_tes_llvm_iface { 98 struct lp_build_tes_iface base; 99 100 struct draw_tes_llvm_variant *variant; 101 LLVMValueRef input; 102}; 103 104static inline const struct draw_tes_llvm_iface * 105draw_tes_llvm_iface(const struct lp_build_tes_iface *iface) 106{ 107 return (const struct draw_tes_llvm_iface *)iface; 108} 109 110/** 111 * Create LLVM type for draw_vertex_buffer. 112 */ 113static LLVMTypeRef 114create_jit_dvbuffer_type(struct gallivm_state *gallivm, 115 const char *struct_name) 116{ 117 LLVMTargetDataRef target = gallivm->target; 118 LLVMTypeRef dvbuffer_type; 119 LLVMTypeRef elem_types[DRAW_JIT_DVBUFFER_NUM_FIELDS]; 120 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 121 122 elem_types[DRAW_JIT_DVBUFFER_MAP] = 123 LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 124 elem_types[DRAW_JIT_DVBUFFER_SIZE] = int32_type; 125 126 dvbuffer_type = LLVMStructTypeInContext(gallivm->context, elem_types, 127 ARRAY_SIZE(elem_types), 0); 128 129 (void) target; /* silence unused var warning for non-debug build */ 130 LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, map, 131 target, dvbuffer_type, 132 DRAW_JIT_DVBUFFER_MAP); 133 LP_CHECK_MEMBER_OFFSET(struct draw_vertex_buffer, size, 134 target, dvbuffer_type, 135 DRAW_JIT_DVBUFFER_SIZE); 136 137 return dvbuffer_type; 138} 139 140/** 141 * Create LLVM type for struct draw_jit_texture 142 */ 143static LLVMTypeRef 144create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name) 145{ 146 LLVMTargetDataRef target = gallivm->target; 147 LLVMTypeRef texture_type; 148 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 149 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 150 151 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 152 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 153 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 154 elem_types[DRAW_JIT_TEXTURE_NUM_SAMPLES] = 155 elem_types[DRAW_JIT_TEXTURE_SAMPLE_STRIDE] = 156 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] = 157 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 158 elem_types[DRAW_JIT_TEXTURE_BASE] = 159 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); 160 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 161 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 162 elem_types[DRAW_JIT_TEXTURE_MIP_OFFSETS] = 163 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 164 165 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 166 ARRAY_SIZE(elem_types), 0); 167 168 (void) target; /* silence unused var warning for non-debug build */ 169 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 170 target, texture_type, 171 DRAW_JIT_TEXTURE_WIDTH); 172 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 173 target, texture_type, 174 DRAW_JIT_TEXTURE_HEIGHT); 175 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 176 target, texture_type, 177 DRAW_JIT_TEXTURE_DEPTH); 178 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, base, 179 target, texture_type, 180 DRAW_JIT_TEXTURE_BASE); 181 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 182 target, texture_type, 183 DRAW_JIT_TEXTURE_ROW_STRIDE); 184 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 185 target, texture_type, 186 DRAW_JIT_TEXTURE_IMG_STRIDE); 187 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level, 188 target, texture_type, 189 DRAW_JIT_TEXTURE_FIRST_LEVEL); 190 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 191 target, texture_type, 192 DRAW_JIT_TEXTURE_LAST_LEVEL); 193 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, mip_offsets, 194 target, texture_type, 195 DRAW_JIT_TEXTURE_MIP_OFFSETS); 196 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, num_samples, 197 target, texture_type, 198 DRAW_JIT_TEXTURE_NUM_SAMPLES); 199 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, sample_stride, 200 target, texture_type, 201 DRAW_JIT_TEXTURE_SAMPLE_STRIDE); 202 203 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 204 205 return texture_type; 206} 207 208 209/** 210 * Create LLVM type for struct draw_jit_sampler 211 */ 212static LLVMTypeRef 213create_jit_sampler_type(struct gallivm_state *gallivm, const char *struct_name) 214{ 215 LLVMTargetDataRef target = gallivm->target; 216 LLVMTypeRef sampler_type; 217 LLVMTypeRef elem_types[DRAW_JIT_SAMPLER_NUM_FIELDS]; 218 219 elem_types[DRAW_JIT_SAMPLER_MIN_LOD] = 220 elem_types[DRAW_JIT_SAMPLER_MAX_LOD] = 221 elem_types[DRAW_JIT_SAMPLER_LOD_BIAS] = 222 elem_types[DRAW_JIT_SAMPLER_MAX_ANISO] = LLVMFloatTypeInContext(gallivm->context); 223 elem_types[DRAW_JIT_SAMPLER_BORDER_COLOR] = 224 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 225 226 sampler_type = LLVMStructTypeInContext(gallivm->context, elem_types, 227 ARRAY_SIZE(elem_types), 0); 228 229 (void) target; /* silence unused var warning for non-debug build */ 230 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, min_lod, 231 target, sampler_type, 232 DRAW_JIT_SAMPLER_MIN_LOD); 233 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_lod, 234 target, sampler_type, 235 DRAW_JIT_SAMPLER_MAX_LOD); 236 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, lod_bias, 237 target, sampler_type, 238 DRAW_JIT_SAMPLER_LOD_BIAS); 239 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, border_color, 240 target, sampler_type, 241 DRAW_JIT_SAMPLER_BORDER_COLOR); 242 LP_CHECK_MEMBER_OFFSET(struct draw_jit_sampler, max_aniso, 243 target, sampler_type, 244 DRAW_JIT_SAMPLER_MAX_ANISO); 245 246 LP_CHECK_STRUCT_SIZE(struct draw_jit_sampler, target, sampler_type); 247 248 return sampler_type; 249} 250 251/** 252 * Create LLVM type for struct draw_jit_texture 253 */ 254static LLVMTypeRef 255create_jit_image_type(struct gallivm_state *gallivm, const char *struct_name) 256{ 257 LLVMTargetDataRef target = gallivm->target; 258 LLVMTypeRef image_type; 259 LLVMTypeRef elem_types[DRAW_JIT_IMAGE_NUM_FIELDS]; 260 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 261 262 elem_types[DRAW_JIT_IMAGE_WIDTH] = 263 elem_types[DRAW_JIT_IMAGE_HEIGHT] = 264 elem_types[DRAW_JIT_IMAGE_DEPTH] = 265 elem_types[DRAW_JIT_IMAGE_ROW_STRIDE] = 266 elem_types[DRAW_JIT_IMAGE_IMG_STRIDE] = 267 elem_types[DRAW_JIT_IMAGE_NUM_SAMPLES] = 268 elem_types[DRAW_JIT_IMAGE_SAMPLE_STRIDE] = int32_type; 269 elem_types[DRAW_JIT_IMAGE_BASE] = 270 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); 271 272 image_type = LLVMStructTypeInContext(gallivm->context, elem_types, 273 ARRAY_SIZE(elem_types), 0); 274 275 (void) target; /* silence unused var warning for non-debug build */ 276 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, width, 277 target, image_type, 278 DRAW_JIT_IMAGE_WIDTH); 279 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, height, 280 target, image_type, 281 DRAW_JIT_IMAGE_HEIGHT); 282 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, depth, 283 target, image_type, 284 DRAW_JIT_IMAGE_DEPTH); 285 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, base, 286 target, image_type, 287 DRAW_JIT_IMAGE_BASE); 288 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, row_stride, 289 target, image_type, 290 DRAW_JIT_IMAGE_ROW_STRIDE); 291 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, img_stride, 292 target, image_type, 293 DRAW_JIT_IMAGE_IMG_STRIDE); 294 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, num_samples, 295 target, image_type, 296 DRAW_JIT_IMAGE_NUM_SAMPLES); 297 LP_CHECK_MEMBER_OFFSET(struct draw_jit_image, sample_stride, 298 target, image_type, 299 DRAW_JIT_IMAGE_SAMPLE_STRIDE); 300 301 LP_CHECK_STRUCT_SIZE(struct draw_jit_image, target, image_type); 302 303 return image_type; 304} 305 306/** 307 * Create LLVM type for struct draw_jit_context 308 */ 309static LLVMTypeRef 310create_jit_context_type(struct gallivm_state *gallivm, const char *struct_name) 311{ 312 LLVMTypeRef texture_type = create_jit_texture_type(gallivm, "texture"); 313 LLVMTypeRef sampler_type = create_jit_sampler_type(gallivm, "sampler"); 314 LLVMTypeRef image_type = create_jit_image_type(gallivm, "image"); 315 316 LLVMTargetDataRef target = gallivm->target; 317 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 318 LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context); 319 LLVMTypeRef elem_types[DRAW_JIT_CTX_NUM_FIELDS]; 320 321 elem_types[DRAW_JIT_CTX_CONSTANTS] = LLVMArrayType(LLVMPointerType(float_type, 0), LP_MAX_TGSI_CONST_BUFFERS); 322 elem_types[DRAW_JIT_CTX_NUM_CONSTANTS] = LLVMArrayType(int_type, LP_MAX_TGSI_CONST_BUFFERS); 323 elem_types[DRAW_JIT_CTX_PLANES] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), DRAW_TOTAL_CLIP_PLANES), 0); 324 elem_types[DRAW_JIT_CTX_VIEWPORT] = LLVMPointerType(float_type, 0); 325 elem_types[DRAW_JIT_CTX_TEXTURES] = LLVMArrayType(texture_type, PIPE_MAX_SHADER_SAMPLER_VIEWS); 326 elem_types[DRAW_JIT_CTX_SAMPLERS] = LLVMArrayType(sampler_type, PIPE_MAX_SAMPLERS); 327 elem_types[DRAW_JIT_CTX_IMAGES] = LLVMArrayType(image_type, PIPE_MAX_SHADER_IMAGES); 328 elem_types[DRAW_JIT_CTX_SSBOS] = LLVMArrayType(LLVMPointerType(int_type, 0), LP_MAX_TGSI_SHADER_BUFFERS); 329 elem_types[DRAW_JIT_CTX_NUM_SSBOS] = LLVMArrayType(int_type, LP_MAX_TGSI_SHADER_BUFFERS); 330 elem_types[DRAW_JIT_CTX_ANISO_FILTER_TABLE] = LLVMPointerType(float_type, 0); 331 LLVMTypeRef context_type = LLVMStructTypeInContext(gallivm->context, elem_types, ARRAY_SIZE(elem_types), 0); 332 333 (void) target; /* silence unused var warning for non-debug build */ 334 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 335 target, context_type, DRAW_JIT_CTX_CONSTANTS); 336 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_constants, 337 target, context_type, DRAW_JIT_CTX_NUM_CONSTANTS); 338 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 339 target, context_type, DRAW_JIT_CTX_PLANES); 340 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, viewports, 341 target, context_type, DRAW_JIT_CTX_VIEWPORT); 342 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 343 target, context_type, 344 DRAW_JIT_CTX_TEXTURES); 345 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers, 346 target, context_type, 347 DRAW_JIT_CTX_SAMPLERS); 348 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, images, 349 target, context_type, DRAW_JIT_CTX_IMAGES); 350 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_ssbos, 351 target, context_type, DRAW_JIT_CTX_SSBOS); 352 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_ssbos, 353 target, context_type, DRAW_JIT_CTX_NUM_SSBOS); 354 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, aniso_filter_table, 355 target, context_type, DRAW_JIT_CTX_ANISO_FILTER_TABLE); 356 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 357 target, context_type); 358 359 return context_type; 360} 361 362 363/** 364 * Create LLVM type for struct draw_gs_jit_context 365 */ 366static LLVMTypeRef 367create_gs_jit_context_type(struct gallivm_state *gallivm, 368 unsigned vector_length, 369 LLVMTypeRef texture_type, LLVMTypeRef sampler_type, 370 LLVMTypeRef image_type, 371 const char *struct_name) 372{ 373 LLVMTargetDataRef target = gallivm->target; 374 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 375 LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context); 376 LLVMTypeRef elem_types[DRAW_GS_JIT_CTX_NUM_FIELDS]; 377 LLVMTypeRef context_type; 378 379 elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */ 380 LP_MAX_TGSI_CONST_BUFFERS); 381 elem_types[1] = LLVMArrayType(int_type, /* num_constants */ 382 LP_MAX_TGSI_CONST_BUFFERS); 383 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 384 DRAW_TOTAL_CLIP_PLANES), 0); 385 elem_types[3] = LLVMPointerType(float_type, 0); /* viewports */ 386 387 elem_types[4] = LLVMArrayType(texture_type, 388 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */ 389 elem_types[5] = LLVMArrayType(sampler_type, 390 PIPE_MAX_SAMPLERS); /* samplers */ 391 elem_types[6] = LLVMArrayType(image_type, 392 PIPE_MAX_SHADER_IMAGES); /* images */ 393 elem_types[7] = LLVMPointerType(LLVMPointerType(int_type, 0), 0); 394 elem_types[8] = LLVMPointerType(LLVMVectorType(int_type, 395 vector_length), 0); 396 elem_types[9] = LLVMPointerType(LLVMVectorType(int_type, 397 vector_length), 0); 398 399 elem_types[10] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */ 400 LP_MAX_TGSI_SHADER_BUFFERS); 401 elem_types[11] = LLVMArrayType(int_type, /* num_ssbos */ 402 LP_MAX_TGSI_SHADER_BUFFERS); 403 elem_types[12] = LLVMPointerType(float_type, 0); /* aniso table */ 404 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 405 ARRAY_SIZE(elem_types), 0); 406 407 (void) target; /* silence unused var warning for non-debug build */ 408 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, constants, 409 target, context_type, DRAW_GS_JIT_CTX_CONSTANTS); 410 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_constants, 411 target, context_type, DRAW_GS_JIT_CTX_NUM_CONSTANTS); 412 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, planes, 413 target, context_type, DRAW_GS_JIT_CTX_PLANES); 414 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, viewports, 415 target, context_type, DRAW_GS_JIT_CTX_VIEWPORT); 416 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, textures, 417 target, context_type, 418 DRAW_GS_JIT_CTX_TEXTURES); 419 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, samplers, 420 target, context_type, 421 DRAW_GS_JIT_CTX_SAMPLERS); 422 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, prim_lengths, 423 target, context_type, 424 DRAW_GS_JIT_CTX_PRIM_LENGTHS); 425 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_vertices, 426 target, context_type, 427 DRAW_GS_JIT_CTX_EMITTED_VERTICES); 428 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_prims, 429 target, context_type, 430 DRAW_GS_JIT_CTX_EMITTED_PRIMS); 431 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, ssbos, 432 target, context_type, DRAW_GS_JIT_CTX_SSBOS); 433 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_ssbos, 434 target, context_type, DRAW_GS_JIT_CTX_NUM_SSBOS); 435 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, images, 436 target, context_type, DRAW_GS_JIT_CTX_IMAGES); 437 LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, aniso_filter_table, 438 target, context_type, DRAW_GS_JIT_CTX_ANISO_FILTER_TABLE); 439 LP_CHECK_STRUCT_SIZE(struct draw_gs_jit_context, 440 target, context_type); 441 442 return context_type; 443} 444 445 446static LLVMTypeRef 447create_gs_jit_input_type(struct gallivm_state *gallivm) 448{ 449 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 450 LLVMTypeRef input_array; 451 452 input_array = LLVMVectorType(float_type, TGSI_NUM_CHANNELS); /* num primitives */ 453 input_array = LLVMArrayType(input_array, TGSI_NUM_CHANNELS); /* num channels */ 454 input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */ 455 input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */ 456 457 return input_array; 458} 459 460/** 461 * Create LLVM type for struct pipe_vertex_buffer 462 */ 463static LLVMTypeRef 464create_jit_vertex_buffer_type(struct gallivm_state *gallivm, 465 const char *struct_name) 466{ 467 LLVMTargetDataRef target = gallivm->target; 468 LLVMTypeRef elem_types[4]; 469 LLVMTypeRef vb_type; 470 471 elem_types[0] = LLVMInt16TypeInContext(gallivm->context); 472 elem_types[1] = LLVMInt8TypeInContext(gallivm->context); 473 elem_types[2] = LLVMInt32TypeInContext(gallivm->context); 474 elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); 475 476 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 477 ARRAY_SIZE(elem_types), 0); 478 479 (void) target; /* silence unused var warning for non-debug build */ 480 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 481 target, vb_type, 0); 482 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, is_user_buffer, 483 target, vb_type, 1); 484 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 485 target, vb_type, 2); 486 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer.resource, 487 target, vb_type, 3); 488 489 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 490 491 return vb_type; 492} 493 494 495/** 496 * Create LLVM type for struct vertex_header; 497 */ 498static LLVMTypeRef 499create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 500{ 501 LLVMTargetDataRef target = gallivm->target; 502 LLVMTypeRef elem_types[3]; 503 LLVMTypeRef vertex_header; 504 char struct_name[24]; 505 506 snprintf(struct_name, 23, "vertex_header%d", data_elems); 507 508 elem_types[DRAW_JIT_VERTEX_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32); 509 elem_types[DRAW_JIT_VERTEX_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 510 elem_types[DRAW_JIT_VERTEX_DATA] = LLVMArrayType(elem_types[1], data_elems); 511 512 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 513 ARRAY_SIZE(elem_types), 0); 514 515 /* these are bit-fields and we can't take address of them 516 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 517 target, vertex_header, 518 DRAW_JIT_VERTEX_CLIPMASK); 519 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 520 target, vertex_header, 521 DRAW_JIT_VERTEX_EDGEFLAG); 522 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 523 target, vertex_header, 524 DRAW_JIT_VERTEX_PAD); 525 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 526 target, vertex_header, 527 DRAW_JIT_VERTEX_VERTEX_ID); 528 */ 529 (void) target; /* silence unused var warning for non-debug build */ 530 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip_pos, 531 target, vertex_header, 532 DRAW_JIT_VERTEX_CLIP_POS); 533 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 534 target, vertex_header, 535 DRAW_JIT_VERTEX_DATA); 536 537 assert(LLVMABISizeOfType(target, vertex_header) == 538 offsetof(struct vertex_header, data[data_elems])); 539 540 return vertex_header; 541} 542 543/** 544 * Create LLVM type for struct draw_tcs_jit_context 545 */ 546static LLVMTypeRef 547create_tcs_jit_context_type(struct gallivm_state *gallivm, 548 unsigned vector_length, 549 LLVMTypeRef texture_type, LLVMTypeRef sampler_type, 550 LLVMTypeRef image_type, 551 const char *struct_name) 552{ 553 LLVMTargetDataRef target = gallivm->target; 554 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 555 LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context); 556 LLVMTypeRef elem_types[DRAW_TCS_JIT_CTX_NUM_FIELDS]; 557 LLVMTypeRef context_type; 558 559 elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */ 560 LP_MAX_TGSI_CONST_BUFFERS); 561 elem_types[1] = LLVMArrayType(int_type, /* num_constants */ 562 LP_MAX_TGSI_CONST_BUFFERS); 563 elem_types[2] = LLVMInt32TypeInContext(gallivm->context); 564 elem_types[3] = LLVMInt32TypeInContext(gallivm->context); 565 566 elem_types[4] = LLVMArrayType(texture_type, 567 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */ 568 elem_types[5] = LLVMArrayType(sampler_type, 569 PIPE_MAX_SAMPLERS); /* samplers */ 570 elem_types[6] = LLVMArrayType(image_type, 571 PIPE_MAX_SHADER_IMAGES); /* images */ 572 573 elem_types[7] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */ 574 LP_MAX_TGSI_SHADER_BUFFERS); 575 elem_types[8] = LLVMArrayType(int_type, /* num_ssbos */ 576 LP_MAX_TGSI_SHADER_BUFFERS); 577 elem_types[9] = LLVMPointerType(float_type, 0); /* aniso table */ 578 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 579 ARRAY_SIZE(elem_types), 0); 580 581 (void) target; /* silence unused var warning for non-debug build */ 582 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, constants, 583 target, context_type, DRAW_TCS_JIT_CTX_CONSTANTS); 584 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, num_constants, 585 target, context_type, DRAW_TCS_JIT_CTX_NUM_CONSTANTS); 586 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, textures, 587 target, context_type, 588 DRAW_TCS_JIT_CTX_TEXTURES); 589 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, samplers, 590 target, context_type, 591 DRAW_TCS_JIT_CTX_SAMPLERS); 592 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, ssbos, 593 target, context_type, DRAW_TCS_JIT_CTX_SSBOS); 594 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, num_ssbos, 595 target, context_type, DRAW_TCS_JIT_CTX_NUM_SSBOS); 596 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, images, 597 target, context_type, DRAW_TCS_JIT_CTX_IMAGES); 598 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, aniso_filter_table, 599 target, context_type, DRAW_TCS_JIT_CTX_ANISO_FILTER_TABLE); 600 LP_CHECK_STRUCT_SIZE(struct draw_tcs_jit_context, 601 target, context_type); 602 603 return context_type; 604} 605 606static LLVMTypeRef 607create_tcs_jit_input_type(struct gallivm_state *gallivm) 608{ 609 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 610 LLVMTypeRef input_array; 611 612 input_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */ 613 input_array = LLVMArrayType(input_array, NUM_TCS_INPUTS); /* num attrs per vertex */ 614 input_array = LLVMPointerType(input_array, 0); /* num vertices per prim */ 615 616 return input_array; 617} 618 619static LLVMTypeRef 620create_tcs_jit_output_type(struct gallivm_state *gallivm) 621{ 622 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 623 LLVMTypeRef output_array; 624 625 output_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */ 626 output_array = LLVMArrayType(output_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */ 627 output_array = LLVMPointerType(output_array, 0); /* num vertices per prim */ 628 629 return output_array; 630} 631 632static LLVMTypeRef 633create_tes_jit_input_deref_type(struct gallivm_state *gallivm) 634{ 635 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 636 LLVMTypeRef input_array; 637 638 input_array = LLVMArrayType(float_type, TGSI_NUM_CHANNELS); /* num channels */ 639 input_array = LLVMArrayType(input_array, PIPE_MAX_SHADER_INPUTS); /* num attrs per vertex */ 640 641 return input_array; 642} 643 644/** 645 * Create LLVM type for struct draw_tes_jit_context 646 */ 647static LLVMTypeRef 648create_tes_jit_context_type(struct gallivm_state *gallivm, 649 unsigned vector_length, 650 LLVMTypeRef texture_type, LLVMTypeRef sampler_type, 651 LLVMTypeRef image_type, 652 const char *struct_name) 653{ 654 LLVMTargetDataRef target = gallivm->target; 655 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 656 LLVMTypeRef int_type = LLVMInt32TypeInContext(gallivm->context); 657 LLVMTypeRef elem_types[DRAW_TCS_JIT_CTX_NUM_FIELDS]; 658 LLVMTypeRef context_type; 659 660 elem_types[0] = LLVMArrayType(LLVMPointerType(float_type, 0), /* constants */ 661 LP_MAX_TGSI_CONST_BUFFERS); 662 elem_types[1] = LLVMArrayType(int_type, /* num_constants */ 663 LP_MAX_TGSI_CONST_BUFFERS); 664 elem_types[2] = LLVMInt32TypeInContext(gallivm->context); 665 elem_types[3] = LLVMInt32TypeInContext(gallivm->context); 666 667 elem_types[4] = LLVMArrayType(texture_type, 668 PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */ 669 elem_types[5] = LLVMArrayType(sampler_type, 670 PIPE_MAX_SAMPLERS); /* samplers */ 671 elem_types[6] = LLVMArrayType(image_type, 672 PIPE_MAX_SHADER_IMAGES); /* images */ 673 674 elem_types[7] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */ 675 LP_MAX_TGSI_SHADER_BUFFERS); 676 elem_types[8] = LLVMArrayType(int_type, /* num_ssbos */ 677 LP_MAX_TGSI_SHADER_BUFFERS); 678 elem_types[9] = LLVMPointerType(float_type, 0); /* aniso table */ 679 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 680 ARRAY_SIZE(elem_types), 0); 681 682 (void) target; /* silence unused var warning for non-debug build */ 683 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, constants, 684 target, context_type, DRAW_TCS_JIT_CTX_CONSTANTS); 685 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, num_constants, 686 target, context_type, DRAW_TCS_JIT_CTX_NUM_CONSTANTS); 687 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, textures, 688 target, context_type, 689 DRAW_TCS_JIT_CTX_TEXTURES); 690 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, samplers, 691 target, context_type, 692 DRAW_TCS_JIT_CTX_SAMPLERS); 693 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, ssbos, 694 target, context_type, DRAW_TCS_JIT_CTX_SSBOS); 695 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, num_ssbos, 696 target, context_type, DRAW_TCS_JIT_CTX_NUM_SSBOS); 697 LP_CHECK_MEMBER_OFFSET(struct draw_tes_jit_context, images, 698 target, context_type, DRAW_TCS_JIT_CTX_IMAGES); 699 LP_CHECK_MEMBER_OFFSET(struct draw_tcs_jit_context, aniso_filter_table, 700 target, context_type, DRAW_TCS_JIT_CTX_ANISO_FILTER_TABLE); 701 LP_CHECK_STRUCT_SIZE(struct draw_tes_jit_context, 702 target, context_type); 703 704 return context_type; 705} 706 707/** 708 * Create LLVM types for various structures. 709 */ 710static void 711create_jit_types(struct draw_llvm_variant *variant) 712{ 713 struct gallivm_state *gallivm = variant->gallivm; 714 715 variant->context_type = create_jit_context_type(gallivm, "draw_jit_context"); 716 variant->context_ptr_type = LLVMPointerType(variant->context_type, 0); 717 718 variant->buffer_type = create_jit_dvbuffer_type(gallivm, "draw_vertex_buffer"); 719 variant->buffer_ptr_type = LLVMPointerType(variant->buffer_type, 0); 720 721 variant->vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer"); 722 variant->vb_ptr_type = LLVMPointerType(variant->vb_type, 0); 723} 724 725 726static LLVMTypeRef 727get_context_ptr_type(struct draw_llvm_variant *variant) 728{ 729 if (!variant->context_ptr_type) 730 create_jit_types(variant); 731 return variant->context_ptr_type; 732} 733 734 735static LLVMTypeRef 736get_buffer_ptr_type(struct draw_llvm_variant *variant) 737{ 738 if (!variant->buffer_ptr_type) 739 create_jit_types(variant); 740 return variant->buffer_ptr_type; 741} 742 743 744static LLVMTypeRef 745get_vb_ptr_type(struct draw_llvm_variant *variant) 746{ 747 if (!variant->vb_ptr_type) 748 create_jit_types(variant); 749 return variant->vb_ptr_type; 750} 751 752static LLVMTypeRef 753get_vertex_header_ptr_type(struct draw_llvm_variant *variant) 754{ 755 assert(variant->vertex_header_ptr_type); 756 return variant->vertex_header_ptr_type; 757} 758 759 760/** 761 * Create per-context LLVM info. 762 */ 763struct draw_llvm * 764draw_llvm_create(struct draw_context *draw, LLVMContextRef context) 765{ 766 struct draw_llvm *llvm; 767 768 if (!lp_build_init()) 769 return NULL; 770 771 llvm = CALLOC_STRUCT( draw_llvm ); 772 if (!llvm) 773 return NULL; 774 775 llvm->draw = draw; 776 777 llvm->context = context; 778 if (!llvm->context) { 779 llvm->context = LLVMContextCreate(); 780 781#if LLVM_VERSION_MAJOR >= 15 782 LLVMContextSetOpaquePointers(llvm->context, false); 783#endif 784 785 llvm->context_owned = true; 786 } 787 if (!llvm->context) 788 goto fail; 789 790 llvm->nr_variants = 0; 791 list_inithead(&llvm->vs_variants_list.list); 792 793 llvm->nr_gs_variants = 0; 794 list_inithead(&llvm->gs_variants_list.list); 795 796 llvm->nr_tcs_variants = 0; 797 list_inithead(&llvm->tcs_variants_list.list); 798 799 llvm->nr_tes_variants = 0; 800 list_inithead(&llvm->tes_variants_list.list); 801 802 return llvm; 803 804fail: 805 draw_llvm_destroy(llvm); 806 return NULL; 807} 808 809 810/** 811 * Free per-context LLVM info. 812 */ 813void 814draw_llvm_destroy(struct draw_llvm *llvm) 815{ 816 if (llvm->context_owned) 817 LLVMContextDispose(llvm->context); 818 llvm->context = NULL; 819 820 /* XXX free other draw_llvm data? */ 821 FREE(llvm); 822} 823 824static void 825draw_get_ir_cache_key(struct nir_shader *nir, 826 const void *key, size_t key_size, 827 uint32_t val_32bit, 828 unsigned char ir_sha1_cache_key[20]) 829{ 830 struct blob blob = { 0 }; 831 unsigned ir_size; 832 void *ir_binary; 833 834 blob_init(&blob); 835 nir_serialize(&blob, nir, true); 836 ir_binary = blob.data; 837 ir_size = blob.size; 838 839 struct mesa_sha1 ctx; 840 _mesa_sha1_init(&ctx); 841 _mesa_sha1_update(&ctx, key, key_size); 842 _mesa_sha1_update(&ctx, ir_binary, ir_size); 843 _mesa_sha1_update(&ctx, &val_32bit, 4); 844 _mesa_sha1_final(&ctx, ir_sha1_cache_key); 845 846 blob_finish(&blob); 847} 848 849/** 850 * Create LLVM-generated code for a vertex shader. 851 */ 852struct draw_llvm_variant * 853draw_llvm_create_variant(struct draw_llvm *llvm, 854 unsigned num_inputs, 855 const struct draw_llvm_variant_key *key) 856{ 857 struct draw_llvm_variant *variant; 858 struct llvm_vertex_shader *shader = 859 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 860 char module_name[64]; 861 unsigned char ir_sha1_cache_key[20]; 862 struct lp_cached_code cached = { 0 }; 863 bool needs_caching = false; 864 variant = MALLOC(sizeof *variant + 865 shader->variant_key_size - 866 sizeof variant->key); 867 if (!variant) 868 return NULL; 869 870 variant->llvm = llvm; 871 variant->shader = shader; 872 memcpy(&variant->key, key, shader->variant_key_size); 873 874 snprintf(module_name, sizeof(module_name), "draw_llvm_vs_variant%u", 875 variant->shader->variants_cached); 876 877 if (shader->base.state.ir.nir && llvm->draw->disk_cache_cookie) { 878 draw_get_ir_cache_key(shader->base.state.ir.nir, 879 key, 880 shader->variant_key_size, 881 num_inputs, 882 ir_sha1_cache_key); 883 884 llvm->draw->disk_cache_find_shader(llvm->draw->disk_cache_cookie, 885 &cached, 886 ir_sha1_cache_key); 887 if (!cached.data_size) 888 needs_caching = true; 889 } 890 variant->gallivm = gallivm_create(module_name, llvm->context, &cached); 891 892 create_jit_types(variant); 893 894 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 895 if (llvm->draw->vs.vertex_shader->state.type == PIPE_SHADER_IR_TGSI) 896 tgsi_dump(llvm->draw->vs.vertex_shader->state.tokens, 0); 897 else 898 nir_print_shader(llvm->draw->vs.vertex_shader->state.ir.nir, stderr); 899 draw_llvm_dump_variant_key(&variant->key); 900 } 901 902 variant->vertex_header_type = create_jit_vertex_header(variant->gallivm, num_inputs); 903 variant->vertex_header_ptr_type = LLVMPointerType(variant->vertex_header_type, 0); 904 905 draw_llvm_generate(llvm, variant); 906 907 gallivm_compile_module(variant->gallivm); 908 909 variant->jit_func = (draw_jit_vert_func) 910 gallivm_jit_function(variant->gallivm, variant->function); 911 912 if (needs_caching) 913 llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, 914 &cached, 915 ir_sha1_cache_key); 916 gallivm_free_ir(variant->gallivm); 917 918 variant->list_item_global.base = variant; 919 variant->list_item_local.base = variant; 920 /*variant->no = */shader->variants_created++; 921 variant->list_item_global.base = variant; 922 923 return variant; 924} 925 926static void 927do_clamp_vertex_color(struct gallivm_state *gallivm, 928 struct lp_type type, 929 const struct tgsi_shader_info *info, 930 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS]) 931{ 932 LLVMBuilderRef builder = gallivm->builder; 933 LLVMValueRef out; 934 unsigned chan, attrib; 935 struct lp_build_context bld; 936 lp_build_context_init(&bld, gallivm, type); 937 938 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 939 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 940 if (outputs[attrib][chan]) { 941 switch (info->output_semantic_name[attrib]) { 942 case TGSI_SEMANTIC_COLOR: 943 case TGSI_SEMANTIC_BCOLOR: 944 out = LLVMBuildLoad2(builder, LLVMTypeOf(bld.zero), outputs[attrib][chan], ""); 945 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 946 LLVMBuildStore(builder, out, outputs[attrib][chan]); 947 break; 948 } 949 } 950 } 951 } 952} 953 954static void 955generate_vs(struct draw_llvm_variant *variant, 956 LLVMBuilderRef builder, 957 struct lp_type vs_type, 958 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 959 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS], 960 const struct lp_bld_tgsi_system_values *system_values, 961 LLVMValueRef context_ptr, 962 const struct lp_build_sampler_soa *draw_sampler, 963 const struct lp_build_image_soa *draw_image, 964 boolean clamp_vertex_color, 965 struct lp_build_mask_context *bld_mask) 966{ 967 struct draw_llvm *llvm = variant->llvm; 968 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 969 LLVMValueRef consts_ptr = 970 draw_jit_context_vs_constants(variant, context_ptr); 971 LLVMValueRef num_consts_ptr = 972 draw_jit_context_num_vs_constants(variant, context_ptr); 973 LLVMValueRef ssbos_ptr = 974 draw_jit_context_vs_ssbos(variant, context_ptr); 975 LLVMValueRef num_ssbos_ptr = 976 draw_jit_context_num_vs_ssbos(variant, context_ptr); 977 978 struct lp_build_tgsi_params params; 979 memset(¶ms, 0, sizeof(params)); 980 981 params.type = vs_type; 982 params.mask = bld_mask; 983 params.consts_ptr = consts_ptr; 984 params.const_sizes_ptr = num_consts_ptr; 985 params.system_values = system_values; 986 params.inputs = inputs; 987 params.context_ptr = context_ptr; 988 params.sampler = draw_sampler; 989 params.info = &llvm->draw->vs.vertex_shader->info; 990 params.ssbo_ptr = ssbos_ptr; 991 params.ssbo_sizes_ptr = num_ssbos_ptr; 992 params.image = draw_image; 993 params.aniso_filter_table = draw_jit_context_aniso_filter_table(variant, context_ptr); 994 995 if (llvm->draw->vs.vertex_shader->state.ir.nir && 996 llvm->draw->vs.vertex_shader->state.type == PIPE_SHADER_IR_NIR) 997 lp_build_nir_soa(variant->gallivm, 998 llvm->draw->vs.vertex_shader->state.ir.nir, 999 ¶ms, 1000 outputs); 1001 else 1002 lp_build_tgsi_soa(variant->gallivm, 1003 tokens, 1004 ¶ms, 1005 outputs); 1006 1007 if (clamp_vertex_color) { 1008 const struct tgsi_shader_info *info = &llvm->draw->vs.vertex_shader->info; 1009 do_clamp_vertex_color(variant->gallivm, 1010 vs_type, info, 1011 outputs); 1012 } 1013} 1014 1015 1016static void 1017fetch_instanced(struct gallivm_state *gallivm, 1018 const struct util_format_description *format_desc, 1019 struct lp_type vs_type, 1020 LLVMValueRef vb_stride, 1021 LLVMValueRef map_ptr, 1022 LLVMValueRef buffer_size_adj, 1023 LLVMValueRef *inputs, 1024 LLVMValueRef index) 1025{ 1026 LLVMTypeRef i32_t = LLVMInt32TypeInContext(gallivm->context); 1027 LLVMTypeRef aosf_t, aosi_t; 1028 LLVMValueRef zero = LLVMConstNull(i32_t); 1029 LLVMBuilderRef builder = gallivm->builder; 1030 LLVMValueRef stride, buffer_overflowed, aos, index_valid; 1031 unsigned i; 1032 1033 aosf_t = lp_build_vec_type(gallivm, lp_float32_vec4_type()); 1034 aosi_t = lp_build_vec_type(gallivm, lp_int32_vec4_type()); 1035 1036 /* This mul can overflow. Wraparound is ok. */ 1037 stride = LLVMBuildMul(builder, vb_stride, index, ""); 1038 1039 buffer_overflowed = LLVMBuildICmp(builder, LLVMIntUGE, 1040 stride, buffer_size_adj, 1041 "buffer_overflowed"); 1042 1043 if (0) { 1044 lp_build_print_value(gallivm, " instance index = ", index); 1045 lp_build_print_value(gallivm, " buffer overflowed = ", buffer_overflowed); 1046 } 1047 1048 index_valid = LLVMBuildNot(builder, buffer_overflowed, ""); 1049 index_valid = LLVMBuildSExt(builder, index_valid, i32_t, ""); 1050 stride = LLVMBuildAnd(builder, stride, index_valid, ""); 1051 1052 aos = lp_build_fetch_rgba_aos(gallivm, 1053 format_desc, 1054 lp_float32_vec4_type(), 1055 FALSE, 1056 map_ptr, 1057 stride, zero, zero, 1058 NULL); 1059 1060 index_valid = lp_build_broadcast(gallivm, aosi_t, index_valid); 1061 aos = LLVMBuildBitCast(builder, aos, aosi_t, ""); 1062 aos = LLVMBuildAnd(builder, aos, index_valid, ""); 1063 aos = LLVMBuildBitCast(builder, aos, aosf_t, ""); 1064 1065 for (i = 0; i < TGSI_NUM_CHANNELS; i++) { 1066 LLVMValueRef index = lp_build_const_int32(gallivm, i); 1067 inputs[i] = lp_build_extract_broadcast(gallivm, 1068 lp_float32_vec4_type(), 1069 vs_type, aos, index); 1070 } 1071} 1072 1073 1074static void 1075fetch_vector(struct gallivm_state *gallivm, 1076 const struct util_format_description *format_desc, 1077 struct lp_type vs_type, 1078 LLVMValueRef vb_stride, 1079 LLVMValueRef map_ptr, 1080 LLVMValueRef buffer_size_adj, 1081 LLVMValueRef *inputs, 1082 LLVMValueRef indices) 1083{ 1084 LLVMBuilderRef builder = gallivm->builder; 1085 struct lp_build_context blduivec; 1086 struct lp_type fetch_type = vs_type; 1087 LLVMValueRef offset, valid_mask; 1088 unsigned i; 1089 1090 lp_build_context_init(&blduivec, gallivm, lp_uint_type(vs_type)); 1091 1092 vb_stride = lp_build_broadcast_scalar(&blduivec, vb_stride); 1093 buffer_size_adj = lp_build_broadcast_scalar(&blduivec, buffer_size_adj); 1094 1095 /* This mul can overflow. Wraparound is ok. */ 1096 offset = lp_build_mul(&blduivec, vb_stride, indices); 1097 1098 valid_mask = lp_build_compare(gallivm, blduivec.type, 1099 PIPE_FUNC_LESS, offset, buffer_size_adj); 1100 1101 /* not valid elements use offset 0 */ 1102 offset = LLVMBuildAnd(builder, offset, valid_mask, ""); 1103 1104 if (0) { 1105 lp_build_print_value(gallivm, " indices = ", indices); 1106 lp_build_print_value(gallivm, " offsets = ", offset); 1107 lp_build_print_value(gallivm, " valid_mask = ", valid_mask); 1108 } 1109 1110 /* 1111 * Unlike fetch_instanced, use SoA fetch instead of multiple AoS fetches. 1112 * This should always produce better code. 1113 */ 1114 1115 /* The type handling is annoying here... */ 1116 if (format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB && 1117 format_desc->channel[0].pure_integer) { 1118 if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) { 1119 fetch_type = lp_type_int_vec(vs_type.width, vs_type.width * vs_type.length); 1120 } 1121 else if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) { 1122 fetch_type = lp_type_uint_vec(vs_type.width, vs_type.width * vs_type.length); 1123 } 1124 } 1125 1126 lp_build_fetch_rgba_soa(gallivm, format_desc, 1127 fetch_type, FALSE, map_ptr, offset, 1128 blduivec.zero, blduivec.zero, 1129 NULL, inputs); 1130 1131 for (i = 0; i < TGSI_NUM_CHANNELS; i++) { 1132 inputs[i] = LLVMBuildBitCast(builder, inputs[i], 1133 lp_build_vec_type(gallivm, vs_type), ""); 1134 } 1135 1136 /* out-of-bound fetches return all zeros */ 1137 for (i = 0; i < format_desc->nr_channels; i++) { 1138 inputs[i] = LLVMBuildBitCast(builder, inputs[i], blduivec.vec_type, ""); 1139 inputs[i] = LLVMBuildAnd(builder, inputs[i], valid_mask, ""); 1140 inputs[i] = LLVMBuildBitCast(builder, inputs[i], 1141 lp_build_vec_type(gallivm, vs_type), ""); 1142 } 1143} 1144 1145 1146static void 1147store_aos(struct gallivm_state *gallivm, 1148 LLVMTypeRef io_type, 1149 LLVMValueRef io_ptr, 1150 LLVMValueRef index, 1151 LLVMValueRef value) 1152{ 1153 LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0); 1154 LLVMBuilderRef builder = gallivm->builder; 1155 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_type, io_ptr); 1156 LLVMValueRef indices[3]; 1157 1158 indices[0] = lp_build_const_int32(gallivm, 0); 1159 indices[1] = index; 1160 indices[2] = lp_build_const_int32(gallivm, 0); 1161 1162 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 1163 data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, ""); 1164 1165#if DEBUG_STORE 1166 lp_build_printf(gallivm, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 1167#endif 1168 1169 /* Unaligned store due to the vertex header */ 1170 LLVMSetAlignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float)); 1171} 1172 1173/** 1174 * Adjust the mask to architecture endianess. The mask will the store in struct: 1175 * 1176 * struct vertex_header { 1177 * unsigned clipmask:DRAW_TOTAL_CLIP_PLANES; 1178 * unsigned edgeflag:1; 1179 * unsigned pad:1; 1180 * unsigned vertex_id:16; 1181 * [...] 1182 * } 1183 * 1184 * On little-endian machine nothing needs to done, however on bit-endian machine 1185 * the mask's fields need to be adjusted with the algorithm: 1186 * 1187 * uint32_t reverse (uint32_t x) 1188 * { 1189 * return (x >> 16) | // vertex_id 1190 * ((x & 0x3fff) << 18) | // clipmask 1191 * ((x & 0x4000) << 3) | // edgeflag 1192 * ((x & 0x8000) << 1); // pad 1193 * } 1194 */ 1195static LLVMValueRef 1196adjust_mask(struct gallivm_state *gallivm, 1197 LLVMValueRef mask) 1198{ 1199#if UTIL_ARCH_BIG_ENDIAN 1200 LLVMBuilderRef builder = gallivm->builder; 1201 LLVMValueRef vertex_id; 1202 LLVMValueRef clipmask; 1203 LLVMValueRef pad; 1204 LLVMValueRef edgeflag; 1205 1206 vertex_id = LLVMBuildLShr(builder, mask, lp_build_const_int32(gallivm, 16), ""); 1207 clipmask = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x3fff), ""); 1208 clipmask = LLVMBuildShl(builder, clipmask, lp_build_const_int32(gallivm, 18), ""); 1209 if (0) { 1210 pad = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x8000), ""); 1211 pad = LLVMBuildShl(builder, pad, lp_build_const_int32(gallivm, 1), ""); 1212 } 1213 edgeflag = LLVMBuildAnd(builder, mask, lp_build_const_int32(gallivm, 0x4000), ""); 1214 edgeflag = LLVMBuildShl(builder, edgeflag, lp_build_const_int32(gallivm, 3), ""); 1215 1216 mask = LLVMBuildOr(builder, vertex_id, clipmask, ""); 1217 if (0) { 1218 mask = LLVMBuildOr(builder, mask, pad, ""); 1219 } 1220 mask = LLVMBuildOr(builder, mask, edgeflag, ""); 1221#endif 1222 return mask; 1223} 1224 1225static void 1226store_aos_array(struct gallivm_state *gallivm, 1227 struct lp_type soa_type, 1228 LLVMTypeRef io_type, 1229 LLVMValueRef io_ptr, 1230 LLVMValueRef *indices, 1231 LLVMValueRef* aos, 1232 int attrib, 1233 int num_outputs, 1234 LLVMValueRef clipmask, 1235 boolean need_edgeflag) 1236{ 1237 LLVMBuilderRef builder = gallivm->builder; 1238 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 1239 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32]; 1240 LLVMValueRef linear_inds[LP_MAX_VECTOR_WIDTH / 32]; 1241 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 1242 int vector_length = soa_type.length; 1243 int i; 1244 1245 assert(TGSI_NUM_CHANNELS == 4); 1246 1247 for (i = 0; i < vector_length; i++) { 1248 linear_inds[i] = lp_build_const_int32(gallivm, i); 1249 if (indices) { 1250 inds[i] = indices[i]; 1251 } else { 1252 inds[i] = linear_inds[i]; 1253 } 1254 io_ptrs[i] = LLVMBuildGEP2(builder, io_type, io_ptr, &inds[i], 1, ""); 1255 } 1256 1257 if (attrib == 0) { 1258 /* store vertex header for each of the n vertices */ 1259 LLVMValueRef val, cliptmp; 1260 int vertex_id_pad_edgeflag; 1261 1262 /* If this assertion fails, it means we need to update the bit twidding 1263 * code here. See struct vertex_header in draw_private.h. 1264 */ 1265 assert(DRAW_TOTAL_CLIP_PLANES==14); 1266 /* initialize vertex id:16 = 0xffff, pad:1 = 0, edgeflag:1 = 1 */ 1267 if (!need_edgeflag) { 1268 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES); 1269 } 1270 else { 1271 vertex_id_pad_edgeflag = (0xffff << 16); 1272 } 1273 val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), 1274 vertex_id_pad_edgeflag); 1275 /* OR with the clipmask */ 1276 cliptmp = LLVMBuildOr(builder, val, clipmask, ""); 1277 for (i = 0; i < vector_length; i++) { 1278 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_type, io_ptrs[i]); 1279 val = LLVMBuildExtractElement(builder, cliptmp, linear_inds[i], ""); 1280 val = adjust_mask(gallivm, val); 1281#if DEBUG_STORE 1282 lp_build_printf(gallivm, "io = %p, index %d, clipmask = %x\n", 1283 io_ptrs[i], inds[i], val); 1284#endif 1285 LLVMBuildStore(builder, val, id_ptr); 1286 } 1287 } 1288 1289 /* store for each of the n vertices */ 1290 for (i = 0; i < vector_length; i++) { 1291 store_aos(gallivm, io_type, io_ptrs[i], attr_index, aos[i]); 1292 } 1293} 1294 1295 1296static void 1297convert_to_aos(struct gallivm_state *gallivm, 1298 LLVMTypeRef io_type, 1299 LLVMValueRef io, 1300 LLVMValueRef *indices, 1301 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1302 LLVMValueRef clipmask, 1303 int num_outputs, 1304 struct lp_type soa_type, 1305 boolean need_edgeflag) 1306{ 1307 LLVMBuilderRef builder = gallivm->builder; 1308 unsigned chan, attrib, i; 1309 1310#if DEBUG_STORE 1311 lp_build_printf(gallivm, " # storing begin\n"); 1312#endif 1313 for (attrib = 0; attrib < num_outputs; ++attrib) { 1314 LLVMValueRef soa[TGSI_NUM_CHANNELS]; 1315 LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32]; 1316 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 1317 if (outputs[attrib][chan]) { 1318 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 1319 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 1320#if DEBUG_STORE 1321 lp_build_printf(gallivm, "output %d : %d ", 1322 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 1323 attrib, 0), 1324 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 1325 chan, 0)); 1326 lp_build_print_value(gallivm, "val = ", out); 1327 { 1328 LLVMValueRef iv = 1329 LLVMBuildBitCast(builder, out, lp_build_int_vec_type(gallivm, soa_type), ""); 1330 1331 lp_build_print_value(gallivm, " ival = ", iv); 1332 } 1333#endif 1334 soa[chan] = out; 1335 } 1336 else { 1337 soa[chan] = 0; 1338 } 1339 } 1340 1341 1342 if (soa_type.length == TGSI_NUM_CHANNELS) { 1343 lp_build_transpose_aos(gallivm, soa_type, soa, aos); 1344 } else { 1345 lp_build_transpose_aos(gallivm, soa_type, soa, soa); 1346 1347 for (i = 0; i < soa_type.length; ++i) { 1348 aos[i] = lp_build_extract_range(gallivm, 1349 soa[i % TGSI_NUM_CHANNELS], 1350 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS, 1351 TGSI_NUM_CHANNELS); 1352 } 1353 } 1354 1355 store_aos_array(gallivm, 1356 soa_type, 1357 io_type, 1358 io, 1359 indices, 1360 aos, 1361 attrib, 1362 num_outputs, 1363 clipmask, 1364 need_edgeflag); 1365 } 1366#if DEBUG_STORE 1367 lp_build_printf(gallivm, " # storing end\n"); 1368#endif 1369} 1370 1371 1372/** 1373 * Stores original vertex positions in clip coordinates 1374 */ 1375static void 1376store_clip(struct gallivm_state *gallivm, 1377 const struct lp_type vs_type, 1378 LLVMTypeRef io_type, 1379 LLVMValueRef io_ptr, 1380 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1381 int idx) 1382{ 1383 LLVMBuilderRef builder = gallivm->builder; 1384 LLVMValueRef soa[4]; 1385 LLVMValueRef aos[LP_MAX_VECTOR_LENGTH]; 1386 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 1387 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32]; 1388 LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 1389 LLVMTypeRef clip_ptr_type = 1390 LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 1391 4), 0); 1392 int i, j; 1393 1394 for (i = 0; i < vs_type.length; i++) { 1395 inds[i] = lp_build_const_int32(gallivm, i); 1396 io_ptrs[i] = LLVMBuildGEP2(builder, io_type, io_ptr, &inds[i], 1, ""); 1397 } 1398 1399 soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/ 1400 soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/ 1401 soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/ 1402 soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/ 1403 1404 for (i = 0; i < vs_type.length; i++) { 1405 clip_ptrs[i] = draw_jit_header_clip_pos(gallivm, io_type, io_ptrs[i]); 1406 } 1407 1408 lp_build_transpose_aos(gallivm, vs_type, soa, soa); 1409 for (i = 0; i < vs_type.length; ++i) { 1410 aos[i] = lp_build_extract_range(gallivm, 1411 soa[i % TGSI_NUM_CHANNELS], 1412 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS, 1413 TGSI_NUM_CHANNELS); 1414 } 1415 1416 for (j = 0; j < vs_type.length; j++) { 1417 LLVMValueRef clip_ptr; 1418 1419 clip_ptr = LLVMBuildPointerCast(builder, clip_ptrs[j], clip_ptr_type, ""); 1420 1421 /* Unaligned store */ 1422 LLVMSetAlignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float)); 1423 } 1424} 1425 1426 1427/** 1428 * Transforms the outputs for viewport mapping 1429 */ 1430static void 1431generate_viewport(struct draw_llvm_variant *variant, 1432 LLVMBuilderRef builder, 1433 struct lp_type vs_type, 1434 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1435 LLVMValueRef context_ptr) 1436{ 1437 int i; 1438 struct gallivm_state *gallivm = variant->gallivm; 1439 struct lp_type f32_type = vs_type; 1440 const unsigned pos = variant->llvm->draw->vs.position_output; 1441 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type); 1442 LLVMValueRef out3 = LLVMBuildLoad2(builder, vs_type_llvm, outputs[pos][3], ""); /*w0 w1 .. wn*/ 1443 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 1444 LLVMValueRef vp_ptr = draw_jit_context_viewports(variant, context_ptr); 1445 1446 /* We treat pipe_viewport_state as a float array */ 1447 const int scale_index_offset = offsetof(struct pipe_viewport_state, scale) / sizeof(float); 1448 const int trans_index_offset = offsetof(struct pipe_viewport_state, translate) / sizeof(float); 1449 1450 /* for 1/w convention*/ 1451 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 1452 LLVMBuildStore(builder, out3, outputs[pos][3]); 1453 1454 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, vs_type); 1455 1456 /* Viewport Mapping */ 1457 for (i=0; i<3; i++) { 1458 LLVMValueRef out = LLVMBuildLoad2(builder, vs_type_llvm, outputs[pos][i], ""); /*x0 x1 .. xn*/ 1459 LLVMValueRef scale; 1460 LLVMValueRef trans; 1461 LLVMValueRef scale_i; 1462 LLVMValueRef trans_i; 1463 LLVMValueRef index; 1464 1465 index = lp_build_const_int32(gallivm, i + scale_index_offset); 1466 scale_i = LLVMBuildGEP2(builder, elem_type, vp_ptr, &index, 1, ""); 1467 1468 index = lp_build_const_int32(gallivm, i + trans_index_offset); 1469 trans_i = LLVMBuildGEP2(builder, elem_type, vp_ptr, &index, 1, ""); 1470 1471 scale = lp_build_broadcast(gallivm, vs_type_llvm, 1472 LLVMBuildLoad2(builder, elem_type, scale_i, "scale")); 1473 trans = lp_build_broadcast(gallivm, vs_type_llvm, 1474 LLVMBuildLoad2(builder, elem_type, trans_i, "trans")); 1475 1476 /* divide by w */ 1477 out = LLVMBuildFMul(builder, out, out3, ""); 1478 /* mult by scale, add translation */ 1479 out = lp_build_fmuladd(builder, out, scale, trans); 1480 1481 /* store transformed outputs */ 1482 LLVMBuildStore(builder, out, outputs[pos][i]); 1483 } 1484 1485} 1486 1487 1488/** 1489 * Returns clipmask as nxi32 bitmask for the n vertices 1490 */ 1491static LLVMValueRef 1492generate_clipmask(struct draw_llvm *llvm, 1493 struct gallivm_state *gallivm, 1494 struct lp_type vs_type, 1495 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1496 struct draw_llvm_variant_key *key, 1497 LLVMTypeRef context_type, 1498 LLVMValueRef context_ptr, 1499 boolean *have_clipdist) 1500{ 1501 LLVMBuilderRef builder = gallivm->builder; 1502 LLVMValueRef mask; /* stores the <nxi32> clipmasks */ 1503 LLVMValueRef test, temp; 1504 LLVMValueRef zero, shift; 1505 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 1506 LLVMValueRef cv_x, cv_y, cv_z, cv_w; 1507 LLVMValueRef plane1, planes, plane_ptr, sum; 1508 struct lp_type f32_type = vs_type; 1509 struct lp_type i32_type = lp_int_type(vs_type); 1510 const unsigned pos = llvm->draw->vs.position_output; 1511 const unsigned cv = llvm->draw->vs.clipvertex_output; 1512 int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance; 1513 boolean have_cd = false; 1514 boolean clip_user = key->clip_user; 1515 unsigned ucp_enable = key->ucp_enable; 1516 unsigned cd[2]; 1517 1518 cd[0] = llvm->draw->vs.ccdistance_output[0]; 1519 cd[1] = llvm->draw->vs.ccdistance_output[1]; 1520 1521 if (cd[0] != pos || cd[1] != pos) 1522 have_cd = true; 1523 1524 if (num_written_clipdistance && !clip_user) { 1525 clip_user = true; 1526 ucp_enable = (1 << num_written_clipdistance) - 1; 1527 } 1528 1529 mask = lp_build_const_int_vec(gallivm, i32_type, 0); 1530 temp = lp_build_const_int_vec(gallivm, i32_type, 0); 1531 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 1532 shift = lp_build_const_int_vec(gallivm, i32_type, 1); /* 1 1 1 1 */ 1533 1534 LLVMTypeRef vec_type = LLVMTypeOf(zero); 1535 1536 /* 1537 * load clipvertex and position from correct locations. 1538 * if they are the same just load them once. 1539 */ 1540 pos_x = LLVMBuildLoad2(builder, vec_type, outputs[pos][0], ""); /*x0 x1 .. xn */ 1541 pos_y = LLVMBuildLoad2(builder, vec_type, outputs[pos][1], ""); /*y0 y1 .. yn */ 1542 pos_z = LLVMBuildLoad2(builder, vec_type, outputs[pos][2], ""); /*z0 z1 .. zn */ 1543 pos_w = LLVMBuildLoad2(builder, vec_type, outputs[pos][3], ""); /*w0 w1 .. wn */ 1544 1545 if (clip_user && cv != pos) { 1546 cv_x = LLVMBuildLoad2(builder, vec_type, outputs[cv][0], ""); /*x0 x1 .. xn */ 1547 cv_y = LLVMBuildLoad2(builder, vec_type, outputs[cv][1], ""); /*y0 y1 .. yn */ 1548 cv_z = LLVMBuildLoad2(builder, vec_type, outputs[cv][2], ""); /*z0 z1 .. zn */ 1549 cv_w = LLVMBuildLoad2(builder, vec_type, outputs[cv][3], ""); /*w0 w1 .. wn */ 1550 } else { 1551 cv_x = pos_x; 1552 cv_y = pos_y; 1553 cv_z = pos_z; 1554 cv_w = pos_w; 1555 } 1556 1557 /* 1558 * Be careful with the comparisons and NaNs (using llvm's unordered 1559 * comparisons here). 1560 */ 1561 /* Cliptest, for hardwired planes */ 1562 /* 1563 * XXX should take guardband into account (currently not in key). 1564 * Otherwise might run the draw pipeline stages for nothing. 1565 */ 1566 if (key->clip_xy) { 1567 /* plane 1 */ 1568 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 1569 temp = shift; 1570 test = LLVMBuildAnd(builder, test, temp, ""); 1571 mask = test; 1572 1573 /* plane 2 */ 1574 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 1575 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1576 temp = LLVMBuildShl(builder, temp, shift, ""); 1577 test = LLVMBuildAnd(builder, test, temp, ""); 1578 mask = LLVMBuildOr(builder, mask, test, ""); 1579 1580 /* plane 3 */ 1581 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 1582 temp = LLVMBuildShl(builder, temp, shift, ""); 1583 test = LLVMBuildAnd(builder, test, temp, ""); 1584 mask = LLVMBuildOr(builder, mask, test, ""); 1585 1586 /* plane 4 */ 1587 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 1588 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1589 temp = LLVMBuildShl(builder, temp, shift, ""); 1590 test = LLVMBuildAnd(builder, test, temp, ""); 1591 mask = LLVMBuildOr(builder, mask, test, ""); 1592 } 1593 1594 if (key->clip_z) { 1595 temp = lp_build_const_int_vec(gallivm, i32_type, 16); 1596 if (key->clip_halfz) { 1597 /* plane 5 */ 1598 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 1599 test = LLVMBuildAnd(builder, test, temp, ""); 1600 mask = LLVMBuildOr(builder, mask, test, ""); 1601 } 1602 else { 1603 /* plane 5 */ 1604 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 1605 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1606 test = LLVMBuildAnd(builder, test, temp, ""); 1607 mask = LLVMBuildOr(builder, mask, test, ""); 1608 } 1609 /* plane 6 */ 1610 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 1611 temp = LLVMBuildShl(builder, temp, shift, ""); 1612 test = LLVMBuildAnd(builder, test, temp, ""); 1613 mask = LLVMBuildOr(builder, mask, test, ""); 1614 } 1615 1616 if (clip_user) { 1617 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_type, context_ptr); 1618 LLVMValueRef indices[3]; 1619 LLVMValueRef is_nan_or_inf; 1620 1621 /* userclip planes */ 1622 while (ucp_enable) { 1623 unsigned plane_idx = ffs(ucp_enable)-1; 1624 ucp_enable &= ~(1 << plane_idx); 1625 plane_idx += 6; 1626 1627 if (have_cd && num_written_clipdistance) { 1628 LLVMValueRef clipdist; 1629 int i; 1630 i = plane_idx - 6; 1631 1632 *have_clipdist = TRUE; 1633 if (i < 4) { 1634 clipdist = LLVMBuildLoad2(builder, vec_type, outputs[cd[0]][i], ""); 1635 } else { 1636 clipdist = LLVMBuildLoad2(builder, vec_type, outputs[cd[1]][i-4], ""); 1637 } 1638 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist); 1639 is_nan_or_inf = lp_build_is_inf_or_nan(gallivm, vs_type, clipdist); 1640 test = LLVMBuildOr(builder, test, is_nan_or_inf, ""); 1641 temp = lp_build_const_int_vec(gallivm, i32_type, 1LL << plane_idx); 1642 test = LLVMBuildAnd(builder, test, temp, ""); 1643 mask = LLVMBuildOr(builder, mask, test, ""); 1644 } else { 1645 LLVMTypeRef vs_elem_type = lp_build_elem_type(gallivm, vs_type); 1646 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type); 1647 indices[0] = lp_build_const_int32(gallivm, 0); 1648 indices[1] = lp_build_const_int32(gallivm, plane_idx); 1649 1650 for (int i = 0; i < 4; ++i) { 1651 indices[2] = lp_build_const_int32(gallivm, i); 1652 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1653 plane1 = LLVMBuildLoad2(builder, vs_elem_type, plane_ptr, 1654 (const char *[]){"plane_x", "plane_y", "plane_z", "plane_w"}[i]); 1655 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1); 1656 if (i == 0) { 1657 sum = LLVMBuildFMul(builder, planes, cv_x, ""); 1658 } else { 1659 sum = lp_build_fmuladd(builder, planes, 1660 (LLVMValueRef[]){cv_x, cv_y, cv_z, cv_w}[i], sum); 1661 } 1662 } 1663 1664 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1665 temp = lp_build_const_int_vec(gallivm, i32_type, 1LL << plane_idx); 1666 test = LLVMBuildAnd(builder, test, temp, ""); 1667 mask = LLVMBuildOr(builder, mask, test, ""); 1668 } 1669 } 1670 } 1671 if (key->need_edgeflags) { 1672 /* 1673 * This isn't really part of clipmask but stored the same in vertex 1674 * header later, so do it here. 1675 */ 1676 unsigned edge_attr = llvm->draw->vs.edgeflag_output; 1677 LLVMValueRef one = lp_build_const_vec(gallivm, f32_type, 1.0); 1678 LLVMValueRef edgeflag = LLVMBuildLoad2(builder, vec_type, outputs[edge_attr][0], ""); 1679 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_EQUAL, one, edgeflag); 1680 temp = lp_build_const_int_vec(gallivm, i32_type, 1681 1LL << DRAW_TOTAL_CLIP_PLANES); 1682 test = LLVMBuildAnd(builder, test, temp, ""); 1683 mask = LLVMBuildOr(builder, mask, test, ""); 1684 } 1685 return mask; 1686} 1687 1688 1689/** 1690 * Returns boolean if any clipping has occurred 1691 * Used zero/one i8 value to represent boolean 1692 */ 1693static LLVMValueRef 1694clipmask_booli8(struct gallivm_state *gallivm, 1695 const struct lp_type vs_type, 1696 LLVMValueRef clipmask_bool_ptr, 1697 boolean edgeflag_in_clipmask) 1698{ 1699 LLVMBuilderRef builder = gallivm->builder; 1700 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context); 1701 LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, ""); 1702 LLVMValueRef ret; 1703 struct lp_build_context bldivec; 1704 1705 lp_build_context_init(&bldivec, gallivm, lp_int_type(vs_type)); 1706 1707 /* 1708 * We need to invert the edgeflag bit from the clipmask here 1709 * (because the result is really if we want to run the pipeline or not 1710 * and we (may) need it if edgeflag was 0). 1711 */ 1712 if (edgeflag_in_clipmask) { 1713 LLVMValueRef edge = lp_build_const_int_vec(gallivm, bldivec.type, 1714 1LL << DRAW_TOTAL_CLIP_PLANES); 1715 clipmask_bool = LLVMBuildXor(builder, clipmask_bool, edge, ""); 1716 } 1717 1718 /* 1719 * XXX: probably should mask off bits from the mask which come from 1720 * vertices which were beyond the count (i.e. indices_valid for 1721 * linear fetches, for elts ones we don't have the correct mask 1722 * right now). Otherwise might run the pipeline for nothing, 1723 * though everything should still work. 1724 */ 1725 ret = lp_build_any_true_range(&bldivec, vs_type.length, clipmask_bool); 1726 ret = LLVMBuildZExt(builder, ret, int8_type, ""); 1727 return ret; 1728} 1729 1730static LLVMValueRef 1731draw_gs_llvm_fetch_input(const struct lp_build_gs_iface *gs_iface, 1732 struct lp_build_context * bld, 1733 boolean is_vindex_indirect, 1734 LLVMValueRef vertex_index, 1735 boolean is_aindex_indirect, 1736 LLVMValueRef attrib_index, 1737 LLVMValueRef swizzle_index) 1738{ 1739 const struct draw_gs_llvm_iface *gs = draw_gs_llvm_iface(gs_iface); 1740 struct gallivm_state *gallivm = bld->gallivm; 1741 LLVMBuilderRef builder = gallivm->builder; 1742 LLVMValueRef indices[3]; 1743 LLVMValueRef res; 1744 struct lp_type type = bld->type; 1745 1746 if (is_vindex_indirect || is_aindex_indirect) { 1747 int i; 1748 res = bld->zero; 1749 for (i = 0; i < type.length; ++i) { 1750 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 1751 LLVMValueRef vert_chan_index = vertex_index; 1752 LLVMValueRef attr_chan_index = attrib_index; 1753 LLVMValueRef channel_vec, value; 1754 1755 if (is_vindex_indirect) { 1756 vert_chan_index = LLVMBuildExtractElement(builder, 1757 vertex_index, idx, ""); 1758 } 1759 if (is_aindex_indirect) { 1760 attr_chan_index = LLVMBuildExtractElement(builder, 1761 attrib_index, idx, ""); 1762 } 1763 1764 indices[0] = vert_chan_index; 1765 indices[1] = attr_chan_index; 1766 indices[2] = swizzle_index; 1767 1768 channel_vec = LLVMBuildGEP(builder, gs->input, indices, 3, ""); 1769 channel_vec = LLVMBuildLoad(builder, channel_vec, ""); 1770 value = LLVMBuildExtractElement(builder, channel_vec, idx, ""); 1771 1772 res = LLVMBuildInsertElement(builder, res, value, idx, ""); 1773 } 1774 } else { 1775 indices[0] = vertex_index; 1776 indices[1] = attrib_index; 1777 indices[2] = swizzle_index; 1778 1779 res = LLVMBuildGEP(builder, gs->input, indices, 3, ""); 1780 res = LLVMBuildLoad(builder, res, ""); 1781 } 1782 1783 return res; 1784} 1785 1786static void 1787draw_gs_llvm_emit_vertex(const struct lp_build_gs_iface *gs_base, 1788 struct lp_build_context * bld, 1789 LLVMValueRef (*outputs)[4], 1790 LLVMValueRef emitted_vertices_vec, 1791 LLVMValueRef mask_vec, LLVMValueRef stream_id) 1792{ 1793 const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base); 1794 struct draw_gs_llvm_variant *variant = gs_iface->variant; 1795 struct gallivm_state *gallivm = variant->gallivm; 1796 LLVMBuilderRef builder = gallivm->builder; 1797 struct lp_type gs_type = bld->type; 1798 LLVMValueRef clipmask = lp_build_const_int_vec(gallivm, 1799 lp_int_type(gs_type), 0); 1800 LLVMValueRef indices[LP_MAX_VECTOR_LENGTH]; 1801 LLVMValueRef next_prim_offset = 1802 lp_build_const_int32(gallivm, variant->shader->base.primitive_boundary); 1803 LLVMValueRef io = variant->io_ptr; 1804 unsigned i; 1805 const struct tgsi_shader_info *gs_info = &variant->shader->base.info; 1806 1807 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, mask_vec, lp_build_const_int_vec(gallivm, bld->type, 0), ""); 1808 for (i = 0; i < gs_type.length; ++i) { 1809 LLVMValueRef ind = lp_build_const_int32(gallivm, i); 1810 LLVMValueRef currently_emitted = 1811 LLVMBuildExtractElement(builder, emitted_vertices_vec, ind, ""); 1812 indices[i] = LLVMBuildMul(builder, ind, next_prim_offset, ""); 1813 indices[i] = LLVMBuildAdd(builder, indices[i], currently_emitted, ""); 1814 indices[i] = LLVMBuildSelect(builder, LLVMBuildExtractElement(builder, cond, ind, ""), indices[i], 1815 lp_build_const_int32(gallivm, variant->shader->base.primitive_boundary - 1), ""); 1816 } 1817 1818 LLVMValueRef stream_idx = LLVMBuildExtractElement(builder, stream_id, lp_build_const_int32(gallivm, 0), ""); 1819 LLVMValueRef cnd = LLVMBuildICmp(builder, LLVMIntULT, stream_idx, lp_build_const_int32(gallivm, variant->shader->base.num_vertex_streams), ""); 1820 struct lp_build_if_state if_ctx; 1821 lp_build_if(&if_ctx, gallivm, cnd); 1822 io = lp_build_pointer_get(builder, io, LLVMBuildExtractElement(builder, stream_id, lp_build_const_int32(gallivm, 0), "")); 1823 1824 if (variant->key.clamp_vertex_color) { 1825 do_clamp_vertex_color(gallivm, gs_type, 1826 gs_info, outputs); 1827 } 1828 convert_to_aos(gallivm, variant->vertex_header_type, 1829 io, indices, 1830 outputs, clipmask, 1831 gs_info->num_outputs, gs_type, 1832 FALSE); 1833 lp_build_endif(&if_ctx); 1834} 1835 1836static void 1837draw_gs_llvm_end_primitive(const struct lp_build_gs_iface *gs_base, 1838 struct lp_build_context * bld, 1839 LLVMValueRef total_emitted_vertices_vec_ptr, 1840 LLVMValueRef verts_per_prim_vec, 1841 LLVMValueRef emitted_prims_vec, 1842 LLVMValueRef mask_vec, unsigned stream) 1843{ 1844 const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base); 1845 struct draw_gs_llvm_variant *variant = gs_iface->variant; 1846 struct gallivm_state *gallivm = variant->gallivm; 1847 LLVMBuilderRef builder = gallivm->builder; 1848 LLVMValueRef prim_lengts_ptr = 1849 draw_gs_jit_prim_lengths(variant, variant->context_ptr); 1850 unsigned i; 1851 1852 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, mask_vec, lp_build_const_int_vec(gallivm, bld->type, 0), ""); 1853 for (i = 0; i < bld->type.length; ++i) { 1854 LLVMValueRef ind = lp_build_const_int32(gallivm, i); 1855 LLVMValueRef prims_emitted = 1856 LLVMBuildExtractElement(builder, emitted_prims_vec, ind, ""); 1857 LLVMValueRef store_ptr; 1858 LLVMValueRef num_vertices = 1859 LLVMBuildExtractElement(builder, verts_per_prim_vec, ind, ""); 1860 1861 LLVMValueRef this_cond = LLVMBuildExtractElement(gallivm->builder, cond, ind, ""); 1862 struct lp_build_if_state ifthen; 1863 lp_build_if(&ifthen, gallivm, this_cond); 1864 prims_emitted = LLVMBuildMul(gallivm->builder, prims_emitted, lp_build_const_int32(gallivm, variant->shader->base.num_vertex_streams), ""); 1865 prims_emitted = LLVMBuildAdd(gallivm->builder, prims_emitted, lp_build_const_int32(gallivm, stream), ""); 1866 store_ptr = LLVMBuildGEP(builder, prim_lengts_ptr, &prims_emitted, 1, ""); 1867 store_ptr = LLVMBuildLoad(builder, store_ptr, ""); 1868 store_ptr = LLVMBuildGEP(builder, store_ptr, &ind, 1, ""); 1869 LLVMBuildStore(builder, num_vertices, store_ptr); 1870 lp_build_endif(&ifthen); 1871 } 1872} 1873 1874static void 1875draw_gs_llvm_epilogue(const struct lp_build_gs_iface *gs_base, 1876 LLVMValueRef total_emitted_vertices_vec, 1877 LLVMValueRef emitted_prims_vec, unsigned stream) 1878{ 1879 const struct draw_gs_llvm_iface *gs_iface = draw_gs_llvm_iface(gs_base); 1880 struct draw_gs_llvm_variant *variant = gs_iface->variant; 1881 struct gallivm_state *gallivm = variant->gallivm; 1882 LLVMBuilderRef builder = gallivm->builder; 1883 LLVMValueRef emitted_verts_ptr = 1884 draw_gs_jit_emitted_vertices(variant, variant->context_ptr); 1885 LLVMValueRef emitted_prims_ptr = 1886 draw_gs_jit_emitted_prims(variant, variant->context_ptr); 1887 LLVMValueRef stream_val = lp_build_const_int32(gallivm, stream); 1888 1889 emitted_verts_ptr = LLVMBuildGEP(builder, emitted_verts_ptr, &stream_val, 1, ""); 1890 emitted_prims_ptr = LLVMBuildGEP(builder, emitted_prims_ptr, &stream_val, 1, ""); 1891 1892 LLVMBuildStore(builder, total_emitted_vertices_vec, emitted_verts_ptr); 1893 LLVMBuildStore(builder, emitted_prims_vec, emitted_prims_ptr); 1894} 1895 1896static void 1897draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 1898{ 1899 struct gallivm_state *gallivm = variant->gallivm; 1900 LLVMContextRef context = gallivm->context; 1901 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1902 LLVMTypeRef arg_types[13]; 1903 unsigned num_arg_types = ARRAY_SIZE(arg_types); 1904 LLVMTypeRef func_type; 1905 LLVMValueRef context_ptr; 1906 LLVMBasicBlockRef block; 1907 LLVMBuilderRef builder; 1908 char func_name[64]; 1909 struct lp_type vs_type; 1910 LLVMValueRef count, fetch_elts, start_or_maxelt; 1911 LLVMValueRef vertex_id_offset; 1912 LLVMValueRef stride, step, io_itr; 1913 LLVMValueRef ind_vec, start_vec, have_elts, fetch_max, tmp; 1914 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1915 LLVMValueRef vb_stride[PIPE_MAX_ATTRIBS]; 1916 LLVMValueRef map_ptr[PIPE_MAX_ATTRIBS]; 1917 LLVMValueRef buffer_size_adj[PIPE_MAX_ATTRIBS]; 1918 LLVMValueRef instance_index[PIPE_MAX_ATTRIBS]; 1919 LLVMValueRef fake_buf_ptr, fake_buf; 1920 1921 struct draw_context *draw = llvm->draw; 1922 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1923 unsigned i, j; 1924 struct lp_build_context bld, blduivec; 1925 struct lp_build_loop_state lp_loop; 1926 struct lp_build_if_state if_ctx; 1927 const int vector_length = lp_native_vector_width / 32; 1928 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 1929 struct lp_build_sampler_soa *sampler = 0; 1930 struct lp_build_image_soa *image = NULL; 1931 LLVMValueRef ret, clipmask_bool_ptr; 1932 struct draw_llvm_variant_key *key = &variant->key; 1933 /* If geometry shader is present we need to skip both the viewport 1934 * transformation and clipping otherwise the inputs to the geometry 1935 * shader will be incorrect. 1936 * The code can't handle vp transform when vs writes vp index neither 1937 * (though this would be fixable here, but couldn't just broadcast 1938 * the values). 1939 */ 1940 const boolean bypass_viewport = key->has_gs_or_tes || key->bypass_viewport || 1941 vs_info->writes_viewport_index; 1942 const boolean enable_cliptest = !key->has_gs_or_tes && (key->clip_xy || 1943 key->clip_z || 1944 key->clip_user || 1945 key->need_edgeflags); 1946 LLVMValueRef variant_func; 1947 const unsigned pos = draw->vs.position_output; 1948 const unsigned cv = draw->vs.clipvertex_output; 1949 boolean have_clipdist = FALSE; 1950 struct lp_bld_tgsi_system_values system_values; 1951 1952 memset(&system_values, 0, sizeof(system_values)); 1953 memset(&outputs, 0, sizeof(outputs)); 1954 snprintf(func_name, sizeof(func_name), "draw_llvm_vs_variant"); 1955 1956 i = 0; 1957 arg_types[i++] = get_context_ptr_type(variant); /* context */ 1958 arg_types[i++] = get_vertex_header_ptr_type(variant); /* vertex_header */ 1959 arg_types[i++] = get_buffer_ptr_type(variant); /* vbuffers */ 1960 arg_types[i++] = int32_type; /* count */ 1961 arg_types[i++] = int32_type; /* start/fetch_elt_max */ 1962 arg_types[i++] = int32_type; /* stride */ 1963 arg_types[i++] = get_vb_ptr_type(variant); /* pipe_vertex_buffer's */ 1964 arg_types[i++] = int32_type; /* instance_id */ 1965 arg_types[i++] = int32_type; /* vertex_id_offset */ 1966 arg_types[i++] = int32_type; /* start_instance */ 1967 arg_types[i++] = LLVMPointerType(int32_type, 0); /* fetch_elts */ 1968 arg_types[i++] = int32_type; /* draw_id */ 1969 arg_types[i++] = int32_type; /* view_id */ 1970 1971 func_type = LLVMFunctionType(LLVMInt8TypeInContext(context), 1972 arg_types, num_arg_types, 0); 1973 1974 variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); 1975 variant->function = variant_func; 1976 1977 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 1978 for (i = 0; i < num_arg_types; ++i) 1979 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1980 lp_add_function_attr(variant_func, i + 1, LP_FUNC_ATTR_NOALIAS); 1981 1982 if (gallivm->cache && gallivm->cache->data_size) 1983 return; 1984 context_ptr = LLVMGetParam(variant_func, 0); 1985 io_ptr = LLVMGetParam(variant_func, 1); 1986 vbuffers_ptr = LLVMGetParam(variant_func, 2); 1987 count = LLVMGetParam(variant_func, 3); 1988 /* 1989 * XXX: the maxelt part is unused. Not really useful, since we cannot 1990 * get index buffer overflows due to vsplit (which provides its own 1991 * elts buffer, with a different size than what's passed in here). 1992 */ 1993 start_or_maxelt = LLVMGetParam(variant_func, 4); 1994 /* 1995 * XXX: stride is actually unused. The stride we use is strictly calculated 1996 * from the number of outputs (including the draw_extra outputs). 1997 * Should probably fix some day (we need a new vs just because of extra 1998 * outputs which the generated vs won't touch). 1999 */ 2000 stride = LLVMGetParam(variant_func, 5); 2001 vb_ptr = LLVMGetParam(variant_func, 6); 2002 system_values.instance_id = LLVMGetParam(variant_func, 7); 2003 vertex_id_offset = LLVMGetParam(variant_func, 8); 2004 system_values.base_instance = LLVMGetParam(variant_func, 9); 2005 fetch_elts = LLVMGetParam(variant_func, 10); 2006 system_values.draw_id = LLVMGetParam(variant_func, 11); 2007 system_values.view_index = LLVMGetParam(variant_func, 12); 2008 2009 lp_build_name(context_ptr, "context"); 2010 lp_build_name(io_ptr, "io"); 2011 lp_build_name(vbuffers_ptr, "vbuffers"); 2012 lp_build_name(count, "count"); 2013 lp_build_name(start_or_maxelt, "start_or_maxelt"); 2014 lp_build_name(stride, "stride"); 2015 lp_build_name(vb_ptr, "vb"); 2016 lp_build_name(system_values.instance_id, "instance_id"); 2017 lp_build_name(vertex_id_offset, "vertex_id_offset"); 2018 lp_build_name(system_values.base_instance, "start_instance"); 2019 lp_build_name(fetch_elts, "fetch_elts"); 2020 lp_build_name(system_values.draw_id, "draw_id"); 2021 2022 /* 2023 * Function body 2024 */ 2025 2026 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 2027 builder = gallivm->builder; 2028 LLVMPositionBuilderAtEnd(builder, block); 2029 2030 memset(&vs_type, 0, sizeof vs_type); 2031 vs_type.floating = TRUE; /* floating point values */ 2032 vs_type.sign = TRUE; /* values are signed */ 2033 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 2034 vs_type.width = 32; /* 32-bit float */ 2035 vs_type.length = vector_length; 2036 2037 lp_build_context_init(&bld, gallivm, lp_type_uint(32)); 2038 lp_build_context_init(&blduivec, gallivm, lp_uint_type(vs_type)); 2039 2040 /* hold temporary "bool" clipmask */ 2041 clipmask_bool_ptr = lp_build_alloca(gallivm, blduivec.vec_type, ""); 2042 2043 fake_buf = lp_build_alloca_undef(gallivm, 2044 LLVMVectorType(LLVMInt64TypeInContext(context), 4), ""); 2045 fake_buf = LLVMBuildBitCast(builder, fake_buf, 2046 LLVMPointerType(LLVMInt8TypeInContext(context), 0), ""); 2047 fake_buf_ptr = LLVMBuildGEP2(builder, LLVMInt8TypeInContext(context), fake_buf, &bld.zero, 1, ""); 2048 2049 /* code generated texture sampling */ 2050 sampler = draw_llvm_sampler_soa_create(draw_llvm_variant_key_samplers(key), 2051 MAX2(key->nr_samplers, 2052 key->nr_sampler_views)); 2053 image = draw_llvm_image_soa_create(draw_llvm_variant_key_images(key), 2054 key->nr_images); 2055 2056 step = lp_build_const_int32(gallivm, vector_length); 2057 2058 ind_vec = blduivec.undef; 2059 for (i = 0; i < vs_type.length; i++) { 2060 LLVMValueRef index = lp_build_const_int32(gallivm, i); 2061 ind_vec = LLVMBuildInsertElement(builder, ind_vec, index, index, ""); 2062 } 2063 2064 have_elts = LLVMBuildICmp(builder, LLVMIntNE, 2065 LLVMConstPointerNull(arg_types[10]), fetch_elts, ""); 2066 2067 fetch_max = LLVMBuildSub(builder, count, bld.one, "fetch_max"); 2068 fetch_max = lp_build_broadcast_scalar(&blduivec, fetch_max); 2069 /* 2070 * Only needed for non-indexed path. 2071 */ 2072 start_vec = lp_build_broadcast_scalar(&blduivec, start_or_maxelt); 2073 2074 /* 2075 * Pre-calculate everything which is constant per shader invocation. 2076 */ 2077 for (j = 0; j < key->nr_vertex_elements; ++j) { 2078 LLVMValueRef vb_buffer_offset, buffer_size, temp_ptr; 2079 LLVMValueRef vb_info, vbuffer_ptr, buf_offset, ofbit; 2080 struct pipe_vertex_element *velem = &key->vertex_element[j]; 2081 LLVMValueRef vb_index = 2082 lp_build_const_int32(gallivm, velem->vertex_buffer_index); 2083 LLVMValueRef bsize = lp_build_const_int32(gallivm, 2084 util_format_get_blocksize(velem->src_format)); 2085 LLVMValueRef src_offset = lp_build_const_int32(gallivm, 2086 velem->src_offset); 2087 struct lp_build_if_state if_ctx; 2088 2089 if (velem->src_format != PIPE_FORMAT_NONE) { 2090 vbuffer_ptr = LLVMBuildGEP2(builder, variant->buffer_type, vbuffers_ptr, &vb_index, 1, ""); 2091 vb_info = LLVMBuildGEP2(builder, variant->vb_type, vb_ptr, &vb_index, 1, ""); 2092 vb_stride[j] = draw_jit_vbuffer_stride(gallivm, variant->vb_type, vb_info); 2093 vb_stride[j] = LLVMBuildZExt(gallivm->builder, vb_stride[j], 2094 LLVMInt32TypeInContext(context), ""); 2095 vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, variant->vb_type, vb_info); 2096 map_ptr[j] = draw_jit_dvbuffer_map(gallivm, variant->buffer_type, vbuffer_ptr); 2097 buffer_size = draw_jit_dvbuffer_size(gallivm, variant->buffer_type, vbuffer_ptr); 2098 2099 ofbit = NULL; 2100 /* 2101 * We'll set buffer_size_adj to zero if we have of, so it will 2102 * always overflow later automatically without having to keep ofbit. 2103 * Overflows (with normal wraparound) doing the actual offset 2104 * calculation should be ok, just not for the buffer size calc. 2105 * It would also be possible to detect such overflows and return 2106 * zeros if that happens, but this would be more complex. 2107 */ 2108 buf_offset = lp_build_add(&bld, vb_buffer_offset, src_offset); 2109 tmp = lp_build_sub(&bld, bsize, bld.one); 2110 buffer_size_adj[j] = lp_build_usub_overflow(gallivm, buffer_size, tmp, 2111 &ofbit); 2112 buffer_size_adj[j] = lp_build_usub_overflow(gallivm, buffer_size_adj[j], 2113 buf_offset, &ofbit); 2114 2115 /* 2116 * We can't easily set fake vertex buffers outside the generated code. 2117 * Hence, set fake vertex buffers here instead basically, so fetch 2118 * code can always fetch using offset 0, eliminating all control flow 2119 * inside the main loop. 2120 * (Alternatively, could have control flow per vector skipping fetch 2121 * if ofbit is true.) 2122 */ 2123 if (velem->instance_divisor) { 2124 /* 2125 * Index is equal to the start instance plus the number of current 2126 * instance divided by the divisor. In this case we compute it as: 2127 * index = start_instance + (instance_id / divisor). 2128 * Note we could actually do the fetch here, outside the loop - 2129 * it's all constant, hopefully llvm recognizes this. 2130 */ 2131 LLVMValueRef current_instance; 2132 current_instance = LLVMBuildUDiv(builder, system_values.instance_id, 2133 lp_build_const_int32(gallivm, 2134 velem->instance_divisor), 2135 "instance_divisor"); 2136 instance_index[j] = lp_build_uadd_overflow(gallivm, system_values.base_instance, 2137 current_instance, &ofbit); 2138 } 2139 2140 buffer_size_adj[j] = LLVMBuildSelect(builder, ofbit, bld.zero, 2141 buffer_size_adj[j], ""); 2142 2143 LLVMTypeRef byte_type = LLVMInt8TypeInContext(context); 2144 LLVMTypeRef byte_ptr_type = LLVMPointerType(byte_type, 0); 2145 temp_ptr = lp_build_alloca_undef(gallivm, byte_ptr_type, ""); 2146 2147 lp_build_if(&if_ctx, gallivm, ofbit); 2148 { 2149 LLVMBuildStore(builder, fake_buf_ptr, temp_ptr); 2150 } 2151 lp_build_else(&if_ctx); 2152 { 2153 map_ptr[j] = LLVMBuildGEP2(builder, byte_type, map_ptr[j], &buf_offset, 1, ""); 2154 LLVMBuildStore(builder, map_ptr[j], temp_ptr); 2155 } 2156 lp_build_endif(&if_ctx); 2157 map_ptr[j] = LLVMBuildLoad2(builder, byte_ptr_type, temp_ptr, "map_ptr"); 2158 2159 if (0) { 2160 lp_build_printf(gallivm, "velem %d, vbuf index = %u, vb_stride = %u\n", 2161 lp_build_const_int32(gallivm, j), 2162 vb_index, vb_stride[j]); 2163 lp_build_printf(gallivm, 2164 " vb_buffer_offset = %u, src_offset = %u, buf_offset = %u\n", 2165 vb_buffer_offset, src_offset, buf_offset); 2166 lp_build_printf(gallivm, " buffer size = %u, blocksize = %u\n", 2167 buffer_size, bsize); 2168 lp_build_printf(gallivm, " instance_id = %u\n", system_values.instance_id); 2169 } 2170 } 2171 } 2172 2173 lp_build_loop_begin(&lp_loop, gallivm, bld.zero); 2174 { 2175 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS]; 2176 LLVMValueRef io; 2177 LLVMValueRef clipmask; /* holds the clipmask value */ 2178 LLVMValueRef true_index_array, index_store; 2179 const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS]; 2180 2181 io_itr = lp_loop.counter; 2182 2183 io = LLVMBuildGEP2(builder, variant->vertex_header_type, io_ptr, &io_itr, 1, ""); 2184#if DEBUG_STORE 2185 lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n", 2186 io_itr, io, lp_loop.counter); 2187#endif 2188 2189 true_index_array = lp_build_broadcast_scalar(&blduivec, lp_loop.counter); 2190 true_index_array = LLVMBuildAdd(builder, true_index_array, ind_vec, ""); 2191 2192 LLVMValueRef exec_mask = lp_build_cmp(&blduivec, PIPE_FUNC_LEQUAL, true_index_array, fetch_max); 2193 /* 2194 * Limit indices to fetch_max, otherwise might try to access indices 2195 * beyond index buffer (or rather vsplit elt buffer) size. 2196 * Could probably safely (?) skip this for non-indexed draws and 2197 * simplify things minimally (by removing it could combine the ind_vec 2198 * and start_vec adds). I think the only effect for non-indexed draws will 2199 * be that for the invalid elements they will be all fetched from the 2200 * same location as the last valid one, but noone should really care. 2201 */ 2202 true_index_array = lp_build_min(&blduivec, true_index_array, fetch_max); 2203 2204 index_store = lp_build_alloca_undef(gallivm, blduivec.vec_type, "index_store"); 2205 2206 lp_build_if(&if_ctx, gallivm, have_elts); 2207 { 2208 /* 2209 * Note: you'd expect some comparison/clamp against fetch_elt_max 2210 * here. 2211 * There used to be one here but it was incorrect: overflow was 2212 * detected if index > fetch_elt_max - but the correct condition 2213 * would be index >= fetch_elt_max (since this is just size of elts 2214 * buffer / element size). 2215 * Using the correct condition however will cause failures - due to 2216 * vsplit/vcache code which rebases indices. So, as an example, if 2217 * fetch_elt_max is just 1 and fetch_count 2, vsplit cache will 2218 * replace all invalid indices with 0 - which in case of elt_bias 2219 * not being zero will get a different fetch index than the valid 2220 * index 0. So, just rely on vsplit code preventing out-of-bounds 2221 * fetches. This is also why it's safe to do elts fetch even if there 2222 * was no index buffer bound - the real buffer is never seen here, at 2223 * least not if there are index buffer overflows... 2224 */ 2225 2226 /* 2227 * XXX should not have to do this, as scale can be handled 2228 * natively by loads (hits asserts though). 2229 */ 2230 tmp = lp_build_shl_imm(&blduivec, true_index_array, 2); 2231 fetch_elts = LLVMBuildBitCast(builder, fetch_elts, 2232 LLVMPointerType(LLVMInt8TypeInContext(context), 2233 0), ""); 2234 tmp = lp_build_gather(gallivm, vs_type.length, 2235 32, bld.type, TRUE, 2236 fetch_elts, tmp, FALSE); 2237 LLVMBuildStore(builder, tmp, index_store); 2238 } 2239 lp_build_else(&if_ctx); 2240 { 2241 tmp = LLVMBuildAdd(builder, true_index_array, start_vec, ""); 2242 LLVMBuildStore(builder, tmp, index_store); 2243 } 2244 lp_build_endif(&if_ctx); 2245 2246 true_index_array = LLVMBuildLoad2(builder, blduivec.vec_type, index_store, ""); 2247 2248 for (j = 0; j < key->nr_vertex_elements; ++j) { 2249 struct pipe_vertex_element *velem = &key->vertex_element[j]; 2250 const struct util_format_description *format_desc = 2251 util_format_description(velem->src_format); 2252 2253 if (format_desc->format == PIPE_FORMAT_NONE) { 2254 for (i = 0; i < TGSI_NUM_CHANNELS; i++) { 2255 inputs[j][i] = lp_build_zero(gallivm, vs_type); 2256 } 2257 } 2258 else if (velem->instance_divisor) { 2259 fetch_instanced(gallivm, format_desc, vs_type, 2260 vb_stride[j], map_ptr[j], 2261 buffer_size_adj[j], 2262 inputs[j], instance_index[j]); 2263 } 2264 else { 2265 fetch_vector(gallivm, format_desc, vs_type, 2266 vb_stride[j], map_ptr[j], 2267 buffer_size_adj[j], 2268 inputs[j], true_index_array); 2269 } 2270 } 2271 2272 struct lp_build_mask_context mask; 2273 2274 lp_build_mask_begin(&mask, gallivm, vs_type, exec_mask); 2275 /* In the paths with elts vertex id has to be unaffected by the 2276 * index bias and because indices inside our elements array have 2277 * already had index bias applied we need to subtract it here to 2278 * get back to the original index. 2279 * in the linear paths vertex id has to be unaffected by the 2280 * original start index and because we abuse the 'start' variable 2281 * to either represent the actual start index or the index at which 2282 * the primitive was split (we split rendering into chunks of at 2283 * most 4095-vertices) we need to back out the original start 2284 * index out of our vertex id here. 2285 * for ARB_shader_draw_parameters, base_vertex should be 0 for non-indexed draws. 2286 */ 2287 LLVMValueRef base_vertex = lp_build_select(&bld, have_elts, vertex_id_offset, lp_build_const_int32(gallivm, 0)); 2288 system_values.basevertex = lp_build_broadcast_scalar(&blduivec, base_vertex); 2289 /* first vertex is for Vulkan base vertex support */ 2290 LLVMValueRef first_vertex = lp_build_select(&bld, have_elts, vertex_id_offset, start_or_maxelt); 2291 system_values.firstvertex = lp_build_broadcast_scalar(&blduivec, first_vertex); 2292 system_values.vertex_id = true_index_array; 2293 system_values.vertex_id_nobase = LLVMBuildSub(builder, true_index_array, 2294 lp_build_broadcast_scalar(&blduivec, vertex_id_offset), ""); 2295 2296 ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs; 2297 generate_vs(variant, 2298 builder, 2299 vs_type, 2300 outputs, 2301 ptr_aos, 2302 &system_values, 2303 context_ptr, 2304 sampler, 2305 image, 2306 key->clamp_vertex_color, 2307 &mask); 2308 2309 lp_build_mask_end(&mask); 2310 if (pos != -1 && cv != -1) { 2311 /* store original positions in clip before further manipulation */ 2312 store_clip(gallivm, vs_type, variant->vertex_header_type, io, outputs, pos); 2313 2314 /* do cliptest */ 2315 if (enable_cliptest) { 2316 LLVMValueRef temp = LLVMBuildLoad2(builder, blduivec.vec_type, clipmask_bool_ptr, ""); 2317 /* allocate clipmask, assign it integer type */ 2318 clipmask = generate_clipmask(llvm, 2319 gallivm, 2320 vs_type, 2321 outputs, 2322 key, 2323 variant->context_type, 2324 context_ptr, &have_clipdist); 2325 temp = LLVMBuildOr(builder, clipmask, temp, ""); 2326 /* store temporary clipping boolean value */ 2327 LLVMBuildStore(builder, temp, clipmask_bool_ptr); 2328 } 2329 else { 2330 clipmask = blduivec.zero; 2331 } 2332 2333 /* do viewport mapping */ 2334 if (!bypass_viewport) { 2335 generate_viewport(variant, builder, vs_type, outputs, context_ptr); 2336 } 2337 } 2338 else { 2339 clipmask = blduivec.zero; 2340 } 2341 2342 /* store clipmask in vertex header, 2343 * original positions in clip 2344 * and transformed positions in data 2345 */ 2346 convert_to_aos(gallivm, variant->vertex_header_type, io, NULL, outputs, clipmask, 2347 vs_info->num_outputs, vs_type, 2348 enable_cliptest && key->need_edgeflags); 2349 } 2350 lp_build_loop_end_cond(&lp_loop, count, step, LLVMIntUGE); 2351 2352 sampler->destroy(sampler); 2353 image->destroy(image); 2354 2355 /* return clipping boolean value for function */ 2356 ret = clipmask_booli8(gallivm, vs_type, clipmask_bool_ptr, 2357 enable_cliptest && key->need_edgeflags); 2358 2359 LLVMBuildRet(builder, ret); 2360 2361 gallivm_verify_function(gallivm, variant_func); 2362} 2363 2364 2365struct draw_llvm_variant_key * 2366draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 2367{ 2368 unsigned i; 2369 struct draw_llvm_variant_key *key; 2370 struct draw_sampler_static_state *draw_sampler; 2371 struct draw_image_static_state *draw_image; 2372 2373 key = (struct draw_llvm_variant_key *)store; 2374 2375 memset(key, 0, offsetof(struct draw_llvm_variant_key, vertex_element[0])); 2376 2377 2378 /* will have to rig this up properly later */ 2379 key->clip_xy = llvm->draw->clip_xy; 2380 key->clip_z = llvm->draw->clip_z; 2381 key->clip_user = llvm->draw->clip_user; 2382 key->bypass_viewport = llvm->draw->bypass_viewport; 2383 key->clip_halfz = llvm->draw->rasterizer->clip_halfz; 2384 /* XXX assumes edgeflag output not at 0 */ 2385 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 2386 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable; 2387 key->has_gs_or_tes = llvm->draw->gs.geometry_shader != NULL || llvm->draw->tes.tess_eval_shader != NULL; 2388 key->num_outputs = draw_total_vs_outputs(llvm->draw); 2389 2390 key->clamp_vertex_color = !key->has_gs_or_tes && 2391 llvm->draw->rasterizer->clamp_vertex_color; 2392 2393 /* All variants of this shader will have the same value for 2394 * nr_samplers. Not yet trying to compact away holes in the 2395 * sampler array. 2396 */ 2397 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 2398 if (llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 2399 key->nr_sampler_views = 2400 llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 2401 } 2402 else { 2403 key->nr_sampler_views = key->nr_samplers; 2404 } 2405 2406 key->nr_images = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_IMAGE] + 1; 2407 2408 /* Presumably all variants of the shader should have the same 2409 * number of vertex elements - ie the number of shader inputs. 2410 * NOTE: we NEED to store the needed number of needed inputs 2411 * here, not the number of provided elements to match keysize 2412 * (and the offset of sampler state in the key). 2413 * If we have excess number of vertex elements, this is valid, 2414 * but the excess ones don't matter. 2415 * If we don't have enough vertex elements (which looks not really 2416 * valid but we'll handle it gracefully) fill out missing ones with 2417 * zero (we'll recognize these later by PIPE_FORMAT_NONE). 2418 */ 2419 key->nr_vertex_elements = 2420 llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_INPUT] + 1; 2421 2422 if (llvm->draw->pt.nr_vertex_elements < key->nr_vertex_elements) { 2423 debug_printf("draw: vs with %d inputs but only have %d vertex elements\n", 2424 key->nr_vertex_elements, llvm->draw->pt.nr_vertex_elements); 2425 memset(key->vertex_element, 0, 2426 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 2427 } 2428 memcpy(key->vertex_element, 2429 llvm->draw->pt.vertex_element, 2430 sizeof(struct pipe_vertex_element) * 2431 MIN2(key->nr_vertex_elements, llvm->draw->pt.nr_vertex_elements)); 2432 2433 draw_sampler = draw_llvm_variant_key_samplers(key); 2434 memset(draw_sampler, 0, 2435 MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler); 2436 2437 for (i = 0 ; i < key->nr_samplers; i++) { 2438 lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state, 2439 llvm->draw->samplers[PIPE_SHADER_VERTEX][i]); 2440 } 2441 for (i = 0 ; i < key->nr_sampler_views; i++) { 2442 lp_sampler_static_texture_state(&draw_sampler[i].texture_state, 2443 llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i]); 2444 } 2445 2446 draw_image = draw_llvm_variant_key_images(key); 2447 memset(draw_image, 0, 2448 key->nr_images * sizeof *draw_image); 2449 for (i = 0; i < key->nr_images; i++) { 2450 lp_sampler_static_texture_state_image(&draw_image[i].image_state, 2451 llvm->draw->images[PIPE_SHADER_VERTEX][i]); 2452 } 2453 return key; 2454} 2455 2456 2457void 2458draw_llvm_dump_variant_key(struct draw_llvm_variant_key *key) 2459{ 2460 unsigned i; 2461 struct draw_sampler_static_state *sampler = draw_llvm_variant_key_samplers(key); 2462 struct draw_image_static_state *image = draw_llvm_variant_key_images(key); 2463 debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color); 2464 debug_printf("clip_xy = %u\n", key->clip_xy); 2465 debug_printf("clip_z = %u\n", key->clip_z); 2466 debug_printf("clip_user = %u\n", key->clip_user); 2467 debug_printf("bypass_viewport = %u\n", key->bypass_viewport); 2468 debug_printf("clip_halfz = %u\n", key->clip_halfz); 2469 debug_printf("need_edgeflags = %u\n", key->need_edgeflags); 2470 debug_printf("has_gs_or_tes = %u\n", key->has_gs_or_tes); 2471 debug_printf("ucp_enable = %u\n", key->ucp_enable); 2472 2473 for (i = 0 ; i < key->nr_vertex_elements; i++) { 2474 debug_printf("vertex_element[%i].src_offset = %u\n", i, key->vertex_element[i].src_offset); 2475 debug_printf("vertex_element[%i].instance_divisor = %u\n", i, key->vertex_element[i].instance_divisor); 2476 debug_printf("vertex_element[%i].vertex_buffer_index = %u\n", i, key->vertex_element[i].vertex_buffer_index); 2477 debug_printf("vertex_element[%i].src_format = %s\n", i, util_format_name(key->vertex_element[i].src_format)); 2478 } 2479 2480 for (i = 0 ; i < key->nr_sampler_views; i++) { 2481 debug_printf("sampler[%i].src_format = %s\n", i, util_format_name(sampler[i].texture_state.format)); 2482 } 2483 2484 for (i = 0 ; i < key->nr_images; i++) 2485 debug_printf("images[%i].format = %s\n", i, util_format_name(image[i].image_state.format)); 2486} 2487 2488 2489void 2490draw_llvm_set_mapped_texture(struct draw_context *draw, 2491 enum pipe_shader_type shader_stage, 2492 unsigned sview_idx, 2493 uint32_t width, uint32_t height, uint32_t depth, 2494 uint32_t first_level, uint32_t last_level, 2495 uint32_t num_samples, 2496 uint32_t sample_stride, 2497 const void *base_ptr, 2498 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 2499 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 2500 uint32_t mip_offsets[PIPE_MAX_TEXTURE_LEVELS]) 2501{ 2502 unsigned j; 2503 struct draw_jit_texture *jit_tex; 2504 2505 switch (shader_stage) { 2506 case PIPE_SHADER_VERTEX: 2507 assert(sview_idx < ARRAY_SIZE(draw->llvm->jit_context.textures)); 2508 jit_tex = &draw->llvm->jit_context.textures[sview_idx]; 2509 break; 2510 case PIPE_SHADER_GEOMETRY: 2511 assert(sview_idx < ARRAY_SIZE(draw->llvm->gs_jit_context.textures)); 2512 jit_tex = &draw->llvm->gs_jit_context.textures[sview_idx]; 2513 break; 2514 case PIPE_SHADER_TESS_CTRL: 2515 assert(sview_idx < ARRAY_SIZE(draw->llvm->tcs_jit_context.textures)); 2516 jit_tex = &draw->llvm->tcs_jit_context.textures[sview_idx]; 2517 break; 2518 case PIPE_SHADER_TESS_EVAL: 2519 assert(sview_idx < ARRAY_SIZE(draw->llvm->tes_jit_context.textures)); 2520 jit_tex = &draw->llvm->tes_jit_context.textures[sview_idx]; 2521 break; 2522 default: 2523 assert(0); 2524 return; 2525 } 2526 2527 jit_tex->width = width; 2528 jit_tex->height = height; 2529 jit_tex->depth = depth; 2530 jit_tex->first_level = first_level; 2531 jit_tex->last_level = last_level; 2532 jit_tex->base = base_ptr; 2533 jit_tex->num_samples = num_samples; 2534 jit_tex->sample_stride = sample_stride; 2535 2536 for (j = first_level; j <= last_level; j++) { 2537 jit_tex->mip_offsets[j] = mip_offsets[j]; 2538 jit_tex->row_stride[j] = row_stride[j]; 2539 jit_tex->img_stride[j] = img_stride[j]; 2540 } 2541} 2542 2543void 2544draw_llvm_set_mapped_image(struct draw_context *draw, 2545 enum pipe_shader_type shader_stage, 2546 unsigned idx, 2547 uint32_t width, uint32_t height, uint32_t depth, 2548 const void *base_ptr, 2549 uint32_t row_stride, 2550 uint32_t img_stride, 2551 uint32_t num_samples, 2552 uint32_t sample_stride) 2553{ 2554 struct draw_jit_image *jit_image; 2555 2556 switch (shader_stage) { 2557 case PIPE_SHADER_VERTEX: 2558 assert(idx < ARRAY_SIZE(draw->llvm->jit_context.images)); 2559 jit_image = &draw->llvm->jit_context.images[idx]; 2560 break; 2561 case PIPE_SHADER_GEOMETRY: 2562 assert(idx < ARRAY_SIZE(draw->llvm->gs_jit_context.images)); 2563 jit_image = &draw->llvm->gs_jit_context.images[idx]; 2564 break; 2565 case PIPE_SHADER_TESS_CTRL: 2566 assert(idx < ARRAY_SIZE(draw->llvm->tcs_jit_context.images)); 2567 jit_image = &draw->llvm->tcs_jit_context.images[idx]; 2568 break; 2569 case PIPE_SHADER_TESS_EVAL: 2570 assert(idx < ARRAY_SIZE(draw->llvm->tes_jit_context.images)); 2571 jit_image = &draw->llvm->tes_jit_context.images[idx]; 2572 break; 2573 default: 2574 assert(0); 2575 return; 2576 } 2577 2578 jit_image->width = width; 2579 jit_image->height = height; 2580 jit_image->depth = depth; 2581 jit_image->base = base_ptr; 2582 2583 jit_image->row_stride = row_stride; 2584 jit_image->img_stride = img_stride; 2585 jit_image->num_samples = num_samples; 2586 jit_image->sample_stride = sample_stride; 2587} 2588 2589 2590void 2591draw_llvm_set_sampler_state(struct draw_context *draw, 2592 enum pipe_shader_type shader_type) 2593{ 2594 unsigned i; 2595 2596 switch (shader_type) { 2597 case PIPE_SHADER_VERTEX: 2598 for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) { 2599 struct draw_jit_sampler *jit_sam = &draw->llvm->jit_context.samplers[i]; 2600 2601 if (draw->samplers[PIPE_SHADER_VERTEX][i]) { 2602 const struct pipe_sampler_state *s 2603 = draw->samplers[PIPE_SHADER_VERTEX][i]; 2604 jit_sam->min_lod = s->min_lod; 2605 jit_sam->max_lod = s->max_lod; 2606 jit_sam->lod_bias = s->lod_bias; 2607 jit_sam->max_aniso = s->max_anisotropy; 2608 COPY_4V(jit_sam->border_color, s->border_color.f); 2609 } 2610 } 2611 break; 2612 case PIPE_SHADER_GEOMETRY: 2613 for (i = 0; i < draw->num_samplers[PIPE_SHADER_GEOMETRY]; i++) { 2614 struct draw_jit_sampler *jit_sam = &draw->llvm->gs_jit_context.samplers[i]; 2615 2616 if (draw->samplers[PIPE_SHADER_GEOMETRY][i]) { 2617 const struct pipe_sampler_state *s 2618 = draw->samplers[PIPE_SHADER_GEOMETRY][i]; 2619 jit_sam->min_lod = s->min_lod; 2620 jit_sam->max_lod = s->max_lod; 2621 jit_sam->lod_bias = s->lod_bias; 2622 jit_sam->max_aniso = s->max_anisotropy; 2623 COPY_4V(jit_sam->border_color, s->border_color.f); 2624 } 2625 } 2626 break; 2627 case PIPE_SHADER_TESS_CTRL: 2628 for (i = 0; i < draw->num_samplers[PIPE_SHADER_TESS_CTRL]; i++) { 2629 struct draw_jit_sampler *jit_sam = &draw->llvm->tcs_jit_context.samplers[i]; 2630 2631 if (draw->samplers[PIPE_SHADER_TESS_CTRL][i]) { 2632 const struct pipe_sampler_state *s 2633 = draw->samplers[PIPE_SHADER_TESS_CTRL][i]; 2634 jit_sam->min_lod = s->min_lod; 2635 jit_sam->max_lod = s->max_lod; 2636 jit_sam->lod_bias = s->lod_bias; 2637 jit_sam->max_aniso = s->max_anisotropy; 2638 COPY_4V(jit_sam->border_color, s->border_color.f); 2639 } 2640 } 2641 break; 2642 case PIPE_SHADER_TESS_EVAL: 2643 for (i = 0; i < draw->num_samplers[PIPE_SHADER_TESS_EVAL]; i++) { 2644 struct draw_jit_sampler *jit_sam = &draw->llvm->tes_jit_context.samplers[i]; 2645 2646 if (draw->samplers[PIPE_SHADER_TESS_EVAL][i]) { 2647 const struct pipe_sampler_state *s 2648 = draw->samplers[PIPE_SHADER_TESS_EVAL][i]; 2649 jit_sam->min_lod = s->min_lod; 2650 jit_sam->max_lod = s->max_lod; 2651 jit_sam->lod_bias = s->lod_bias; 2652 jit_sam->max_aniso = s->max_anisotropy; 2653 COPY_4V(jit_sam->border_color, s->border_color.f); 2654 } 2655 } 2656 break; 2657 default: 2658 assert(0); 2659 break; 2660 } 2661} 2662 2663 2664void 2665draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 2666{ 2667 struct draw_llvm *llvm = variant->llvm; 2668 2669 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 2670 debug_printf("Deleting VS variant: %u vs variants,\t%u total variants\n", 2671 variant->shader->variants_cached, llvm->nr_variants); 2672 } 2673 2674 gallivm_destroy(variant->gallivm); 2675 2676 list_del(&variant->list_item_local.list); 2677 variant->shader->variants_cached--; 2678 list_del(&variant->list_item_global.list); 2679 llvm->nr_variants--; 2680 FREE(variant); 2681} 2682 2683 2684/** 2685 * Create LLVM types for various structures. 2686 */ 2687static void 2688create_gs_jit_types(struct draw_gs_llvm_variant *var) 2689{ 2690 struct gallivm_state *gallivm = var->gallivm; 2691 LLVMTypeRef texture_type, sampler_type, image_type; 2692 2693 texture_type = create_jit_texture_type(gallivm, "texture"); 2694 sampler_type = create_jit_sampler_type(gallivm, "sampler"); 2695 image_type = create_jit_image_type(gallivm, "image"); 2696 2697 var->context_type = create_gs_jit_context_type(gallivm, 2698 var->shader->base.vector_length, 2699 texture_type, sampler_type, 2700 image_type, 2701 "draw_gs_jit_context"); 2702 var->context_ptr_type = LLVMPointerType(var->context_type, 0); 2703 2704 var->input_array_type = create_gs_jit_input_type(gallivm); 2705} 2706 2707static LLVMTypeRef 2708get_gs_context_ptr_type(struct draw_gs_llvm_variant *variant) 2709{ 2710 if (!variant->context_ptr_type) 2711 create_gs_jit_types(variant); 2712 return variant->context_ptr_type; 2713} 2714 2715static LLVMValueRef 2716generate_mask_value(struct draw_gs_llvm_variant *variant, 2717 struct lp_type gs_type) 2718{ 2719 struct gallivm_state *gallivm = variant->gallivm; 2720 LLVMBuilderRef builder = gallivm->builder; 2721 struct lp_type mask_type = lp_int_type(gs_type); 2722 LLVMValueRef num_prims; 2723 LLVMValueRef mask_val = lp_build_const_vec(gallivm, mask_type, 0); 2724 unsigned i; 2725 2726 num_prims = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, mask_type), 2727 variant->num_prims); 2728 for (i = 0; i < gs_type.length; i++) { 2729 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 2730 mask_val = LLVMBuildInsertElement(builder, mask_val, idx, idx, ""); 2731 } 2732 mask_val = lp_build_compare(gallivm, mask_type, 2733 PIPE_FUNC_GREATER, num_prims, mask_val); 2734 2735 return mask_val; 2736} 2737 2738static void 2739draw_gs_llvm_generate(struct draw_llvm *llvm, 2740 struct draw_gs_llvm_variant *variant) 2741{ 2742 struct gallivm_state *gallivm = variant->gallivm; 2743 LLVMContextRef context = gallivm->context; 2744 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 2745 LLVMTypeRef arg_types[8]; 2746 LLVMTypeRef func_type; 2747 LLVMValueRef variant_func; 2748 LLVMValueRef context_ptr; 2749 LLVMValueRef prim_id_ptr; 2750 LLVMBasicBlockRef block; 2751 LLVMBuilderRef builder; 2752 LLVMValueRef io_ptr, input_array, num_prims, mask_val; 2753 struct lp_build_sampler_soa *sampler = 0; 2754 struct lp_build_image_soa *image = NULL; 2755 struct lp_build_context bld; 2756 struct lp_bld_tgsi_system_values system_values; 2757 char func_name[64]; 2758 struct lp_type gs_type; 2759 unsigned i; 2760 struct draw_gs_llvm_iface gs_iface; 2761 const struct tgsi_token *tokens = variant->shader->base.state.tokens; 2762 LLVMValueRef consts_ptr, num_consts_ptr; 2763 LLVMValueRef ssbos_ptr, num_ssbos_ptr; 2764 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 2765 struct lp_build_mask_context mask; 2766 const struct tgsi_shader_info *gs_info = &variant->shader->base.info; 2767 unsigned vector_length = variant->shader->base.vector_length; 2768 2769 memset(&system_values, 0, sizeof(system_values)); 2770 memset(&outputs, 0, sizeof(outputs)); 2771 2772 snprintf(func_name, sizeof(func_name), "draw_llvm_gs_variant"); 2773 2774 assert(variant->vertex_header_ptr_type); 2775 2776 LLVMTypeRef prim_id_type = LLVMVectorType(int32_type, vector_length); 2777 arg_types[0] = get_gs_context_ptr_type(variant); /* context */ 2778 arg_types[1] = variant->input_array_type; /* input */ 2779 arg_types[2] = LLVMPointerType(variant->vertex_header_ptr_type, 0); /* vertex_header */ 2780 arg_types[3] = int32_type; /* num_prims */ 2781 arg_types[4] = int32_type; /* instance_id */ 2782 arg_types[5] = LLVMPointerType(prim_id_type, 0); /* prim_id_ptr */ 2783 arg_types[6] = int32_type; 2784 arg_types[7] = int32_type; 2785 2786 func_type = LLVMFunctionType(int32_type, arg_types, ARRAY_SIZE(arg_types), 0); 2787 2788 variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); 2789 2790 variant->function = variant_func; 2791 2792 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 2793 2794 for (i = 0; i < ARRAY_SIZE(arg_types); ++i) 2795 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 2796 lp_add_function_attr(variant_func, i + 1, LP_FUNC_ATTR_NOALIAS); 2797 2798 if (gallivm->cache && gallivm->cache->data_size) 2799 return; 2800 context_ptr = LLVMGetParam(variant_func, 0); 2801 input_array = LLVMGetParam(variant_func, 1); 2802 io_ptr = LLVMGetParam(variant_func, 2); 2803 num_prims = LLVMGetParam(variant_func, 3); 2804 system_values.instance_id = LLVMGetParam(variant_func, 4); 2805 prim_id_ptr = LLVMGetParam(variant_func, 5); 2806 system_values.invocation_id = LLVMGetParam(variant_func, 6); 2807 system_values.view_index = LLVMGetParam(variant_func, 7); 2808 2809 lp_build_name(context_ptr, "context"); 2810 lp_build_name(input_array, "input"); 2811 lp_build_name(io_ptr, "io"); 2812 lp_build_name(num_prims, "num_prims"); 2813 lp_build_name(system_values.instance_id, "instance_id"); 2814 lp_build_name(prim_id_ptr, "prim_id_ptr"); 2815 lp_build_name(system_values.invocation_id, "invocation_id"); 2816 lp_build_name(system_values.view_index, "view_index"); 2817 2818 variant->context_ptr = context_ptr; 2819 variant->io_ptr = io_ptr; 2820 variant->num_prims = num_prims; 2821 2822 gs_iface.base.fetch_input = draw_gs_llvm_fetch_input; 2823 gs_iface.base.emit_vertex = draw_gs_llvm_emit_vertex; 2824 gs_iface.base.end_primitive = draw_gs_llvm_end_primitive; 2825 gs_iface.base.gs_epilogue = draw_gs_llvm_epilogue; 2826 gs_iface.input = input_array; 2827 gs_iface.variant = variant; 2828 2829 /* 2830 * Function body 2831 */ 2832 2833 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 2834 builder = gallivm->builder; 2835 LLVMPositionBuilderAtEnd(builder, block); 2836 2837 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 2838 2839 memset(&gs_type, 0, sizeof gs_type); 2840 gs_type.floating = TRUE; /* floating point values */ 2841 gs_type.sign = TRUE; /* values are signed */ 2842 gs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 2843 gs_type.width = 32; /* 32-bit float */ 2844 gs_type.length = vector_length; 2845 2846 consts_ptr = draw_gs_jit_context_constants(variant, context_ptr); 2847 num_consts_ptr = 2848 draw_gs_jit_context_num_constants(variant, context_ptr); 2849 2850 ssbos_ptr = draw_gs_jit_context_ssbos(variant, context_ptr); 2851 num_ssbos_ptr = 2852 draw_gs_jit_context_num_ssbos(variant, context_ptr); 2853 2854 /* code generated texture sampling */ 2855 sampler = draw_llvm_sampler_soa_create(variant->key.samplers, 2856 MAX2(variant->key.nr_samplers, 2857 variant->key.nr_sampler_views)); 2858 image = draw_llvm_image_soa_create(draw_gs_llvm_variant_key_images(&variant->key), 2859 variant->key.nr_images); 2860 mask_val = generate_mask_value(variant, gs_type); 2861 lp_build_mask_begin(&mask, gallivm, gs_type, mask_val); 2862 2863 if (gs_info->uses_primid) { 2864 system_values.prim_id = LLVMBuildLoad2(builder, prim_id_type, prim_id_ptr, "prim_id"); 2865 } 2866 2867 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 2868 if (llvm->draw->gs.geometry_shader->state.type == PIPE_SHADER_IR_TGSI) 2869 tgsi_dump(tokens, 0); 2870 else 2871 nir_print_shader(llvm->draw->gs.geometry_shader->state.ir.nir, stderr); 2872 draw_gs_llvm_dump_variant_key(&variant->key); 2873 } 2874 2875 struct lp_build_tgsi_params params; 2876 memset(¶ms, 0, sizeof(params)); 2877 2878 params.type = gs_type; 2879 params.mask = &mask; 2880 params.consts_ptr = consts_ptr; 2881 params.const_sizes_ptr = num_consts_ptr; 2882 params.system_values = &system_values; 2883 params.context_ptr = context_ptr; 2884 params.sampler = sampler; 2885 params.info = &llvm->draw->gs.geometry_shader->info; 2886 params.gs_iface = (const struct lp_build_gs_iface *)&gs_iface; 2887 params.ssbo_ptr = ssbos_ptr; 2888 params.ssbo_sizes_ptr = num_ssbos_ptr; 2889 params.image = image; 2890 params.gs_vertex_streams = variant->shader->base.num_vertex_streams; 2891 params.aniso_filter_table = draw_gs_jit_context_aniso_filter_table(variant, context_ptr); 2892 2893 if (llvm->draw->gs.geometry_shader->state.type == PIPE_SHADER_IR_TGSI) 2894 lp_build_tgsi_soa(variant->gallivm, 2895 tokens, 2896 ¶ms, 2897 outputs); 2898 else 2899 lp_build_nir_soa(variant->gallivm, 2900 llvm->draw->gs.geometry_shader->state.ir.nir, 2901 ¶ms, 2902 outputs); 2903 2904 sampler->destroy(sampler); 2905 image->destroy(image); 2906 2907 lp_build_mask_end(&mask); 2908 2909 LLVMBuildRet(builder, lp_build_zero(gallivm, lp_type_uint(32))); 2910 2911 gallivm_verify_function(gallivm, variant_func); 2912} 2913 2914struct draw_gs_llvm_variant * 2915draw_gs_llvm_create_variant(struct draw_llvm *llvm, 2916 unsigned num_outputs, 2917 const struct draw_gs_llvm_variant_key *key) 2918{ 2919 struct draw_gs_llvm_variant *variant; 2920 struct llvm_geometry_shader *shader = 2921 llvm_geometry_shader(llvm->draw->gs.geometry_shader); 2922 char module_name[64]; 2923 unsigned char ir_sha1_cache_key[20]; 2924 struct lp_cached_code cached = { 0 }; 2925 bool needs_caching = false; 2926 2927 variant = MALLOC(sizeof *variant + 2928 shader->variant_key_size - 2929 sizeof variant->key); 2930 if (!variant) 2931 return NULL; 2932 2933 variant->llvm = llvm; 2934 variant->shader = shader; 2935 2936 snprintf(module_name, sizeof(module_name), "draw_llvm_gs_variant%u", 2937 variant->shader->variants_cached); 2938 2939 memcpy(&variant->key, key, shader->variant_key_size); 2940 2941 if (shader->base.state.ir.nir && llvm->draw->disk_cache_cookie) { 2942 draw_get_ir_cache_key(shader->base.state.ir.nir, 2943 key, 2944 shader->variant_key_size, 2945 num_outputs, 2946 ir_sha1_cache_key); 2947 2948 llvm->draw->disk_cache_find_shader(llvm->draw->disk_cache_cookie, 2949 &cached, 2950 ir_sha1_cache_key); 2951 if (!cached.data_size) 2952 needs_caching = true; 2953 } 2954 variant->gallivm = gallivm_create(module_name, llvm->context, &cached); 2955 2956 create_gs_jit_types(variant); 2957 2958 variant->vertex_header_type = create_jit_vertex_header(variant->gallivm, num_outputs); 2959 variant->vertex_header_ptr_type = LLVMPointerType(variant->vertex_header_type, 0); 2960 2961 draw_gs_llvm_generate(llvm, variant); 2962 2963 gallivm_compile_module(variant->gallivm); 2964 2965 variant->jit_func = (draw_gs_jit_func) 2966 gallivm_jit_function(variant->gallivm, variant->function); 2967 2968 if (needs_caching) 2969 llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, 2970 &cached, 2971 ir_sha1_cache_key); 2972 gallivm_free_ir(variant->gallivm); 2973 2974 variant->list_item_global.base = variant; 2975 variant->list_item_local.base = variant; 2976 /*variant->no = */shader->variants_created++; 2977 variant->list_item_global.base = variant; 2978 2979 return variant; 2980} 2981 2982void 2983draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant) 2984{ 2985 struct draw_llvm *llvm = variant->llvm; 2986 2987 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 2988 debug_printf("Deleting GS variant: %u gs variants,\t%u total variants\n", 2989 variant->shader->variants_cached, llvm->nr_gs_variants); 2990 } 2991 2992 gallivm_destroy(variant->gallivm); 2993 2994 list_del(&variant->list_item_local.list); 2995 variant->shader->variants_cached--; 2996 list_del(&variant->list_item_global.list); 2997 llvm->nr_gs_variants--; 2998 FREE(variant); 2999} 3000 3001struct draw_gs_llvm_variant_key * 3002draw_gs_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 3003{ 3004 unsigned i; 3005 struct draw_gs_llvm_variant_key *key; 3006 struct draw_sampler_static_state *draw_sampler; 3007 struct draw_image_static_state *draw_image; 3008 3009 key = (struct draw_gs_llvm_variant_key *)store; 3010 3011 memset(key, 0, offsetof(struct draw_gs_llvm_variant_key, samplers[0])); 3012 3013 key->num_outputs = draw_total_gs_outputs(llvm->draw); 3014 3015 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; 3016 3017 /* All variants of this shader will have the same value for 3018 * nr_samplers. Not yet trying to compact away holes in the 3019 * sampler array. 3020 */ 3021 key->nr_samplers = llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 3022 if (llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 3023 key->nr_sampler_views = 3024 llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 3025 } 3026 else { 3027 key->nr_sampler_views = key->nr_samplers; 3028 } 3029 3030 key->nr_images = llvm->draw->gs.geometry_shader->info.file_max[TGSI_FILE_IMAGE] + 1; 3031 3032 draw_sampler = key->samplers; 3033 3034 memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler); 3035 3036 for (i = 0 ; i < key->nr_samplers; i++) { 3037 lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state, 3038 llvm->draw->samplers[PIPE_SHADER_GEOMETRY][i]); 3039 } 3040 for (i = 0 ; i < key->nr_sampler_views; i++) { 3041 lp_sampler_static_texture_state(&draw_sampler[i].texture_state, 3042 llvm->draw->sampler_views[PIPE_SHADER_GEOMETRY][i]); 3043 } 3044 3045 draw_image = draw_gs_llvm_variant_key_images(key); 3046 memset(draw_image, 0, 3047 key->nr_images * sizeof *draw_image); 3048 for (i = 0; i < key->nr_images; i++) { 3049 lp_sampler_static_texture_state_image(&draw_image[i].image_state, 3050 llvm->draw->images[PIPE_SHADER_GEOMETRY][i]); 3051 } 3052 return key; 3053} 3054 3055void 3056draw_gs_llvm_dump_variant_key(struct draw_gs_llvm_variant_key *key) 3057{ 3058 unsigned i; 3059 struct draw_sampler_static_state *sampler = key->samplers; 3060 struct draw_image_static_state *image = draw_gs_llvm_variant_key_images(key); 3061 3062 debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color); 3063 for (i = 0 ; i < key->nr_sampler_views; i++) { 3064 debug_printf("sampler[%i].src_format = %s\n", i, 3065 util_format_name(sampler[i].texture_state.format)); 3066 } 3067 3068 for (i = 0 ; i < key->nr_images; i++) 3069 debug_printf("images[%i].format = %s\n", i, util_format_name(image[i].image_state.format)); 3070 3071} 3072 3073static void 3074create_tcs_jit_types(struct draw_tcs_llvm_variant *var) 3075{ 3076 struct gallivm_state *gallivm = var->gallivm; 3077 LLVMTypeRef texture_type, sampler_type, image_type; 3078 3079 texture_type = create_jit_texture_type(gallivm, "texture"); 3080 sampler_type = create_jit_sampler_type(gallivm, "sampler"); 3081 image_type = create_jit_image_type(gallivm, "image"); 3082 3083 var->context_type = create_tcs_jit_context_type(gallivm, 3084 0, 3085 texture_type, sampler_type, 3086 image_type, 3087 "draw_tcs_jit_context"); 3088 var->input_array_type = create_tcs_jit_input_type(gallivm); 3089 var->output_array_type = create_tcs_jit_output_type(gallivm); 3090 var->context_ptr_type = LLVMPointerType(var->context_type, 0); 3091} 3092 3093static LLVMTypeRef 3094get_tcs_context_ptr_type(struct draw_tcs_llvm_variant *variant) 3095{ 3096 if (!variant->context_ptr_type) 3097 create_tcs_jit_types(variant); 3098 return variant->context_ptr_type; 3099} 3100 3101static LLVMValueRef 3102draw_tcs_llvm_emit_fetch_input(const struct lp_build_tcs_iface *tes_iface, 3103 struct lp_build_context *bld, 3104 boolean is_vindex_indirect, 3105 LLVMValueRef vertex_index, 3106 boolean is_aindex_indirect, 3107 LLVMValueRef attrib_index, 3108 boolean is_sindex_indirect, 3109 LLVMValueRef swizzle_index) 3110{ 3111 const struct draw_tcs_llvm_iface *tcs = draw_tcs_llvm_iface(tes_iface); 3112 struct gallivm_state *gallivm = bld->gallivm; 3113 LLVMBuilderRef builder = gallivm->builder; 3114 LLVMValueRef indices[3]; 3115 LLVMValueRef res; 3116 struct lp_type type = bld->type; 3117 3118 if (is_vindex_indirect || is_aindex_indirect || is_sindex_indirect) { 3119 int i; 3120 3121 res = bld->zero; 3122 for (i = 0; i < type.length; ++i) { 3123 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3124 LLVMValueRef vert_chan_index = vertex_index; 3125 LLVMValueRef attr_chan_index = attrib_index; 3126 LLVMValueRef swiz_chan_index = swizzle_index; 3127 LLVMValueRef channel_vec; 3128 3129 if (is_vindex_indirect) { 3130 vert_chan_index = LLVMBuildExtractElement(builder, 3131 vertex_index, idx, ""); 3132 } 3133 if (is_aindex_indirect) { 3134 attr_chan_index = LLVMBuildExtractElement(builder, 3135 attrib_index, idx, ""); 3136 } 3137 if (is_sindex_indirect) { 3138 swiz_chan_index = LLVMBuildExtractElement(builder, 3139 swizzle_index, idx, ""); 3140 } 3141 3142 indices[0] = vert_chan_index; 3143 indices[1] = attr_chan_index; 3144 indices[2] = swiz_chan_index; 3145 3146 channel_vec = LLVMBuildGEP(builder, tcs->input, indices, 3, ""); 3147 channel_vec = LLVMBuildLoad(builder, channel_vec, ""); 3148 3149 res = LLVMBuildInsertElement(builder, res, channel_vec, idx, ""); 3150 } 3151 } else { 3152 indices[0] = vertex_index; 3153 indices[1] = attrib_index; 3154 indices[2] = swizzle_index; 3155 3156 res = LLVMBuildGEP(builder, tcs->input, indices, 3, ""); 3157 res = LLVMBuildLoad(builder, res, ""); 3158 res = lp_build_broadcast_scalar(bld, res); 3159 } 3160 return res; 3161} 3162 3163static LLVMValueRef 3164draw_tcs_llvm_emit_fetch_output(const struct lp_build_tcs_iface *tes_iface, 3165 struct lp_build_context *bld, 3166 boolean is_vindex_indirect, 3167 LLVMValueRef vertex_index, 3168 boolean is_aindex_indirect, 3169 LLVMValueRef attrib_index, 3170 boolean is_sindex_indirect, 3171 LLVMValueRef swizzle_index, 3172 uint32_t name) 3173{ 3174 const struct draw_tcs_llvm_iface *tcs = draw_tcs_llvm_iface(tes_iface); 3175 struct gallivm_state *gallivm = bld->gallivm; 3176 LLVMBuilderRef builder = gallivm->builder; 3177 LLVMValueRef indices[3]; 3178 LLVMValueRef res; 3179 struct lp_type type = bld->type; 3180 3181 if (is_vindex_indirect || is_aindex_indirect || is_sindex_indirect) { 3182 int i; 3183 3184 res = bld->zero; 3185 for (i = 0; i < type.length; ++i) { 3186 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3187 LLVMValueRef vert_chan_index = vertex_index; 3188 LLVMValueRef attr_chan_index = attrib_index; 3189 LLVMValueRef swiz_chan_index = swizzle_index; 3190 LLVMValueRef channel_vec; 3191 3192 if (is_vindex_indirect) { 3193 vert_chan_index = LLVMBuildExtractElement(builder, 3194 vertex_index, idx, ""); 3195 } 3196 if (is_aindex_indirect) { 3197 attr_chan_index = LLVMBuildExtractElement(builder, 3198 attrib_index, idx, ""); 3199 } 3200 if (is_sindex_indirect) { 3201 swiz_chan_index = LLVMBuildExtractElement(builder, 3202 swizzle_index, idx, ""); 3203 } 3204 3205 indices[0] = vert_chan_index; 3206 indices[1] = attr_chan_index; 3207 indices[2] = swiz_chan_index; 3208 3209 channel_vec = LLVMBuildGEP(builder, tcs->output, indices, 3, ""); 3210 channel_vec = LLVMBuildLoad(builder, channel_vec, ""); 3211 3212 res = LLVMBuildInsertElement(builder, res, channel_vec, idx, ""); 3213 } 3214 } else { 3215 indices[0] = vertex_index ? vertex_index : lp_build_const_int32(gallivm, 0); 3216 indices[1] = attrib_index; 3217 indices[2] = swizzle_index; 3218 3219 res = LLVMBuildGEP(builder, tcs->output, indices, 3, ""); 3220 res = LLVMBuildLoad(builder, res, ""); 3221 res = lp_build_broadcast_scalar(bld, res); 3222 } 3223 return res; 3224} 3225 3226static void 3227draw_tcs_llvm_emit_store_output(const struct lp_build_tcs_iface *tes_iface, 3228 struct lp_build_context *bld, 3229 unsigned name, 3230 boolean is_vindex_indirect, 3231 LLVMValueRef vertex_index, 3232 boolean is_aindex_indirect, 3233 LLVMValueRef attrib_index, 3234 boolean is_sindex_indirect, 3235 LLVMValueRef swizzle_index, 3236 LLVMValueRef value, 3237 LLVMValueRef mask_vec) 3238{ 3239 const struct draw_tcs_llvm_iface *tcs = draw_tcs_llvm_iface(tes_iface); 3240 struct gallivm_state *gallivm = bld->gallivm; 3241 LLVMBuilderRef builder = gallivm->builder; 3242 LLVMValueRef indices[3]; 3243 LLVMValueRef res; 3244 struct lp_type type = bld->type; 3245 3246 if (is_vindex_indirect || is_aindex_indirect || is_sindex_indirect) { 3247 int i; 3248 3249 for (i = 0; i < type.length; ++i) { 3250 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3251 LLVMValueRef vert_chan_index = vertex_index ? vertex_index : lp_build_const_int32(gallivm, 0); 3252 LLVMValueRef attr_chan_index = attrib_index; 3253 LLVMValueRef swiz_chan_index = swizzle_index; 3254 LLVMValueRef channel_vec; 3255 3256 if (is_vindex_indirect) { 3257 vert_chan_index = LLVMBuildExtractElement(builder, 3258 vertex_index, idx, ""); 3259 } 3260 if (is_aindex_indirect) { 3261 attr_chan_index = LLVMBuildExtractElement(builder, 3262 attrib_index, idx, ""); 3263 } 3264 3265 if (is_sindex_indirect) { 3266 swiz_chan_index = LLVMBuildExtractElement(builder, 3267 swizzle_index, idx, ""); 3268 } 3269 3270 indices[0] = vert_chan_index; 3271 indices[1] = attr_chan_index; 3272 indices[2] = swiz_chan_index; 3273 3274 channel_vec = LLVMBuildGEP(builder, tcs->output, indices, 3, ""); 3275 3276 res = LLVMBuildExtractElement(builder, value, idx, ""); 3277 3278 struct lp_build_if_state ifthen; 3279 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, mask_vec, lp_build_const_int_vec(gallivm, bld->type, 0), ""); 3280 cond = LLVMBuildExtractElement(gallivm->builder, cond, idx, ""); 3281 lp_build_if(&ifthen, gallivm, cond); 3282 LLVMBuildStore(builder, res, channel_vec); 3283 lp_build_endif(&ifthen); 3284 } 3285 } else { 3286 indices[0] = vertex_index ? vertex_index : lp_build_const_int32(gallivm, 0); 3287 indices[1] = attrib_index; 3288 indices[2] = swizzle_index; 3289 3290 res = LLVMBuildGEP(builder, tcs->output, indices, 3, ""); 3291 for (unsigned i = 0; i < type.length; ++i) { 3292 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3293 LLVMValueRef val = LLVMBuildExtractElement(builder, value, idx, ""); 3294 3295 struct lp_build_if_state ifthen; 3296 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, mask_vec, lp_build_const_int_vec(gallivm, bld->type, 0), ""); 3297 cond = LLVMBuildExtractElement(gallivm->builder, cond, idx, ""); 3298 lp_build_if(&ifthen, gallivm, cond); 3299 LLVMBuildStore(builder, val, res); 3300 lp_build_endif(&ifthen); 3301 } 3302 } 3303} 3304 3305 3306static LLVMValueRef 3307generate_tcs_mask_value(struct draw_tcs_llvm_variant *variant, 3308 struct lp_type tcs_type, LLVMValueRef limit, LLVMValueRef loop_counter) 3309{ 3310 struct gallivm_state *gallivm = variant->gallivm; 3311 LLVMBuilderRef builder = gallivm->builder; 3312 struct lp_type mask_type = lp_int_type(tcs_type); 3313 LLVMValueRef num_vecs; 3314 LLVMValueRef mask_val = lp_build_const_vec(gallivm, mask_type, 0); 3315 unsigned i; 3316 3317 num_vecs = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, mask_type), limit); 3318 for (i = 0; i < tcs_type.length; i++) { 3319 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3320 mask_val = LLVMBuildInsertElement(builder, mask_val, LLVMBuildAdd(builder, loop_counter, idx, ""), idx, ""); 3321 } 3322 mask_val = lp_build_compare(gallivm, mask_type, 3323 PIPE_FUNC_GREATER, num_vecs, mask_val); 3324 3325 return mask_val; 3326} 3327 3328static void 3329draw_tcs_llvm_generate(struct draw_llvm *llvm, 3330 struct draw_tcs_llvm_variant *variant) 3331{ 3332 struct gallivm_state *gallivm = variant->gallivm; 3333 LLVMContextRef context = gallivm->context; 3334 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 3335 LLVMTypeRef arg_types[7]; 3336 LLVMTypeRef func_type, coro_func_type; 3337 LLVMValueRef variant_func, variant_coro; 3338 LLVMValueRef context_ptr; 3339 LLVMValueRef view_index; 3340 LLVMValueRef input_array, output_array, prim_id, patch_vertices_in; 3341 LLVMValueRef mask_val; 3342 LLVMBasicBlockRef block; 3343 LLVMBuilderRef builder; 3344 struct lp_build_context bld, bldvec; 3345 struct lp_build_sampler_soa *sampler = 0; 3346 struct lp_build_image_soa *image = NULL; 3347 struct lp_bld_tgsi_system_values system_values; 3348 char func_name[64], func_name_coro[64]; 3349 unsigned i; 3350 struct draw_tcs_llvm_iface tcs_iface; 3351 struct lp_build_mask_context mask; 3352 LLVMValueRef consts_ptr, num_consts_ptr; 3353 LLVMValueRef ssbos_ptr, num_ssbos_ptr; 3354 struct lp_type tcs_type; 3355 unsigned vector_length = variant->shader->base.vector_length; 3356 3357 memset(&system_values, 0, sizeof(system_values)); 3358 3359 snprintf(func_name, sizeof(func_name), "draw_llvm_tcs_variant"); 3360 3361 snprintf(func_name_coro, sizeof(func_name_coro), "draw_llvm_tcs_coro_variant"); 3362 3363 arg_types[0] = get_tcs_context_ptr_type(variant); /* context */ 3364 arg_types[1] = variant->input_array_type; /* input */ 3365 arg_types[2] = variant->output_array_type; 3366 arg_types[3] = int32_type; 3367 arg_types[4] = int32_type; 3368 arg_types[5] = int32_type; 3369 arg_types[6] = int32_type; /* coroutine only */ 3370 3371 func_type = LLVMFunctionType(int32_type, arg_types, ARRAY_SIZE(arg_types) - 1, 0); 3372 3373 coro_func_type = LLVMFunctionType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), arg_types, ARRAY_SIZE(arg_types), 0); 3374 3375 variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); 3376 3377 variant_coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type); 3378 3379 variant->function = variant_func; 3380 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 3381 3382 LLVMSetFunctionCallConv(variant_coro, LLVMCCallConv); 3383 3384 lp_build_coro_add_presplit(variant_coro); 3385 3386 for (i = 0; i < ARRAY_SIZE(arg_types); ++i) { 3387 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) { 3388 lp_add_function_attr(variant_coro, i + 1, LP_FUNC_ATTR_NOALIAS); 3389 lp_add_function_attr(variant_func, i + 1, LP_FUNC_ATTR_NOALIAS); 3390 } 3391 } 3392 3393 if (gallivm->cache && gallivm->cache->data_size) 3394 return; 3395 context_ptr = LLVMGetParam(variant_func, 0); 3396 input_array = LLVMGetParam(variant_func, 1); 3397 output_array = LLVMGetParam(variant_func, 2); 3398 prim_id = LLVMGetParam(variant_func, 3); 3399 patch_vertices_in = LLVMGetParam(variant_func, 4); 3400 view_index = LLVMGetParam(variant_func, 5); 3401 3402 lp_build_name(context_ptr, "context"); 3403 lp_build_name(input_array, "input"); 3404 lp_build_name(output_array, "output"); 3405 lp_build_name(prim_id, "prim_id"); 3406 lp_build_name(patch_vertices_in, "patch_vertices_in"); 3407 lp_build_name(view_index, "view_index"); 3408 3409 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 3410 builder = gallivm->builder; 3411 LLVMPositionBuilderAtEnd(builder, block); 3412 3413 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 3414 3415 memset(&tcs_type, 0, sizeof tcs_type); 3416 tcs_type.floating = TRUE; /* floating point values */ 3417 tcs_type.sign = TRUE; /* values are signed */ 3418 tcs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 3419 tcs_type.width = 32; /* 32-bit float */ 3420 tcs_type.length = vector_length; 3421 3422 lp_build_context_init(&bldvec, variant->gallivm, lp_int_type(tcs_type)); 3423 3424 LLVMValueRef count = lp_build_const_int32(gallivm, variant->shader->base.vertices_out); 3425 LLVMValueRef step = lp_build_const_int32(gallivm, vector_length); 3426 3427 struct lp_build_loop_state loop_state[2]; 3428 LLVMValueRef num_inner_loop; 3429 unsigned count_align = util_align_npot(variant->shader->base.vertices_out, tcs_type.length); 3430 num_inner_loop = lp_build_const_int32(gallivm, count_align / tcs_type.length); 3431 LLVMTypeRef hdl_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); 3432 LLVMValueRef coro_hdls = LLVMBuildArrayAlloca(gallivm->builder, hdl_ptr_type, num_inner_loop, "coro_hdls"); 3433 unsigned end_coroutine = INT_MAX; 3434 lp_build_loop_begin(&loop_state[1], gallivm, 3435 lp_build_const_int32(gallivm, 0)); /* coroutine reentry loop */ 3436 lp_build_loop_begin(&loop_state[0], gallivm, 3437 lp_build_const_int32(gallivm, 0)); /* inner loop */ 3438 { 3439 LLVMValueRef args[7]; 3440 args[0] = context_ptr; 3441 args[1] = input_array; 3442 args[2] = output_array; 3443 args[3] = prim_id; 3444 args[4] = patch_vertices_in; 3445 args[5] = view_index; 3446 args[6] = loop_state[0].counter; 3447 LLVMValueRef coro_entry = LLVMBuildGEP2(builder, hdl_ptr_type, coro_hdls, &loop_state[0].counter, 1, ""); 3448 LLVMValueRef coro_hdl = LLVMBuildLoad2(builder, hdl_ptr_type, coro_entry, "coro_hdl"); 3449 3450 struct lp_build_if_state ifstate; 3451 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntEQ, loop_state[1].counter, 3452 lp_build_const_int32(gallivm, 0), ""); 3453 /* first time here - call the coroutine function entry point */ 3454 lp_build_if(&ifstate, gallivm, cmp); 3455 LLVMValueRef coro_ret = LLVMBuildCall2(builder, coro_func_type, variant_coro, args, 7, ""); 3456 LLVMBuildStore(builder, coro_ret, coro_entry); 3457 lp_build_else(&ifstate); 3458 /* subsequent calls for this invocation - check if done. */ 3459 LLVMValueRef coro_done = lp_build_coro_done(gallivm, coro_hdl); 3460 struct lp_build_if_state ifstate2; 3461 lp_build_if(&ifstate2, gallivm, coro_done); 3462 /* if done destroy and force loop exit */ 3463 lp_build_coro_destroy(gallivm, coro_hdl); 3464 lp_build_loop_force_set_counter(&loop_state[1], lp_build_const_int32(gallivm, end_coroutine - 1)); 3465 lp_build_else(&ifstate2); 3466 /* otherwise resume the coroutine */ 3467 lp_build_coro_resume(gallivm, coro_hdl); 3468 lp_build_endif(&ifstate2); 3469 lp_build_endif(&ifstate); 3470 lp_build_loop_force_reload_counter(&loop_state[1]); 3471 } 3472 lp_build_loop_end_cond(&loop_state[0], 3473 num_inner_loop, 3474 NULL, LLVMIntUGE); 3475 lp_build_loop_end_cond(&loop_state[1], 3476 lp_build_const_int32(gallivm, end_coroutine), 3477 NULL, LLVMIntEQ); 3478 LLVMBuildRet(builder, lp_build_zero(gallivm, lp_type_uint(32))); 3479 3480 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_coro, "entry"); 3481 LLVMPositionBuilderAtEnd(builder, block); 3482 3483 context_ptr = LLVMGetParam(variant_coro, 0); 3484 input_array = LLVMGetParam(variant_coro, 1); 3485 output_array = LLVMGetParam(variant_coro, 2); 3486 prim_id = LLVMGetParam(variant_coro, 3); 3487 patch_vertices_in = LLVMGetParam(variant_coro, 4); 3488 view_index = LLVMGetParam(variant_coro, 5); 3489 3490 consts_ptr = draw_tcs_jit_context_constants(variant, context_ptr); 3491 num_consts_ptr = 3492 draw_tcs_jit_context_num_constants(variant, context_ptr); 3493 3494 ssbos_ptr = draw_tcs_jit_context_ssbos(variant, context_ptr); 3495 num_ssbos_ptr = 3496 draw_tcs_jit_context_num_ssbos(variant, context_ptr); 3497 sampler = draw_llvm_sampler_soa_create(variant->key.samplers, 3498 MAX2(variant->key.nr_samplers, 3499 variant->key.nr_sampler_views)); 3500 image = draw_llvm_image_soa_create(draw_tcs_llvm_variant_key_images(&variant->key), 3501 variant->key.nr_images); 3502 3503 LLVMValueRef counter = LLVMGetParam(variant_coro, 6); 3504 LLVMValueRef invocvec = LLVMGetUndef(LLVMVectorType(int32_type, vector_length)); 3505 for (i = 0; i < vector_length; i++) { 3506 LLVMValueRef loop_iter = lp_build_const_int32(gallivm, i); 3507 LLVMValueRef idx = LLVMBuildAdd(builder, LLVMBuildMul(builder, counter, step, ""), loop_iter, ""); 3508 invocvec = LLVMBuildInsertElement(builder, invocvec, idx, loop_iter, ""); 3509 } 3510 3511 system_values.invocation_id = invocvec; 3512 system_values.prim_id = lp_build_broadcast_scalar(&bldvec, prim_id); 3513 system_values.view_index = view_index; 3514 system_values.vertices_in = lp_build_broadcast_scalar(&bldvec, patch_vertices_in); 3515 tcs_iface.input = input_array; 3516 tcs_iface.output = output_array; 3517 tcs_iface.base.emit_fetch_input = draw_tcs_llvm_emit_fetch_input; 3518 tcs_iface.base.emit_fetch_output = draw_tcs_llvm_emit_fetch_output; 3519 tcs_iface.base.emit_store_output = draw_tcs_llvm_emit_store_output; 3520 3521 3522 { 3523 LLVMValueRef coro_id = lp_build_coro_id(gallivm); 3524 LLVMValueRef coro_hdl = lp_build_coro_begin_alloc_mem(gallivm, coro_id); 3525 3526 mask_val = generate_tcs_mask_value(variant, tcs_type, count, LLVMBuildMul(builder, counter, step, "")); 3527 lp_build_mask_begin(&mask, gallivm, tcs_type, mask_val); 3528 3529 struct lp_build_coro_suspend_info coro_info; 3530 3531 LLVMBasicBlockRef sus_block = LLVMAppendBasicBlockInContext(gallivm->context, variant_coro, "suspend"); 3532 LLVMBasicBlockRef clean_block = LLVMAppendBasicBlockInContext(gallivm->context, variant_coro, "cleanup"); 3533 3534 coro_info.suspend = sus_block; 3535 coro_info.cleanup = clean_block; 3536 3537 struct lp_build_tgsi_params params; 3538 memset(¶ms, 0, sizeof(params)); 3539 3540 params.type = tcs_type; 3541 params.mask = &mask; 3542 params.consts_ptr = consts_ptr; 3543 params.const_sizes_ptr = num_consts_ptr; 3544 params.system_values = &system_values; 3545 params.context_ptr = context_ptr; 3546 params.sampler = sampler; 3547 params.info = &llvm->draw->tcs.tess_ctrl_shader->info; 3548 params.ssbo_ptr = ssbos_ptr; 3549 params.ssbo_sizes_ptr = num_ssbos_ptr; 3550 params.image = image; 3551 params.coro = &coro_info; 3552 params.tcs_iface = &tcs_iface.base; 3553 params.aniso_filter_table = draw_tcs_jit_context_aniso_filter_table(variant, context_ptr); 3554 3555 lp_build_nir_soa(variant->gallivm, 3556 llvm->draw->tcs.tess_ctrl_shader->state.ir.nir, 3557 ¶ms, NULL); 3558 3559 lp_build_mask_end(&mask); 3560 3561 lp_build_coro_suspend_switch(gallivm, &coro_info, NULL, true); 3562 LLVMPositionBuilderAtEnd(builder, clean_block); 3563 3564 lp_build_coro_free_mem(gallivm, coro_id, coro_hdl); 3565 3566 LLVMBuildBr(builder, sus_block); 3567 LLVMPositionBuilderAtEnd(builder, sus_block); 3568 3569 lp_build_coro_end(gallivm, coro_hdl); 3570 LLVMBuildRet(builder, coro_hdl); 3571 } 3572 3573 sampler->destroy(sampler); 3574 image->destroy(image); 3575 gallivm_verify_function(gallivm, variant_func); 3576 gallivm_verify_function(gallivm, variant_coro); 3577} 3578 3579struct draw_tcs_llvm_variant * 3580draw_tcs_llvm_create_variant(struct draw_llvm *llvm, 3581 unsigned num_outputs, 3582 const struct draw_tcs_llvm_variant_key *key) 3583{ 3584 struct draw_tcs_llvm_variant *variant; 3585 struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(llvm->draw->tcs.tess_ctrl_shader); 3586 char module_name[64]; 3587 unsigned char ir_sha1_cache_key[20]; 3588 struct lp_cached_code cached = { 0 }; 3589 bool needs_caching = false; 3590 3591 variant = MALLOC(sizeof *variant + 3592 shader->variant_key_size - sizeof variant->key); 3593 if (!variant) 3594 return NULL; 3595 3596 variant->llvm = llvm; 3597 variant->shader = shader; 3598 3599 snprintf(module_name, sizeof(module_name), "draw_llvm_tcs_variant%u", 3600 variant->shader->variants_cached); 3601 3602 memcpy(&variant->key, key, shader->variant_key_size); 3603 3604 if (shader->base.state.ir.nir && llvm->draw->disk_cache_cookie) { 3605 draw_get_ir_cache_key(shader->base.state.ir.nir, 3606 key, 3607 shader->variant_key_size, 3608 num_outputs, 3609 ir_sha1_cache_key); 3610 3611 llvm->draw->disk_cache_find_shader(llvm->draw->disk_cache_cookie, 3612 &cached, 3613 ir_sha1_cache_key); 3614 if (!cached.data_size) 3615 needs_caching = true; 3616 } 3617 3618 variant->gallivm = gallivm_create(module_name, llvm->context, &cached); 3619 3620 create_tcs_jit_types(variant); 3621 3622 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 3623 nir_print_shader(llvm->draw->tcs.tess_ctrl_shader->state.ir.nir, stderr); 3624 draw_tcs_llvm_dump_variant_key(&variant->key); 3625 } 3626 3627 draw_tcs_llvm_generate(llvm, variant); 3628 3629 gallivm_compile_module(variant->gallivm); 3630 3631 variant->jit_func = (draw_tcs_jit_func) 3632 gallivm_jit_function(variant->gallivm, variant->function); 3633 3634 if (needs_caching) 3635 llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, 3636 &cached, 3637 ir_sha1_cache_key); 3638 gallivm_free_ir(variant->gallivm); 3639 3640 variant->list_item_global.base = variant; 3641 variant->list_item_local.base = variant; 3642 /*variant->no = */shader->variants_created++; 3643 variant->list_item_global.base = variant; 3644 3645 return variant; 3646} 3647 3648void 3649draw_tcs_llvm_destroy_variant(struct draw_tcs_llvm_variant *variant) 3650{ 3651 struct draw_llvm *llvm = variant->llvm; 3652 3653 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 3654 debug_printf("Deleting TCS variant: %u tcs variants,\t%u total variants\n", 3655 variant->shader->variants_cached, llvm->nr_tcs_variants); 3656 } 3657 3658 gallivm_destroy(variant->gallivm); 3659 3660 list_del(&variant->list_item_local.list); 3661 variant->shader->variants_cached--; 3662 list_del(&variant->list_item_global.list); 3663 llvm->nr_tcs_variants--; 3664 FREE(variant); 3665} 3666 3667struct draw_tcs_llvm_variant_key * 3668draw_tcs_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 3669{ 3670 unsigned i; 3671 struct draw_tcs_llvm_variant_key *key; 3672 struct draw_sampler_static_state *draw_sampler; 3673 struct draw_image_static_state *draw_image; 3674 3675 key = (struct draw_tcs_llvm_variant_key *)store; 3676 3677 memset(key, 0, offsetof(struct draw_tcs_llvm_variant_key, samplers[0])); 3678 3679 /* All variants of this shader will have the same value for 3680 * nr_samplers. Not yet trying to compact away holes in the 3681 * sampler array. 3682 */ 3683 key->nr_samplers = llvm->draw->tcs.tess_ctrl_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 3684 if (llvm->draw->tcs.tess_ctrl_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 3685 key->nr_sampler_views = 3686 llvm->draw->tcs.tess_ctrl_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 3687 } 3688 else { 3689 key->nr_sampler_views = key->nr_samplers; 3690 } 3691 3692 key->nr_images = llvm->draw->tcs.tess_ctrl_shader->info.file_max[TGSI_FILE_IMAGE] + 1; 3693 3694 draw_sampler = key->samplers; 3695 3696 memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler); 3697 3698 for (i = 0 ; i < key->nr_samplers; i++) { 3699 lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state, 3700 llvm->draw->samplers[PIPE_SHADER_TESS_CTRL][i]); 3701 } 3702 for (i = 0 ; i < key->nr_sampler_views; i++) { 3703 lp_sampler_static_texture_state(&draw_sampler[i].texture_state, 3704 llvm->draw->sampler_views[PIPE_SHADER_TESS_CTRL][i]); 3705 } 3706 3707 draw_image = draw_tcs_llvm_variant_key_images(key); 3708 memset(draw_image, 0, 3709 key->nr_images * sizeof *draw_image); 3710 for (i = 0; i < key->nr_images; i++) { 3711 lp_sampler_static_texture_state_image(&draw_image[i].image_state, 3712 llvm->draw->images[PIPE_SHADER_TESS_CTRL][i]); 3713 } 3714 return key; 3715} 3716 3717void 3718draw_tcs_llvm_dump_variant_key(struct draw_tcs_llvm_variant_key *key) 3719{ 3720 unsigned i; 3721 struct draw_sampler_static_state *sampler = key->samplers; 3722 struct draw_image_static_state *image = draw_tcs_llvm_variant_key_images(key); 3723 for (i = 0 ; i < key->nr_sampler_views; i++) { 3724 debug_printf("sampler[%i].src_format = %s\n", i, 3725 util_format_name(sampler[i].texture_state.format)); 3726 } 3727 3728 for (i = 0 ; i < key->nr_images; i++) 3729 debug_printf("images[%i].format = %s\n", i, util_format_name(image[i].image_state.format)); 3730 3731} 3732 3733static void 3734create_tes_jit_types(struct draw_tes_llvm_variant *var) 3735{ 3736 struct gallivm_state *gallivm = var->gallivm; 3737 LLVMTypeRef texture_type, sampler_type, image_type; 3738 3739 texture_type = create_jit_texture_type(gallivm, "texture"); 3740 sampler_type = create_jit_sampler_type(gallivm, "sampler"); 3741 image_type = create_jit_image_type(gallivm, "image"); 3742 3743 var->context_type = create_tes_jit_context_type(gallivm, 3744 0, 3745 texture_type, sampler_type, 3746 image_type, 3747 "draw_tes_jit_context"); 3748 var->context_ptr_type = LLVMPointerType(var->context_type, 0); 3749 3750 var->input_array_deref_type = create_tes_jit_input_deref_type(gallivm); 3751 var->input_array_type = LLVMPointerType(var->input_array_deref_type, 0); /* num vertices per prim */ 3752} 3753 3754static LLVMTypeRef 3755get_tes_context_ptr_type(struct draw_tes_llvm_variant *variant) 3756{ 3757 if (!variant->context_ptr_type) 3758 create_tes_jit_types(variant); 3759 return variant->context_ptr_type; 3760} 3761 3762static LLVMValueRef 3763generate_tes_mask_value(struct draw_tes_llvm_variant *variant, 3764 struct lp_type tes_type, LLVMValueRef limit, LLVMValueRef loop_counter) 3765{ 3766 struct gallivm_state *gallivm = variant->gallivm; 3767 LLVMBuilderRef builder = gallivm->builder; 3768 struct lp_type mask_type = lp_int_type(tes_type); 3769 LLVMValueRef num_prims; 3770 LLVMValueRef mask_val = lp_build_const_vec(gallivm, mask_type, 0); 3771 unsigned i; 3772 3773 num_prims = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, mask_type), limit); 3774 for (i = 0; i < tes_type.length; i++) { 3775 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3776 mask_val = LLVMBuildInsertElement(builder, mask_val, LLVMBuildAdd(builder, loop_counter, idx, ""), idx, ""); 3777 } 3778 mask_val = lp_build_compare(gallivm, mask_type, 3779 PIPE_FUNC_GREATER, num_prims, mask_val); 3780 3781 return mask_val; 3782} 3783 3784static LLVMValueRef 3785draw_tes_llvm_fetch_vertex_input(const struct lp_build_tes_iface *tes_iface, 3786 struct lp_build_context *bld, 3787 boolean is_vindex_indirect, 3788 LLVMValueRef vertex_index, 3789 boolean is_aindex_indirect, 3790 LLVMValueRef attrib_index, 3791 boolean is_sindex_indirect, 3792 LLVMValueRef swizzle_index) 3793{ 3794 const struct draw_tes_llvm_iface *tes = draw_tes_llvm_iface(tes_iface); 3795 struct gallivm_state *gallivm = bld->gallivm; 3796 LLVMBuilderRef builder = gallivm->builder; 3797 LLVMValueRef indices[3]; 3798 LLVMValueRef res; 3799 struct lp_type type = bld->type; 3800 3801 if (is_vindex_indirect || is_aindex_indirect || is_sindex_indirect) { 3802 int i; 3803 3804 res = bld->zero; 3805 3806 for (i = 0; i < type.length; ++i) { 3807 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3808 LLVMValueRef vert_chan_index = vertex_index; 3809 LLVMValueRef attr_chan_index = attrib_index; 3810 LLVMValueRef swiz_chan_index = swizzle_index; 3811 LLVMValueRef channel_vec; 3812 3813 if (is_vindex_indirect) { 3814 vert_chan_index = LLVMBuildExtractElement(builder, 3815 vertex_index, idx, ""); 3816 } 3817 if (is_aindex_indirect) { 3818 attr_chan_index = LLVMBuildExtractElement(builder, 3819 attrib_index, idx, ""); 3820 } 3821 if (is_sindex_indirect) { 3822 swiz_chan_index = LLVMBuildExtractElement(builder, 3823 swizzle_index, idx, ""); 3824 } 3825 3826 indices[0] = vert_chan_index; 3827 indices[1] = attr_chan_index; 3828 indices[2] = swiz_chan_index; 3829 3830 channel_vec = LLVMBuildGEP2(builder, tes->variant->input_array_deref_type, tes->input, indices, 3, ""); 3831 channel_vec = LLVMBuildLoad2(builder, LLVMFloatTypeInContext(gallivm->context), channel_vec, ""); 3832 3833 res = LLVMBuildInsertElement(builder, res, channel_vec, idx, ""); 3834 } 3835 } else { 3836 indices[0] = vertex_index; 3837 indices[1] = attrib_index; 3838 indices[2] = swizzle_index; 3839 3840 res = LLVMBuildGEP2(builder, tes->variant->input_array_deref_type, tes->input, indices, 3, ""); 3841 res = LLVMBuildLoad2(builder, LLVMFloatTypeInContext(gallivm->context), res, ""); 3842 res = lp_build_broadcast_scalar(bld, res); 3843 } 3844 return res; 3845} 3846 3847static LLVMValueRef 3848draw_tes_llvm_fetch_patch_input(const struct lp_build_tes_iface *tes_iface, 3849 struct lp_build_context *bld, 3850 boolean is_aindex_indirect, 3851 LLVMValueRef attrib_index, 3852 LLVMValueRef swizzle_index) 3853{ 3854 const struct draw_tes_llvm_iface *tes = draw_tes_llvm_iface(tes_iface); 3855 struct gallivm_state *gallivm = bld->gallivm; 3856 LLVMBuilderRef builder = gallivm->builder; 3857 LLVMValueRef indices[3]; 3858 LLVMValueRef res; 3859 struct lp_type type = bld->type; 3860 3861 if (is_aindex_indirect) { 3862 int i; 3863 3864 res = bld->zero; 3865 3866 for (i = 0; i < type.length; ++i) { 3867 LLVMValueRef idx = lp_build_const_int32(gallivm, i); 3868 LLVMValueRef attr_chan_index = attrib_index; 3869 LLVMValueRef channel_vec; 3870 3871 if (is_aindex_indirect) { 3872 attr_chan_index = LLVMBuildExtractElement(builder, 3873 attrib_index, idx, ""); 3874 } 3875 3876 indices[0] = lp_build_const_int32(gallivm, 0); 3877 indices[1] = attr_chan_index; 3878 indices[2] = swizzle_index; 3879 3880 channel_vec = LLVMBuildGEP2(builder, tes->variant->input_array_deref_type, tes->input, indices, 3, ""); 3881 channel_vec = LLVMBuildLoad2(builder, LLVMFloatTypeInContext(gallivm->context), channel_vec, ""); 3882 3883 res = LLVMBuildInsertElement(builder, res, channel_vec, idx, ""); 3884 } 3885 } else { 3886 indices[0] = lp_build_const_int32(gallivm, 0); 3887 indices[1] = attrib_index; 3888 indices[2] = swizzle_index; 3889 3890 res = LLVMBuildGEP2(builder, tes->variant->input_array_deref_type, tes->input, indices, 3, ""); 3891 res = LLVMBuildLoad2(builder, LLVMFloatTypeInContext(gallivm->context), res, ""); 3892 res = lp_build_broadcast_scalar(bld, res); 3893 } 3894 return res; 3895} 3896 3897static void 3898draw_tes_llvm_generate(struct draw_llvm *llvm, 3899 struct draw_tes_llvm_variant *variant) 3900{ 3901 struct gallivm_state *gallivm = variant->gallivm; 3902 LLVMContextRef context = gallivm->context; 3903 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 3904 LLVMTypeRef flt_type = LLVMFloatTypeInContext(context); 3905 LLVMTypeRef arg_types[11]; 3906 LLVMTypeRef func_type; 3907 LLVMValueRef variant_func; 3908 LLVMValueRef context_ptr; 3909 LLVMValueRef tess_coord[2], io_ptr, input_array, num_tess_coord; 3910 LLVMValueRef view_index; 3911 LLVMValueRef tess_inner, tess_outer, prim_id, patch_vertices_in; 3912 LLVMBasicBlockRef block; 3913 LLVMBuilderRef builder; 3914 LLVMValueRef mask_val; 3915 struct lp_build_context bld, bldvec; 3916 struct lp_build_sampler_soa *sampler = 0; 3917 struct lp_build_image_soa *image = NULL; 3918 struct lp_bld_tgsi_system_values system_values; 3919 char func_name[64]; 3920 unsigned i; 3921 struct draw_tes_llvm_iface tes_iface; 3922 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 3923 struct lp_build_mask_context mask; 3924 LLVMValueRef consts_ptr, num_consts_ptr; 3925 LLVMValueRef ssbos_ptr, num_ssbos_ptr; 3926 LLVMValueRef step; 3927 struct lp_type tes_type; 3928 unsigned vector_length = variant->shader->base.vector_length; 3929 3930 memset(&system_values, 0, sizeof(system_values)); 3931 memset(&outputs, 0, sizeof(outputs)); 3932 3933 snprintf(func_name, sizeof(func_name), "draw_llvm_tes_variant"); 3934 3935 LLVMTypeRef tess_outer_deref_type = LLVMArrayType(flt_type, 4); 3936 LLVMTypeRef tess_inner_deref_type = LLVMArrayType(flt_type, 2); 3937 3938 arg_types[0] = get_tes_context_ptr_type(variant); /* context */ 3939 arg_types[1] = variant->input_array_type; /* input */ 3940 arg_types[2] = variant->vertex_header_ptr_type; 3941 arg_types[3] = int32_type; 3942 arg_types[4] = int32_type; 3943 arg_types[5] = LLVMPointerType(flt_type, 0); 3944 arg_types[6] = LLVMPointerType(flt_type, 0); 3945 arg_types[7] = LLVMPointerType(tess_outer_deref_type, 0); 3946 arg_types[8] = LLVMPointerType(tess_inner_deref_type, 0); 3947 arg_types[9] = int32_type; 3948 arg_types[10] = int32_type; 3949 3950 func_type = LLVMFunctionType(int32_type, arg_types, ARRAY_SIZE(arg_types), 0); 3951 variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); 3952 3953 variant->function = variant_func; 3954 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 3955 3956 for (i = 0; i < ARRAY_SIZE(arg_types); ++i) 3957 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 3958 lp_add_function_attr(variant_func, i + 1, LP_FUNC_ATTR_NOALIAS); 3959 3960 if (gallivm->cache && gallivm->cache->data_size) 3961 return; 3962 context_ptr = LLVMGetParam(variant_func, 0); 3963 input_array = LLVMGetParam(variant_func, 1); 3964 io_ptr = LLVMGetParam(variant_func, 2); 3965 prim_id = LLVMGetParam(variant_func, 3); 3966 num_tess_coord = LLVMGetParam(variant_func, 4); 3967 tess_coord[0] = LLVMGetParam(variant_func, 5); 3968 tess_coord[1] = LLVMGetParam(variant_func, 6); 3969 tess_outer = LLVMGetParam(variant_func, 7); 3970 tess_inner = LLVMGetParam(variant_func, 8); 3971 patch_vertices_in = LLVMGetParam(variant_func, 9); 3972 view_index = LLVMGetParam(variant_func, 10); 3973 3974 lp_build_name(context_ptr, "context"); 3975 lp_build_name(input_array, "input"); 3976 lp_build_name(io_ptr, "io"); 3977 lp_build_name(prim_id, "prim_id"); 3978 lp_build_name(num_tess_coord, "num_tess_coord"); 3979 lp_build_name(tess_coord[0], "tess_coord[0]"); 3980 lp_build_name(tess_coord[1], "tess_coord[1]"); 3981 lp_build_name(tess_outer, "tess_outer"); 3982 lp_build_name(tess_inner, "tess_inner"); 3983 lp_build_name(patch_vertices_in, "patch_vertices_in"); 3984 lp_build_name(view_index, "view_index"); 3985 3986 tes_iface.base.fetch_vertex_input = draw_tes_llvm_fetch_vertex_input; 3987 tes_iface.base.fetch_patch_input = draw_tes_llvm_fetch_patch_input; 3988 tes_iface.input = input_array; 3989 tes_iface.variant = variant; 3990 3991 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 3992 builder = gallivm->builder; 3993 LLVMPositionBuilderAtEnd(builder, block); 3994 3995 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 3996 3997 memset(&tes_type, 0, sizeof tes_type); 3998 tes_type.floating = TRUE; /* floating point values */ 3999 tes_type.sign = TRUE; /* values are signed */ 4000 tes_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 4001 tes_type.width = 32; /* 32-bit float */ 4002 tes_type.length = vector_length; 4003 4004 lp_build_context_init(&bldvec, variant->gallivm, lp_int_type(tes_type)); 4005 consts_ptr = draw_tes_jit_context_constants(variant, context_ptr); 4006 num_consts_ptr = 4007 draw_tes_jit_context_num_constants(variant, context_ptr); 4008 4009 ssbos_ptr = draw_tes_jit_context_ssbos(variant, context_ptr); 4010 num_ssbos_ptr = 4011 draw_tes_jit_context_num_ssbos(variant, context_ptr); 4012 sampler = draw_llvm_sampler_soa_create(variant->key.samplers, 4013 MAX2(variant->key.nr_samplers, 4014 variant->key.nr_sampler_views)); 4015 image = draw_llvm_image_soa_create(draw_tes_llvm_variant_key_images(&variant->key), 4016 variant->key.nr_images); 4017 step = lp_build_const_int32(gallivm, vector_length); 4018 4019 system_values.tess_outer = LLVMBuildLoad2(builder, tess_outer_deref_type, tess_outer, ""); 4020 system_values.tess_inner = LLVMBuildLoad2(builder, tess_inner_deref_type, tess_inner, ""); 4021 4022 system_values.prim_id = lp_build_broadcast_scalar(&bldvec, prim_id); 4023 4024 system_values.view_index = view_index; 4025 4026 system_values.vertices_in = lp_build_broadcast_scalar(&bldvec, patch_vertices_in); 4027 4028 if (variant->key.primid_needed) { 4029 int slot = variant->key.primid_output; 4030 for (unsigned i = 0; i < 4; i++) { 4031 outputs[slot][i] = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, tes_type), "primid"); 4032 LLVMBuildStore(builder, system_values.prim_id, outputs[slot][i]); 4033 } 4034 } 4035 struct lp_build_loop_state lp_loop; 4036 lp_build_loop_begin(&lp_loop, gallivm, bld.zero); 4037 { 4038 LLVMValueRef io; 4039 4040 io = LLVMBuildGEP2(builder, variant->vertex_header_type, io_ptr, &lp_loop.counter, 1, ""); 4041 mask_val = generate_tes_mask_value(variant, tes_type, num_tess_coord, lp_loop.counter); 4042 lp_build_mask_begin(&mask, gallivm, tes_type, mask_val); 4043 4044 system_values.tess_coord = LLVMGetUndef(LLVMArrayType(LLVMVectorType(flt_type, vector_length), 3)); 4045 for (i = 0; i < 3; i++) { 4046 LLVMValueRef tess_coord_chan = LLVMGetUndef(LLVMVectorType(flt_type, vector_length)); 4047 for (unsigned j = 0; j < vector_length; j++) { 4048 LLVMValueRef idx = LLVMBuildAdd(builder, lp_loop.counter, lp_build_const_int32(gallivm, j), ""); 4049 LLVMValueRef tc_val; 4050 if (i == 2) { 4051 if (variant->shader->base.prim_mode == PIPE_PRIM_TRIANGLES) { 4052 tc_val = lp_build_const_float(gallivm, 1.0); 4053 tc_val = LLVMBuildFSub(builder, tc_val, lp_build_pointer_get(builder, tess_coord[0], idx), ""); 4054 tc_val = LLVMBuildFSub(builder, tc_val, lp_build_pointer_get(builder, tess_coord[1], idx), ""); 4055 } else 4056 tc_val = lp_build_const_float(gallivm, 0.0); 4057 } else 4058 tc_val = lp_build_pointer_get(builder, tess_coord[i], idx); 4059 4060 tess_coord_chan = LLVMBuildInsertElement(builder, tess_coord_chan, tc_val, lp_build_const_int32(gallivm, j), ""); 4061 } 4062 system_values.tess_coord = LLVMBuildInsertValue(builder, system_values.tess_coord, tess_coord_chan, i, ""); 4063 } 4064 4065 struct lp_build_tgsi_params params; 4066 memset(¶ms, 0, sizeof(params)); 4067 4068 params.type = tes_type; 4069 params.mask = &mask; 4070 params.consts_ptr = consts_ptr; 4071 params.const_sizes_ptr = num_consts_ptr; 4072 params.system_values = &system_values; 4073 params.context_ptr = context_ptr; 4074 params.sampler = sampler; 4075 params.info = &llvm->draw->tes.tess_eval_shader->info; 4076 params.ssbo_ptr = ssbos_ptr; 4077 params.ssbo_sizes_ptr = num_ssbos_ptr; 4078 params.image = image; 4079 params.tes_iface = &tes_iface.base; 4080 params.aniso_filter_table = draw_tes_jit_context_aniso_filter_table(variant, context_ptr); 4081 4082 lp_build_nir_soa(variant->gallivm, 4083 llvm->draw->tes.tess_eval_shader->state.ir.nir, 4084 ¶ms, 4085 outputs); 4086 4087 lp_build_mask_end(&mask); 4088 4089 if (variant->key.clamp_vertex_color) { 4090 const struct tgsi_shader_info *info = &llvm->draw->tes.tess_eval_shader->info; 4091 do_clamp_vertex_color(variant->gallivm, 4092 tes_type, info, 4093 outputs); 4094 } 4095 LLVMValueRef clipmask = lp_build_const_int_vec(gallivm, 4096 lp_int_type(tes_type), 0); 4097 4098 convert_to_aos(gallivm, variant->vertex_header_type, io, NULL, outputs, clipmask, 4099 draw_total_tes_outputs(llvm->draw), tes_type, FALSE); 4100 } 4101 lp_build_loop_end_cond(&lp_loop, num_tess_coord, step, LLVMIntUGE); 4102 sampler->destroy(sampler); 4103 image->destroy(image); 4104 4105 LLVMBuildRet(builder, lp_build_zero(gallivm, lp_type_uint(32))); 4106 gallivm_verify_function(gallivm, variant_func); 4107} 4108 4109struct draw_tes_llvm_variant * 4110draw_tes_llvm_create_variant(struct draw_llvm *llvm, 4111 unsigned num_outputs, 4112 const struct draw_tes_llvm_variant_key *key) 4113{ 4114 struct draw_tes_llvm_variant *variant; 4115 struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(llvm->draw->tes.tess_eval_shader); 4116 char module_name[64]; 4117 unsigned char ir_sha1_cache_key[20]; 4118 struct lp_cached_code cached = { 0 }; 4119 bool needs_caching = false; 4120 4121 variant = MALLOC(sizeof *variant + 4122 shader->variant_key_size - sizeof variant->key); 4123 if (!variant) 4124 return NULL; 4125 4126 variant->llvm = llvm; 4127 variant->shader = shader; 4128 4129 snprintf(module_name, sizeof(module_name), "draw_llvm_tes_variant%u", 4130 variant->shader->variants_cached); 4131 4132 memcpy(&variant->key, key, shader->variant_key_size); 4133 if (shader->base.state.ir.nir && llvm->draw->disk_cache_cookie) { 4134 draw_get_ir_cache_key(shader->base.state.ir.nir, 4135 key, 4136 shader->variant_key_size, 4137 num_outputs, 4138 ir_sha1_cache_key); 4139 4140 llvm->draw->disk_cache_find_shader(llvm->draw->disk_cache_cookie, 4141 &cached, 4142 ir_sha1_cache_key); 4143 if (!cached.data_size) 4144 needs_caching = true; 4145 } 4146 variant->gallivm = gallivm_create(module_name, llvm->context, &cached); 4147 4148 create_tes_jit_types(variant); 4149 4150 variant->vertex_header_type = create_jit_vertex_header(variant->gallivm, num_outputs); 4151 variant->vertex_header_ptr_type = LLVMPointerType(variant->vertex_header_type, 0); 4152 4153 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 4154 nir_print_shader(llvm->draw->tes.tess_eval_shader->state.ir.nir, stderr); 4155 draw_tes_llvm_dump_variant_key(&variant->key); 4156 } 4157 4158 draw_tes_llvm_generate(llvm, variant); 4159 4160 gallivm_compile_module(variant->gallivm); 4161 4162 variant->jit_func = (draw_tes_jit_func) 4163 gallivm_jit_function(variant->gallivm, variant->function); 4164 4165 if (needs_caching) 4166 llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, 4167 &cached, 4168 ir_sha1_cache_key); 4169 gallivm_free_ir(variant->gallivm); 4170 4171 variant->list_item_global.base = variant; 4172 variant->list_item_local.base = variant; 4173 /*variant->no = */shader->variants_created++; 4174 variant->list_item_global.base = variant; 4175 4176 return variant; 4177} 4178 4179void 4180draw_tes_llvm_destroy_variant(struct draw_tes_llvm_variant *variant) 4181{ 4182 struct draw_llvm *llvm = variant->llvm; 4183 4184 if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) { 4185 debug_printf("Deleting TES variant: %u tes variants,\t%u total variants\n", 4186 variant->shader->variants_cached, llvm->nr_tes_variants); 4187 } 4188 4189 gallivm_destroy(variant->gallivm); 4190 4191 list_del(&variant->list_item_local.list); 4192 variant->shader->variants_cached--; 4193 list_del(&variant->list_item_global.list); 4194 llvm->nr_tes_variants--; 4195 FREE(variant); 4196} 4197 4198struct draw_tes_llvm_variant_key * 4199draw_tes_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 4200{ 4201 unsigned i; 4202 struct draw_tes_llvm_variant_key *key; 4203 struct draw_sampler_static_state *draw_sampler; 4204 struct draw_image_static_state *draw_image; 4205 4206 key = (struct draw_tes_llvm_variant_key *)store; 4207 4208 memset(key, 0, offsetof(struct draw_tes_llvm_variant_key, samplers[0])); 4209 4210 int primid_output = draw_find_shader_output(llvm->draw, TGSI_SEMANTIC_PRIMID, 0); 4211 if (primid_output >= 0) { 4212 key->primid_output = primid_output; 4213 key->primid_needed = true; 4214 } 4215 4216 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color && 4217 llvm->draw->gs.geometry_shader == NULL; 4218 4219 /* All variants of this shader will have the same value for 4220 * nr_samplers. Not yet trying to compact away holes in the 4221 * sampler array. 4222 */ 4223 key->nr_samplers = llvm->draw->tes.tess_eval_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 4224 if (llvm->draw->tes.tess_eval_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 4225 key->nr_sampler_views = 4226 llvm->draw->tes.tess_eval_shader->info.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 4227 } 4228 else { 4229 key->nr_sampler_views = key->nr_samplers; 4230 } 4231 4232 key->nr_images = llvm->draw->tes.tess_eval_shader->info.file_max[TGSI_FILE_IMAGE] + 1; 4233 4234 draw_sampler = key->samplers; 4235 4236 memset(draw_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *draw_sampler); 4237 4238 for (i = 0 ; i < key->nr_samplers; i++) { 4239 lp_sampler_static_sampler_state(&draw_sampler[i].sampler_state, 4240 llvm->draw->samplers[PIPE_SHADER_TESS_EVAL][i]); 4241 } 4242 for (i = 0 ; i < key->nr_sampler_views; i++) { 4243 lp_sampler_static_texture_state(&draw_sampler[i].texture_state, 4244 llvm->draw->sampler_views[PIPE_SHADER_TESS_EVAL][i]); 4245 } 4246 4247 draw_image = draw_tes_llvm_variant_key_images(key); 4248 memset(draw_image, 0, 4249 key->nr_images * sizeof *draw_image); 4250 for (i = 0; i < key->nr_images; i++) { 4251 lp_sampler_static_texture_state_image(&draw_image[i].image_state, 4252 llvm->draw->images[PIPE_SHADER_TESS_EVAL][i]); 4253 } 4254 return key; 4255} 4256 4257void 4258draw_tes_llvm_dump_variant_key(struct draw_tes_llvm_variant_key *key) 4259{ 4260 unsigned i; 4261 struct draw_sampler_static_state *sampler = key->samplers; 4262 struct draw_image_static_state *image = draw_tes_llvm_variant_key_images(key); 4263 4264 if (key->primid_needed) 4265 debug_printf("prim id output %d\n", key->primid_output); 4266 debug_printf("clamp_vertex_color = %u\n", key->clamp_vertex_color); 4267 for (i = 0 ; i < key->nr_sampler_views; i++) { 4268 debug_printf("sampler[%i].src_format = %s\n", i, 4269 util_format_name(sampler[i].texture_state.format)); 4270 } 4271 4272 for (i = 0 ; i < key->nr_images; i++) 4273 debug_printf("images[%i].format = %s\n", i, util_format_name(image[i].image_state.format)); 4274 4275} 4276