1/* 2 * Copyright © 2019 Valve Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include "nir.h" 25 26/* This pass optimizes GL access qualifiers. So far it does three things: 27 * 28 * - Infer readonly when it's missing. 29 * - Infer writeonly when it's missing. 30 * - Infer ACCESS_CAN_REORDER when the following are true: 31 * - Either there are no writes, or ACCESS_NON_WRITEABLE is set. In either 32 * case there are no writes to the underlying memory. 33 * - ACCESS_VOLATILE is not set. 34 * 35 * If these conditions are true, then image and buffer reads may be treated as 36 * if they were uniform buffer reads, i.e. they may be arbitrarily moved, 37 * combined, rematerialized etc. 38 */ 39 40struct access_state { 41 nir_shader *shader; 42 bool infer_non_readable; 43 44 struct set *vars_written; 45 struct set *vars_read; 46 bool images_written; 47 bool buffers_written; 48 bool images_read; 49 bool buffers_read; 50}; 51 52static void 53gather_buffer_access(struct access_state *state, nir_ssa_def *def, bool read, bool write) 54{ 55 state->buffers_read |= read; 56 state->buffers_written |= write; 57 58 if (!def) 59 return; 60 61 const nir_variable *var = nir_get_binding_variable( 62 state->shader, nir_chase_binding(nir_src_for_ssa(def))); 63 if (var) { 64 if (read) 65 _mesa_set_add(state->vars_read, var); 66 if (write) 67 _mesa_set_add(state->vars_written, var); 68 } else { 69 nir_foreach_variable_with_modes(possible_var, state->shader, nir_var_mem_ssbo) { 70 if (read) 71 _mesa_set_add(state->vars_read, possible_var); 72 if (write) 73 _mesa_set_add(state->vars_written, possible_var); 74 } 75 } 76} 77 78static void 79gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) 80{ 81 const nir_variable *var; 82 bool read, write; 83 switch (instr->intrinsic) { 84 case nir_intrinsic_image_deref_load: 85 case nir_intrinsic_image_deref_store: 86 case nir_intrinsic_image_deref_sparse_load: 87 case nir_intrinsic_image_deref_atomic_add: 88 case nir_intrinsic_image_deref_atomic_imin: 89 case nir_intrinsic_image_deref_atomic_umin: 90 case nir_intrinsic_image_deref_atomic_imax: 91 case nir_intrinsic_image_deref_atomic_umax: 92 case nir_intrinsic_image_deref_atomic_and: 93 case nir_intrinsic_image_deref_atomic_or: 94 case nir_intrinsic_image_deref_atomic_xor: 95 case nir_intrinsic_image_deref_atomic_exchange: 96 case nir_intrinsic_image_deref_atomic_comp_swap: 97 case nir_intrinsic_image_deref_atomic_fadd: 98 case nir_intrinsic_image_deref_atomic_fmin: 99 case nir_intrinsic_image_deref_atomic_fmax: 100 var = nir_intrinsic_get_var(instr, 0); 101 read = instr->intrinsic != nir_intrinsic_image_deref_store; 102 write = instr->intrinsic != nir_intrinsic_image_deref_load && 103 instr->intrinsic != nir_intrinsic_image_deref_sparse_load; 104 105 /* In OpenGL, buffer images use normal buffer objects, whereas other 106 * image types use textures which cannot alias with buffer objects. 107 * Therefore we have to group buffer samplers together with SSBO's. 108 */ 109 if (glsl_get_sampler_dim(glsl_without_array(var->type)) == 110 GLSL_SAMPLER_DIM_BUF) { 111 state->buffers_read |= read; 112 state->buffers_written |= write; 113 } else { 114 state->images_read |= read; 115 state->images_written |= write; 116 } 117 118 if ((var->data.mode == nir_var_uniform || 119 var->data.mode == nir_var_image) && read) 120 _mesa_set_add(state->vars_read, var); 121 if ((var->data.mode == nir_var_uniform || 122 var->data.mode == nir_var_image) && write) 123 _mesa_set_add(state->vars_written, var); 124 break; 125 126 case nir_intrinsic_bindless_image_load: 127 case nir_intrinsic_bindless_image_store: 128 case nir_intrinsic_bindless_image_sparse_load: 129 case nir_intrinsic_bindless_image_atomic_add: 130 case nir_intrinsic_bindless_image_atomic_imin: 131 case nir_intrinsic_bindless_image_atomic_umin: 132 case nir_intrinsic_bindless_image_atomic_imax: 133 case nir_intrinsic_bindless_image_atomic_umax: 134 case nir_intrinsic_bindless_image_atomic_and: 135 case nir_intrinsic_bindless_image_atomic_or: 136 case nir_intrinsic_bindless_image_atomic_xor: 137 case nir_intrinsic_bindless_image_atomic_exchange: 138 case nir_intrinsic_bindless_image_atomic_comp_swap: 139 case nir_intrinsic_bindless_image_atomic_fadd: 140 case nir_intrinsic_bindless_image_atomic_fmin: 141 case nir_intrinsic_bindless_image_atomic_fmax: 142 read = instr->intrinsic != nir_intrinsic_bindless_image_store; 143 write = instr->intrinsic != nir_intrinsic_bindless_image_load && 144 instr->intrinsic != nir_intrinsic_bindless_image_sparse_load; 145 146 if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_BUF) { 147 state->buffers_read |= read; 148 state->buffers_written |= write; 149 } else { 150 state->images_read |= read; 151 state->images_written |= write; 152 } 153 break; 154 155 case nir_intrinsic_load_deref: 156 case nir_intrinsic_store_deref: 157 case nir_intrinsic_deref_atomic_add: 158 case nir_intrinsic_deref_atomic_imin: 159 case nir_intrinsic_deref_atomic_umin: 160 case nir_intrinsic_deref_atomic_imax: 161 case nir_intrinsic_deref_atomic_umax: 162 case nir_intrinsic_deref_atomic_and: 163 case nir_intrinsic_deref_atomic_or: 164 case nir_intrinsic_deref_atomic_xor: 165 case nir_intrinsic_deref_atomic_exchange: 166 case nir_intrinsic_deref_atomic_comp_swap: 167 case nir_intrinsic_deref_atomic_fadd: 168 case nir_intrinsic_deref_atomic_fmin: 169 case nir_intrinsic_deref_atomic_fmax: 170 case nir_intrinsic_deref_atomic_fcomp_swap: { 171 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]); 172 if (!nir_deref_mode_may_be(deref, nir_var_mem_ssbo | nir_var_mem_global)) 173 break; 174 175 bool ssbo = nir_deref_mode_is(deref, nir_var_mem_ssbo); 176 gather_buffer_access(state, ssbo ? instr->src[0].ssa : NULL, 177 instr->intrinsic != nir_intrinsic_store_deref, 178 instr->intrinsic != nir_intrinsic_load_deref); 179 break; 180 } 181 182 default: 183 break; 184 } 185} 186 187static bool 188process_variable(struct access_state *state, nir_variable *var) 189{ 190 const struct glsl_type *type = glsl_without_array(var->type); 191 if (var->data.mode != nir_var_mem_ssbo && 192 !(var->data.mode == nir_var_uniform && glsl_type_is_image(type)) && 193 var->data.mode != nir_var_image) 194 return false; 195 196 /* Ignore variables we've already marked */ 197 if (var->data.access & ACCESS_CAN_REORDER) 198 return false; 199 200 unsigned access = var->data.access; 201 bool is_buffer = var->data.mode == nir_var_mem_ssbo || 202 glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF; 203 204 if (!(access & ACCESS_NON_WRITEABLE)) { 205 if (is_buffer ? !state->buffers_written : !state->images_written) 206 access |= ACCESS_NON_WRITEABLE; 207 else if ((access & ACCESS_RESTRICT) && !_mesa_set_search(state->vars_written, var)) 208 access |= ACCESS_NON_WRITEABLE; 209 } 210 211 if (state->infer_non_readable && !(access & ACCESS_NON_READABLE)) { 212 if (is_buffer ? !state->buffers_read : !state->images_read) 213 access |= ACCESS_NON_READABLE; 214 else if ((access & ACCESS_RESTRICT) && !_mesa_set_search(state->vars_read, var)) 215 access |= ACCESS_NON_READABLE; 216 } 217 218 bool changed = var->data.access != access; 219 var->data.access = access; 220 return changed; 221} 222 223static bool 224update_access(struct access_state *state, nir_intrinsic_instr *instr, bool is_buffer, bool is_global) 225{ 226 enum gl_access_qualifier access = nir_intrinsic_access(instr); 227 228 bool is_memory_readonly = access & ACCESS_NON_WRITEABLE; 229 bool is_memory_writeonly = access & ACCESS_NON_READABLE; 230 231 if (instr->intrinsic != nir_intrinsic_bindless_image_load && 232 instr->intrinsic != nir_intrinsic_bindless_image_store && 233 instr->intrinsic != nir_intrinsic_bindless_image_sparse_load && 234 !is_global) { 235 const nir_variable *var = nir_get_binding_variable( 236 state->shader, nir_chase_binding(instr->src[0])); 237 is_memory_readonly |= var && (var->data.access & ACCESS_NON_WRITEABLE); 238 is_memory_writeonly |= var && (var->data.access & ACCESS_NON_READABLE); 239 } 240 241 if (is_global) { 242 is_memory_readonly |= !state->buffers_written && !state->images_written; 243 is_memory_writeonly |= !state->buffers_read && !state->images_read; 244 } else { 245 is_memory_readonly |= is_buffer ? !state->buffers_written : !state->images_written; 246 is_memory_writeonly |= is_buffer ? !state->buffers_read : !state->images_read; 247 } 248 249 if (is_memory_readonly) 250 access |= ACCESS_NON_WRITEABLE; 251 if (state->infer_non_readable && is_memory_writeonly) 252 access |= ACCESS_NON_READABLE; 253 if (!(access & ACCESS_VOLATILE) && is_memory_readonly) 254 access |= ACCESS_CAN_REORDER; 255 256 bool progress = nir_intrinsic_access(instr) != access; 257 nir_intrinsic_set_access(instr, access); 258 return progress; 259} 260 261static bool 262process_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) 263{ 264 switch (instr->intrinsic) { 265 case nir_intrinsic_bindless_image_load: 266 case nir_intrinsic_bindless_image_store: 267 case nir_intrinsic_bindless_image_sparse_load: 268 return update_access(state, instr, nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_BUF, 269 false); 270 271 case nir_intrinsic_load_deref: 272 case nir_intrinsic_store_deref: { 273 if (nir_deref_mode_is(nir_src_as_deref(instr->src[0]), nir_var_mem_global)) 274 return update_access(state, instr, false, true); 275 else if (nir_deref_mode_is(nir_src_as_deref(instr->src[0]), nir_var_mem_ssbo)) 276 return update_access(state, instr, true, false); 277 else 278 return false; 279 } 280 281 case nir_intrinsic_image_deref_load: 282 case nir_intrinsic_image_deref_store: 283 case nir_intrinsic_image_deref_sparse_load: { 284 nir_variable *var = nir_intrinsic_get_var(instr, 0); 285 286 bool is_buffer = 287 glsl_get_sampler_dim(glsl_without_array(var->type)) == GLSL_SAMPLER_DIM_BUF; 288 289 return update_access(state, instr, is_buffer, false); 290 } 291 292 default: 293 return false; 294 } 295} 296 297static bool 298opt_access_impl(struct access_state *state, 299 nir_function_impl *impl) 300{ 301 bool progress = false; 302 303 nir_foreach_block(block, impl) { 304 nir_foreach_instr(instr, block) { 305 if (instr->type == nir_instr_type_intrinsic) 306 progress |= process_intrinsic(state, 307 nir_instr_as_intrinsic(instr)); 308 } 309 } 310 311 if (progress) { 312 nir_metadata_preserve(impl, 313 nir_metadata_block_index | 314 nir_metadata_dominance | 315 nir_metadata_live_ssa_defs | 316 nir_metadata_loop_analysis); 317 } 318 319 320 return progress; 321} 322 323bool 324nir_opt_access(nir_shader *shader, const nir_opt_access_options *options) 325{ 326 struct access_state state = { 327 .shader = shader, 328 .infer_non_readable = options->infer_non_readable, 329 .vars_written = _mesa_pointer_set_create(NULL), 330 .vars_read = _mesa_pointer_set_create(NULL), 331 }; 332 333 bool var_progress = false; 334 bool progress = false; 335 336 nir_foreach_function(func, shader) { 337 if (func->impl) { 338 nir_foreach_block(block, func->impl) { 339 nir_foreach_instr(instr, block) { 340 if (instr->type == nir_instr_type_intrinsic) 341 gather_intrinsic(&state, nir_instr_as_intrinsic(instr)); 342 } 343 } 344 } 345 } 346 347 /* In Vulkan, buffers and images can alias. */ 348 if (options->is_vulkan) { 349 state.buffers_written |= state.images_written; 350 state.images_written |= state.buffers_written; 351 state.buffers_read |= state.images_read; 352 state.images_read |= state.buffers_read; 353 } 354 355 nir_foreach_variable_with_modes(var, shader, nir_var_uniform | 356 nir_var_mem_ubo | 357 nir_var_mem_ssbo | 358 nir_var_image) 359 var_progress |= process_variable(&state, var); 360 361 nir_foreach_function(func, shader) { 362 if (func->impl) { 363 progress |= opt_access_impl(&state, func->impl); 364 365 /* If we make a change to the uniforms, update all the impls. */ 366 if (var_progress) { 367 nir_metadata_preserve(func->impl, 368 nir_metadata_block_index | 369 nir_metadata_dominance | 370 nir_metadata_live_ssa_defs | 371 nir_metadata_loop_analysis); 372 } 373 } 374 } 375 376 progress |= var_progress; 377 378 _mesa_set_destroy(state.vars_read, NULL); 379 _mesa_set_destroy(state.vars_written, NULL); 380 return progress; 381} 382