1bf215546Sopenharmony_ci/* 2bf215546Sopenharmony_ci * Copyright © 2019 Collabora, Ltd. 3bf215546Sopenharmony_ci * 4bf215546Sopenharmony_ci * Permission is hereby granted, free of charge, to any person obtaining a 5bf215546Sopenharmony_ci * copy of this software and associated documentation files (the "Software"), 6bf215546Sopenharmony_ci * to deal in the Software without restriction, including without limitation 7bf215546Sopenharmony_ci * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8bf215546Sopenharmony_ci * and/or sell copies of the Software, and to permit persons to whom the 9bf215546Sopenharmony_ci * Software is furnished to do so, subject to the following conditions: 10bf215546Sopenharmony_ci * 11bf215546Sopenharmony_ci * The above copyright notice and this permission notice (including the next 12bf215546Sopenharmony_ci * paragraph) shall be included in all copies or substantial portions of the 13bf215546Sopenharmony_ci * Software. 14bf215546Sopenharmony_ci * 15bf215546Sopenharmony_ci * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16bf215546Sopenharmony_ci * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17bf215546Sopenharmony_ci * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18bf215546Sopenharmony_ci * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19bf215546Sopenharmony_ci * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20bf215546Sopenharmony_ci * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21bf215546Sopenharmony_ci * IN THE SOFTWARE. 22bf215546Sopenharmony_ci * 23bf215546Sopenharmony_ci * Authors (Collabora): 24bf215546Sopenharmony_ci * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com> 25bf215546Sopenharmony_ci */ 26bf215546Sopenharmony_ci 27bf215546Sopenharmony_ci#include "nir.h" 28bf215546Sopenharmony_ci#include "nir_builder.h" 29bf215546Sopenharmony_ci 30bf215546Sopenharmony_ci/* 31bf215546Sopenharmony_ci * Lowers SSBOs to globals, for hardware that lack native SSBO support. When 32bf215546Sopenharmony_ci * lowering, *_ssbo_* instructions will become *_global_* instructions, 33bf215546Sopenharmony_ci * augmented with load_ssbo_address. 34bf215546Sopenharmony_ci * 35bf215546Sopenharmony_ci * DOES NOT PERFORM BOUNDS CHECKING. DO NOT USE IN PRODUCTION ON UNTRUSTED 36bf215546Sopenharmony_ci * CONTEXTS INCLUDING WEBGL 2. 37bf215546Sopenharmony_ci */ 38bf215546Sopenharmony_ci 39bf215546Sopenharmony_cistatic nir_intrinsic_op 40bf215546Sopenharmony_cilower_ssbo_op(nir_intrinsic_op op) 41bf215546Sopenharmony_ci{ 42bf215546Sopenharmony_ci switch (op) { 43bf215546Sopenharmony_ci case nir_intrinsic_load_ssbo: 44bf215546Sopenharmony_ci return nir_intrinsic_load_global; 45bf215546Sopenharmony_ci 46bf215546Sopenharmony_ci case nir_intrinsic_store_ssbo: 47bf215546Sopenharmony_ci return nir_intrinsic_store_global; 48bf215546Sopenharmony_ci 49bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_add: 50bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_add; 51bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_imin: 52bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_imin; 53bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_umin: 54bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_umin; 55bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_imax: 56bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_imax; 57bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_umax: 58bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_umax; 59bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_and: 60bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_and; 61bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_or: 62bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_or; 63bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_xor: 64bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_xor; 65bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_exchange: 66bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_exchange; 67bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_comp_swap: 68bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_comp_swap; 69bf215546Sopenharmony_ci 70bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fadd: 71bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_fadd; 72bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fmin: 73bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_fmin; 74bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fmax: 75bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_fmax; 76bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fcomp_swap: 77bf215546Sopenharmony_ci return nir_intrinsic_global_atomic_fcomp_swap; 78bf215546Sopenharmony_ci 79bf215546Sopenharmony_ci default: 80bf215546Sopenharmony_ci unreachable("Invalid SSBO op"); 81bf215546Sopenharmony_ci } 82bf215546Sopenharmony_ci} 83bf215546Sopenharmony_ci 84bf215546Sopenharmony_ci/* Like SSBO property sysvals, though SSBO index may be indirect. C.f. 85bf215546Sopenharmony_ci * nir_load_system_value */ 86bf215546Sopenharmony_ci 87bf215546Sopenharmony_cistatic inline nir_ssa_def * 88bf215546Sopenharmony_cinir_load_ssbo_prop(nir_builder *b, nir_intrinsic_op op, 89bf215546Sopenharmony_ci nir_src *idx, unsigned bitsize) 90bf215546Sopenharmony_ci{ 91bf215546Sopenharmony_ci nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op); 92bf215546Sopenharmony_ci load->num_components = 1; 93bf215546Sopenharmony_ci nir_src_copy(&load->src[0], idx); 94bf215546Sopenharmony_ci nir_ssa_dest_init(&load->instr, &load->dest, 1, bitsize, NULL); 95bf215546Sopenharmony_ci nir_builder_instr_insert(b, &load->instr); 96bf215546Sopenharmony_ci return &load->dest.ssa; 97bf215546Sopenharmony_ci} 98bf215546Sopenharmony_ci 99bf215546Sopenharmony_ci#define nir_ssbo_prop(b, prop, index, bitsize) \ 100bf215546Sopenharmony_ci nir_load_ssbo_prop(b, nir_intrinsic_##prop, index, bitsize) 101bf215546Sopenharmony_ci 102bf215546Sopenharmony_cistatic nir_ssa_def * 103bf215546Sopenharmony_cilower_ssbo_instr(nir_builder *b, nir_intrinsic_instr *intr) 104bf215546Sopenharmony_ci{ 105bf215546Sopenharmony_ci nir_intrinsic_op op = lower_ssbo_op(intr->intrinsic); 106bf215546Sopenharmony_ci bool is_store = op == nir_intrinsic_store_global; 107bf215546Sopenharmony_ci bool is_atomic = !is_store && op != nir_intrinsic_load_global; 108bf215546Sopenharmony_ci 109bf215546Sopenharmony_ci /* We have to calculate the address: 110bf215546Sopenharmony_ci * 111bf215546Sopenharmony_ci * &(SSBO[offset]) = &SSBO + offset 112bf215546Sopenharmony_ci */ 113bf215546Sopenharmony_ci 114bf215546Sopenharmony_ci nir_src index = intr->src[is_store ? 1 : 0]; 115bf215546Sopenharmony_ci nir_src *offset_src = nir_get_io_offset_src(intr); 116bf215546Sopenharmony_ci nir_ssa_def *offset = nir_ssa_for_src(b, *offset_src, 1); 117bf215546Sopenharmony_ci 118bf215546Sopenharmony_ci nir_ssa_def *address = 119bf215546Sopenharmony_ci nir_iadd(b, 120bf215546Sopenharmony_ci nir_ssbo_prop(b, load_ssbo_address, &index, 64), 121bf215546Sopenharmony_ci nir_u2u64(b, offset)); 122bf215546Sopenharmony_ci 123bf215546Sopenharmony_ci /* Create the replacement intrinsic */ 124bf215546Sopenharmony_ci 125bf215546Sopenharmony_ci nir_intrinsic_instr *global = 126bf215546Sopenharmony_ci nir_intrinsic_instr_create(b->shader, op); 127bf215546Sopenharmony_ci 128bf215546Sopenharmony_ci global->num_components = intr->num_components; 129bf215546Sopenharmony_ci global->src[is_store ? 1 : 0] = nir_src_for_ssa(address); 130bf215546Sopenharmony_ci 131bf215546Sopenharmony_ci if (!is_atomic) { 132bf215546Sopenharmony_ci nir_intrinsic_set_align_mul(global, nir_intrinsic_align_mul(intr)); 133bf215546Sopenharmony_ci nir_intrinsic_set_align_offset(global, nir_intrinsic_align_offset(intr)); 134bf215546Sopenharmony_ci } 135bf215546Sopenharmony_ci 136bf215546Sopenharmony_ci if (is_store) { 137bf215546Sopenharmony_ci nir_src_copy(&global->src[0], &intr->src[0]); 138bf215546Sopenharmony_ci nir_intrinsic_set_write_mask(global, nir_intrinsic_write_mask(intr)); 139bf215546Sopenharmony_ci } else { 140bf215546Sopenharmony_ci nir_ssa_dest_init(&global->instr, &global->dest, 141bf215546Sopenharmony_ci intr->dest.ssa.num_components, 142bf215546Sopenharmony_ci intr->dest.ssa.bit_size, NULL); 143bf215546Sopenharmony_ci 144bf215546Sopenharmony_ci if (is_atomic) { 145bf215546Sopenharmony_ci nir_src_copy(&global->src[1], &intr->src[2]); 146bf215546Sopenharmony_ci if (nir_intrinsic_infos[op].num_srcs > 2) 147bf215546Sopenharmony_ci nir_src_copy(&global->src[2], &intr->src[3]); 148bf215546Sopenharmony_ci } 149bf215546Sopenharmony_ci } 150bf215546Sopenharmony_ci 151bf215546Sopenharmony_ci nir_builder_instr_insert(b, &global->instr); 152bf215546Sopenharmony_ci return is_store ? NULL : &global->dest.ssa; 153bf215546Sopenharmony_ci} 154bf215546Sopenharmony_ci 155bf215546Sopenharmony_cistatic bool 156bf215546Sopenharmony_cishould_lower_ssbo_instr(const nir_instr *instr) 157bf215546Sopenharmony_ci{ 158bf215546Sopenharmony_ci if (instr->type != nir_instr_type_intrinsic) 159bf215546Sopenharmony_ci return false; 160bf215546Sopenharmony_ci 161bf215546Sopenharmony_ci const nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); 162bf215546Sopenharmony_ci 163bf215546Sopenharmony_ci switch (intr->intrinsic) { 164bf215546Sopenharmony_ci case nir_intrinsic_load_ssbo: 165bf215546Sopenharmony_ci case nir_intrinsic_store_ssbo: 166bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_add: 167bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_imin: 168bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_umin: 169bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_imax: 170bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_umax: 171bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_and: 172bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_or: 173bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_xor: 174bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_exchange: 175bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_comp_swap: 176bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fadd: 177bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fmin: 178bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fmax: 179bf215546Sopenharmony_ci case nir_intrinsic_ssbo_atomic_fcomp_swap: 180bf215546Sopenharmony_ci return true; 181bf215546Sopenharmony_ci default: 182bf215546Sopenharmony_ci return false; 183bf215546Sopenharmony_ci } 184bf215546Sopenharmony_ci 185bf215546Sopenharmony_ci return false; 186bf215546Sopenharmony_ci} 187bf215546Sopenharmony_ci 188bf215546Sopenharmony_cibool 189bf215546Sopenharmony_cinir_lower_ssbo(nir_shader *shader) 190bf215546Sopenharmony_ci{ 191bf215546Sopenharmony_ci bool progress = false; 192bf215546Sopenharmony_ci 193bf215546Sopenharmony_ci nir_foreach_function(function, shader) { 194bf215546Sopenharmony_ci nir_function_impl *impl = function->impl; 195bf215546Sopenharmony_ci nir_builder b; 196bf215546Sopenharmony_ci nir_builder_init(&b, impl); 197bf215546Sopenharmony_ci 198bf215546Sopenharmony_ci nir_foreach_block(block, impl) { 199bf215546Sopenharmony_ci nir_foreach_instr_safe(instr, block) { 200bf215546Sopenharmony_ci if (!should_lower_ssbo_instr(instr)) continue; 201bf215546Sopenharmony_ci progress = true; 202bf215546Sopenharmony_ci b.cursor = nir_before_instr(instr); 203bf215546Sopenharmony_ci 204bf215546Sopenharmony_ci nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); 205bf215546Sopenharmony_ci nir_ssa_def *replace = lower_ssbo_instr(&b, intr); 206bf215546Sopenharmony_ci 207bf215546Sopenharmony_ci if (replace) { 208bf215546Sopenharmony_ci nir_ssa_def_rewrite_uses(&intr->dest.ssa, 209bf215546Sopenharmony_ci replace); 210bf215546Sopenharmony_ci } 211bf215546Sopenharmony_ci 212bf215546Sopenharmony_ci nir_instr_remove(instr); 213bf215546Sopenharmony_ci } 214bf215546Sopenharmony_ci } 215bf215546Sopenharmony_ci } 216bf215546Sopenharmony_ci 217bf215546Sopenharmony_ci return progress; 218bf215546Sopenharmony_ci} 219