xref: /third_party/mesa3d/src/freedreno/ir3/ir3_legalize.c

/*
 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Rob Clark <robclark@freedesktop.org>
 */

#include "util/ralloc.h"
#include "util/u_math.h"

#include "ir3.h"
#include "ir3_shader.h"

/*
 * Legalize:
 *
 * The legalize pass handles ensuring sufficient nop's and sync flags for
 * correct execution.
 *
 * 1) Iteratively determine where sync ((sy)/(ss)) flags are needed,
 *    based on state flowing out of predecessor blocks until there is
 *    no further change.  In some cases this requires inserting nops.
 * 2) Mark (ei) on last varying input
 * 3) Final nop scheduling for instruction latency
 * 4) Resolve jumps and schedule blocks, marking potential convergence
 *    points with (jp)
 */

struct ir3_legalize_ctx {
   struct ir3_compiler *compiler;
   struct ir3_shader_variant *so;
   gl_shader_stage type;
   int max_bary;
   bool early_input_release;
   bool has_inputs;
};

struct ir3_legalize_state {
   regmask_t needs_ss;
   regmask_t needs_ss_war; /* write after read */
   regmask_t needs_sy;
};

struct ir3_legalize_block_data {
   bool valid;
   struct ir3_legalize_state state;
};

/* We want to evaluate each block from the position of any other
 * predecessor block, in order that the flags set are the union of
 * all possible program paths.
 *
 * To do this, we need to know the output state (needs_ss/ss_war/sy)
 * of all predecessor blocks.  The tricky thing is loops, which mean
 * that we can't simply recursively process each predecessor block
 * before legalizing the current block.
 *
 * How we handle that is by looping over all the blocks until the
 * results converge.  If the output state of a given block changes
 * in a given pass, this means that all successor blocks are not
 * yet fully legalized.
 */

static bool
legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block)
{
   struct ir3_legalize_block_data *bd = block->data;

   if (bd->valid)
      return false;

   struct ir3_instruction *last_n = NULL;
   struct list_head instr_list;
   struct ir3_legalize_state prev_state = bd->state;
   struct ir3_legalize_state *state = &bd->state;
   bool last_input_needs_ss = false;
   bool has_tex_prefetch = false;
   bool mergedregs = ctx->so->mergedregs;

   /* our input state is the OR of all predecessor blocks' state: */
   for (unsigned i = 0; i < block->predecessors_count; i++) {
      struct ir3_block *predecessor = block->predecessors[i];
      struct ir3_legalize_block_data *pbd = predecessor->data;
      struct ir3_legalize_state *pstate = &pbd->state;

      /* Our input (ss)/(sy) state is based on OR'ing the output
       * state of all our predecessor blocks
       */
      regmask_or(&state->needs_ss, &state->needs_ss, &pstate->needs_ss);
      regmask_or(&state->needs_ss_war, &state->needs_ss_war,
                 &pstate->needs_ss_war);
      regmask_or(&state->needs_sy, &state->needs_sy, &pstate->needs_sy);
   }

   /* We need to take phsyical-only edges into account when tracking shared
    * registers.
    */
   for (unsigned i = 0; i < block->physical_predecessors_count; i++) {
      struct ir3_block *predecessor = block->physical_predecessors[i];
      struct ir3_legalize_block_data *pbd = predecessor->data;
      struct ir3_legalize_state *pstate = &pbd->state;

      regmask_or_shared(&state->needs_ss, &state->needs_ss, &pstate->needs_ss);
   }

   unsigned input_count = 0;

   foreach_instr (n, &block->instr_list) {
      if (is_input(n)) {
         input_count++;
      }
   }

   unsigned inputs_remaining = input_count;

   /* Either inputs are in the first block or we expect inputs to be released
    * with the end of the program.
    */
   assert(input_count == 0 || !ctx->early_input_release ||
          block == ir3_after_preamble(block->shader));

   /* remove all the instructions from the list, we'll be adding
    * them back in as we go
    */
   list_replace(&block->instr_list, &instr_list);
   list_inithead(&block->instr_list);

   foreach_instr_safe (n, &instr_list) {
      unsigned i;

      n->flags &= ~(IR3_INSTR_SS | IR3_INSTR_SY);

      /* _meta::tex_prefetch instructions removed later in
       * collect_tex_prefetches()
       */
      if (is_meta(n) && (n->opc != OPC_META_TEX_PREFETCH))
         continue;

      if (is_input(n)) {
         struct ir3_register *inloc = n->srcs[0];
         assert(inloc->flags & IR3_REG_IMMED);
         ctx->max_bary = MAX2(ctx->max_bary, inloc->iim_val);
      }

      if ((last_n && is_barrier(last_n)) || n->opc == OPC_SHPE) {
         n->flags |= IR3_INSTR_SS | IR3_INSTR_SY;
         last_input_needs_ss = false;
         regmask_init(&state->needs_ss_war, mergedregs);
         regmask_init(&state->needs_ss, mergedregs);
         regmask_init(&state->needs_sy, mergedregs);
      }

      if (last_n && (last_n->opc == OPC_PREDT)) {
         n->flags |= IR3_INSTR_SS;
         regmask_init(&state->needs_ss_war, mergedregs);
         regmask_init(&state->needs_ss, mergedregs);
      }

      /* NOTE: consider dst register too.. it could happen that
       * texture sample instruction (for example) writes some
       * components which are unused.  A subsequent instruction
       * that writes the same register can race w/ the sam instr
       * resulting in undefined results:
       */
      for (i = 0; i < n->dsts_count + n->srcs_count; i++) {
         struct ir3_register *reg;
         if (i < n->dsts_count)
            reg = n->dsts[i];
         else
            reg = n->srcs[i - n->dsts_count];

         if (reg_gpr(reg)) {

            /* TODO: we probably only need (ss) for alu
             * instr consuming sfu result.. need to make
             * some tests for both this and (sy)..
             */
            if (regmask_get(&state->needs_ss, reg)) {
               n->flags |= IR3_INSTR_SS;
               last_input_needs_ss = false;
               regmask_init(&state->needs_ss_war, mergedregs);
               regmask_init(&state->needs_ss, mergedregs);
            }

            if (regmask_get(&state->needs_sy, reg)) {
               n->flags |= IR3_INSTR_SY;
               regmask_init(&state->needs_sy, mergedregs);
            }
         }
      }

      foreach_dst (reg, n) {
         if (regmask_get(&state->needs_ss_war, reg)) {
            n->flags |= IR3_INSTR_SS;
            last_input_needs_ss = false;
            regmask_init(&state->needs_ss_war, mergedregs);
            regmask_init(&state->needs_ss, mergedregs);
         }
      }

      /* cat5+ does not have an (ss) bit, if needed we need to
       * insert a nop to carry the sync flag.  Would be kinda
       * clever if we were aware of this during scheduling, but
       * this should be a pretty rare case:
       */
      if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) {
         struct ir3_instruction *nop;
         nop = ir3_NOP(block);
         nop->flags |= IR3_INSTR_SS;
         n->flags &= ~IR3_INSTR_SS;
      }

      /* need to be able to set (ss) on first instruction: */
      if (list_is_empty(&block->instr_list) && (opc_cat(n->opc) >= 5))
         ir3_NOP(block);

      if (ctx->compiler->samgq_workaround &&
          ctx->type != MESA_SHADER_FRAGMENT &&
          ctx->type != MESA_SHADER_COMPUTE && n->opc == OPC_SAMGQ) {
         struct ir3_instruction *samgp;

         list_delinit(&n->node);

         for (i = 0; i < 4; i++) {
            samgp = ir3_instr_clone(n);
            samgp->opc = OPC_SAMGP0 + i;
            if (i > 1)
               samgp->flags |= IR3_INSTR_SY;
         }
      } else {
         list_delinit(&n->node);
         list_addtail(&n->node, &block->instr_list);
      }

      if (is_sfu(n))
         regmask_set(&state->needs_ss, n->dsts[0]);

      foreach_dst (dst, n) {
         if (dst->flags & IR3_REG_SHARED)
            regmask_set(&state->needs_ss, dst);
      }

      if (is_tex_or_prefetch(n)) {
         regmask_set(&state->needs_sy, n->dsts[0]);
         if (n->opc == OPC_META_TEX_PREFETCH)
            has_tex_prefetch = true;
      } else if (n->opc == OPC_RESINFO) {
         regmask_set(&state->needs_ss, n->dsts[0]);
         ir3_NOP(block)->flags |= IR3_INSTR_SS;
         last_input_needs_ss = false;
      } else if (is_load(n)) {
         if (is_local_mem_load(n))
            regmask_set(&state->needs_ss, n->dsts[0]);
         else
            regmask_set(&state->needs_sy, n->dsts[0]);
      } else if (is_atomic(n->opc)) {
         if (is_bindless_atomic(n->opc)) {
            regmask_set(&state->needs_sy, n->srcs[2]);
         } else if (is_global_a3xx_atomic(n->opc) ||
                    is_global_a6xx_atomic(n->opc)) {
            regmask_set(&state->needs_sy, n->dsts[0]);
         } else {
            regmask_set(&state->needs_ss, n->dsts[0]);
         }
      }

      if (is_ssbo(n->opc) || is_global_a3xx_atomic(n->opc) ||
          is_bindless_atomic(n->opc))
         ctx->so->has_ssbo = true;

      /* both tex/sfu appear to not always immediately consume
       * their src register(s):
       */
      if (is_tex(n) || is_sfu(n) || is_mem(n)) {
         foreach_src (reg, n) {
            regmask_set(&state->needs_ss_war, reg);
         }
      }

      if (ctx->early_input_release && is_input(n)) {
         last_input_needs_ss |= (n->opc == OPC_LDLV);

         assert(inputs_remaining > 0);
         inputs_remaining--;
         if (inputs_remaining == 0) {
            /* This is the last input. We add the (ei) flag to release
             * varying memory after this executes. If it's an ldlv,
             * however, we need to insert a dummy bary.f on which we can
             * set the (ei) flag. We may also need to insert an (ss) to
             * guarantee that all ldlv's have finished fetching their
             * results before releasing the varying memory.
             */
            struct ir3_instruction *last_input = n;
            if (n->opc == OPC_LDLV) {
               struct ir3_instruction *baryf;

               /* (ss)bary.f (ei)r63.x, 0, r0.x */
               baryf = ir3_instr_create(block, OPC_BARY_F, 1, 2);
               ir3_dst_create(baryf, regid(63, 0), 0);
               ir3_src_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0;
               ir3_src_create(baryf, regid(0, 0), 0);

               last_input = baryf;
            }

            last_input->dsts[0]->flags |= IR3_REG_EI;
            if (last_input_needs_ss) {
               last_input->flags |= IR3_INSTR_SS;
               regmask_init(&state->needs_ss_war, mergedregs);
               regmask_init(&state->needs_ss, mergedregs);
            }
         }
      }

      last_n = n;
   }

   assert(inputs_remaining == 0 || !ctx->early_input_release);

   if (has_tex_prefetch && !ctx->has_inputs) {
      /* texture prefetch, but *no* inputs.. we need to insert a
       * dummy bary.f at the top of the shader to unblock varying
       * storage:
       */
      struct ir3_instruction *baryf;

      /* (ss)bary.f (ei)r63.x, 0, r0.x */
      baryf = ir3_instr_create(block, OPC_BARY_F, 1, 2);
      ir3_dst_create(baryf, regid(63, 0), 0)->flags |= IR3_REG_EI;
      ir3_src_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0;
      ir3_src_create(baryf, regid(0, 0), 0);

      /* insert the dummy bary.f at head: */
      list_delinit(&baryf->node);
      list_add(&baryf->node, &block->instr_list);
   }

   bd->valid = true;

   if (memcmp(&prev_state, state, sizeof(*state))) {
      /* our output state changed, this invalidates all of our
       * successors:
       */
      for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) {
         if (!block->successors[i])
            break;
         struct ir3_legalize_block_data *pbd = block->successors[i]->data;
         pbd->valid = false;
      }
   }

   return true;
}

/* Expands dsxpp and dsypp macros to:
 *
 * dsxpp.1 dst, src
 * dsxpp.1.p dst, src
 *
 * We apply this after flags syncing, as we don't want to sync in between the
 * two (which might happen if dst == src).  We do it before nop scheduling
 * because that needs to count actual instructions.
 */
static bool
apply_fine_deriv_macro(struct ir3_legalize_ctx *ctx, struct ir3_block *block)
{
   struct list_head instr_list;

   /* remove all the instructions from the list, we'll be adding
    * them back in as we go
    */
   list_replace(&block->instr_list, &instr_list);
   list_inithead(&block->instr_list);

   foreach_instr_safe (n, &instr_list) {
      list_addtail(&n->node, &block->instr_list);

      if (n->opc == OPC_DSXPP_MACRO || n->opc == OPC_DSYPP_MACRO) {
         n->opc = (n->opc == OPC_DSXPP_MACRO) ? OPC_DSXPP_1 : OPC_DSYPP_1;

         struct ir3_instruction *op_p = ir3_instr_clone(n);
         op_p->flags = IR3_INSTR_P;

         ctx->so->need_fine_derivatives = true;
      }
   }

   return true;
}

/* NOTE: branch instructions are always the last instruction(s)
 * in the block.  We take advantage of this as we resolve the
 * branches, since "if (foo) break;" constructs turn into
 * something like:
 *
 *   block3 {
 *   	...
 *   	0029:021: mov.s32s32 r62.x, r1.y
 *   	0082:022: br !p0.x, target=block5
 *   	0083:023: br p0.x, target=block4
 *   	// succs: if _[0029:021: mov.s32s32] block4; else block5;
 *   }
 *   block4 {
 *   	0084:024: jump, target=block6
 *   	// succs: block6;
 *   }
 *   block5 {
 *   	0085:025: jump, target=block7
 *   	// succs: block7;
 *   }
 *
 * ie. only instruction in block4/block5 is a jump, so when
 * resolving branches we can easily detect this by checking
 * that the first instruction in the target block is itself
 * a jump, and setup the br directly to the jump's target
 * (and strip back out the now unreached jump)
 *
 * TODO sometimes we end up with things like:
 *
 *    br !p0.x, #2
 *    br p0.x, #12
 *    add.u r0.y, r0.y, 1
 *
 * If we swapped the order of the branches, we could drop one.
 */
static struct ir3_block *
resolve_dest_block(struct ir3_block *block)
{
   /* special case for last block: */
   if (!block->successors[0])
      return block;

   /* NOTE that we may or may not have inserted the jump
    * in the target block yet, so conditions to resolve
    * the dest to the dest block's successor are:
    *
    *   (1) successor[1] == NULL &&
    *   (2) (block-is-empty || only-instr-is-jump)
    */
   if (block->successors[1] == NULL) {
      if (list_is_empty(&block->instr_list)) {
         return block->successors[0];
      } else if (list_length(&block->instr_list) == 1) {
         struct ir3_instruction *instr =
            list_first_entry(&block->instr_list, struct ir3_instruction, node);
         if (instr->opc == OPC_JUMP) {
            /* If this jump is backwards, then we will probably convert
             * the jump being resolved to a backwards jump, which will
             * change a loop-with-continue or loop-with-if into a
             * doubly-nested loop and change the convergence behavior.
             * Disallow this here.
             */
            if (block->successors[0]->index <= block->index)
               return block;
            return block->successors[0];
         }
      }
   }
   return block;
}

static void
remove_unused_block(struct ir3_block *old_target)
{
   list_delinit(&old_target->node);

   /* If there are any physical predecessors due to fallthroughs, then they may
    * fall through to any of the physical successors of this block. But we can
    * only fit two, so just pick the "earliest" one, i.e. the fallthrough if
    * possible.
    *
    * TODO: we really ought to have unlimited numbers of physical successors,
    * both because of this and because we currently don't model some scenarios
    * with nested break/continue correctly.
    */
   struct ir3_block *new_target;
   if (old_target->physical_successors[1] &&
       old_target->physical_successors[1]->start_ip <
       old_target->physical_successors[0]->start_ip) {
      new_target = old_target->physical_successors[1];
   } else {
      new_target = old_target->physical_successors[0];
   }

   for (unsigned i = 0; i < old_target->physical_predecessors_count; i++) {
      struct ir3_block *pred = old_target->physical_predecessors[i];
      if (pred->physical_successors[0] == old_target) {
         if (!new_target) {
            /* If we remove a physical successor, make sure the only physical
             * successor is the first one.
             */
            pred->physical_successors[0] = pred->physical_successors[1];
            pred->physical_successors[1] = NULL;
         } else {
            pred->physical_successors[0] = new_target;
         }
      } else {
         assert(pred->physical_successors[1] == old_target);
         pred->physical_successors[1] = new_target;
      }
      if (new_target)
         ir3_block_add_physical_predecessor(new_target, pred);
   }

   /* cleanup dangling predecessors: */
   for (unsigned i = 0; i < ARRAY_SIZE(old_target->successors); i++) {
      if (old_target->successors[i]) {
         struct ir3_block *succ = old_target->successors[i];
         ir3_block_remove_predecessor(succ, old_target);
      }
   }

   for (unsigned i = 0; i < ARRAY_SIZE(old_target->physical_successors); i++) {
      if (old_target->physical_successors[i]) {
         struct ir3_block *succ = old_target->physical_successors[i];
         ir3_block_remove_physical_predecessor(succ, old_target);
      }
   }
}

static bool
retarget_jump(struct ir3_instruction *instr, struct ir3_block *new_target)
{
   struct ir3_block *old_target = instr->cat0.target;
   struct ir3_block *cur_block = instr->block;

   /* update current blocks successors to reflect the retargetting: */
   if (cur_block->successors[0] == old_target) {
      cur_block->successors[0] = new_target;
   } else {
      assert(cur_block->successors[1] == old_target);
      cur_block->successors[1] = new_target;
   }

   /* also update physical_successors: */
   if (cur_block->physical_successors[0] == old_target) {
      cur_block->physical_successors[0] = new_target;
   } else {
      assert(cur_block->physical_successors[1] == old_target);
      cur_block->physical_successors[1] = new_target;
   }

   /* update new target's predecessors: */
   ir3_block_add_predecessor(new_target, cur_block);
   ir3_block_add_physical_predecessor(new_target, cur_block);

   /* and remove old_target's predecessor: */
   ir3_block_remove_predecessor(old_target, cur_block);
   ir3_block_remove_physical_predecessor(old_target, cur_block);

   instr->cat0.target = new_target;

   if (old_target->predecessors_count == 0) {
      remove_unused_block(old_target);
      return true;
   }

   return false;
}

static bool
opt_jump(struct ir3 *ir)
{
   bool progress = false;

   unsigned index = 0;
   foreach_block (block, &ir->block_list)
      block->index = index++;

   foreach_block (block, &ir->block_list) {
      foreach_instr (instr, &block->instr_list) {
         if (!is_flow(instr) || !instr->cat0.target)
            continue;

         struct ir3_block *tblock = resolve_dest_block(instr->cat0.target);
         if (tblock != instr->cat0.target) {
            progress = true;

            /* Exit early if we deleted a block to avoid iterator
             * weirdness/assert fails
             */
            if (retarget_jump(instr, tblock))
               return true;
         }
      }

      /* Detect the case where the block ends either with:
       * - A single unconditional jump to the next block.
       * - Two jump instructions with opposite conditions, and one of the
       *   them jumps to the next block.
       * We can remove the one that jumps to the next block in either case.
       */
      if (list_is_empty(&block->instr_list))
         continue;

      struct ir3_instruction *jumps[2] = {NULL, NULL};
      jumps[0] =
         list_last_entry(&block->instr_list, struct ir3_instruction, node);
      if (!list_is_singular(&block->instr_list))
         jumps[1] =
            list_last_entry(&jumps[0]->node, struct ir3_instruction, node);

      if (jumps[0]->opc == OPC_JUMP)
         jumps[1] = NULL;
      else if (jumps[0]->opc != OPC_B || !jumps[1] || jumps[1]->opc != OPC_B)
         continue;

      for (unsigned i = 0; i < 2; i++) {
         if (!jumps[i])
            continue;

         struct ir3_block *tblock = jumps[i]->cat0.target;
         if (&tblock->node == block->node.next) {
            list_delinit(&jumps[i]->node);
            progress = true;
            break;
         }
      }
   }

   return progress;
}

static void
resolve_jumps(struct ir3 *ir)
{
   foreach_block (block, &ir->block_list)
      foreach_instr (instr, &block->instr_list)
         if (is_flow(instr) && instr->cat0.target) {
            struct ir3_instruction *target = list_first_entry(
               &instr->cat0.target->instr_list, struct ir3_instruction, node);

            instr->cat0.immed = (int)target->ip - (int)instr->ip;
         }
}

static void
mark_jp(struct ir3_block *block)
{
   /* We only call this on the end block (in kill_sched) or after retargeting
    * all jumps to empty blocks (in mark_xvergence_points) so there's no need to
    * worry about empty blocks.
    */
   assert(!list_is_empty(&block->instr_list));

   struct ir3_instruction *target =
      list_first_entry(&block->instr_list, struct ir3_instruction, node);
   target->flags |= IR3_INSTR_JP;
}

/* Mark points where control flow converges or diverges.
 *
 * Divergence points could actually be re-convergence points where
 * "parked" threads are recoverged with threads that took the opposite
 * path last time around.  Possibly it is easier to think of (jp) as
 * "the execution mask might have changed".
 */
static void
mark_xvergence_points(struct ir3 *ir)
{
   foreach_block (block, &ir->block_list) {
      /* We need to insert (jp) if an entry in the "branch stack" is created for
       * our block. This happens if there is a predecessor to our block that may
       * fallthrough to an earlier block in the physical CFG, either because it
       * ends in a non-uniform conditional branch or because there's a
       * fallthrough for an block in-between that also starts with (jp) and was
       * pushed on the branch stack already.
       */
      for (unsigned i = 0; i < block->predecessors_count; i++) {
         struct ir3_block *pred = block->predecessors[i];

         for (unsigned j = 0; j < ARRAY_SIZE(pred->physical_successors); j++) {
            if (pred->physical_successors[j] != NULL &&
                pred->physical_successors[j]->start_ip < block->start_ip)
               mark_jp(block);

            /* If the predecessor just falls through to this block, we still
             * need to check if it "falls through" by jumping to the block. This
             * can happen if opt_jump fails and the block ends in two branches,
             * or if there's an empty if-statement (which currently can happen
             * with binning shaders after dead-code elimination) and the block
             * before ends with a conditional branch directly to this block.
             */
            if (pred->physical_successors[j] == block) {
               foreach_instr_rev (instr, &pred->instr_list) {
                  if (!is_flow(instr))
                     break;
                  if (instr->cat0.target == block) {
                     mark_jp(block);
                     break;
                  }
               }
            }
         }
      }
   }
}

/* Insert the branch/jump instructions for flow control between blocks.
 * Initially this is done naively, without considering if the successor
 * block immediately follows the current block (ie. so no jump required),
 * but that is cleaned up in opt_jump().
 *
 * TODO what ensures that the last write to p0.x in a block is the
 * branch condition?  Have we been getting lucky all this time?
 */
static void
block_sched(struct ir3 *ir)
{
   foreach_block (block, &ir->block_list) {
      if (block->successors[1]) {
         /* if/else, conditional branches to "then" or "else": */
         struct ir3_instruction *br1, *br2;

         if (block->brtype == IR3_BRANCH_GETONE ||
             block->brtype == IR3_BRANCH_SHPS) {
            /* getone/shps can't be inverted, and it wouldn't even make sense
             * to follow it with an inverted branch, so follow it by an
             * unconditional branch.
             */
            assert(!block->condition);
            if (block->brtype == IR3_BRANCH_GETONE)
               br1 = ir3_GETONE(block);
            else
               br1 = ir3_SHPS(block);
            br1->cat0.target = block->successors[1];

            br2 = ir3_JUMP(block);
            br2->cat0.target = block->successors[0];
         } else {
            assert(block->condition);

            /* create "else" branch first (since "then" block should
             * frequently/always end up being a fall-thru):
             */
            br1 = ir3_instr_create(block, OPC_B, 0, 1);
            ir3_src_create(br1, regid(REG_P0, 0), 0)->def =
               block->condition->dsts[0];
            br1->cat0.inv1 = true;
            br1->cat0.target = block->successors[1];

            /* "then" branch: */
            br2 = ir3_instr_create(block, OPC_B, 0, 1);
            ir3_src_create(br2, regid(REG_P0, 0), 0)->def =
               block->condition->dsts[0];
            br2->cat0.target = block->successors[0];

            switch (block->brtype) {
            case IR3_BRANCH_COND:
               br1->cat0.brtype = br2->cat0.brtype = BRANCH_PLAIN;
               break;
            case IR3_BRANCH_ALL:
               br1->cat0.brtype = BRANCH_ANY;
               br2->cat0.brtype = BRANCH_ALL;
               break;
            case IR3_BRANCH_ANY:
               br1->cat0.brtype = BRANCH_ALL;
               br2->cat0.brtype = BRANCH_ANY;
               break;
            case IR3_BRANCH_GETONE:
            case IR3_BRANCH_SHPS:
               unreachable("can't get here");
            }
         }
      } else if (block->successors[0]) {
         /* otherwise unconditional jump to next block: */
         struct ir3_instruction *jmp;

         jmp = ir3_JUMP(block);
         jmp->cat0.target = block->successors[0];
      }
   }
}

/* Here we workaround the fact that kill doesn't actually kill the thread as
 * GL expects. The last instruction always needs to be an end instruction,
 * which means that if we're stuck in a loop where kill is the only way out,
 * then we may have to jump out to the end. kill may also have the d3d
 * semantics of converting the thread to a helper thread, rather than setting
 * the exec mask to 0, in which case the helper thread could get stuck in an
 * infinite loop.
 *
 * We do this late, both to give the scheduler the opportunity to reschedule
 * kill instructions earlier and to avoid having to create a separate basic
 * block.
 *
 * TODO: Assuming that the wavefront doesn't stop as soon as all threads are
 * killed, we might benefit by doing this more aggressively when the remaining
 * part of the program after the kill is large, since that would let us
 * skip over the instructions when there are no non-killed threads left.
 */
static void
kill_sched(struct ir3 *ir, struct ir3_shader_variant *so)
{
   /* True if we know that this block will always eventually lead to the end
    * block:
    */
   bool always_ends = true;
   bool added = false;
   struct ir3_block *last_block =
      list_last_entry(&ir->block_list, struct ir3_block, node);

   foreach_block_rev (block, &ir->block_list) {
      for (unsigned i = 0; i < 2 && block->successors[i]; i++) {
         if (block->successors[i]->start_ip <= block->end_ip)
            always_ends = false;
      }

      if (always_ends)
         continue;

      foreach_instr_safe (instr, &block->instr_list) {
         if (instr->opc != OPC_KILL)
            continue;

         struct ir3_instruction *br = ir3_instr_create(block, OPC_B, 0, 1);
         ir3_src_create(br, instr->srcs[0]->num, instr->srcs[0]->flags)->wrmask =
            1;
         br->cat0.target =
            list_last_entry(&ir->block_list, struct ir3_block, node);

         list_del(&br->node);
         list_add(&br->node, &instr->node);

         added = true;
      }
   }

   if (added) {
      /* I'm not entirely sure how the branchstack works, but we probably
       * need to add at least one entry for the divergence which is resolved
       * at the end:
       */
      so->branchstack++;

      /* We don't update predecessors/successors, so we have to do this
       * manually:
       */
      mark_jp(last_block);
   }
}

/* Insert nop's required to make this a legal/valid shader program: */
static void
nop_sched(struct ir3 *ir, struct ir3_shader_variant *so)
{
   foreach_block (block, &ir->block_list) {
      struct ir3_instruction *last = NULL;
      struct list_head instr_list;

      /* remove all the instructions from the list, we'll be adding
       * them back in as we go
       */
      list_replace(&block->instr_list, &instr_list);
      list_inithead(&block->instr_list);

      foreach_instr_safe (instr, &instr_list) {
         unsigned delay = ir3_delay_calc(block, instr, so->mergedregs);

         /* NOTE: I think the nopN encoding works for a5xx and
          * probably a4xx, but not a3xx.  So far only tested on
          * a6xx.
          */

         if ((delay > 0) && (ir->compiler->gen >= 6) && last &&
             ((opc_cat(last->opc) == 2) || (opc_cat(last->opc) == 3)) &&
             (last->repeat == 0)) {
            /* the previous cat2/cat3 instruction can encode at most 3 nop's: */
            unsigned transfer = MIN2(delay, 3 - last->nop);
            last->nop += transfer;
            delay -= transfer;
         }

         if ((delay > 0) && last && (last->opc == OPC_NOP)) {
            /* the previous nop can encode at most 5 repeats: */
            unsigned transfer = MIN2(delay, 5 - last->repeat);
            last->repeat += transfer;
            delay -= transfer;
         }

         if (delay > 0) {
            assert(delay <= 6);
            ir3_NOP(block)->repeat = delay - 1;
         }

         list_addtail(&instr->node, &block->instr_list);
         last = instr;
      }
   }
}

bool
ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary)
{
   struct ir3_legalize_ctx *ctx = rzalloc(ir, struct ir3_legalize_ctx);
   bool mergedregs = so->mergedregs;
   bool progress;

   ctx->so = so;
   ctx->max_bary = -1;
   ctx->compiler = ir->compiler;
   ctx->type = ir->type;

   /* allocate per-block data: */
   foreach_block (block, &ir->block_list) {
      struct ir3_legalize_block_data *bd =
         rzalloc(ctx, struct ir3_legalize_block_data);

      regmask_init(&bd->state.needs_ss_war, mergedregs);
      regmask_init(&bd->state.needs_ss, mergedregs);
      regmask_init(&bd->state.needs_sy, mergedregs);

      block->data = bd;
   }

   /* We may have failed to pull all input loads into the first block.
    * In such case at the moment we aren't able to find a better place
    * to for (ei) than the end of the program.
    * a5xx and a6xx do automatically release varying storage at the end.
    */
   ctx->early_input_release = true;
   struct ir3_block *start_block = ir3_after_preamble(ir);
   foreach_block (block, &ir->block_list) {
      foreach_instr (instr, &block->instr_list) {
         if (is_input(instr)) {
            ctx->has_inputs = true;
            if (block != start_block) {
               ctx->early_input_release = false;
               break;
            }
         }
      }
   }

   assert(ctx->early_input_release || ctx->compiler->gen >= 5);

   /* process each block: */
   do {
      progress = false;
      foreach_block (block, &ir->block_list) {
         progress |= legalize_block(ctx, block);
      }
   } while (progress);

   *max_bary = ctx->max_bary;

   block_sched(ir);
   if (so->type == MESA_SHADER_FRAGMENT)
      kill_sched(ir, so);

   foreach_block (block, &ir->block_list) {
      progress |= apply_fine_deriv_macro(ctx, block);
   }

   nop_sched(ir, so);

   while (opt_jump(ir))
      ;

   ir3_count_instructions(ir);
   resolve_jumps(ir);

   mark_xvergence_points(ir);

   ralloc_free(ctx);

   return true;
}