Lines Matching refs:ctx
145 add_base_reg(struct ra_spill_ctx *ctx, struct ir3 *ir)
164 ctx->base_reg = mov->dsts[0];
170 ctx->limit_pressure.full -= reg_size(ctx->base_reg);
211 compute_block_next_distance(struct ra_spill_ctx *ctx, struct ir3_block *block,
214 struct ra_spill_block_state *state = &ctx->blocks[block->index];
216 ctx->live->definitions_count * sizeof(*tmp_next_use));
252 ctx->live->definitions_count * sizeof(*tmp_next_use));
257 struct ra_spill_block_state *pred_state = &ctx->blocks[pred->index];
267 for (unsigned j = 0; j < ctx->live->definitions_count; j++) {
297 compute_next_distance(struct ra_spill_ctx *ctx, struct ir3 *ir)
299 for (unsigned i = 0; i < ctx->live->block_count; i++) {
300 ctx->blocks[i].next_use_start =
301 ralloc_array(ctx, unsigned, ctx->live->definitions_count);
302 ctx->blocks[i].next_use_end =
303 ralloc_array(ctx, unsigned, ctx->live->definitions_count);
305 for (unsigned j = 0; j < ctx->live->definitions_count; j++) {
306 ctx->blocks[i].next_use_start[j] = UINT_MAX;
307 ctx->blocks[i].next_use_end[j] = UINT_MAX;
312 struct ra_spill_block_state *state = &ctx->blocks[block->index];
323 ralloc_array(ctx, unsigned, ctx->live->definitions_count);
329 progress |= compute_block_next_distance(ctx, block, tmp_next_use);
406 ir3_reg_ctx_to_ctx(struct ir3_reg_ctx *ctx)
408 return rb_node_data(struct ra_spill_ctx, ctx, reg_ctx);
448 struct ra_spill_ctx *ctx = ir3_reg_ctx_to_ctx(_ctx);
452 ctx->cur_pressure.shared += size;
455 ctx->cur_pressure.half += size;
456 if (ctx->spilling) {
457 rb_tree_insert(&ctx->half_live_intervals, &interval->half_node,
461 if (ctx->merged_regs || !(interval->interval.reg->flags & IR3_REG_HALF)) {
462 ctx->cur_pressure.full += size;
463 if (ctx->spilling) {
464 rb_tree_insert(&ctx->full_live_intervals, &interval->node,
475 struct ra_spill_ctx *ctx = ir3_reg_ctx_to_ctx(_ctx);
479 ctx->cur_pressure.shared -= size;
482 ctx->cur_pressure.half -= size;
483 if (ctx->spilling) {
484 rb_tree_remove(&ctx->half_live_intervals, &interval->half_node);
487 if (ctx->merged_regs || !(interval->interval.reg->flags & IR3_REG_HALF)) {
488 ctx->cur_pressure.full -= size;
489 if (ctx->spilling) {
490 rb_tree_remove(&ctx->full_live_intervals, &interval->node);
504 spill_ctx_init(struct ra_spill_ctx *ctx, struct ir3_shader_variant *v,
507 ctx->live = live;
508 ctx->intervals = ralloc_array(ctx, struct ra_spill_interval *,
509 ctx->live->definitions_count);
511 rzalloc_array(ctx, struct ra_spill_interval,
512 ctx->live->definitions_count);
513 for (unsigned i = 0; i < ctx->live->definitions_count; i++)
514 ctx->intervals[i] = &intervals[i];
516 ctx->intervals_count = ctx->live->definitions_count;
517 ctx->compiler = v->compiler;
518 ctx->merged_regs = v->mergedregs;
520 rb_tree_init(&ctx->reg_ctx.intervals);
521 ctx->reg_ctx.interval_add = interval_add;
522 ctx->reg_ctx.interval_delete = interval_delete;
523 ctx->reg_ctx.interval_readd = interval_readd;
527 ra_spill_ctx_insert(struct ra_spill_ctx *ctx,
530 ir3_reg_interval_insert(&ctx->reg_ctx, &interval->interval);
534 ra_spill_ctx_remove(struct ra_spill_ctx *ctx,
537 ir3_reg_interval_remove(&ctx->reg_ctx, &interval->interval);
541 init_dst(struct ra_spill_ctx *ctx, struct ir3_register *dst)
543 struct ra_spill_interval *interval = ctx->intervals[dst->name];
545 if (ctx->spilling) {
558 insert_dst(struct ra_spill_ctx *ctx, struct ir3_register *dst)
560 struct ra_spill_interval *interval = ctx->intervals[dst->name];
564 ra_spill_ctx_insert(ctx, interval);
578 ctx->max_pressure.shared = MAX2(ctx->max_pressure.shared, max);
580 ctx->max_pressure.half = MAX2(ctx->max_pressure.half, max);
582 ctx->max_pressure.full = MAX2(ctx->max_pressure.full, max);
587 insert_src(struct ra_spill_ctx *ctx, struct ir3_register *src)
589 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
592 ra_spill_ctx_insert(ctx, interval);
602 remove_src_early(struct ra_spill_ctx *ctx, struct ir3_instruction *instr,
605 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
611 ra_spill_ctx_remove(ctx, interval);
615 remove_src(struct ra_spill_ctx *ctx, struct ir3_instruction *instr,
618 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
623 ra_spill_ctx_remove(ctx, interval);
627 finish_dst(struct ra_spill_ctx *ctx, struct ir3_register *dst)
629 struct ra_spill_interval *interval = ctx->intervals[dst->name];
634 remove_dst(struct ra_spill_ctx *ctx, struct ir3_register *dst)
636 struct ra_spill_interval *interval = ctx->intervals[dst->name];
641 ra_spill_ctx_remove(ctx, interval);
645 update_src_next_use(struct ra_spill_ctx *ctx, struct ir3_register *src)
647 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
658 rb_tree_remove(&ctx->half_live_intervals, &interval->half_node);
659 rb_tree_insert(&ctx->half_live_intervals, &interval->half_node,
662 if (ctx->merged_regs || !(src->flags & IR3_REG_HALF)) {
663 rb_tree_remove(&ctx->full_live_intervals, &interval->node);
664 rb_tree_insert(&ctx->full_live_intervals, &interval->node,
671 get_spill_slot(struct ra_spill_ctx *ctx, struct ir3_register *reg)
675 reg->merge_set->spill_slot = ALIGN_POT(ctx->spill_slot,
677 ctx->spill_slot = reg->merge_set->spill_slot + reg->merge_set->size * 2;
682 reg->spill_slot = ALIGN_POT(ctx->spill_slot, reg_elem_size(reg));
683 ctx->spill_slot = reg->spill_slot + reg_size(reg) * 2;
725 spill(struct ra_spill_ctx *ctx, const struct reg_or_immed *val,
746 ir3_src_create(spill, INVALID_REG, ctx->base_reg->flags)->def = ctx->base_reg;
769 spill_interval(struct ra_spill_ctx *ctx, struct ra_spill_interval *interval,
775 spill(ctx, &interval->dst, get_spill_slot(ctx, interval->interval.reg),
781 limit(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
783 if (ctx->cur_pressure.half > ctx->limit_pressure.half) {
784 d("cur half pressure %u exceeds %u", ctx->cur_pressure.half,
785 ctx->limit_pressure.half);
787 &ctx->half_live_intervals, half_node) {
792 spill_interval(ctx, interval, instr, instr->block);
793 ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
794 if (ctx->cur_pressure.half <= ctx->limit_pressure.half)
799 assert(ctx->cur_pressure.half <= ctx->limit_pressure.half);
802 if (ctx->cur_pressure.full > ctx->limit_pressure.full) {
803 d("cur full pressure %u exceeds %u", ctx->cur_pressure.full,
804 ctx->limit_pressure.full);
806 &ctx->full_live_intervals, node) {
811 spill_interval(ctx, interval, instr, instr->block);
812 ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
813 if (ctx->cur_pressure.full <= ctx->limit_pressure.full)
820 assert(ctx->cur_pressure.full <= ctx->limit_pressure.full);
899 reload(struct ra_spill_ctx *ctx, struct ir3_register *reg,
902 unsigned spill_slot = get_spill_slot(ctx, reg);
920 ir3_src_create(reload, INVALID_REG, ctx->base_reg->flags)->def = ctx->base_reg;
947 rewrite_src_interval(struct ra_spill_ctx *ctx,
964 rewrite_src_interval(ctx, child, child_def, instr, block);
969 reload_def(struct ra_spill_ctx *ctx, struct ir3_register *def,
973 struct ra_spill_interval *interval = ctx->intervals[def->name];
993 dst = reload(ctx, def, instr, block);
995 rewrite_src_interval(ctx, interval, dst, instr, block);
999 reload_src(struct ra_spill_ctx *ctx, struct ir3_instruction *instr,
1002 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
1005 reload_def(ctx, src->def, instr, instr->block);
1012 rewrite_src(struct ra_spill_ctx *ctx, struct ir3_instruction *instr,
1015 struct ra_spill_interval *interval = ctx->intervals[src->def->name];
1021 update_max_pressure(struct ra_spill_ctx *ctx)
1024 d("\tfull: %u", ctx->cur_pressure.full);
1025 d("\thalf: %u", ctx->cur_pressure.half);
1026 d("\tshared: %u", ctx->cur_pressure.shared);
1028 ctx->max_pressure.full =
1029 MAX2(ctx->max_pressure.full, ctx->cur_pressure.full);
1030 ctx->max_pressure.half =
1031 MAX2(ctx->max_pressure.half, ctx->cur_pressure.half);
1032 ctx->max_pressure.shared =
1033 MAX2(ctx->max_pressure.shared, ctx->cur_pressure.shared);
1037 handle_instr(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
1040 init_dst(ctx, dst);
1043 if (ctx->spilling) {
1045 insert_src(ctx, src);
1059 insert_dst(ctx, dst);
1062 if (ctx->spilling)
1063 limit(ctx, instr);
1065 update_max_pressure(ctx);
1067 if (ctx->spilling) {
1069 reload_src(ctx, instr, src);
1070 update_src_next_use(ctx, src);
1076 remove_src_early(ctx, instr, src);
1080 insert_dst(ctx, dst);
1083 if (ctx->spilling)
1084 limit(ctx, instr);
1086 update_max_pressure(ctx);
1093 remove_src(ctx, instr, src);
1096 if (ctx->spilling) {
1098 rewrite_src(ctx, instr, src);
1103 finish_dst(ctx, dst);
1109 remove_dst(ctx, instr->dsts[i]);
1114 create_temp_interval(struct ra_spill_ctx *ctx, struct ir3_register *def)
1116 unsigned name = ctx->intervals_count++;
1117 unsigned offset = ctx->live->interval_offset;
1122 struct ir3_register *reg = rzalloc(ctx, struct ir3_register);
1129 ctx->intervals = reralloc(ctx, ctx->intervals, struct ra_spill_interval *,
1130 ctx->intervals_count);
1131 struct ra_spill_interval *interval = rzalloc(ctx, struct ra_spill_interval);
1133 ctx->intervals[name] = interval;
1134 ctx->live->interval_offset += reg_size(def);
1194 handle_pcopy(struct ra_spill_ctx *ctx, struct ir3_instruction *pcopy)
1197 struct ra_spill_interval *dst_interval = ctx->intervals[dst->name];
1212 struct ra_spill_interval *src_interval = ctx->intervals[src->def->name];
1213 struct ra_spill_interval *dst_interval = ctx->intervals[dst->name];
1215 update_src_next_use(ctx, src);
1217 ra_spill_ctx_remove(ctx, src_interval);
1219 ra_spill_ctx_insert(ctx, dst_interval);
1220 limit(ctx, pcopy);
1226 create_temp_interval(ctx, dst);
1230 insert_src(ctx, src);
1231 limit(ctx, pcopy);
1232 reload_src(ctx, pcopy, src);
1233 update_src_next_use(ctx, src);
1235 remove_src(ctx, pcopy, src);
1237 ctx->intervals[src->def->name];
1241 ra_spill_ctx_insert(ctx, temp_interval);
1242 limit(ctx, pcopy);
1260 struct ra_spill_interval *dst_interval = ctx->intervals[dst->name];
1264 ra_spill_ctx_insert(ctx, dst_interval);
1265 limit(ctx, pcopy);
1277 struct ra_spill_interval *temp_interval = ctx->intervals[src->def->name];
1279 insert_src(ctx, src);
1280 limit(ctx, pcopy);
1281 reload_src(ctx, pcopy, src);
1282 remove_src(ctx, pcopy, src);
1285 ra_spill_ctx_insert(ctx, dst_interval);
1293 handle_input_phi(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
1295 init_dst(ctx, instr->dsts[0]);
1296 insert_dst(ctx, instr->dsts[0]);
1297 finish_dst(ctx, instr->dsts[0]);
1301 remove_input_phi(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
1305 remove_src(ctx, instr, src);
1308 remove_dst(ctx, instr->dsts[0]);
1312 handle_live_in(struct ra_spill_ctx *ctx, struct ir3_block *block,
1315 struct ra_spill_interval *interval = ctx->intervals[def->name];
1317 if (ctx->spilling) {
1319 ctx->blocks[block->index].next_use_start[def->name];
1322 ra_spill_ctx_insert(ctx, interval);
1332 is_live_in_pred(struct ra_spill_ctx *ctx, struct ir3_register *def,
1336 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1357 read_live_in(struct ra_spill_ctx *ctx, struct ir3_register *def,
1361 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1377 is_live_in_all_preds(struct ra_spill_ctx *ctx, struct ir3_register *def,
1381 if (!is_live_in_pred(ctx, def, block, i))
1389 spill_live_in(struct ra_spill_ctx *ctx, struct ir3_register *def,
1394 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1399 struct reg_or_immed *pred_def = read_live_in(ctx, def, block, i);
1401 spill(ctx, pred_def, get_spill_slot(ctx, def), NULL, pred);
1407 spill_live_ins(struct ra_spill_ctx *ctx, struct ir3_block *block)
1412 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1428 if (ctx->cur_pressure.half > ctx->limit_pressure.half) {
1430 &ctx->half_live_intervals, half_node) {
1432 is_live_in_all_preds(ctx, interval->interval.reg, block))
1436 spill_live_in(ctx, interval->interval.reg, block);
1437 ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
1438 if (ctx->cur_pressure.half <= ctx->limit_pressure.half)
1443 if (ctx->cur_pressure.full > ctx->limit_pressure.full) {
1445 &ctx->full_live_intervals, node) {
1447 is_live_in_all_preds(ctx, interval->interval.reg, block))
1449 spill_live_in(ctx, interval->interval.reg, block);
1450 ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
1451 if (ctx->cur_pressure.full <= ctx->limit_pressure.full)
1458 live_in_rewrite(struct ra_spill_ctx *ctx,
1464 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1481 struct reg_or_immed *child_val = ralloc(ctx, struct reg_or_immed);
1484 live_in_rewrite(ctx, child, child_val, block, pred_idx);
1489 reload_live_in(struct ra_spill_ctx *ctx, struct ir3_register *def,
1492 struct ra_spill_interval *interval = ctx->intervals[def->name];
1495 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1502 struct reg_or_immed *new_val = read_live_in(ctx, def, block, i);
1505 new_val = ralloc(ctx, struct reg_or_immed);
1509 new_val->def = reload(ctx, def, NULL, pred);
1512 live_in_rewrite(ctx, interval, new_val, block, i);
1517 reload_live_ins(struct ra_spill_ctx *ctx, struct ir3_block *block)
1519 rb_tree_foreach (struct ra_spill_interval, interval, &ctx->reg_ctx.intervals,
1521 reload_live_in(ctx, interval->interval.reg, block);
1526 add_live_in_phi(struct ra_spill_ctx *ctx, struct ir3_register *def,
1529 struct ra_spill_interval *interval = ctx->intervals[def->name];
1537 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1577 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1606 spill_single_pred_live_in(struct ra_spill_ctx *ctx,
1610 BITSET_FOREACH_SET (name, ctx->live->live_in[block->index],
1611 ctx->live->definitions_count) {
1612 struct ir3_register *reg = ctx->live->definitions[name];
1613 struct ra_spill_interval *interval = ctx->intervals[reg->name];
1614 struct reg_or_immed *val = read_live_in(ctx, reg, block, 0);
1618 ra_spill_ctx_remove(ctx, interval);
1623 rewrite_phi(struct ra_spill_ctx *ctx, struct ir3_instruction *phi,
1626 if (!ctx->intervals[phi->dsts[0]->name]->interval.inserted) {
1633 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1651 spill_live_out(struct ra_spill_ctx *ctx, struct ra_spill_interval *interval,
1658 spill(ctx, &interval->dst, get_spill_slot(ctx, def), NULL, block);
1659 ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
1663 spill_live_outs(struct ra_spill_ctx *ctx, struct ir3_block *block)
1665 struct ra_spill_block_state *state = &ctx->blocks[block->index];
1667 &ctx->reg_ctx.intervals, interval.node) {
1669 spill_live_out(ctx, interval, block);
1675 reload_live_out(struct ra_spill_ctx *ctx, struct ir3_register *def,
1678 struct ra_spill_interval *interval = ctx->intervals[def->name];
1679 ir3_reg_interval_insert(&ctx->reg_ctx, &interval->interval);
1681 reload_def(ctx, def, NULL, block);
1685 reload_live_outs(struct ra_spill_ctx *ctx, struct ir3_block *block)
1687 struct ra_spill_block_state *state = &ctx->blocks[block->index];
1689 BITSET_FOREACH_SET (name, state->live_out, ctx->live->definitions_count) {
1690 struct ir3_register *reg = ctx->live->definitions[name];
1691 struct ra_spill_interval *interval = ctx->intervals[name];
1693 reload_live_out(ctx, reg, block);
1698 update_live_out_phis(struct ra_spill_ctx *ctx, struct ir3_block *block)
1712 struct ra_spill_interval *interval = ctx->intervals[def->name];
1720 record_pred_live_out(struct ra_spill_ctx *ctx,
1725 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1735 record_pred_live_out(ctx, child, block, pred_idx);
1740 record_pred_live_outs(struct ra_spill_ctx *ctx, struct ir3_block *block)
1744 struct ra_spill_block_state *state = &ctx->blocks[pred->index];
1748 state->live_out = rzalloc_array(ctx, BITSET_WORD,
1749 BITSET_WORDS(ctx->live->definitions_count));
1753 &ctx->reg_ctx.intervals, interval.node) {
1754 record_pred_live_out(ctx, interval, block, i);
1760 record_live_out(struct ra_spill_ctx *ctx,
1766 struct reg_or_immed *val = ralloc(ctx, struct reg_or_immed);
1772 record_live_out(ctx, state, child);
1777 record_live_outs(struct ra_spill_ctx *ctx, struct ir3_block *block)
1779 struct ra_spill_block_state *state = &ctx->blocks[block->index];
1780 state->remap = _mesa_pointer_hash_table_create(ctx);
1782 rb_tree_foreach (struct ra_spill_interval, interval, &ctx->reg_ctx.intervals,
1784 record_live_out(ctx, state, interval);
1789 handle_block(struct ra_spill_ctx *ctx, struct ir3_block *block)
1791 memset(&ctx->cur_pressure, 0, sizeof(ctx->cur_pressure));
1792 rb_tree_init(&ctx->reg_ctx.intervals);
1793 rb_tree_init(&ctx->full_live_intervals);
1794 rb_tree_init(&ctx->half_live_intervals);
1797 BITSET_FOREACH_SET (name, ctx->live->live_in[block->index],
1798 ctx->live->definitions_count) {
1799 struct ir3_register *reg = ctx->live->definitions[name];
1800 handle_live_in(ctx, block, reg);
1807 handle_input_phi(ctx, instr);
1810 if (ctx->spilling) {
1812 spill_single_pred_live_in(ctx, block);
1814 spill_live_ins(ctx, block);
1815 reload_live_ins(ctx, block);
1816 record_pred_live_outs(ctx, block);
1820 rewrite_phi(ctx, instr, block);
1822 BITSET_FOREACH_SET (name, ctx->live->live_in[block->index],
1823 ctx->live->definitions_count) {
1824 struct ir3_register *reg = ctx->live->definitions[name];
1825 add_live_in_phi(ctx, reg, block);
1829 update_max_pressure(ctx);
1837 remove_input_phi(ctx, instr);
1838 else if (ctx->spilling && instr->opc == OPC_META_PARALLEL_COPY)
1839 handle_pcopy(ctx, instr);
1840 else if (ctx->spilling && instr->opc == OPC_MOV &&
1841 instr->dsts[0] == ctx->base_reg)
1844 handle_instr(ctx, instr);
1847 if (ctx->spilling && block->successors[0]) {
1849 &ctx->blocks[block->successors[0]->index];
1853 spill_live_outs(ctx, block);
1854 reload_live_outs(ctx, block);
1855 update_live_out_phis(ctx, block);
1859 if (ctx->spilling) {
1860 record_live_outs(ctx, block);
1861 ctx->blocks[block->index].visited = true;
2065 struct ra_spill_ctx *ctx = rzalloc(NULL, struct ra_spill_ctx);
2066 spill_ctx_init(ctx, v, live);
2069 handle_block(ctx, block);
2072 assert(ctx->cur_pressure.full == 0);
2073 assert(ctx->cur_pressure.half == 0);
2074 assert(ctx->cur_pressure.shared == 0);
2076 *max_pressure = ctx->max_pressure;
2077 ralloc_free(ctx);
2086 struct ra_spill_ctx *ctx = rzalloc(mem_ctx, struct ra_spill_ctx);
2087 spill_ctx_init(ctx, v, *live);
2089 ctx->spilling = true;
2091 ctx->blocks = rzalloc_array(ctx, struct ra_spill_block_state,
2092 ctx->live->block_count);
2093 rb_tree_init(&ctx->full_live_intervals);
2094 rb_tree_init(&ctx->half_live_intervals);
2096 ctx->limit_pressure = *limit_pressure;
2097 ctx->spill_slot = v->pvtmem_size;
2099 add_base_reg(ctx, ir);
2100 compute_next_distance(ctx, ir);
2105 handle_block(ctx, block);
2117 ralloc_free(ctx->live);
2122 v->pvtmem_size = ctx->spill_slot;
2123 ralloc_free(ctx);