1/*
2 * Copyright (c) 2017 Lima Project
3 * Copyright (c) 2013 Connor Abbott
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a copy
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *
23 */
24
25#ifndef LIMA_IR_PP_PPIR_H
26#define LIMA_IR_PP_PPIR_H
27
28#include "util/u_math.h"
29#include "util/list.h"
30#include "util/set.h"
31
32#include "ir/lima_ir.h"
33
34typedef enum {
35   ppir_op_unsupported = 0,
36   ppir_op_mov,
37   ppir_op_abs,
38   ppir_op_neg,
39   ppir_op_sat,
40   ppir_op_add,
41
42   ppir_op_ddx,
43   ppir_op_ddy,
44
45   ppir_op_mul,
46   ppir_op_rcp,
47
48   ppir_op_sin_lut,
49   ppir_op_cos_lut,
50
51   ppir_op_sum3,
52   ppir_op_sum4,
53
54   ppir_op_normalize2,
55   ppir_op_normalize3,
56   ppir_op_normalize4,
57
58   ppir_op_select,
59
60   ppir_op_sin,
61   ppir_op_cos,
62   ppir_op_tan,
63   ppir_op_asin,
64   ppir_op_acos,
65
66   ppir_op_atan,
67   ppir_op_atan2,
68   ppir_op_atan_pt1,
69   ppir_op_atan2_pt1,
70   ppir_op_atan_pt2,
71
72   ppir_op_exp,
73   ppir_op_log,
74   ppir_op_exp2,
75   ppir_op_log2,
76   ppir_op_sqrt,
77   ppir_op_rsqrt,
78
79   ppir_op_sign,
80   ppir_op_floor,
81   ppir_op_ceil,
82   ppir_op_fract,
83   ppir_op_mod,
84   ppir_op_min,
85   ppir_op_max,
86   ppir_op_trunc,
87
88   ppir_op_and,
89   ppir_op_or,
90   ppir_op_xor,
91
92   ppir_op_lt,
93   ppir_op_gt,
94   ppir_op_le,
95   ppir_op_ge,
96   ppir_op_eq,
97   ppir_op_ne,
98   ppir_op_not,
99
100   ppir_op_load_uniform,
101   ppir_op_load_varying,
102   ppir_op_load_coords,
103   ppir_op_load_coords_reg,
104   ppir_op_load_fragcoord,
105   ppir_op_load_pointcoord,
106   ppir_op_load_frontface,
107   ppir_op_load_texture,
108   ppir_op_load_temp,
109
110   ppir_op_store_temp,
111
112   ppir_op_const,
113
114   ppir_op_discard,
115   ppir_op_branch,
116
117   ppir_op_undef,
118   ppir_op_dummy,
119
120   ppir_op_num,
121} ppir_op;
122
123typedef enum {
124   ppir_node_type_alu,
125   ppir_node_type_const,
126   ppir_node_type_load,
127   ppir_node_type_store,
128   ppir_node_type_load_texture,
129   ppir_node_type_discard,
130   ppir_node_type_branch,
131} ppir_node_type;
132
133typedef struct {
134   char *name;
135   ppir_node_type type;
136   int *slots;
137} ppir_op_info;
138
139extern const ppir_op_info ppir_op_infos[];
140
141typedef enum {
142   ppir_dep_src,
143   ppir_dep_write_after_read,
144   ppir_dep_sequence,
145} ppir_dep_type;
146
147typedef struct {
148   void *pred, *succ;
149   ppir_dep_type type;
150   struct list_head pred_link;
151   struct list_head succ_link;
152} ppir_dep;
153
154typedef struct ppir_node {
155   struct list_head list;
156   struct list_head sched_list;
157   ppir_op op;
158   ppir_node_type type;
159   int index;
160   char name[16];
161   bool printed;
162   struct ppir_instr *instr;
163   int instr_pos;
164   struct ppir_block *block;
165   bool is_out;
166   bool succ_different_block;
167
168   /* for scheduler */
169   struct list_head succ_list;
170   struct list_head pred_list;
171} ppir_node;
172
173typedef enum {
174   ppir_pipeline_reg_const0,
175   ppir_pipeline_reg_const1,
176   ppir_pipeline_reg_sampler,
177   ppir_pipeline_reg_uniform,
178   ppir_pipeline_reg_vmul,
179   ppir_pipeline_reg_fmul,
180   ppir_pipeline_reg_discard, /* varying load */
181} ppir_pipeline;
182
183typedef enum {
184   ppir_output_color0,
185   ppir_output_color1,
186   ppir_output_depth,
187   ppir_output_num,
188   ppir_output_invalid = -1,
189} ppir_output_type;
190
191static inline const char *ppir_output_type_to_str(ppir_output_type type)
192{
193   switch (type) {
194   case ppir_output_color0:
195      return "OUTPUT_COLOR0";
196   case ppir_output_color1:
197      return "OUTPUT_COLOR1";
198   case ppir_output_depth:
199      return "OUTPUT_DEPTH";
200   default:
201      return "INVALID";
202   }
203}
204
205static inline ppir_output_type ppir_nir_output_to_ppir(gl_frag_result res, int dual_src_index)
206{
207   switch (res) {
208   case FRAG_RESULT_COLOR:
209   case FRAG_RESULT_DATA0:
210      return ppir_output_color0 + dual_src_index;
211   case FRAG_RESULT_DEPTH:
212      return ppir_output_depth;
213   default:
214      return ppir_output_invalid;
215   }
216}
217
218typedef struct ppir_reg {
219   struct list_head list;
220   int index;
221   ppir_output_type out_type;
222   int regalloc_index;
223   int num_components;
224
225   /* whether this reg has to start from the x component
226    * of a full physical reg, this is true for reg used
227    * in load/store instr which has no swizzle field */
228   bool is_head;
229   bool spilled;
230   bool undef;
231   bool out_reg;
232} ppir_reg;
233
234typedef enum {
235   ppir_target_ssa,
236   ppir_target_pipeline,
237   ppir_target_register,
238} ppir_target;
239
240typedef struct ppir_src {
241   ppir_target type;
242   ppir_node *node;
243
244   union {
245      ppir_reg *ssa;
246      ppir_reg *reg;
247      ppir_pipeline pipeline;
248   };
249
250   uint8_t swizzle[4];
251   bool absolute, negate;
252} ppir_src;
253
254typedef enum {
255   ppir_outmod_none,
256   ppir_outmod_clamp_fraction,
257   ppir_outmod_clamp_positive,
258   ppir_outmod_round,
259} ppir_outmod;
260
261typedef struct ppir_dest {
262   ppir_target type;
263
264   union {
265      ppir_reg ssa;
266      ppir_reg *reg;
267      ppir_pipeline pipeline;
268   };
269
270   ppir_outmod modifier;
271   unsigned write_mask : 4;
272} ppir_dest;
273
274typedef struct {
275   ppir_node node;
276   ppir_dest dest;
277   ppir_src src[3];
278   int num_src;
279   int shift : 3; /* Only used for ppir_op_mul */
280} ppir_alu_node;
281
282typedef struct ppir_const {
283   union fi value[4];
284   int num;
285} ppir_const;
286
287typedef struct {
288   ppir_node node;
289   ppir_const constant;
290   ppir_dest dest;
291} ppir_const_node;
292
293typedef enum {
294   ppir_perspective_none = 0,
295   ppir_perspective_z,
296   ppir_perspective_w,
297} ppir_perspective;
298
299typedef struct {
300   ppir_node node;
301   int index;
302   int num_components;
303   ppir_dest dest;
304   ppir_src src;
305   int num_src;
306   ppir_perspective perspective;
307   int sampler_dim;
308} ppir_load_node;
309
310typedef struct {
311   ppir_node node;
312   int index;
313   int num_components;
314   ppir_src src;
315} ppir_store_node;
316
317typedef struct {
318   ppir_node node;
319   ppir_dest dest;
320   ppir_src src[2];
321   int num_src;
322   int sampler;
323   int sampler_dim;
324   bool lod_bias_en;
325   bool explicit_lod;
326} ppir_load_texture_node;
327
328typedef struct {
329   ppir_node node;
330} ppir_discard_node;
331
332enum ppir_instr_slot {
333   PPIR_INSTR_SLOT_VARYING,
334   PPIR_INSTR_SLOT_TEXLD,
335   PPIR_INSTR_SLOT_UNIFORM,
336   PPIR_INSTR_SLOT_ALU_VEC_MUL,
337   PPIR_INSTR_SLOT_ALU_SCL_MUL,
338   PPIR_INSTR_SLOT_ALU_VEC_ADD,
339   PPIR_INSTR_SLOT_ALU_SCL_ADD,
340   PPIR_INSTR_SLOT_ALU_COMBINE,
341   PPIR_INSTR_SLOT_STORE_TEMP,
342   PPIR_INSTR_SLOT_BRANCH,
343   PPIR_INSTR_SLOT_NUM,
344   PPIR_INSTR_SLOT_END,
345   PPIR_INSTR_SLOT_ALU_START = PPIR_INSTR_SLOT_ALU_VEC_MUL,
346   PPIR_INSTR_SLOT_ALU_END = PPIR_INSTR_SLOT_ALU_COMBINE,
347};
348
349typedef struct ppir_instr {
350   struct list_head list;
351   int index;
352   bool printed;
353   int seq; /* command sequence after schedule */
354
355   ppir_node *slots[PPIR_INSTR_SLOT_NUM];
356   ppir_const constant[2];
357   bool stop;
358
359   /* for scheduler */
360   struct list_head succ_list;
361   struct list_head pred_list;
362   float reg_pressure;
363   int est; /* earliest start time */
364   int parent_index;
365   bool scheduled;
366   int offset;
367   int encode_size;
368
369   /* for liveness analysis */
370   BITSET_WORD *live_set;
371   uint8_t *live_mask; /* mask for non-ssa registers */
372   /* live_internal is to mark registers only live within an
373    * instruction, without propagation */
374   BITSET_WORD *live_internal;
375} ppir_instr;
376
377typedef struct ppir_block {
378   struct list_head list;
379   struct list_head node_list;
380   struct list_head instr_list;
381   bool stop;
382
383   struct ppir_block *successors[2];
384
385   struct ppir_compiler *comp;
386
387   /* for scheduler */
388   int sched_instr_index;
389   int sched_instr_base;
390   int index;
391} ppir_block;
392
393typedef struct {
394   ppir_node node;
395   ppir_src src[2];
396   int num_src;
397   bool cond_gt;
398   bool cond_eq;
399   bool cond_lt;
400   bool negate;
401   ppir_block *target;
402} ppir_branch_node;
403
404struct ra_regs;
405struct lima_fs_compiled_shader;
406
407typedef struct ppir_compiler {
408   struct list_head block_list;
409   struct hash_table_u64 *blocks;
410   int cur_index;
411   int cur_instr_index;
412   int *out_type_to_reg;
413
414   struct list_head reg_list;
415   int reg_num;
416
417   /* array for searching ssa/reg node */
418   ppir_node **var_nodes;
419   unsigned reg_base;
420
421   struct ra_regs *ra;
422   struct lima_fs_compiled_shader *prog;
423   bool uses_discard;
424   bool dual_source_blend;
425
426   /* for scheduler */
427   int sched_instr_base;
428
429   /* for regalloc spilling debug */
430   int force_spilling;
431
432   /* shaderdb */
433   int num_loops;
434   int num_spills;
435   int num_fills;
436
437   ppir_block *discard_block;
438   ppir_block *current_block;
439   ppir_block *loop_break_block;
440   ppir_block *loop_cont_block;
441} ppir_compiler;
442
443void *ppir_node_create(ppir_block *block, ppir_op op, int index, unsigned mask);
444void ppir_node_add_dep(ppir_node *succ, ppir_node *pred, ppir_dep_type type);
445void ppir_node_remove_dep(ppir_dep *dep);
446void ppir_node_delete(ppir_node *node);
447void ppir_node_print_prog(ppir_compiler *comp);
448void ppir_node_replace_child(ppir_node *parent, ppir_node *old_child, ppir_node *new_child);
449void ppir_node_replace_all_succ(ppir_node *dst, ppir_node *src);
450void ppir_node_replace_pred(ppir_dep *dep, ppir_node *new_pred);
451ppir_dep *ppir_dep_for_pred(ppir_node *node, ppir_node *pred);
452/* Assumes that node successors are in the same block */
453ppir_node *ppir_node_insert_mov(ppir_node *node);
454
455static inline bool ppir_node_is_root(ppir_node *node)
456{
457   return list_is_empty(&node->succ_list);
458}
459
460static inline bool ppir_node_is_leaf(ppir_node *node)
461{
462   return list_is_empty(&node->pred_list);
463}
464
465static inline bool ppir_node_has_single_succ(ppir_node *node)
466{
467   return list_is_singular(&node->succ_list)
468      && !node->succ_different_block;
469}
470
471bool ppir_node_has_single_src_succ(ppir_node *node);
472
473static inline ppir_node *ppir_node_first_succ(ppir_node *node)
474{
475   return list_first_entry(&node->succ_list, ppir_dep, succ_link)->succ;
476}
477
478static inline bool ppir_node_has_single_pred(ppir_node *node)
479{
480   return list_is_singular(&node->pred_list);
481}
482
483static inline ppir_node *ppir_node_first_pred(ppir_node *node)
484{
485   return list_first_entry(&node->pred_list, ppir_dep, pred_link)->pred;
486}
487
488#define ppir_node_foreach_succ(node, dep) \
489   list_for_each_entry(ppir_dep, dep, &node->succ_list, succ_link)
490#define ppir_node_foreach_succ_safe(node, dep) \
491   list_for_each_entry_safe(ppir_dep, dep, &node->succ_list, succ_link)
492#define ppir_node_foreach_pred(node, dep) \
493   list_for_each_entry(ppir_dep, dep, &node->pred_list, pred_link)
494#define ppir_node_foreach_pred_safe(node, dep) \
495   list_for_each_entry_safe(ppir_dep, dep, &node->pred_list, pred_link)
496
497#define ppir_node_to_alu(node) ((ppir_alu_node *)(node))
498#define ppir_node_to_const(node) ((ppir_const_node *)(node))
499#define ppir_node_to_load(node) ((ppir_load_node *)(node))
500#define ppir_node_to_store(node) ((ppir_store_node *)(node))
501#define ppir_node_to_load_texture(node) ((ppir_load_texture_node *)(node))
502#define ppir_node_to_discard(node) ((ppir_discard_node *)(node))
503#define ppir_node_to_branch(node) ((ppir_branch_node *)(node))
504
505static inline ppir_dest *ppir_node_get_dest(ppir_node *node)
506{
507   assert(node);
508   switch (node->type) {
509   case ppir_node_type_alu:
510      return &ppir_node_to_alu(node)->dest;
511   case ppir_node_type_load:
512      return &ppir_node_to_load(node)->dest;
513   case ppir_node_type_const:
514      return &ppir_node_to_const(node)->dest;
515   case ppir_node_type_load_texture:
516      return &ppir_node_to_load_texture(node)->dest;
517   default:
518      return NULL;
519   }
520}
521
522static inline int ppir_node_get_src_num(ppir_node *node)
523{
524   assert(node);
525   switch (node->type) {
526   case ppir_node_type_alu:
527      return ppir_node_to_alu(node)->num_src;
528   case ppir_node_type_branch:
529      return ppir_node_to_branch(node)->num_src;
530   case ppir_node_type_load:
531      return ppir_node_to_load(node)->num_src;
532   case ppir_node_type_load_texture:
533      return ppir_node_to_load_texture(node)->num_src;
534   case ppir_node_type_store:
535      return 1;
536   default:
537      return 0;
538   }
539
540   return 0;
541}
542
543static inline ppir_src *ppir_node_get_src(ppir_node *node, int idx)
544{
545   if (idx < 0 || idx >= ppir_node_get_src_num(node))
546      return NULL;
547
548   switch (node->type) {
549   case ppir_node_type_alu:
550      return &ppir_node_to_alu(node)->src[idx];
551   case ppir_node_type_branch:
552      return &ppir_node_to_branch(node)->src[idx];
553   case ppir_node_type_load_texture:
554      return &ppir_node_to_load_texture(node)->src[idx];
555   case ppir_node_type_load:
556      return &ppir_node_to_load(node)->src;
557   case ppir_node_type_store:
558      return &ppir_node_to_store(node)->src;
559   default:
560      break;
561   }
562
563   return NULL;
564}
565
566static inline ppir_reg *ppir_src_get_reg(ppir_src *src)
567{
568   switch (src->type) {
569   case ppir_target_ssa:
570      return src->ssa;
571   case ppir_target_register:
572      return src->reg;
573   default:
574      return NULL;
575   }
576}
577
578static inline ppir_reg *ppir_dest_get_reg(ppir_dest *dest)
579{
580   switch (dest->type) {
581   case ppir_target_ssa:
582      return &dest->ssa;
583   case ppir_target_register:
584      return dest->reg;
585   default:
586      return NULL;
587   }
588}
589
590static inline void ppir_node_target_assign(ppir_src *src, ppir_node *node)
591{
592   ppir_dest *dest = ppir_node_get_dest(node);
593   src->type = dest->type;
594   switch (src->type) {
595   case ppir_target_ssa:
596      src->ssa = &dest->ssa;
597      src->node = node;
598      break;
599   case ppir_target_register:
600      src->reg = dest->reg;
601      /* Registers can be assigned from multiple nodes, so don't keep
602       * pointer to the node here
603       */
604      src->node = NULL;
605      break;
606   case ppir_target_pipeline:
607      src->pipeline = dest->pipeline;
608      src->node = node;
609      break;
610   }
611}
612
613static inline bool ppir_node_target_equal(ppir_src *src, ppir_dest *dest)
614{
615   if (src->type != dest->type ||
616       (src->type == ppir_target_ssa && src->ssa != &dest->ssa) ||
617       (src->type == ppir_target_register && src->reg != dest->reg) ||
618       (src->type == ppir_target_pipeline && src->pipeline != dest->pipeline))
619      return false;
620
621   return true;
622}
623
624static inline int ppir_target_get_src_reg_index(ppir_src *src)
625{
626   switch (src->type) {
627   case ppir_target_ssa:
628      if (src->ssa)
629         return src->ssa->index;
630      break;
631   case ppir_target_register:
632      if (src->reg)
633         return src->reg->index;
634      break;
635   case ppir_target_pipeline:
636      if (src->pipeline == ppir_pipeline_reg_discard)
637         return 15 * 4;
638      return (src->pipeline + 12) * 4;
639   }
640
641   return -1;
642}
643
644static inline int ppir_target_get_dest_reg_index(ppir_dest *dest)
645{
646   switch (dest->type) {
647   case ppir_target_ssa:
648      return dest->ssa.index;
649   case ppir_target_register:
650      return dest->reg->index;
651   case ppir_target_pipeline:
652      if (dest->pipeline == ppir_pipeline_reg_discard)
653         return 15 * 4;
654      return (dest->pipeline + 12) * 4;
655   }
656
657   return -1;
658}
659
660static inline int ppir_src_get_mask(ppir_src *src)
661{
662   ppir_reg *reg = ppir_src_get_reg(src);
663   int mask = 0;
664
665   for (int i = 0; i < reg->num_components; i++)
666      mask |= (1 << src->swizzle[i]);
667
668   return mask;
669}
670
671static inline bool ppir_target_is_scalar(ppir_dest *dest)
672{
673   switch (dest->type) {
674   case ppir_target_ssa:
675      return dest->ssa.num_components == 1;
676   case ppir_target_register:
677      /* only one bit in mask is set */
678      if ((dest->write_mask & 0x3) == 0x3 ||
679          (dest->write_mask & 0x5) == 0x5 ||
680          (dest->write_mask & 0x9) == 0x9 ||
681          (dest->write_mask & 0x6) == 0x6 ||
682          (dest->write_mask & 0xa) == 0xa ||
683          (dest->write_mask & 0xc) == 0xc)
684         return false;
685      else
686         return true;
687   case ppir_target_pipeline:
688      if (dest->pipeline == ppir_pipeline_reg_fmul)
689         return true;
690      else
691         return false;
692   default:
693      return false;
694   }
695}
696
697static inline bool ppir_node_schedulable_slot(ppir_node *node,
698                                              enum ppir_instr_slot slot)
699{
700   int *slots = ppir_op_infos[node->op].slots;
701   for (int i = 0; slots[i] != PPIR_INSTR_SLOT_END; i++)
702      if (slots[i] == slot)
703         return true;
704
705   return false;
706}
707
708ppir_instr *ppir_instr_create(ppir_block *block);
709bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node);
710void ppir_instr_add_dep(ppir_instr *succ, ppir_instr *pred);
711void ppir_instr_print_list(ppir_compiler *comp);
712void ppir_instr_print_dep(ppir_compiler *comp);
713void ppir_instr_insert_mul_node(ppir_node *add, ppir_node *mul);
714
715#define ppir_instr_foreach_succ(instr, dep) \
716   list_for_each_entry(ppir_dep, dep, &instr->succ_list, succ_link)
717#define ppir_instr_foreach_succ_safe(instr, dep) \
718   list_for_each_entry_safe(ppir_dep, dep, &instr->succ_list, succ_link)
719#define ppir_instr_foreach_pred(instr, dep) \
720   list_for_each_entry(ppir_dep, dep, &instr->pred_list, pred_link)
721#define ppir_instr_foreach_pred_safe(instr, dep) \
722   list_for_each_entry_safe(ppir_dep, dep, &instr->pred_list, pred_link)
723
724static inline bool ppir_instr_is_root(ppir_instr *instr)
725{
726   return list_is_empty(&instr->succ_list);
727}
728
729static inline bool ppir_instr_is_leaf(ppir_instr *instr)
730{
731   return list_is_empty(&instr->pred_list);
732}
733
734bool ppir_lower_prog(ppir_compiler *comp);
735bool ppir_node_to_instr(ppir_compiler *comp);
736bool ppir_schedule_prog(ppir_compiler *comp);
737bool ppir_regalloc_prog(ppir_compiler *comp);
738bool ppir_codegen_prog(ppir_compiler *comp);
739void ppir_liveness_analysis(ppir_compiler *comp);
740
741static inline unsigned int reg_mask_size(unsigned int num_reg)
742{
743   return (num_reg + 1) / 2;
744}
745
746static inline uint8_t get_reg_mask(uint8_t *set, unsigned index)
747{
748   unsigned int i = index / 2;
749   unsigned int shift = index % 2 ? 4 : 0;
750   uint8_t mask = 0x0f << shift;
751   return (set[i] & mask) >> shift;
752}
753
754static inline void set_reg_mask(uint8_t *set, unsigned int index, uint8_t bits)
755{
756   unsigned int i = index / 2;
757   unsigned int shift = index % 2 ? 4 : 0;
758   uint8_t mask = 0x0f << shift;
759   set[i] &= ~mask;
760   set[i] |= (bits << shift);
761}
762
763#endif
764