1/*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include "nv50_ir.h"
24#include "nv50_ir_build_util.h"
25
26namespace nv50_ir {
27
28BuildUtil::BuildUtil()
29{
30   init(NULL);
31}
32
33BuildUtil::BuildUtil(Program *prog)
34{
35   init(prog);
36}
37
38void
39BuildUtil::init(Program *prog)
40{
41   this->prog = prog;
42
43   func = NULL;
44   bb = NULL;
45   pos = NULL;
46
47   tail = false;
48
49   memset(imms, 0, sizeof(imms));
50   immCount = 0;
51}
52
53void
54BuildUtil::addImmediate(ImmediateValue *imm)
55{
56   if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
57      return;
58
59   unsigned int pos = u32Hash(imm->reg.data.u32);
60
61   while (imms[pos])
62      pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
63   imms[pos] = imm;
64   immCount++;
65}
66
67Instruction *
68BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
69{
70   Instruction *insn = new_Instruction(func, op, ty);
71
72   insn->setDef(0, dst);
73   insn->setSrc(0, src);
74
75   insert(insn);
76   return insn;
77}
78
79Instruction *
80BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
81                 Value *src0, Value *src1)
82{
83   Instruction *insn = new_Instruction(func, op, ty);
84
85   insn->setDef(0, dst);
86   insn->setSrc(0, src0);
87   insn->setSrc(1, src1);
88
89   insert(insn);
90   return insn;
91}
92
93Instruction *
94BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
95                 Value *src0, Value *src1, Value *src2)
96{
97   Instruction *insn = new_Instruction(func, op, ty);
98
99   insn->setDef(0, dst);
100   insn->setSrc(0, src0);
101   insn->setSrc(1, src1);
102   insn->setSrc(2, src2);
103
104   insert(insn);
105   return insn;
106}
107
108Instruction *
109BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
110{
111   Instruction *insn = new_Instruction(func, OP_LOAD, ty);
112
113   insn->setDef(0, dst);
114   insn->setSrc(0, mem);
115   if (ptr)
116      insn->setIndirect(0, 0, ptr);
117
118   insert(insn);
119   return insn;
120}
121
122Instruction *
123BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
124                   Value *stVal)
125{
126   Instruction *insn = new_Instruction(func, op, ty);
127
128   insn->setSrc(0, mem);
129   insn->setSrc(1, stVal);
130   if (ptr)
131      insn->setIndirect(0, 0, ptr);
132
133   insert(insn);
134   return insn;
135}
136
137Instruction *
138BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
139                   Value *attrRel, Value *primRel)
140{
141   Symbol *sym = mkSymbol(file, 0, ty, offset);
142
143   Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
144
145   insn->setIndirect(0, 0, attrRel);
146   insn->setIndirect(0, 1, primRel);
147
148   // already inserted
149   return insn;
150}
151
152Instruction *
153BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
154{
155   operation op = OP_LINTERP;
156   DataType ty = TYPE_F32;
157
158   if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
159      ty = TYPE_U32;
160   else
161   if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
162      op = OP_PINTERP;
163
164   Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
165
166   Instruction *insn = mkOp1(op, ty, dst, sym);
167   insn->setIndirect(0, 0, rel);
168   insn->setInterpolate(mode);
169   return insn;
170}
171
172Instruction *
173BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
174{
175   Instruction *insn = new_Instruction(func, OP_MOV, ty);
176
177   insn->setDef(0, dst);
178   insn->setSrc(0, src);
179
180   insert(insn);
181   return insn;
182}
183
184Instruction *
185BuildUtil::mkMovToReg(int id, Value *src)
186{
187   Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
188
189   insn->setDef(0, new_LValue(func, FILE_GPR));
190   insn->getDef(0)->reg.data.id = id;
191   insn->setSrc(0, src);
192
193   insert(insn);
194   return insn;
195}
196
197Instruction *
198BuildUtil::mkMovFromReg(Value *dst, int id)
199{
200   Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
201
202   insn->setDef(0, dst);
203   insn->setSrc(0, new_LValue(func, FILE_GPR));
204   insn->getSrc(0)->reg.data.id = id;
205
206   insert(insn);
207   return insn;
208}
209
210Instruction *
211BuildUtil::mkCvt(operation op,
212                 DataType dstTy, Value *dst, DataType srcTy, Value *src)
213{
214   Instruction *insn = new_Instruction(func, op, dstTy);
215
216   insn->setType(dstTy, srcTy);
217   insn->setDef(0, dst);
218   insn->setSrc(0, src);
219
220   insert(insn);
221   return insn;
222}
223
224CmpInstruction *
225BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst,
226                 DataType srcTy, Value *src0, Value *src1, Value *src2)
227{
228   CmpInstruction *insn = new_CmpInstruction(func, op);
229
230   insn->setType((dst->reg.file == FILE_PREDICATE ||
231                  dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy);
232   insn->setCondition(cc);
233   insn->setDef(0, dst);
234   insn->setSrc(0, src0);
235   insn->setSrc(1, src1);
236   if (src2)
237      insn->setSrc(2, src2);
238
239   if (dst->reg.file == FILE_FLAGS)
240      insn->flagsDef = 0;
241
242   insert(insn);
243   return insn;
244}
245
246TexInstruction *
247BuildUtil::mkTex(operation op, TexTarget targ,
248                 uint16_t tic, uint16_t tsc,
249                 const std::vector<Value *> &def,
250                 const std::vector<Value *> &src)
251{
252   TexInstruction *tex = new_TexInstruction(func, op);
253
254   for (size_t d = 0; d < def.size() && def[d]; ++d)
255      tex->setDef(d, def[d]);
256   for (size_t s = 0; s < src.size() && src[s]; ++s)
257      tex->setSrc(s, src[s]);
258
259   tex->setTexture(targ, tic, tsc);
260
261   insert(tex);
262   return tex;
263}
264
265Instruction *
266BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
267{
268   Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
269   quadop->subOp = q;
270   quadop->lanes = l;
271   return quadop;
272}
273
274Instruction *
275BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
276{
277   LValue *def0 = getSSA();
278   LValue *def1 = getSSA();
279
280   mkMov(def0, trSrc)->setPredicate(CC_P, pred);
281   mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
282
283   return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
284}
285
286Instruction *
287BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
288{
289   Instruction *insn = NULL;
290
291   const DataType fTy = typeOfSize(halfSize * 2);
292
293   if (val->reg.file == FILE_IMMEDIATE)
294      val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
295
296   if (isMemoryFile(val->reg.file)) {
297      h[0] = cloneShallow(getFunction(), val);
298      h[1] = cloneShallow(getFunction(), val);
299      h[0]->reg.size = halfSize;
300      h[1]->reg.size = halfSize;
301      h[1]->reg.data.offset += halfSize;
302   } else {
303      h[0] = getSSA(halfSize, val->reg.file);
304      h[1] = getSSA(halfSize, val->reg.file);
305      insn = mkOp1(OP_SPLIT, fTy, h[0], val);
306      insn->setDef(1, h[1]);
307   }
308   return insn;
309}
310
311FlowInstruction *
312BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
313{
314   FlowInstruction *insn = new_FlowInstruction(func, op, targ);
315
316   if (pred)
317      insn->setPredicate(cc, pred);
318
319   insert(insn);
320   return insn;
321}
322
323void
324BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
325{
326   static const uint16_t baseSize2[16] =
327   {
328      0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
329      0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
330   };
331
332   int base = 0;
333
334   for (; rMask; rMask >>= 4, base += 4) {
335      const uint32_t mask = rMask & 0xf;
336      if (!mask)
337         continue;
338      int base1 = (baseSize2[mask] >>  0) & 0xf;
339      int size1 = (baseSize2[mask] >>  4) & 0xf;
340      int base2 = (baseSize2[mask] >>  8) & 0xf;
341      int size2 = (baseSize2[mask] >> 12) & 0xf;
342      Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
343      if (true) { // size1 can't be 0
344         LValue *reg = new_LValue(func, f);
345         reg->reg.size = size1 << unit;
346         reg->reg.data.id = base + base1;
347         insn->setDef(0, reg);
348      }
349      if (size2) {
350         LValue *reg = new_LValue(func, f);
351         reg->reg.size = size2 << unit;
352         reg->reg.data.id = base + base2;
353         insn->setDef(1, reg);
354      }
355   }
356}
357
358ImmediateValue *
359BuildUtil::mkImm(uint16_t u)
360{
361   ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
362
363   imm->reg.size = 2;
364   imm->reg.type = TYPE_U16;
365   imm->reg.data.u32 = u;
366
367   return imm;
368}
369
370ImmediateValue *
371BuildUtil::mkImm(uint32_t u)
372{
373   unsigned int pos = u32Hash(u);
374
375   while (imms[pos] && imms[pos]->reg.data.u32 != u)
376      pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
377
378   ImmediateValue *imm = imms[pos];
379   if (!imm) {
380      imm = new_ImmediateValue(prog, u);
381      addImmediate(imm);
382   }
383   return imm;
384}
385
386ImmediateValue *
387BuildUtil::mkImm(uint64_t u)
388{
389   ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
390
391   imm->reg.size = 8;
392   imm->reg.type = TYPE_U64;
393   imm->reg.data.u64 = u;
394
395   return imm;
396}
397
398ImmediateValue *
399BuildUtil::mkImm(float f)
400{
401   union {
402      float f32;
403      uint32_t u32;
404   } u;
405   u.f32 = f;
406   return mkImm(u.u32);
407}
408
409ImmediateValue *
410BuildUtil::mkImm(double d)
411{
412   return new_ImmediateValue(prog, d);
413}
414
415Value *
416BuildUtil::loadImm(Value *dst, float f)
417{
418   return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
419}
420
421Value *
422BuildUtil::loadImm(Value *dst, double d)
423{
424   return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d));
425}
426
427Value *
428BuildUtil::loadImm(Value *dst, uint16_t u)
429{
430   return mkOp1v(OP_MOV, TYPE_U16, dst ? dst : getScratch(2), mkImm(u));
431}
432
433Value *
434BuildUtil::loadImm(Value *dst, uint32_t u)
435{
436   return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
437}
438
439Value *
440BuildUtil::loadImm(Value *dst, uint64_t u)
441{
442   return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
443}
444
445Symbol *
446BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
447                    uint32_t baseAddr)
448{
449   Symbol *sym = new_Symbol(prog, file, fileIndex);
450
451   sym->setOffset(baseAddr);
452   sym->reg.type = ty;
453   sym->reg.size = typeSizeof(ty);
454
455   return sym;
456}
457
458Symbol *
459BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
460{
461   Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
462
463   assert(svIndex < 4 || svName == SV_CLIP_DISTANCE);
464
465   switch (svName) {
466   case SV_POSITION:
467   case SV_FACE:
468   case SV_YDIR:
469   case SV_POINT_SIZE:
470   case SV_POINT_COORD:
471   case SV_CLIP_DISTANCE:
472   case SV_TESS_OUTER:
473   case SV_TESS_INNER:
474   case SV_TESS_COORD:
475      sym->reg.type = TYPE_F32;
476      break;
477   default:
478      sym->reg.type = TYPE_U32;
479      break;
480   }
481   sym->reg.size = typeSizeof(sym->reg.type);
482
483   sym->reg.data.sv.sv = svName;
484   sym->reg.data.sv.index = svIndex;
485
486   return sym;
487}
488
489Symbol *
490BuildUtil::mkTSVal(TSSemantic tsName)
491{
492   Symbol *sym = new_Symbol(prog, FILE_THREAD_STATE, 0);
493   sym->reg.type = TYPE_U32;
494   sym->reg.size = typeSizeof(sym->reg.type);
495   sym->reg.data.ts = tsName;
496   return sym;
497}
498
499void
500BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx,
501                            uint32_t base, int len, int vecDim, int eltSize,
502                            DataFile file, int8_t fileIdx)
503{
504   this->array = array;
505   this->arrayIdx = arrayIdx;
506   this->baseAddr = base;
507   this->arrayLen = len;
508   this->vecDim = vecDim;
509   this->eltSize = eltSize;
510   this->file = file;
511   this->regOnly = !isMemoryFile(file);
512
513   if (!regOnly) {
514      baseSym = new_Symbol(up->getProgram(), file, fileIdx);
515      baseSym->setOffset(baseAddr);
516      baseSym->reg.size = eltSize;
517   } else {
518      baseSym = NULL;
519   }
520}
521
522Value *
523BuildUtil::DataArray::acquire(ValueMap &m, int i, int c)
524{
525   if (regOnly) {
526      Value *v = lookup(m, i, c);
527      if (!v)
528         v = insert(m, i, c, new_LValue(up->getFunction(), file));
529
530      return v;
531   } else {
532      return up->getScratch(eltSize);
533   }
534}
535
536Value *
537BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr)
538{
539   if (regOnly) {
540      Value *v = lookup(m, i, c);
541      if (!v)
542         v = insert(m, i, c, new_LValue(up->getFunction(), file));
543
544      return v;
545   } else {
546      Value *sym = lookup(m, i, c);
547      if (!sym)
548         sym = insert(m, i, c, mkSymbol(i, c));
549
550      return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr);
551   }
552}
553
554void
555BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value)
556{
557   if (regOnly) {
558      assert(!ptr);
559      if (!lookup(m, i, c))
560         insert(m, i, c, value);
561
562      assert(lookup(m, i, c) == value);
563   } else {
564      Value *sym = lookup(m, i, c);
565      if (!sym)
566         sym = insert(m, i, c, mkSymbol(i, c));
567
568      const DataType stTy = typeOfSize(value->reg.size);
569
570      up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value);
571   }
572}
573
574Symbol *
575BuildUtil::DataArray::mkSymbol(int i, int c)
576{
577   const unsigned int idx = i * vecDim + c;
578   Symbol *sym = new_Symbol(up->getProgram(), file, 0);
579
580   assert(baseSym || (idx < arrayLen && c < vecDim));
581
582   sym->reg.size = eltSize;
583   sym->reg.type = typeOfSize(eltSize);
584   sym->setAddress(baseSym, baseAddr + idx * eltSize);
585   return sym;
586}
587
588
589Instruction *
590BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
591                              Value *zero,
592                              Value *carry)
593{
594   DataType hTy;
595   int srcNr;
596
597   switch (i->dType) {
598   case TYPE_U64: hTy = TYPE_U32; break;
599   case TYPE_S64: hTy = TYPE_S32; break;
600   case TYPE_F64:
601      if (i->op == OP_MOV) {
602         hTy = TYPE_U32;
603         break;
604      }
605      FALLTHROUGH;
606   default:
607      return NULL;
608   }
609
610   switch (i->op) {
611   case OP_MOV: srcNr = 1; break;
612   case OP_ADD:
613   case OP_SUB:
614      if (!carry)
615         return NULL;
616      srcNr = 2;
617      break;
618   case OP_SELP: srcNr = 3; break;
619   default:
620      // TODO when needed
621      return NULL;
622   }
623
624   i->setType(hTy);
625   i->setDef(0, cloneShallow(fn, i->getDef(0)));
626   i->getDef(0)->reg.size = 4;
627   Instruction *lo = i;
628   Instruction *hi = cloneForward(fn, i);
629   lo->bb->insertAfter(lo, hi);
630
631   hi->getDef(0)->reg.data.id++;
632
633   for (int s = 0; s < srcNr; ++s) {
634      if (lo->getSrc(s)->reg.size < 8) {
635         if (s == 2)
636            hi->setSrc(s, lo->getSrc(s));
637         else
638            hi->setSrc(s, zero);
639      } else {
640         if (lo->getSrc(s)->refCount() > 1)
641            lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
642         lo->getSrc(s)->reg.size /= 2;
643         hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
644
645         switch (hi->src(s).getFile()) {
646         case FILE_IMMEDIATE:
647            hi->getSrc(s)->reg.data.u64 >>= 32;
648            break;
649         case FILE_MEMORY_CONST:
650         case FILE_MEMORY_SHARED:
651         case FILE_SHADER_INPUT:
652         case FILE_SHADER_OUTPUT:
653            hi->getSrc(s)->reg.data.offset += 4;
654            break;
655         default:
656            assert(hi->src(s).getFile() == FILE_GPR);
657            hi->getSrc(s)->reg.data.id++;
658            break;
659         }
660      }
661   }
662   if (srcNr == 2) {
663      lo->setFlagsDef(1, carry);
664      hi->setFlagsSrc(hi->srcCount(), carry);
665   }
666   return hi;
667}
668
669} // namespace nv50_ir
670