1/* -*- mesa-c++  -*-
2 *
3 * Copyright (c) 2021 Collabora LTD
4 *
5 * Author: Gert Wollny <gert.wollny@collabora.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * on the rights to use, copy, modify, merge, publish, distribute, sub
11 * license, and/or sell copies of the Software, and to permit persons to whom
12 * the Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include "sfn_instr_alugroup.h"
28#include "sfn_instr_export.h"
29#include "sfn_instr_fetch.h"
30#include "sfn_instr_mem.h"
31#include "sfn_instr_lds.h"
32#include "sfn_instr_tex.h"
33#include "sfn_instr_controlflow.h"
34
35#include <iostream>
36#include <sstream>
37#include <numeric>
38
39namespace r600 {
40
41using std::string;
42using std::vector;
43
44Instr::Instr():
45   m_use_count(0),
46   m_block_id(std::numeric_limits<int>::max()),
47   m_index(std::numeric_limits<int>::max())
48{
49}
50
51Instr::~Instr()
52{
53
54}
55
56void Instr::print(std::ostream& os) const
57{
58   do_print(os);
59}
60
61bool Instr::ready() const
62{
63   for (auto& i : m_required_instr)
64      if (!i->ready())
65         return false;
66   return do_ready();
67}
68
69int int_from_string_with_prefix(const std::string& str, const std::string& prefix)
70{
71   if (str.substr(0, prefix.length()) != prefix) {
72      std::cerr << "Expect '" << prefix << "' as start of '" << str << "'\n";
73      assert(0);
74   }
75
76   std::stringstream help(str.substr(prefix.length()));
77   int retval;
78   help >> retval;
79   return retval;
80}
81
82int sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle &swz, bool& is_ssa)
83{
84   assert(str[0] == 'R' || str[0] == '_' || str[0] == 'S');
85   int sel = 0;
86
87   auto istr = str.begin() + 1;
88
89   if (str[0] == '_') {
90      while (istr != str.end() && *istr == '_')
91         ++istr;
92      sel = std::numeric_limits<int>::max();
93   } else {
94      while (istr != str.end() && isdigit(*istr)) {
95         sel *= 10;
96         sel += *istr - '0';
97         ++istr;
98      }
99   }
100
101   assert(*istr == '.');
102   istr++;
103
104   int i = 0;
105   while (istr != str.end()) {
106      switch (*istr) {
107      case 'x': swz[i] = 0; break;
108      case 'y': swz[i] = 1; break;
109      case 'z': swz[i] = 2; break;
110      case 'w': swz[i] = 3; break;
111      case '0': swz[i] = 4; break;
112      case '1': swz[i] = 5; break;
113      case '_': swz[i] = 7; break;
114      default:
115         unreachable("Unknown swizzle character");
116      }
117      ++istr;
118      ++i;
119   }
120
121   is_ssa = str[0] == 'S';
122
123   return sel;
124}
125
126bool Instr::is_last() const
127{
128   return true;
129}
130
131bool Instr::set_dead()
132{
133   if (m_instr_flags.test(always_keep))
134      return false;
135   bool is_dead = propagate_death();
136   m_instr_flags.set(dead);
137   return is_dead;
138}
139
140bool Instr::propagate_death()
141{
142   return true;
143}
144
145bool Instr::replace_source(PRegister old_src, PVirtualValue new_src)
146{
147   (void)old_src;
148   (void)new_src;
149   return false;
150}
151
152void Instr::add_required_instr(Instr *instr)
153{
154   assert(instr);
155   m_required_instr.push_back(instr);
156   instr->m_dependend_instr.push_back(this);
157}
158
159void Instr::replace_required_instr(Instr *old_instr, Instr *new_instr)
160{
161
162   for (auto i = m_required_instr.begin(); i != m_required_instr.end(); ++i) {
163      if (*i == old_instr)
164         *i = new_instr;
165   }
166}
167
168bool Instr::replace_dest(PRegister new_dest, r600::AluInstr *move_instr)
169{
170   (void)new_dest;
171   (void)move_instr;
172   return false;
173}
174
175void Instr::set_blockid(int id, int index)
176{
177   m_block_id = id;
178   m_index = index;
179   forward_set_blockid(id, index);
180}
181
182
183void Instr::forward_set_blockid(int id, int index)
184{
185   (void)id;
186   (void)index;
187}
188
189InstrWithVectorResult::InstrWithVectorResult(const RegisterVec4& dest,
190                                             const RegisterVec4::Swizzle& dest_swizzle):
191   m_dest(dest),
192   m_dest_swizzle(dest_swizzle)
193{
194   for (int i = 0; i < 4; ++i) {
195      if (m_dest_swizzle[i] < 6)
196         m_dest[i]->add_parent(this);
197   }
198}
199
200void InstrWithVectorResult::print_dest(std::ostream& os) const
201{
202   os << (m_dest[0]->is_ssa() ? 'S' : 'R' ) << m_dest.sel();
203   os << ".";
204   for (int i = 0; i < 4; ++i)
205      os << VirtualValue::chanchar[m_dest_swizzle[i]];
206}
207
208bool InstrWithVectorResult::comp_dest(const RegisterVec4& dest,
209                                      const RegisterVec4::Swizzle& dest_swizzle) const
210{
211   for(int i = 0; i < 4; ++i) {
212      if (!m_dest[i]->equal_to(*dest[i])) {
213         return false;
214      }
215      if (m_dest_swizzle[i] != dest_swizzle[i])
216         return false;
217   }
218   return true;
219}
220
221void Block::do_print(std::ostream& os) const
222{
223   for (int j = 0; j < 2 * m_nesting_depth; ++j)
224      os << ' ';
225   os << "BLOCK START\n";
226   for (auto& i : m_instructions) {
227      for (int j = 0; j < 2 * (m_nesting_depth + i->nesting_corr()) + 2; ++j)
228         os << ' ';
229      os << *i << "\n";
230   }
231   for (int j = 0; j < 2 * m_nesting_depth; ++j)
232      os << ' ';
233   os << "BLOCK END\n";
234}
235
236bool Block::is_equal_to(const Block& lhs) const
237{
238   if (m_id != lhs.m_id || m_nesting_depth != lhs.m_nesting_depth)
239      return false;
240
241   if (m_instructions.size() != lhs.m_instructions.size())
242      return false;
243
244   return std::inner_product(m_instructions.begin(), m_instructions.end(), lhs.m_instructions.begin(),
245                             true,
246                             [] (bool l, bool r) { return l && r;},
247   [](PInst l, PInst r) { return l->equal_to(*r);});
248}
249
250inline bool operator != (const Block& lhs, const Block& rhs)
251{
252   return !lhs.is_equal_to(rhs);
253}
254
255void Block::erase(iterator node)
256{
257   m_instructions.erase(node);
258}
259
260void Block::set_type(Type t)
261{
262   m_blocK_type = t;
263   switch (t) {
264   case vtx:
265   case gds:
266   case tex: m_remaining_slots = 8; break; /* TODO: 16 for >= EVERGREEN */
267   default:
268      m_remaining_slots = 0xffff;
269   }
270}
271
272Block::Block(int nesting_depth, int id):
273   m_nesting_depth(nesting_depth),
274   m_id(id),
275   m_next_index(0)
276{
277   assert(!has_instr_flag(force_cf));
278}
279
280void Block::accept(ConstInstrVisitor& visitor) const
281{
282   visitor.visit(*this);
283}
284
285void Block::accept(InstrVisitor& visitor)
286{
287   visitor.visit(this);
288}
289
290void Block::push_back(PInst instr)
291{
292   instr->set_blockid(m_id, m_next_index++);
293   if (m_remaining_slots != 0xffff) {
294      uint32_t new_slots = instr->slots();
295      m_remaining_slots -= new_slots;
296   }
297   if (m_lds_group_start)
298      m_lds_group_requirement += instr->slots();
299
300   m_instructions.push_back(instr);
301}
302
303bool Block::try_reserve_kcache(const AluGroup& group)
304{
305   auto kcache = m_kcache;
306
307   auto kcache_constants = group.get_kconsts();
308   for (auto& kc : kcache_constants)  {
309      auto u = kc->as_uniform();
310      assert(u);
311      if (!try_reserve_kcache(*u, kcache)) {
312         m_kcache_alloc_failed = true;
313         return false;
314      }
315   }
316
317   m_kcache = kcache;
318   m_kcache_alloc_failed = false;
319   return true;
320}
321
322bool Block::try_reserve_kcache(const AluInstr& instr)
323{
324   auto kcache = m_kcache;
325
326   for (auto& src : instr.sources()) {
327      auto u = src->as_uniform();
328      if (u) {
329         if (!try_reserve_kcache(*u, kcache)) {
330            m_kcache_alloc_failed = true;
331            return false;
332         }
333      }
334   }
335   m_kcache = kcache;
336   m_kcache_alloc_failed = false;
337   return true;
338}
339
340void Block::set_chipclass(r600_chip_class chip_class)
341{
342   if (chip_class < ISA_CC_EVERGREEN)
343      s_max_kcache_banks = 2;
344   else
345      s_max_kcache_banks = 4;
346}
347
348unsigned Block::s_max_kcache_banks = 4;
349
350bool Block::try_reserve_kcache(const UniformValue& u,
351                               std::array<KCacheLine, 4>& kcache) const
352{
353   const int kcache_banks = s_max_kcache_banks; // TODO: handle pre-evergreen
354
355   int bank = u.kcache_bank();
356   int sel  = (u.sel() - 512);
357   int line = sel >> 4;
358
359   bool found = false;
360
361   for (int i = 0; i < kcache_banks && !found; ++i) {
362      if (kcache[i].mode) {
363         if (kcache[i].bank < bank)
364            continue;
365
366         if ((kcache[i].bank == bank &&
367              kcache[i].addr > line  + 1) ||
368             kcache[i].bank > bank) {
369            if (kcache[kcache_banks - 1].mode)
370               return false;
371
372            memmove(&kcache[i+1],&kcache[i], (kcache_banks-i-1)*sizeof(KCacheLine));
373            kcache[i].mode = KCacheLine::lock_1;
374            kcache[i].bank = bank;
375            kcache[i].addr = line;
376            return true;
377         }
378
379         int d = line - kcache[i].addr;
380
381         if (d == -1) {
382            kcache[i].addr--;
383            if (kcache[i].mode == KCacheLine::lock_2) {
384               /* we are prepending the line to the current set,
385                * discarding the existing second line,
386                * so we'll have to insert line+2 after it */
387               line += 2;
388               continue;
389            } else if (kcache[i].mode == KCacheLine::lock_1) {
390               kcache[i].mode = KCacheLine::lock_2;
391               return true;
392            } else {
393               /* V_SQ_CF_KCACHE_LOCK_LOOP_INDEX is not supported */
394               return false;
395            }
396         } else if (d == 1) {
397            kcache[i].mode = KCacheLine::lock_2;
398            return true;
399         } else if (d == 0) {
400            return true;
401         }
402      } else { /* free kcache set - use it */
403         kcache[i].mode = KCacheLine::lock_1;
404         kcache[i].bank = bank;
405         kcache[i].addr = line;
406         return true;
407      }
408   }
409   return false;
410}
411
412void Block::lds_group_start(AluInstr *alu)
413{
414   assert(!m_lds_group_start);
415   m_lds_group_start = alu;
416   m_lds_group_requirement = 0;
417}
418
419void Block::lds_group_end()
420{
421   assert(m_lds_group_start);
422   m_lds_group_start->set_required_slots(m_lds_group_requirement);
423   m_lds_group_start = 0;
424}
425
426InstrWithVectorResult::InstrWithVectorResult(const InstrWithVectorResult& orig):
427   m_dest(orig.m_dest),
428   m_dest_swizzle(orig.m_dest_swizzle)
429{
430}
431
432class InstrComparer : public ConstInstrVisitor {
433public:
434   InstrComparer() = default;
435   bool result {false};
436
437#define DECLARE_MEMBER(TYPE)         \
438    InstrComparer(const TYPE *instr) \
439    {                                \
440       this_ ## TYPE = instr;        \
441    }                                \
442                                     \
443    void visit(const TYPE& instr)    \
444    {                                \
445       result = false;               \
446       if (!this_ ## TYPE)           \
447         return;                     \
448      result = this_ ## TYPE->is_equal_to(instr); \
449   }                                 \
450                                     \
451   const TYPE *this_ ## TYPE{nullptr};
452
453   DECLARE_MEMBER(AluInstr);
454   DECLARE_MEMBER(AluGroup);
455   DECLARE_MEMBER(TexInstr);
456   DECLARE_MEMBER(ExportInstr);
457   DECLARE_MEMBER(FetchInstr);
458   DECLARE_MEMBER(Block);
459   DECLARE_MEMBER(ControlFlowInstr);
460   DECLARE_MEMBER(IfInstr);
461   DECLARE_MEMBER(ScratchIOInstr);
462   DECLARE_MEMBER(StreamOutInstr);
463   DECLARE_MEMBER(MemRingOutInstr);
464   DECLARE_MEMBER(EmitVertexInstr);
465   DECLARE_MEMBER(GDSInstr);
466   DECLARE_MEMBER(WriteTFInstr);
467   DECLARE_MEMBER(LDSAtomicInstr);
468   DECLARE_MEMBER(LDSReadInstr);
469   DECLARE_MEMBER(RatInstr);
470};
471
472class InstrCompareForward: public ConstInstrVisitor {
473public:
474
475   void visit(const AluInstr& instr) override {
476      m_comparer = InstrComparer(&instr);
477   }
478
479   void visit(const AluGroup& instr) override {
480      m_comparer = InstrComparer(&instr);
481   }
482
483   void visit(const TexInstr& instr) override {
484      m_comparer = InstrComparer(&instr);
485   }
486
487   void visit(const ExportInstr& instr) override {
488      m_comparer = InstrComparer(&instr);
489   }
490
491   void visit(const FetchInstr& instr) override {
492      m_comparer = InstrComparer(&instr);
493   }
494
495   void visit(const Block& instr) override {
496      m_comparer = InstrComparer(&instr);
497   }
498
499   void visit(const ControlFlowInstr& instr) override {
500      m_comparer = InstrComparer(&instr);
501   }
502
503   void visit(const IfInstr& instr) override {
504      m_comparer = InstrComparer(&instr);
505   }
506
507   void visit(const ScratchIOInstr& instr) override {
508      m_comparer = InstrComparer(&instr);
509   }
510
511   void visit(const StreamOutInstr& instr) override {
512      m_comparer = InstrComparer(&instr);
513   }
514
515   void visit(const MemRingOutInstr& instr) override {
516      m_comparer = InstrComparer(&instr);
517   }
518
519   void visit(const EmitVertexInstr& instr) override {
520      m_comparer = InstrComparer(&instr);
521   }
522
523   void visit(const GDSInstr& instr) override {
524      m_comparer = InstrComparer(&instr);
525   }
526
527   void visit(const WriteTFInstr& instr) override {
528      m_comparer = InstrComparer(&instr);
529   }
530
531   void visit(const LDSAtomicInstr& instr) override {
532      m_comparer = InstrComparer(&instr);
533   }
534
535   void visit(const LDSReadInstr& instr) override {
536         m_comparer = InstrComparer(&instr);
537   }
538
539   void visit(const RatInstr& instr) override {
540         m_comparer = InstrComparer(&instr);
541   }
542
543   InstrComparer m_comparer;
544};
545
546
547bool Instr::equal_to(const Instr& lhs) const
548{
549   InstrCompareForward cmp;
550   accept(cmp);
551   lhs.accept(cmp.m_comparer);
552
553   return cmp.m_comparer.result;
554}
555
556
557
558
559} // ns r600
560