1/*
2 * Copyright © 2018 Google
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "aco_interface.h"
26
27#include "aco_ir.h"
28
29#include "vulkan/radv_shader_args.h"
30
31#include "util/memstream.h"
32
33#include <array>
34#include <iostream>
35#include <vector>
36
37static const std::array<aco_compiler_statistic_info, aco::num_statistics> statistic_infos = []()
38{
39   std::array<aco_compiler_statistic_info, aco::num_statistics> ret{};
40   ret[aco::statistic_hash] =
41      aco_compiler_statistic_info{"Hash", "CRC32 hash of code and constant data"};
42   ret[aco::statistic_instructions] =
43      aco_compiler_statistic_info{"Instructions", "Instruction count"};
44   ret[aco::statistic_copies] =
45      aco_compiler_statistic_info{"Copies", "Copy instructions created for pseudo-instructions"};
46   ret[aco::statistic_branches] = aco_compiler_statistic_info{"Branches", "Branch instructions"};
47   ret[aco::statistic_latency] =
48      aco_compiler_statistic_info{"Latency", "Issue cycles plus stall cycles"};
49   ret[aco::statistic_inv_throughput] = aco_compiler_statistic_info{
50      "Inverse Throughput", "Estimated busy cycles to execute one wave"};
51   ret[aco::statistic_vmem_clauses] = aco_compiler_statistic_info{
52      "VMEM Clause", "Number of VMEM clauses (includes 1-sized clauses)"};
53   ret[aco::statistic_smem_clauses] = aco_compiler_statistic_info{
54      "SMEM Clause", "Number of SMEM clauses (includes 1-sized clauses)"};
55   ret[aco::statistic_sgpr_presched] =
56      aco_compiler_statistic_info{"Pre-Sched SGPRs", "SGPR usage before scheduling"};
57   ret[aco::statistic_vgpr_presched] =
58      aco_compiler_statistic_info{"Pre-Sched VGPRs", "VGPR usage before scheduling"};
59   return ret;
60}();
61
62const unsigned aco_num_statistics = aco::num_statistics;
63const aco_compiler_statistic_info* aco_statistic_infos = statistic_infos.data();
64
65uint64_t
66aco_get_codegen_flags()
67{
68   aco::init();
69   /* Exclude flags which don't affect code generation. */
70   uint64_t exclude = aco::DEBUG_VALIDATE_IR | aco::DEBUG_VALIDATE_RA | aco::DEBUG_PERFWARN |
71                      aco::DEBUG_PERF_INFO | aco::DEBUG_LIVE_INFO;
72   return aco::debug_flags & ~exclude;
73}
74
75static void
76validate(aco::Program* program)
77{
78   if (!(aco::debug_flags & aco::DEBUG_VALIDATE_IR))
79      return;
80
81   ASSERTED bool is_valid = aco::validate_ir(program);
82   assert(is_valid);
83}
84
85static std::string
86get_disasm_string(aco::Program* program, std::vector<uint32_t>& code,
87                  unsigned exec_size)
88{
89   std::string disasm;
90
91   if (check_print_asm_support(program)) {
92      char* data = NULL;
93      size_t disasm_size = 0;
94      struct u_memstream mem;
95      if (u_memstream_open(&mem, &data, &disasm_size)) {
96         FILE* const memf = u_memstream_get(&mem);
97         aco::print_asm(program, code, exec_size / 4u, memf);
98         fputc(0, memf);
99         u_memstream_close(&mem);
100      }
101
102      disasm = std::string(data, data + disasm_size);
103      free(data);
104   } else {
105      disasm = "Shader disassembly is not supported in the current configuration"
106#ifndef LLVM_AVAILABLE
107               " (LLVM not available)"
108#endif
109               ".\n";
110   }
111
112   return disasm;
113}
114
115static std::string
116aco_postprocess_shader(const struct aco_compiler_options* options,
117                       const struct radv_shader_args *args,
118                       std::unique_ptr<aco::Program>& program)
119{
120   std::string llvm_ir;
121
122   if (options->dump_preoptir)
123      aco_print_program(program.get(), stderr);
124
125   aco::live live_vars;
126   if (!args->is_trap_handler_shader) {
127      /* Phi lowering */
128      aco::lower_phis(program.get());
129      aco::dominator_tree(program.get());
130      validate(program.get());
131
132      /* Optimization */
133      if (!options->key.optimisations_disabled) {
134         if (!(aco::debug_flags & aco::DEBUG_NO_VN))
135            aco::value_numbering(program.get());
136         if (!(aco::debug_flags & aco::DEBUG_NO_OPT))
137            aco::optimize(program.get());
138      }
139
140      /* cleanup and exec mask handling */
141      aco::setup_reduce_temp(program.get());
142      aco::insert_exec_mask(program.get());
143      validate(program.get());
144
145      /* spilling and scheduling */
146      live_vars = aco::live_var_analysis(program.get());
147      aco::spill(program.get(), live_vars);
148   }
149
150   if (options->record_ir) {
151      char* data = NULL;
152      size_t size = 0;
153      u_memstream mem;
154      if (u_memstream_open(&mem, &data, &size)) {
155         FILE* const memf = u_memstream_get(&mem);
156         aco_print_program(program.get(), memf);
157         fputc(0, memf);
158         u_memstream_close(&mem);
159      }
160
161      llvm_ir = std::string(data, data + size);
162      free(data);
163   }
164
165   if (program->collect_statistics)
166      aco::collect_presched_stats(program.get());
167
168   if ((aco::debug_flags & aco::DEBUG_LIVE_INFO) && options->dump_shader)
169      aco_print_program(program.get(), stderr, live_vars, aco::print_live_vars | aco::print_kill);
170
171   if (!args->is_trap_handler_shader) {
172      if (!options->key.optimisations_disabled && !(aco::debug_flags & aco::DEBUG_NO_SCHED))
173         aco::schedule_program(program.get(), live_vars);
174      validate(program.get());
175
176      /* Register Allocation */
177      aco::register_allocation(program.get(), live_vars.live_out);
178
179      if (aco::validate_ra(program.get())) {
180         aco_print_program(program.get(), stderr);
181         abort();
182      } else if (options->dump_shader) {
183         aco_print_program(program.get(), stderr);
184      }
185
186      validate(program.get());
187
188      /* Optimization */
189      if (!options->key.optimisations_disabled && !(aco::debug_flags & aco::DEBUG_NO_OPT)) {
190         aco::optimize_postRA(program.get());
191         validate(program.get());
192      }
193
194      aco::ssa_elimination(program.get());
195   }
196
197   /* Lower to HW Instructions */
198   aco::lower_to_hw_instr(program.get());
199
200   /* Insert Waitcnt */
201   aco::insert_wait_states(program.get());
202   aco::insert_NOPs(program.get());
203
204   if (program->gfx_level >= GFX10)
205      aco::form_hard_clauses(program.get());
206
207   if (program->collect_statistics || (aco::debug_flags & aco::DEBUG_PERF_INFO))
208      aco::collect_preasm_stats(program.get());
209
210   return llvm_ir;
211}
212
213void
214aco_compile_shader(const struct aco_compiler_options* options,
215                   const struct aco_shader_info* info,
216                   unsigned shader_count, struct nir_shader* const* shaders,
217                   const struct radv_shader_args *args,
218                   aco_callback *build_binary,
219                   void **binary)
220{
221   aco::init();
222
223   ac_shader_config config = {0};
224   std::unique_ptr<aco::Program> program{new aco::Program};
225
226   program->collect_statistics = options->record_stats;
227   if (program->collect_statistics)
228      memset(program->statistics, 0, sizeof(program->statistics));
229
230   program->debug.func = options->debug.func;
231   program->debug.private_data = options->debug.private_data;
232
233   /* Instruction Selection */
234   if (args->is_gs_copy_shader)
235      aco::select_gs_copy_shader(program.get(), shaders[0], &config, options, info, args);
236   else if (args->is_trap_handler_shader)
237      aco::select_trap_handler_shader(program.get(), shaders[0], &config, options, info, args);
238   else
239      aco::select_program(program.get(), shader_count, shaders, &config, options, info, args);
240
241   std::string llvm_ir = aco_postprocess_shader(options, args, program);
242
243   /* assembly */
244   std::vector<uint32_t> code;
245   unsigned exec_size = aco::emit_program(program.get(), code);
246
247   if (program->collect_statistics)
248      aco::collect_postasm_stats(program.get(), code);
249
250   bool get_disasm = options->dump_shader || options->record_ir;
251
252   std::string disasm;
253   if (get_disasm)
254      disasm = get_disasm_string(program.get(), code, exec_size);
255
256   size_t stats_size = 0;
257   if (program->collect_statistics)
258      stats_size = aco::num_statistics * sizeof(uint32_t);
259
260   (*build_binary)(binary,
261                   shaders[shader_count - 1]->info.stage,
262                   args->is_gs_copy_shader,
263                   &config,
264                   llvm_ir.c_str(),
265                   llvm_ir.size(),
266                   disasm.c_str(),
267                   disasm.size(),
268                   program->statistics,
269                   stats_size,
270                   exec_size,
271                   code.data(),
272                   code.size());
273}
274
275void
276aco_compile_vs_prolog(const struct aco_compiler_options* options,
277                      const struct aco_shader_info* info,
278                      const struct aco_vs_prolog_key* key,
279                      const struct radv_shader_args* args,
280                      aco_shader_part_callback *build_prolog,
281                      void **binary)
282{
283   aco::init();
284
285   /* create program */
286   ac_shader_config config = {0};
287   std::unique_ptr<aco::Program> program{new aco::Program};
288   program->collect_statistics = false;
289   program->debug.func = NULL;
290   program->debug.private_data = NULL;
291
292   /* create IR */
293   unsigned num_preserved_sgprs;
294   aco::select_vs_prolog(program.get(), key, &config, options, info, args, &num_preserved_sgprs);
295   aco::insert_NOPs(program.get());
296
297   if (options->dump_shader)
298      aco_print_program(program.get(), stderr);
299
300   /* assembly */
301   std::vector<uint32_t> code;
302   code.reserve(align(program->blocks[0].instructions.size() * 2, 16));
303   unsigned exec_size = aco::emit_program(program.get(), code);
304
305   bool get_disasm = options->dump_shader || options->record_ir;
306
307   std::string disasm;
308   if (get_disasm)
309      disasm = get_disasm_string(program.get(), code, exec_size);
310
311   (*build_prolog)(binary,
312                   config.num_sgprs,
313                   config.num_vgprs,
314                   num_preserved_sgprs,
315                   code.data(),
316                   code.size(),
317                   disasm.data(),
318                   disasm.size());
319}
320
321void
322aco_compile_ps_epilog(const struct aco_compiler_options* options,
323                      const struct aco_shader_info* info,
324                      const struct aco_ps_epilog_key* key,
325                      const struct radv_shader_args* args,
326                      aco_shader_part_callback* build_epilog,
327                      void** binary)
328{
329   aco::init();
330
331   ac_shader_config config = {0};
332   std::unique_ptr<aco::Program> program{new aco::Program};
333
334   program->collect_statistics = options->record_stats;
335   if (program->collect_statistics)
336      memset(program->statistics, 0, sizeof(program->statistics));
337
338   program->debug.func = options->debug.func;
339   program->debug.private_data = options->debug.private_data;
340
341   /* Instruction selection */
342   aco::select_ps_epilog(program.get(), key, &config, options, info, args);
343
344   aco_postprocess_shader(options, args, program);
345
346   /* assembly */
347   std::vector<uint32_t> code;
348   unsigned exec_size = aco::emit_program(program.get(), code);
349
350   bool get_disasm = options->dump_shader || options->record_ir;
351
352   std::string disasm;
353   if (get_disasm)
354      disasm = get_disasm_string(program.get(), code, exec_size);
355
356   (*build_epilog)(binary,
357                   config.num_sgprs,
358                   config.num_vgprs,
359                   0,
360                   code.data(),
361                   code.size(),
362                   disasm.data(),
363                   disasm.size());
364}
365