1/**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29/**
30 * @file
31 * Helpers for emiting intrinsic calls.
32 *
33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
34 * about, and it is often necessary to resort target-specific intrinsics for
35 * performance, convenience.
36 *
37 * Ideally we would like to stay away from target specific intrinsics and
38 * move all the instruction selection logic into upstream LLVM where it belongs.
39 *
40 * These functions are also used for calling C functions provided by us from
41 * generated LLVM code.
42 *
43 * @author Jose Fonseca <jfonseca@vmware.com>
44 */
45
46#include <llvm/Config/llvm-config.h>
47
48#include "util/u_debug.h"
49#include "util/u_string.h"
50#include "util/bitscan.h"
51
52#include "lp_bld_const.h"
53#include "lp_bld_intr.h"
54#include "lp_bld_type.h"
55#include "lp_bld_pack.h"
56#include "lp_bld_debug.h"
57
58
59void
60lp_format_intrinsic(char *name,
61                    size_t size,
62                    const char *name_root,
63                    LLVMTypeRef type)
64{
65   unsigned length = 0;
66   unsigned width;
67   char c;
68
69   LLVMTypeKind kind = LLVMGetTypeKind(type);
70   if (kind == LLVMVectorTypeKind) {
71      length = LLVMGetVectorSize(type);
72      type = LLVMGetElementType(type);
73      kind = LLVMGetTypeKind(type);
74   }
75
76   switch (kind) {
77   case LLVMIntegerTypeKind:
78      c = 'i';
79      width = LLVMGetIntTypeWidth(type);
80      break;
81   case LLVMFloatTypeKind:
82      c = 'f';
83      width = 32;
84      break;
85   case LLVMDoubleTypeKind:
86      c = 'f';
87      width = 64;
88      break;
89   case LLVMHalfTypeKind:
90      c = 'f';
91      width = 16;
92      break;
93   default:
94      unreachable("unexpected LLVMTypeKind");
95   }
96
97   if (length) {
98      snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
99   } else {
100      snprintf(name, size, "%s.%c%u", name_root, c, width);
101   }
102}
103
104
105LLVMValueRef
106lp_declare_intrinsic_with_type(LLVMModuleRef module,
107                               const char *name,
108                               LLVMTypeRef function_type)
109{
110   assert(!LLVMGetNamedFunction(module, name));
111
112   LLVMValueRef function = LLVMAddFunction(module, name, function_type);
113
114   LLVMSetFunctionCallConv(function, LLVMCCallConv);
115   LLVMSetLinkage(function, LLVMExternalLinkage);
116
117   assert(LLVMIsDeclaration(function));
118
119   return function;
120}
121
122
123LLVMValueRef
124lp_declare_intrinsic(LLVMModuleRef module,
125                     const char *name,
126                     LLVMTypeRef ret_type,
127                     LLVMTypeRef *arg_types,
128                     unsigned num_args)
129{
130   LLVMTypeRef function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
131   return lp_declare_intrinsic_with_type(module, name, function_type);
132}
133
134
135#if LLVM_VERSION_MAJOR < 4
136static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
137{
138   switch (attr) {
139   case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
140   case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
141   case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
142   case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
143   case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
144   case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
145   default:
146      _debug_printf("Unhandled function attribute: %x\n", attr);
147      return 0;
148   }
149}
150
151#else
152
153static const char *attr_to_str(enum lp_func_attr attr)
154{
155   switch (attr) {
156   case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
157   case LP_FUNC_ATTR_INREG: return "inreg";
158   case LP_FUNC_ATTR_NOALIAS: return "noalias";
159   case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
160   case LP_FUNC_ATTR_READNONE: return "readnone";
161   case LP_FUNC_ATTR_READONLY: return "readonly";
162   case LP_FUNC_ATTR_WRITEONLY: return "writeonly";
163   case LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
164   case LP_FUNC_ATTR_CONVERGENT: return "convergent";
165   case LP_FUNC_ATTR_PRESPLITCORO: return "presplitcoroutine";
166   default:
167      _debug_printf("Unhandled function attribute: %x\n", attr);
168      return 0;
169   }
170}
171
172#endif
173
174void
175lp_add_function_attr(LLVMValueRef function_or_call,
176                     int attr_idx, enum lp_func_attr attr)
177{
178
179#if LLVM_VERSION_MAJOR < 4
180   LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
181   if (LLVMIsAFunction(function_or_call)) {
182      if (attr_idx == -1) {
183         LLVMAddFunctionAttr(function_or_call, llvm_attr);
184      } else {
185         LLVMAddAttribute(LLVMGetParam(function_or_call, attr_idx - 1), llvm_attr);
186      }
187   } else {
188      LLVMAddInstrAttribute(function_or_call, attr_idx, llvm_attr);
189   }
190#else
191
192   LLVMModuleRef module;
193   if (LLVMIsAFunction(function_or_call)) {
194      module = LLVMGetGlobalParent(function_or_call);
195   } else {
196      LLVMBasicBlockRef bb = LLVMGetInstructionParent(function_or_call);
197      LLVMValueRef function = LLVMGetBasicBlockParent(bb);
198      module = LLVMGetGlobalParent(function);
199   }
200   LLVMContextRef ctx = LLVMGetModuleContext(module);
201
202   const char *attr_name = attr_to_str(attr);
203   unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
204                                                      strlen(attr_name));
205   LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
206
207   if (LLVMIsAFunction(function_or_call))
208      LLVMAddAttributeAtIndex(function_or_call, attr_idx, llvm_attr);
209   else
210      LLVMAddCallSiteAttribute(function_or_call, attr_idx, llvm_attr);
211#endif
212}
213
214static void
215lp_add_func_attributes(LLVMValueRef function, unsigned attrib_mask)
216{
217   /* NoUnwind indicates that the intrinsic never raises a C++ exception.
218    * Set it for all intrinsics.
219    */
220   attrib_mask |= LP_FUNC_ATTR_NOUNWIND;
221   attrib_mask &= ~LP_FUNC_ATTR_LEGACY;
222
223   while (attrib_mask) {
224      enum lp_func_attr attr = 1u << u_bit_scan(&attrib_mask);
225      lp_add_function_attr(function, -1, attr);
226   }
227}
228
229LLVMValueRef
230lp_build_intrinsic(LLVMBuilderRef builder,
231                   const char *name,
232                   LLVMTypeRef ret_type,
233                   LLVMValueRef *args,
234                   unsigned num_args,
235                   unsigned attr_mask)
236{
237   LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
238   LLVMValueRef function, call;
239   bool set_callsite_attrs = LLVM_VERSION_MAJOR >= 4 &&
240                             !(attr_mask & LP_FUNC_ATTR_LEGACY);
241
242   LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
243
244   assert(num_args <= LP_MAX_FUNC_ARGS);
245
246   for(unsigned i = 0; i < num_args; ++i) {
247      assert(args[i]);
248      arg_types[i] = LLVMTypeOf(args[i]);
249   }
250
251   LLVMTypeRef function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
252
253   function = LLVMGetNamedFunction(module, name);
254
255   if(!function) {
256      function = lp_declare_intrinsic_with_type(module, name, function_type);
257
258      /*
259       * If llvm removes an intrinsic we use, we'll hit this abort (rather
260       * than a call to address zero in the jited code).
261       */
262      if (LLVMGetIntrinsicID(function) == 0) {
263         _debug_printf("llvm (version " MESA_LLVM_VERSION_STRING
264                       ") found no intrinsic for %s, going to crash...\n",
265                name);
266         abort();
267      }
268
269      if (!set_callsite_attrs)
270         lp_add_func_attributes(function, attr_mask);
271
272      if (gallivm_debug & GALLIVM_DEBUG_IR) {
273         lp_debug_dump_value(function);
274      }
275   }
276
277   call = LLVMBuildCall2(builder, function_type, function, args, num_args, "");
278   if (set_callsite_attrs)
279      lp_add_func_attributes(call, attr_mask);
280   return call;
281}
282
283
284LLVMValueRef
285lp_build_intrinsic_unary(LLVMBuilderRef builder,
286                         const char *name,
287                         LLVMTypeRef ret_type,
288                         LLVMValueRef a)
289{
290   return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
291}
292
293
294LLVMValueRef
295lp_build_intrinsic_binary(LLVMBuilderRef builder,
296                          const char *name,
297                          LLVMTypeRef ret_type,
298                          LLVMValueRef a,
299                          LLVMValueRef b)
300{
301   LLVMValueRef args[2];
302
303   args[0] = a;
304   args[1] = b;
305
306   return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
307}
308
309
310/**
311 * Call intrinsic with arguments adapted to intrinsic vector length.
312 *
313 * Split vectors which are too large for the hw, or expand them if they
314 * are too small, so a caller calling a function which might use intrinsics
315 * doesn't need to do splitting/expansion on its own.
316 * This only supports intrinsics where src and dst types match.
317 */
318LLVMValueRef
319lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
320                                    const char *name,
321                                    struct lp_type src_type,
322                                    unsigned intr_size,
323                                    LLVMValueRef a,
324                                    LLVMValueRef b)
325{
326   unsigned i;
327   struct lp_type intrin_type = src_type;
328   LLVMBuilderRef builder = gallivm->builder;
329   LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
330   LLVMValueRef anative, bnative;
331   unsigned intrin_length = intr_size / src_type.width;
332
333   intrin_type.length = intrin_length;
334
335   if (intrin_length > src_type.length) {
336      LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
337      LLVMValueRef constvec, tmp;
338
339      for (i = 0; i < src_type.length; i++) {
340         elems[i] = lp_build_const_int32(gallivm, i);
341      }
342      for (; i < intrin_length; i++) {
343         elems[i] = i32undef;
344      }
345      if (src_type.length == 1) {
346         LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
347         a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
348         b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
349      }
350      constvec = LLVMConstVector(elems, intrin_length);
351      anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
352      bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
353      tmp = lp_build_intrinsic_binary(builder, name,
354                                      lp_build_vec_type(gallivm, intrin_type),
355                                      anative, bnative);
356      if (src_type.length > 1) {
357         constvec = LLVMConstVector(elems, src_type.length);
358         return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
359      }
360      else {
361         return LLVMBuildExtractElement(builder, tmp, elems[0], "");
362      }
363   }
364   else if (intrin_length < src_type.length) {
365      unsigned num_vec = src_type.length / intrin_length;
366      LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
367
368      /* don't support arbitrary size here as this is so yuck */
369      if (src_type.length % intrin_length) {
370         /* FIXME: This is something which should be supported
371          * but there doesn't seem to be any need for it currently
372          * so crash and burn.
373          */
374         debug_printf("%s: should handle arbitrary vector size\n",
375                      __FUNCTION__);
376         assert(0);
377         return NULL;
378      }
379
380      for (i = 0; i < num_vec; i++) {
381         anative = lp_build_extract_range(gallivm, a, i*intrin_length,
382                                        intrin_length);
383         bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
384                                        intrin_length);
385         tmp[i] = lp_build_intrinsic_binary(builder, name,
386                                            lp_build_vec_type(gallivm, intrin_type),
387                                            anative, bnative);
388      }
389      return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
390   }
391   else {
392      return lp_build_intrinsic_binary(builder, name,
393                                       lp_build_vec_type(gallivm, src_type),
394                                       a, b);
395   }
396}
397
398
399LLVMValueRef
400lp_build_intrinsic_map(struct gallivm_state *gallivm,
401                       const char *name,
402                       LLVMTypeRef ret_type,
403                       LLVMValueRef *args,
404                       unsigned num_args)
405{
406   LLVMBuilderRef builder = gallivm->builder;
407   LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
408   unsigned n = LLVMGetVectorSize(ret_type);
409   unsigned i, j;
410   LLVMValueRef res;
411
412   assert(num_args <= LP_MAX_FUNC_ARGS);
413
414   res = LLVMGetUndef(ret_type);
415   for(i = 0; i < n; ++i) {
416      LLVMValueRef index = lp_build_const_int32(gallivm, i);
417      LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
418      LLVMValueRef res_elem;
419      for(j = 0; j < num_args; ++j)
420         arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
421      res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
422      res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
423   }
424
425   return res;
426}
427
428
429LLVMValueRef
430lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
431                             const char *name,
432                             LLVMTypeRef ret_type,
433                             LLVMValueRef a)
434{
435   return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
436}
437
438
439LLVMValueRef
440lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
441                              const char *name,
442                              LLVMTypeRef ret_type,
443                              LLVMValueRef a,
444                              LLVMValueRef b)
445{
446   LLVMValueRef args[2];
447
448   args[0] = a;
449   args[1] = b;
450
451   return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
452}
453
454
455