1/*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25#ifndef SI_QUERY_H
26#define SI_QUERY_H
27
28#include "util/u_threaded_context.h"
29
30#include "ac_perfcounter.h"
31
32struct pipe_context;
33struct pipe_query;
34struct pipe_resource;
35
36struct si_screen;
37struct si_context;
38struct si_query;
39struct si_query_buffer;
40struct si_query_hw;
41struct si_resource;
42
43#define SI_MAX_STREAMS 4
44
45enum
46{
47   SI_QUERY_DRAW_CALLS = PIPE_QUERY_DRIVER_SPECIFIC,
48   SI_QUERY_DECOMPRESS_CALLS,
49   SI_QUERY_PRIM_RESTART_CALLS,
50   SI_QUERY_COMPUTE_CALLS,
51   SI_QUERY_CP_DMA_CALLS,
52   SI_QUERY_NUM_VS_FLUSHES,
53   SI_QUERY_NUM_PS_FLUSHES,
54   SI_QUERY_NUM_CS_FLUSHES,
55   SI_QUERY_NUM_CB_CACHE_FLUSHES,
56   SI_QUERY_NUM_DB_CACHE_FLUSHES,
57   SI_QUERY_NUM_L2_INVALIDATES,
58   SI_QUERY_NUM_L2_WRITEBACKS,
59   SI_QUERY_NUM_RESIDENT_HANDLES,
60   SI_QUERY_TC_OFFLOADED_SLOTS,
61   SI_QUERY_TC_DIRECT_SLOTS,
62   SI_QUERY_TC_NUM_SYNCS,
63   SI_QUERY_CS_THREAD_BUSY,
64   SI_QUERY_GALLIUM_THREAD_BUSY,
65   SI_QUERY_REQUESTED_VRAM,
66   SI_QUERY_REQUESTED_GTT,
67   SI_QUERY_MAPPED_VRAM,
68   SI_QUERY_MAPPED_GTT,
69   SI_QUERY_SLAB_WASTED_VRAM,
70   SI_QUERY_SLAB_WASTED_GTT,
71   SI_QUERY_BUFFER_WAIT_TIME,
72   SI_QUERY_NUM_MAPPED_BUFFERS,
73   SI_QUERY_NUM_GFX_IBS,
74   SI_QUERY_GFX_BO_LIST_SIZE,
75   SI_QUERY_GFX_IB_SIZE,
76   SI_QUERY_NUM_BYTES_MOVED,
77   SI_QUERY_NUM_EVICTIONS,
78   SI_QUERY_NUM_VRAM_CPU_PAGE_FAULTS,
79   SI_QUERY_VRAM_USAGE,
80   SI_QUERY_VRAM_VIS_USAGE,
81   SI_QUERY_GTT_USAGE,
82   SI_QUERY_GPU_TEMPERATURE,
83   SI_QUERY_CURRENT_GPU_SCLK,
84   SI_QUERY_CURRENT_GPU_MCLK,
85   SI_QUERY_GPU_LOAD,
86   SI_QUERY_GPU_SHADERS_BUSY,
87   SI_QUERY_GPU_TA_BUSY,
88   SI_QUERY_GPU_GDS_BUSY,
89   SI_QUERY_GPU_VGT_BUSY,
90   SI_QUERY_GPU_IA_BUSY,
91   SI_QUERY_GPU_SX_BUSY,
92   SI_QUERY_GPU_WD_BUSY,
93   SI_QUERY_GPU_BCI_BUSY,
94   SI_QUERY_GPU_SC_BUSY,
95   SI_QUERY_GPU_PA_BUSY,
96   SI_QUERY_GPU_DB_BUSY,
97   SI_QUERY_GPU_CP_BUSY,
98   SI_QUERY_GPU_CB_BUSY,
99   SI_QUERY_GPU_SDMA_BUSY,
100   SI_QUERY_GPU_PFP_BUSY,
101   SI_QUERY_GPU_MEQ_BUSY,
102   SI_QUERY_GPU_ME_BUSY,
103   SI_QUERY_GPU_SURF_SYNC_BUSY,
104   SI_QUERY_GPU_CP_DMA_BUSY,
105   SI_QUERY_GPU_SCRATCH_RAM_BUSY,
106   SI_QUERY_NUM_COMPILATIONS,
107   SI_QUERY_NUM_SHADERS_CREATED,
108   SI_QUERY_BACK_BUFFER_PS_DRAW_RATIO,
109   SI_QUERY_GPIN_ASIC_ID,
110   SI_QUERY_GPIN_NUM_SIMD,
111   SI_QUERY_GPIN_NUM_RB,
112   SI_QUERY_GPIN_NUM_SPI,
113   SI_QUERY_GPIN_NUM_SE,
114   SI_QUERY_LIVE_SHADER_CACHE_HITS,
115   SI_QUERY_LIVE_SHADER_CACHE_MISSES,
116   SI_QUERY_MEMORY_SHADER_CACHE_HITS,
117   SI_QUERY_MEMORY_SHADER_CACHE_MISSES,
118   SI_QUERY_DISK_SHADER_CACHE_HITS,
119   SI_QUERY_DISK_SHADER_CACHE_MISSES,
120
121   SI_QUERY_FIRST_PERFCOUNTER = PIPE_QUERY_DRIVER_SPECIFIC + 100,
122};
123
124enum
125{
126   SI_QUERY_GROUP_GPIN = 0,
127   SI_NUM_SW_QUERY_GROUPS
128};
129
130struct si_query_ops {
131   void (*destroy)(struct si_context *, struct si_query *);
132   bool (*begin)(struct si_context *, struct si_query *);
133   bool (*end)(struct si_context *, struct si_query *);
134   bool (*get_result)(struct si_context *, struct si_query *, bool wait,
135                      union pipe_query_result *result);
136   void (*get_result_resource)(struct si_context *, struct si_query *,
137                               enum pipe_query_flags flags,
138                               enum pipe_query_value_type result_type, int index,
139                               struct pipe_resource *resource, unsigned offset);
140
141   void (*suspend)(struct si_context *, struct si_query *);
142   void (*resume)(struct si_context *, struct si_query *);
143};
144
145struct si_query {
146   struct threaded_query b;
147   const struct si_query_ops *ops;
148
149   /* The PIPE_QUERY_xxx type of query */
150   unsigned type;
151
152   /* The number of dwords for suspend. */
153   unsigned num_cs_dw_suspend;
154
155   /* Linked list of queries that must be suspended at end of CS. */
156   struct list_head active_list;
157};
158
159enum
160{
161   SI_QUERY_HW_FLAG_NO_START = (1 << 0),
162   /* gap */
163   /* whether begin_query doesn't clear the result */
164   SI_QUERY_HW_FLAG_BEGIN_RESUMES = (1 << 2),
165   /* whether GS invocations and emitted primitives counters are emulated
166    * using atomic adds.
167    */
168   SI_QUERY_EMULATE_GS_COUNTERS = (1 << 3),
169};
170
171struct si_query_hw_ops {
172   bool (*prepare_buffer)(struct si_context *, struct si_query_buffer *);
173   void (*emit_start)(struct si_context *, struct si_query_hw *, struct si_resource *buffer,
174                      uint64_t va);
175   void (*emit_stop)(struct si_context *, struct si_query_hw *, struct si_resource *buffer,
176                     uint64_t va);
177   void (*clear_result)(struct si_query_hw *, union pipe_query_result *);
178   void (*add_result)(struct si_screen *screen, struct si_query_hw *, void *buffer,
179                      union pipe_query_result *result);
180};
181
182struct si_query_buffer {
183   /* The buffer where query results are stored. */
184   struct si_resource *buf;
185   /* If a query buffer is full, a new buffer is created and the old one
186    * is put in here. When we calculate the result, we sum up the samples
187    * from all buffers. */
188   struct si_query_buffer *previous;
189   /* Offset of the next free result after current query data */
190   unsigned results_end;
191   bool unprepared;
192};
193
194void si_query_buffer_destroy(struct si_screen *sctx, struct si_query_buffer *buffer);
195void si_query_buffer_reset(struct si_context *sctx, struct si_query_buffer *buffer);
196bool si_query_buffer_alloc(struct si_context *sctx, struct si_query_buffer *buffer,
197                           bool (*prepare_buffer)(struct si_context *, struct si_query_buffer *),
198                           unsigned size);
199
200struct si_query_hw {
201   struct si_query b;
202   struct si_query_hw_ops *ops;
203   unsigned flags;
204
205   /* The query buffer and how many results are in it. */
206   struct si_query_buffer buffer;
207   /* Size of the result in memory for both begin_query and end_query,
208    * this can be one or two numbers, or it could even be a size of a structure. */
209   unsigned result_size;
210   union {
211      /* For transform feedback: which stream the query is for */
212      unsigned stream;
213      /* For pipeline stats: which counter is active */
214      unsigned index;
215   };
216
217   /* Workaround via compute shader */
218   struct si_resource *workaround_buf;
219   unsigned workaround_offset;
220};
221
222unsigned si_query_pipestat_end_dw_offset(struct si_screen *sscreen,
223                                         enum pipe_statistics_query_index index);
224void si_query_hw_destroy(struct si_context *sctx, struct si_query *squery);
225bool si_query_hw_begin(struct si_context *sctx, struct si_query *squery);
226bool si_query_hw_end(struct si_context *sctx, struct si_query *squery);
227bool si_query_hw_get_result(struct si_context *sctx, struct si_query *squery, bool wait,
228                            union pipe_query_result *result);
229void si_query_hw_suspend(struct si_context *sctx, struct si_query *query);
230void si_query_hw_resume(struct si_context *sctx, struct si_query *query);
231
232/* Shader-based queries */
233
234/**
235 * The query buffer is written to by ESGS NGG shaders with statistics about
236 * generated and (streamout-)emitted primitives.
237 *
238 * The context maintains a ring of these query buffers, and queries simply
239 * point into the ring, allowing an arbitrary number of queries to be active
240 * without additional GPU cost.
241 */
242struct gfx10_sh_query_buffer {
243   struct list_head list;
244   struct si_resource *buf;
245   unsigned refcount;
246
247   /* Offset into the buffer in bytes; points at the first un-emitted entry. */
248   unsigned head;
249};
250
251/* Memory layout of the query buffer. Must be kept in sync with shaders
252 * (including QBO shaders) and should be aligned to cachelines.
253 *
254 * The somewhat awkward memory layout is for compatibility with the
255 * SET_PREDICATION packet, which also means that we're setting the high bit
256 * of all those values unconditionally.
257 */
258struct gfx10_sh_query_buffer_mem {
259   struct {
260      uint64_t generated_primitives_start_dummy;
261      uint64_t emitted_primitives_start_dummy;
262      uint64_t generated_primitives;
263      uint64_t emitted_primitives;
264   } stream[4];
265   uint32_t fence; /* bottom-of-pipe fence: set to ~0 when draws have finished */
266   uint32_t pad[31];
267};
268
269struct gfx10_sh_query {
270   struct si_query b;
271
272   struct gfx10_sh_query_buffer *first;
273   struct gfx10_sh_query_buffer *last;
274   unsigned first_begin;
275   unsigned last_end;
276
277   unsigned stream;
278};
279
280struct pipe_query *gfx10_sh_query_create(struct si_screen *screen, enum pipe_query_type query_type,
281                                         unsigned index);
282
283/* Performance counters */
284struct si_perfcounters {
285   struct ac_perfcounters base;
286
287   unsigned num_stop_cs_dwords;
288   unsigned num_instance_cs_dwords;
289};
290
291struct pipe_query *si_create_batch_query(struct pipe_context *ctx, unsigned num_queries,
292                                         unsigned *query_types);
293
294int si_get_perfcounter_info(struct si_screen *, unsigned index,
295                            struct pipe_driver_query_info *info);
296int si_get_perfcounter_group_info(struct si_screen *, unsigned index,
297                                  struct pipe_driver_query_group_info *info);
298
299struct si_qbo_state {
300   struct pipe_constant_buffer saved_const0;
301};
302
303#endif /* SI_QUERY_H */
304