1/*
2 * Copyright © Microsoft Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "d3d12_query.h"
25#include "d3d12_compiler.h"
26#include "d3d12_context.h"
27#include "d3d12_resource.h"
28#include "d3d12_screen.h"
29
30#include "util/u_dump.h"
31#include "util/u_inlines.h"
32#include "util/u_memory.h"
33#include "util/u_threaded_context.h"
34
35#include <dxguids/dxguids.h>
36
37constexpr unsigned MAX_SUBQUERIES = 3;
38
39struct d3d12_query_impl {
40   ID3D12QueryHeap *query_heap;
41   unsigned curr_query, num_queries;
42   size_t query_size;
43
44   D3D12_QUERY_TYPE d3d12qtype;
45
46   pipe_resource *buffer;
47   unsigned buffer_offset;
48
49   bool active;
50};
51
52struct d3d12_query {
53   struct threaded_query base;
54   enum pipe_query_type type;
55
56   struct d3d12_query_impl subqueries[MAX_SUBQUERIES];
57
58   struct list_head active_list;
59   struct d3d12_resource *predicate;
60};
61
62static unsigned
63num_sub_queries(unsigned query_type)
64{
65   switch (query_type) {
66   case PIPE_QUERY_PRIMITIVES_GENERATED:
67      return 3;
68   default:
69      return 1;
70   }
71}
72
73static D3D12_QUERY_HEAP_TYPE
74d3d12_query_heap_type(unsigned query_type, unsigned sub_query)
75{
76   switch (query_type) {
77   case PIPE_QUERY_OCCLUSION_COUNTER:
78   case PIPE_QUERY_OCCLUSION_PREDICATE:
79   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
80      return D3D12_QUERY_HEAP_TYPE_OCCLUSION;
81   case PIPE_QUERY_PIPELINE_STATISTICS:
82      return D3D12_QUERY_HEAP_TYPE_PIPELINE_STATISTICS;
83   case PIPE_QUERY_PRIMITIVES_GENERATED:
84      return sub_query == 0 ?
85         D3D12_QUERY_HEAP_TYPE_SO_STATISTICS :
86         D3D12_QUERY_HEAP_TYPE_PIPELINE_STATISTICS;
87   case PIPE_QUERY_PRIMITIVES_EMITTED:
88   case PIPE_QUERY_SO_STATISTICS:
89      return D3D12_QUERY_HEAP_TYPE_SO_STATISTICS;
90   case PIPE_QUERY_TIMESTAMP:
91   case PIPE_QUERY_TIME_ELAPSED:
92      return D3D12_QUERY_HEAP_TYPE_TIMESTAMP;
93
94   default:
95      debug_printf("unknown query: %s\n",
96                   util_str_query_type(query_type, true));
97      unreachable("d3d12: unknown query type");
98   }
99}
100
101static D3D12_QUERY_TYPE
102d3d12_query_type(unsigned query_type, unsigned sub_query, unsigned index)
103{
104   switch (query_type) {
105   case PIPE_QUERY_OCCLUSION_COUNTER:
106      return D3D12_QUERY_TYPE_OCCLUSION;
107   case PIPE_QUERY_OCCLUSION_PREDICATE:
108   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
109      return D3D12_QUERY_TYPE_BINARY_OCCLUSION;
110   case PIPE_QUERY_PIPELINE_STATISTICS:
111      return D3D12_QUERY_TYPE_PIPELINE_STATISTICS;
112   case PIPE_QUERY_PRIMITIVES_GENERATED:
113      return sub_query == 0 ?
114         D3D12_QUERY_TYPE_SO_STATISTICS_STREAM0 :
115         D3D12_QUERY_TYPE_PIPELINE_STATISTICS;
116   case PIPE_QUERY_PRIMITIVES_EMITTED:
117   case PIPE_QUERY_SO_STATISTICS:
118      return (D3D12_QUERY_TYPE)(D3D12_QUERY_TYPE_SO_STATISTICS_STREAM0 + index);
119   case PIPE_QUERY_TIMESTAMP:
120   case PIPE_QUERY_TIME_ELAPSED:
121      return D3D12_QUERY_TYPE_TIMESTAMP;
122   default:
123      debug_printf("unknown query: %s\n",
124                   util_str_query_type(query_type, true));
125      unreachable("d3d12: unknown query type");
126   }
127}
128
129static struct pipe_query *
130d3d12_create_query(struct pipe_context *pctx,
131                   unsigned query_type, unsigned index)
132{
133   struct d3d12_context *ctx = d3d12_context(pctx);
134   struct d3d12_screen *screen = d3d12_screen(pctx->screen);
135   struct d3d12_query *query = CALLOC_STRUCT(d3d12_query);
136   D3D12_QUERY_HEAP_DESC desc = {};
137
138   if (!query)
139      return NULL;
140
141   query->type = (pipe_query_type)query_type;
142   for (unsigned i = 0; i < num_sub_queries(query_type); ++i) {
143      assert(i < MAX_SUBQUERIES);
144      query->subqueries[i].d3d12qtype = d3d12_query_type(query_type, i, index);
145      query->subqueries[i].num_queries = 16;
146
147      /* With timer queries we want a few more queries, especially since we need two slots
148       * per query for TIME_ELAPSED queries
149       * For TIMESTAMP, we don't need more than one slot, since there's nothing to accumulate */
150      if (unlikely(query_type == PIPE_QUERY_TIME_ELAPSED))
151         query->subqueries[i].num_queries = 64;
152      else if (query_type == PIPE_QUERY_TIMESTAMP)
153         query->subqueries[i].num_queries = 1;
154
155      query->subqueries[i].curr_query = 0;
156      desc.Count = query->subqueries[i].num_queries;
157      desc.Type = d3d12_query_heap_type(query_type, i);
158
159      switch (desc.Type) {
160      case D3D12_QUERY_HEAP_TYPE_PIPELINE_STATISTICS:
161         query->subqueries[i].query_size = sizeof(D3D12_QUERY_DATA_PIPELINE_STATISTICS);
162         break;
163      case D3D12_QUERY_HEAP_TYPE_SO_STATISTICS:
164         query->subqueries[i].query_size = sizeof(D3D12_QUERY_DATA_SO_STATISTICS);
165         break;
166      default:
167         query->subqueries[i].query_size = sizeof(uint64_t);
168         break;
169      }
170      if (FAILED(screen->dev->CreateQueryHeap(&desc,
171                                              IID_PPV_ARGS(&query->subqueries[i].query_heap)))) {
172         FREE(query);
173         return NULL;
174      }
175
176      /* Query result goes into a readback buffer */
177      size_t buffer_size = query->subqueries[i].query_size * query->subqueries[i].num_queries;
178      u_suballocator_alloc(&ctx->query_allocator, buffer_size, 256,
179                           &query->subqueries[i].buffer_offset, &query->subqueries[i].buffer);
180
181      query->subqueries[i].active = (query_type == PIPE_QUERY_TIMESTAMP);
182   }
183
184   return (struct pipe_query *)query;
185}
186
187static void
188d3d12_destroy_query(struct pipe_context *pctx,
189                    struct pipe_query *q)
190{
191   struct d3d12_query *query = (struct d3d12_query *)q;
192   pipe_resource *predicate = &query->predicate->base.b;
193   pipe_resource_reference(&predicate, NULL);
194   for (unsigned i = 0; i < num_sub_queries(query->type); ++i) {
195      query->subqueries[i].query_heap->Release();
196      pipe_resource_reference(&query->subqueries[i].buffer, NULL);
197   }
198   FREE(query);
199}
200
201static bool
202accumulate_subresult(struct d3d12_context *ctx, struct d3d12_query *q_parent,
203                     unsigned sub_query,
204                     union pipe_query_result *result, bool write, bool wait)
205{
206   struct pipe_transfer *transfer = NULL;
207   struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
208   struct d3d12_query_impl *q = &q_parent->subqueries[sub_query];
209   unsigned access = PIPE_MAP_READ;
210   void *results;
211
212   if (write)
213      access |= PIPE_MAP_WRITE;
214   if (!wait)
215      access |= PIPE_MAP_DONTBLOCK;
216   results = pipe_buffer_map_range(&ctx->base, q->buffer, q->buffer_offset,
217                                   q->num_queries * q->query_size,
218                                   access, &transfer);
219
220   if (results == NULL)
221      return false;
222
223   uint64_t *results_u64 = (uint64_t *)results;
224   D3D12_QUERY_DATA_PIPELINE_STATISTICS *results_stats = (D3D12_QUERY_DATA_PIPELINE_STATISTICS *)results;
225   D3D12_QUERY_DATA_SO_STATISTICS *results_so = (D3D12_QUERY_DATA_SO_STATISTICS *)results;
226
227   memset(result, 0, sizeof(*result));
228   for (unsigned i = 0; i < q->curr_query; ++i) {
229      switch (q->d3d12qtype) {
230      case D3D12_QUERY_TYPE_BINARY_OCCLUSION:
231         result->b |= results_u64[i] != 0;
232         break;
233
234      case D3D12_QUERY_TYPE_OCCLUSION:
235         result->u64 += results_u64[i];
236         break;
237
238      case D3D12_QUERY_TYPE_TIMESTAMP:
239         if (q_parent->type == PIPE_QUERY_TIME_ELAPSED)
240            result->u64 += results_u64[2 * i + 1] - results_u64[2 * i];
241         else
242            result->u64 = results_u64[i];
243         break;
244
245      case D3D12_QUERY_TYPE_PIPELINE_STATISTICS:
246         result->pipeline_statistics.ia_vertices += results_stats[i].IAVertices;
247         result->pipeline_statistics.ia_primitives += results_stats[i].IAPrimitives;
248         result->pipeline_statistics.vs_invocations += results_stats[i].VSInvocations;
249         result->pipeline_statistics.gs_invocations += results_stats[i].GSInvocations;
250         result->pipeline_statistics.gs_primitives += results_stats[i].GSPrimitives;
251         result->pipeline_statistics.c_invocations += results_stats[i].CInvocations;
252         result->pipeline_statistics.c_primitives += results_stats[i].CPrimitives;
253         result->pipeline_statistics.ps_invocations += results_stats[i].PSInvocations;
254         result->pipeline_statistics.hs_invocations += results_stats[i].HSInvocations;
255         result->pipeline_statistics.ds_invocations += results_stats[i].DSInvocations;
256         result->pipeline_statistics.cs_invocations += results_stats[i].CSInvocations;
257         break;
258
259      case D3D12_QUERY_TYPE_SO_STATISTICS_STREAM0:
260      case D3D12_QUERY_TYPE_SO_STATISTICS_STREAM1:
261      case D3D12_QUERY_TYPE_SO_STATISTICS_STREAM2:
262      case D3D12_QUERY_TYPE_SO_STATISTICS_STREAM3:
263         result->so_statistics.num_primitives_written += results_so[i].NumPrimitivesWritten;
264         result->so_statistics.primitives_storage_needed += results_so[i].PrimitivesStorageNeeded;
265         break;
266
267      default:
268         debug_printf("unsupported query type: %s\n",
269                      util_str_query_type(q_parent->type, true));
270         unreachable("unexpected query type");
271      }
272   }
273
274   if (write) {
275      if (q->d3d12qtype == D3D12_QUERY_TYPE_PIPELINE_STATISTICS) {
276         results_stats[0].IAVertices = result->pipeline_statistics.ia_vertices;
277         results_stats[0].IAPrimitives = result->pipeline_statistics.ia_primitives;
278         results_stats[0].VSInvocations = result->pipeline_statistics.vs_invocations;
279         results_stats[0].GSInvocations = result->pipeline_statistics.gs_invocations;
280         results_stats[0].GSPrimitives = result->pipeline_statistics.gs_primitives;
281         results_stats[0].CInvocations = result->pipeline_statistics.c_invocations;
282         results_stats[0].CPrimitives = result->pipeline_statistics.c_primitives;
283         results_stats[0].PSInvocations = result->pipeline_statistics.ps_invocations;
284         results_stats[0].HSInvocations = result->pipeline_statistics.hs_invocations;
285         results_stats[0].DSInvocations = result->pipeline_statistics.ds_invocations;
286         results_stats[0].CSInvocations = result->pipeline_statistics.cs_invocations;
287      } else if (d3d12_query_heap_type(q_parent->type, sub_query) == D3D12_QUERY_HEAP_TYPE_SO_STATISTICS) {
288         results_so[0].NumPrimitivesWritten = result->so_statistics.num_primitives_written;
289         results_so[0].PrimitivesStorageNeeded = result->so_statistics.primitives_storage_needed;
290      } else {
291         if (unlikely(q->d3d12qtype == D3D12_QUERY_TYPE_TIMESTAMP)) {
292            results_u64[0] = 0;
293            results_u64[1] = result->u64;
294         } else {
295            results_u64[0] = result->u64;
296         }
297      }
298   }
299
300   pipe_buffer_unmap(&ctx->base, transfer);
301
302   if (q->d3d12qtype == D3D12_QUERY_TYPE_TIMESTAMP)
303      result->u64 = static_cast<uint64_t>(screen->timestamp_multiplier * result->u64);
304
305   return true;
306}
307
308static bool
309accumulate_result(struct d3d12_context *ctx, struct d3d12_query *q,
310                  union pipe_query_result *result, bool write, bool wait)
311{
312   union pipe_query_result local_result;
313
314   switch (q->type) {
315   case PIPE_QUERY_PRIMITIVES_GENERATED:
316      if (!accumulate_subresult(ctx, q, 0, &local_result, write, wait))
317         return false;
318      result->u64 = local_result.so_statistics.primitives_storage_needed;
319
320      if (!accumulate_subresult(ctx, q, 1, &local_result, write, wait))
321         return false;
322      result->u64 += local_result.pipeline_statistics.gs_primitives;
323
324      if (!accumulate_subresult(ctx, q, 2, &local_result, write, wait))
325         return false;
326      result->u64 += local_result.pipeline_statistics.ia_primitives;
327      return true;
328   case PIPE_QUERY_PRIMITIVES_EMITTED:
329      if (!accumulate_subresult(ctx, q, 0, &local_result, write, wait))
330         return false;
331      result->u64 = local_result.so_statistics.num_primitives_written;
332      return true;
333   default:
334      assert(num_sub_queries(q->type) == 1);
335      return accumulate_subresult(ctx, q, 0, result, write, wait);
336   }
337}
338
339static bool
340subquery_should_be_active(struct d3d12_context *ctx, struct d3d12_query *q, unsigned sub_query)
341{
342   switch (q->type) {
343   case PIPE_QUERY_PRIMITIVES_GENERATED: {
344      bool has_xfb = !!ctx->gfx_pipeline_state.num_so_targets;
345      struct d3d12_shader_selector *gs = ctx->gfx_stages[PIPE_SHADER_GEOMETRY];
346      bool has_gs = gs && !gs->is_variant;
347      switch (sub_query) {
348      case 0: return has_xfb;
349      case 1: return !has_xfb && has_gs;
350      case 2: return !has_xfb && !has_gs;
351      default: unreachable("Invalid subquery for primitives generated");
352      }
353      break;
354   }
355   default:
356      return true;
357   }
358}
359
360static void
361begin_subquery(struct d3d12_context *ctx, struct d3d12_query *q_parent, unsigned sub_query)
362{
363   struct d3d12_query_impl *q = &q_parent->subqueries[sub_query];
364   if (q->curr_query == q->num_queries) {
365      union pipe_query_result result;
366
367      /* Accumulate current results and store in first slot */
368      accumulate_subresult(ctx, q_parent, sub_query, &result, true, true);
369      q->curr_query = 1;
370   }
371
372   ctx->cmdlist->BeginQuery(q->query_heap, q->d3d12qtype, q->curr_query);
373   q->active = true;
374}
375
376static void
377begin_query(struct d3d12_context *ctx, struct d3d12_query *q_parent, bool restart)
378{
379   for (unsigned i = 0; i < num_sub_queries(q_parent->type); ++i) {
380      if (restart)
381         q_parent->subqueries[i].curr_query = 0;
382
383      if (!subquery_should_be_active(ctx, q_parent, i))
384         continue;
385
386      begin_subquery(ctx, q_parent, i);
387   }
388}
389
390
391static void
392begin_timer_query(struct d3d12_context *ctx, struct d3d12_query *q_parent, bool restart)
393{
394   struct d3d12_query_impl *q = &q_parent->subqueries[0];
395
396   /* For PIPE_QUERY_TIME_ELAPSED we record one time with BeginQuery and one in
397    * EndQuery, so we need two query slots */
398   unsigned query_index = 2 * q->curr_query;
399
400   if (restart) {
401      q->curr_query = 0;
402      query_index = 0;
403   } else if (query_index == q->num_queries) {
404      union pipe_query_result result;
405
406      /* Accumulate current results and store in first slot */
407      d3d12_flush_cmdlist_and_wait(ctx);
408      accumulate_subresult(ctx, q_parent, 0, &result, true, true);
409      q->curr_query = 2;
410   }
411
412   ctx->cmdlist->EndQuery(q->query_heap, q->d3d12qtype, query_index);
413   q->active = true;
414}
415
416static bool
417d3d12_begin_query(struct pipe_context *pctx,
418                  struct pipe_query *q)
419{
420   struct d3d12_context *ctx = d3d12_context(pctx);
421   struct d3d12_query *query = (struct d3d12_query *)q;
422
423   assert(query->type != PIPE_QUERY_TIMESTAMP);
424
425   if (unlikely(query->type == PIPE_QUERY_TIME_ELAPSED))
426      begin_timer_query(ctx, query, true);
427   else {
428      begin_query(ctx, query, true);
429      list_addtail(&query->active_list, &ctx->active_queries);
430   }
431
432   return true;
433}
434
435static void
436end_subquery(struct d3d12_context *ctx, struct d3d12_query *q_parent, unsigned sub_query)
437{
438   struct d3d12_query_impl *q = &q_parent->subqueries[sub_query];
439
440   uint64_t offset = 0;
441   struct d3d12_batch *batch = d3d12_current_batch(ctx);
442   struct d3d12_resource *res = (struct d3d12_resource *)q->buffer;
443   ID3D12Resource *d3d12_res = d3d12_resource_underlying(res, &offset);
444
445   /* For TIMESTAMP, there's only one slot */
446   if (q_parent->type == PIPE_QUERY_TIMESTAMP)
447      q->curr_query = 0;
448
449   /* With QUERY_TIME_ELAPSED we have recorded one value at
450      * (2 * q->curr_query), and now we record a value at (2 * q->curr_query + 1)
451      * and when resolving the query we subtract the latter from the former */
452
453   unsigned resolve_count = q_parent->type == PIPE_QUERY_TIME_ELAPSED ? 2 : 1;
454   unsigned resolve_index = resolve_count * q->curr_query;
455   unsigned end_index = resolve_index + resolve_count - 1;
456
457   offset += q->buffer_offset + resolve_index * q->query_size;
458   ctx->cmdlist->EndQuery(q->query_heap, q->d3d12qtype, end_index);
459   d3d12_transition_resource_state(ctx, res, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
460   d3d12_apply_resource_states(ctx, false);
461   ctx->cmdlist->ResolveQueryData(q->query_heap, q->d3d12qtype, resolve_index,
462      resolve_count, d3d12_res, offset);
463
464   d3d12_batch_reference_object(batch, q->query_heap);
465   d3d12_batch_reference_resource(batch, res, true);
466
467   assert(q->curr_query < q->num_queries);
468   q->curr_query++;
469   q->active = (q_parent->type == PIPE_QUERY_TIMESTAMP);
470}
471
472static void
473end_query(struct d3d12_context *ctx, struct d3d12_query *q_parent)
474{
475   for (unsigned i = 0; i < num_sub_queries(q_parent->type); ++i) {
476      struct d3d12_query_impl *q = &q_parent->subqueries[i];
477      if (!q->active)
478         continue;
479
480      end_subquery(ctx, q_parent, i);
481   }
482}
483
484static bool
485d3d12_end_query(struct pipe_context *pctx,
486               struct pipe_query *q)
487{
488   struct d3d12_context *ctx = d3d12_context(pctx);
489   struct d3d12_query *query = (struct d3d12_query *)q;
490
491   end_query(ctx, query);
492
493   if (query->type != PIPE_QUERY_TIMESTAMP &&
494       query->type != PIPE_QUERY_TIME_ELAPSED)
495      list_delinit(&query->active_list);
496   return true;
497}
498
499static bool
500d3d12_get_query_result(struct pipe_context *pctx,
501                      struct pipe_query *q,
502                      bool wait,
503                      union pipe_query_result *result)
504{
505   struct d3d12_context *ctx = d3d12_context(pctx);
506   struct d3d12_query *query = (struct d3d12_query *)q;
507
508   return accumulate_result(ctx, query, result, false, wait);
509}
510
511void
512d3d12_suspend_queries(struct d3d12_context *ctx)
513{
514   list_for_each_entry(struct d3d12_query, query, &ctx->active_queries, active_list) {
515      end_query(ctx, query);
516   }
517}
518
519void
520d3d12_resume_queries(struct d3d12_context *ctx)
521{
522   list_for_each_entry(struct d3d12_query, query, &ctx->active_queries, active_list) {
523      begin_query(ctx, query, false);
524   }
525}
526
527void
528d3d12_validate_queries(struct d3d12_context *ctx)
529{
530   /* Nothing to do, all queries are suspended */
531   if (ctx->queries_disabled)
532      return;
533
534   list_for_each_entry(struct d3d12_query, query, &ctx->active_queries, active_list) {
535      for (unsigned i = 0; i < num_sub_queries(query->type); ++i) {
536         if (query->subqueries[i].active && !subquery_should_be_active(ctx, query, i))
537            end_subquery(ctx, query, i);
538         else if (!query->subqueries[i].active && subquery_should_be_active(ctx, query, i))
539            begin_subquery(ctx, query, i);
540      }
541   }
542}
543
544static void
545d3d12_set_active_query_state(struct pipe_context *pctx, bool enable)
546{
547   struct d3d12_context *ctx = d3d12_context(pctx);
548   ctx->queries_disabled = !enable;
549
550   if (enable)
551      d3d12_resume_queries(ctx);
552   else
553      d3d12_suspend_queries(ctx);
554}
555
556static void
557d3d12_render_condition(struct pipe_context *pctx,
558                       struct pipe_query *pquery,
559                       bool condition,
560                       enum pipe_render_cond_flag mode)
561{
562   struct d3d12_context *ctx = d3d12_context(pctx);
563   struct d3d12_query *query = (struct d3d12_query *)pquery;
564
565   if (query == nullptr) {
566      ctx->cmdlist->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO);
567      ctx->current_predication = nullptr;
568      return;
569   }
570
571   assert(num_sub_queries(query->type) == 1);
572   if (!query->predicate)
573      query->predicate = d3d12_resource(pipe_buffer_create(pctx->screen, 0,
574                                                           PIPE_USAGE_DEFAULT, sizeof(uint64_t)));
575
576   if (mode == PIPE_RENDER_COND_WAIT) {
577      d3d12_flush_cmdlist_and_wait(ctx);
578      union pipe_query_result result;
579      accumulate_result(ctx, (d3d12_query *)pquery, &result, true, true);
580   }
581
582   struct d3d12_resource *res = (struct d3d12_resource *)query->subqueries[0].buffer;
583   uint64_t source_offset = 0;
584   ID3D12Resource *source = d3d12_resource_underlying(res, &source_offset);
585   source_offset += query->subqueries[0].buffer_offset;
586   d3d12_transition_resource_state(ctx, res, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
587   d3d12_transition_resource_state(ctx, query->predicate, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_TRANSITION_FLAG_NONE);
588   d3d12_apply_resource_states(ctx, false);
589   ctx->cmdlist->CopyBufferRegion(d3d12_resource_resource(query->predicate), 0,
590                                  source, source_offset,
591                                  sizeof(uint64_t));
592
593   d3d12_transition_resource_state(ctx, query->predicate, D3D12_RESOURCE_STATE_PREDICATION, D3D12_TRANSITION_FLAG_NONE);
594   d3d12_apply_resource_states(ctx, false);
595
596   ctx->current_predication = query->predicate;
597   ctx->predication_condition = condition;
598   d3d12_enable_predication(ctx);
599}
600
601void
602d3d12_enable_predication(struct d3d12_context *ctx)
603{
604   /* documentation of ID3D12GraphicsCommandList::SetPredication method:
605      * "resource manipulation commands are _not_ actually performed
606      *  if the resulting predicate data of the predicate is equal to
607      *  the operation specified."
608      */
609   ctx->cmdlist->SetPredication(d3d12_resource_resource(ctx->current_predication), 0,
610                                ctx->predication_condition ? D3D12_PREDICATION_OP_NOT_EQUAL_ZERO :
611                                D3D12_PREDICATION_OP_EQUAL_ZERO);
612}
613
614void
615d3d12_context_query_init(struct pipe_context *pctx)
616{
617   struct d3d12_context *ctx = d3d12_context(pctx);
618   list_inithead(&ctx->active_queries);
619
620   u_suballocator_init(&ctx->query_allocator, &ctx->base, 4096, 0, PIPE_USAGE_STAGING,
621                         0, true);
622
623   pctx->create_query = d3d12_create_query;
624   pctx->destroy_query = d3d12_destroy_query;
625   pctx->begin_query = d3d12_begin_query;
626   pctx->end_query = d3d12_end_query;
627   pctx->get_query_result = d3d12_get_query_result;
628   pctx->set_active_query_state = d3d12_set_active_query_state;
629   pctx->render_condition = d3d12_render_condition;
630}
631