1/**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28/* Helper utility for uploading user buffers & other data, and
29 * coalescing small buffers into larger ones.
30 */
31
32#include "pipe/p_defines.h"
33#include "util/u_inlines.h"
34#include "pipe/p_context.h"
35#include "util/u_memory.h"
36#include "util/u_math.h"
37
38#include "u_upload_mgr.h"
39
40
41struct u_upload_mgr {
42   struct pipe_context *pipe;
43
44   unsigned default_size;  /* Minimum size of the upload buffer, in bytes. */
45   unsigned bind;          /* Bitmask of PIPE_BIND_* flags. */
46   enum pipe_resource_usage usage;
47   unsigned flags;
48   unsigned map_flags;     /* Bitmask of PIPE_MAP_* flags. */
49   boolean map_persistent; /* If persistent mappings are supported. */
50
51   struct pipe_resource *buffer;   /* Upload buffer. */
52   struct pipe_transfer *transfer; /* Transfer object for the upload buffer. */
53   uint8_t *map;    /* Pointer to the mapped upload buffer. */
54   unsigned buffer_size; /* Same as buffer->width0. */
55   unsigned offset; /* Aligned offset to the upload buffer, pointing
56                     * at the first unused byte. */
57   int buffer_private_refcount;
58};
59
60
61struct u_upload_mgr *
62u_upload_create(struct pipe_context *pipe, unsigned default_size,
63                unsigned bind, enum pipe_resource_usage usage, unsigned flags)
64{
65   struct u_upload_mgr *upload = CALLOC_STRUCT(u_upload_mgr);
66   if (!upload)
67      return NULL;
68
69   upload->pipe = pipe;
70   upload->default_size = default_size;
71   upload->bind = bind;
72   upload->usage = usage;
73   upload->flags = flags;
74
75   upload->map_persistent =
76      pipe->screen->get_param(pipe->screen,
77                              PIPE_CAP_BUFFER_MAP_PERSISTENT_COHERENT);
78
79   if (upload->map_persistent) {
80      upload->map_flags = PIPE_MAP_WRITE |
81                          PIPE_MAP_UNSYNCHRONIZED |
82                          PIPE_MAP_PERSISTENT |
83                          PIPE_MAP_COHERENT;
84   }
85   else {
86      upload->map_flags = PIPE_MAP_WRITE |
87                          PIPE_MAP_UNSYNCHRONIZED |
88                          PIPE_MAP_FLUSH_EXPLICIT;
89   }
90
91   return upload;
92}
93
94struct u_upload_mgr *
95u_upload_create_default(struct pipe_context *pipe)
96{
97   return u_upload_create(pipe, 1024 * 1024,
98                          PIPE_BIND_VERTEX_BUFFER |
99                          PIPE_BIND_INDEX_BUFFER |
100                          PIPE_BIND_CONSTANT_BUFFER,
101                          PIPE_USAGE_STREAM, 0);
102}
103
104struct u_upload_mgr *
105u_upload_clone(struct pipe_context *pipe, struct u_upload_mgr *upload)
106{
107   struct u_upload_mgr *result = u_upload_create(pipe, upload->default_size,
108                                                 upload->bind, upload->usage,
109                                                 upload->flags);
110   if (!upload->map_persistent && result->map_persistent)
111      u_upload_disable_persistent(result);
112
113   return result;
114}
115
116void
117u_upload_disable_persistent(struct u_upload_mgr *upload)
118{
119   upload->map_persistent = FALSE;
120   upload->map_flags &= ~(PIPE_MAP_COHERENT | PIPE_MAP_PERSISTENT);
121   upload->map_flags |= PIPE_MAP_FLUSH_EXPLICIT;
122}
123
124static void
125upload_unmap_internal(struct u_upload_mgr *upload, boolean destroying)
126{
127   if ((!destroying && upload->map_persistent) || !upload->transfer)
128      return;
129
130   struct pipe_box *box = &upload->transfer->box;
131
132   if (!upload->map_persistent && (int) upload->offset > box->x) {
133      pipe_buffer_flush_mapped_range(upload->pipe, upload->transfer,
134                                     box->x, upload->offset - box->x);
135   }
136
137   pipe_buffer_unmap(upload->pipe, upload->transfer);
138   upload->transfer = NULL;
139   upload->map = NULL;
140}
141
142
143void
144u_upload_unmap(struct u_upload_mgr *upload)
145{
146   upload_unmap_internal(upload, FALSE);
147}
148
149
150static void
151u_upload_release_buffer(struct u_upload_mgr *upload)
152{
153   /* Unmap and unreference the upload buffer. */
154   upload_unmap_internal(upload, TRUE);
155   if (upload->buffer_private_refcount) {
156      /* Subtract the remaining private references before unreferencing
157       * the buffer. The mega comment below explains it.
158       */
159      assert(upload->buffer_private_refcount > 0);
160      p_atomic_add(&upload->buffer->reference.count,
161                   -upload->buffer_private_refcount);
162      upload->buffer_private_refcount = 0;
163   }
164   pipe_resource_reference(&upload->buffer, NULL);
165   upload->buffer_size = 0;
166}
167
168
169void
170u_upload_destroy(struct u_upload_mgr *upload)
171{
172   u_upload_release_buffer(upload);
173   FREE(upload);
174}
175
176/* Return the allocated buffer size or 0 if it failed. */
177static unsigned
178u_upload_alloc_buffer(struct u_upload_mgr *upload, unsigned min_size)
179{
180   struct pipe_screen *screen = upload->pipe->screen;
181   struct pipe_resource buffer;
182   unsigned size;
183
184   /* Release the old buffer, if present:
185    */
186   u_upload_release_buffer(upload);
187
188   /* Allocate a new one:
189    */
190   size = align(MAX2(upload->default_size, min_size), 4096);
191
192   memset(&buffer, 0, sizeof buffer);
193   buffer.target = PIPE_BUFFER;
194   buffer.format = PIPE_FORMAT_R8_UNORM; /* want TYPELESS or similar */
195   buffer.bind = upload->bind;
196   buffer.usage = upload->usage;
197   buffer.flags = upload->flags | PIPE_RESOURCE_FLAG_SINGLE_THREAD_USE;
198   buffer.width0 = size;
199   buffer.height0 = 1;
200   buffer.depth0 = 1;
201   buffer.array_size = 1;
202
203   if (upload->map_persistent) {
204      buffer.flags |= PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
205                      PIPE_RESOURCE_FLAG_MAP_COHERENT;
206   }
207
208   upload->buffer = screen->resource_create(screen, &buffer);
209   if (upload->buffer == NULL)
210      return 0;
211
212   /* Since atomic operations are very very slow when 2 threads are not
213    * sharing the same L3 cache (which happens on AMD Zen), eliminate all
214    * atomics in u_upload_alloc as follows:
215    *
216    * u_upload_alloc has to return a buffer reference to the caller.
217    * Instead of atomic_inc for every call, it does all possible future
218    * increments in advance here. The maximum number of times u_upload_alloc
219    * can be called per upload buffer is "size", because the minimum
220    * allocation size is 1, thus u_upload_alloc can only return "size" number
221    * of suballocations at most, so we will never need more. This is
222    * the number that is added to reference.count here.
223    *
224    * buffer_private_refcount tracks how many buffer references we can return
225    * without using atomics. If the buffer is full and there are still
226    * references left, they are atomically subtracted from reference.count
227    * before the buffer is unreferenced.
228    *
229    * This technique can increase CPU performance by 10%.
230    *
231    * The caller of u_upload_alloc_buffer will consume min_size bytes,
232    * so init the buffer_private_refcount to 1 + size - min_size, instead
233    * of size to avoid overflowing reference.count when size is huge.
234    */
235   upload->buffer_private_refcount = 1 + (size - min_size);
236   assert(upload->buffer_private_refcount < INT32_MAX / 2);
237   p_atomic_add(&upload->buffer->reference.count, upload->buffer_private_refcount);
238
239   /* Map the new buffer. */
240   upload->map = pipe_buffer_map_range(upload->pipe, upload->buffer,
241                                       0, size, upload->map_flags,
242                                       &upload->transfer);
243   if (upload->map == NULL) {
244      u_upload_release_buffer(upload);
245      return 0;
246   }
247
248   upload->buffer_size = size;
249   upload->offset = 0;
250   return size;
251}
252
253void
254u_upload_alloc(struct u_upload_mgr *upload,
255               unsigned min_out_offset,
256               unsigned size,
257               unsigned alignment,
258               unsigned *out_offset,
259               struct pipe_resource **outbuf,
260               void **ptr)
261{
262   unsigned buffer_size = upload->buffer_size;
263   unsigned offset = MAX2(min_out_offset, upload->offset);
264
265   offset = align(offset, alignment);
266
267   /* Make sure we have enough space in the upload buffer
268    * for the sub-allocation.
269    */
270   if (unlikely(offset + size > buffer_size)) {
271      /* Allocate a new buffer and set the offset to the smallest one. */
272      offset = align(min_out_offset, alignment);
273      buffer_size = u_upload_alloc_buffer(upload, offset + size);
274
275      if (unlikely(!buffer_size)) {
276         *out_offset = ~0;
277         pipe_resource_reference(outbuf, NULL);
278         *ptr = NULL;
279         return;
280      }
281   }
282
283   if (unlikely(!upload->map)) {
284      upload->map = pipe_buffer_map_range(upload->pipe, upload->buffer,
285                                          offset,
286                                          buffer_size - offset,
287                                          upload->map_flags,
288                                          &upload->transfer);
289      if (unlikely(!upload->map)) {
290         upload->transfer = NULL;
291         *out_offset = ~0;
292         pipe_resource_reference(outbuf, NULL);
293         *ptr = NULL;
294         return;
295      }
296
297      upload->map -= offset;
298   }
299
300   assert(offset < buffer_size);
301   assert(offset + size <= buffer_size);
302   assert(size);
303
304   /* Emit the return values: */
305   *ptr = upload->map + offset;
306   *out_offset = offset;
307
308   if (*outbuf != upload->buffer) {
309      pipe_resource_reference(outbuf, NULL);
310      *outbuf = upload->buffer;
311      assert (upload->buffer_private_refcount > 0);
312      upload->buffer_private_refcount--;
313   }
314
315   upload->offset = offset + size;
316}
317
318void
319u_upload_data(struct u_upload_mgr *upload,
320              unsigned min_out_offset,
321              unsigned size,
322              unsigned alignment,
323              const void *data,
324              unsigned *out_offset,
325              struct pipe_resource **outbuf)
326{
327   uint8_t *ptr;
328
329   u_upload_alloc(upload, min_out_offset, size, alignment,
330                  out_offset, outbuf,
331                  (void**)&ptr);
332   if (ptr)
333      memcpy(ptr, data, size);
334}
335