1/*
2 * Copyright 2009 Marek Olšák <maraeo@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE. */
22
23#include "r300_context.h"
24#include "r300_emit.h"
25#include "r300_texture.h"
26#include "r300_reg.h"
27
28#include "util/format/u_format.h"
29#include "util/half_float.h"
30#include "util/u_pack_color.h"
31#include "util/u_surface.h"
32
33enum r300_blitter_op /* bitmask */
34{
35    R300_STOP_QUERY         = 1,
36    R300_SAVE_TEXTURES      = 2,
37    R300_SAVE_FRAMEBUFFER   = 4,
38    R300_IGNORE_RENDER_COND = 8,
39
40    R300_CLEAR         = R300_STOP_QUERY,
41
42    R300_CLEAR_SURFACE = R300_STOP_QUERY | R300_SAVE_FRAMEBUFFER,
43
44    R300_COPY          = R300_STOP_QUERY | R300_SAVE_FRAMEBUFFER |
45                         R300_SAVE_TEXTURES | R300_IGNORE_RENDER_COND,
46
47    R300_BLIT          = R300_STOP_QUERY | R300_SAVE_FRAMEBUFFER |
48                         R300_SAVE_TEXTURES,
49
50    R300_DECOMPRESS    = R300_STOP_QUERY | R300_IGNORE_RENDER_COND,
51};
52
53static void r300_blitter_begin(struct r300_context* r300, enum r300_blitter_op op)
54{
55    if ((op & R300_STOP_QUERY) && r300->query_current) {
56        r300->blitter_saved_query = r300->query_current;
57        r300_stop_query(r300);
58    }
59
60    /* Yeah we have to save all those states to ensure the blitter operation
61     * is really transparent. The states will be restored by the blitter once
62     * copying is done. */
63    util_blitter_save_blend(r300->blitter, r300->blend_state.state);
64    util_blitter_save_depth_stencil_alpha(r300->blitter, r300->dsa_state.state);
65    util_blitter_save_stencil_ref(r300->blitter, &(r300->stencil_ref));
66    util_blitter_save_rasterizer(r300->blitter, r300->rs_state.state);
67    util_blitter_save_fragment_shader(r300->blitter, r300->fs.state);
68    util_blitter_save_vertex_shader(r300->blitter, r300->vs_state.state);
69    util_blitter_save_viewport(r300->blitter, &r300->viewport);
70    util_blitter_save_scissor(r300->blitter, r300->scissor_state.state);
71    util_blitter_save_sample_mask(r300->blitter, *(unsigned*)r300->sample_mask.state, 0);
72    util_blitter_save_vertex_buffer_slot(r300->blitter, r300->vertex_buffer);
73    util_blitter_save_vertex_elements(r300->blitter, r300->velems);
74
75    struct pipe_constant_buffer cb = {
76       /* r300 doesn't use the size for FS at all. The shader determines it.
77        * Set something for blitter.
78        */
79       .buffer_size = 4,
80       .user_buffer = ((struct r300_constant_buffer*)r300->fs_constants.state)->ptr,
81    };
82    util_blitter_save_fragment_constant_buffer_slot(r300->blitter, &cb);
83
84    if (op & R300_SAVE_FRAMEBUFFER) {
85        util_blitter_save_framebuffer(r300->blitter, r300->fb_state.state);
86    }
87
88    if (op & R300_SAVE_TEXTURES) {
89        struct r300_textures_state* state =
90            (struct r300_textures_state*)r300->textures_state.state;
91
92        util_blitter_save_fragment_sampler_states(
93            r300->blitter, state->sampler_state_count,
94            (void**)state->sampler_states);
95
96        util_blitter_save_fragment_sampler_views(
97            r300->blitter, state->sampler_view_count,
98            (struct pipe_sampler_view**)state->sampler_views);
99    }
100
101    if (op & R300_IGNORE_RENDER_COND) {
102        /* Save the flag. */
103        r300->blitter_saved_skip_rendering = r300->skip_rendering+1;
104        r300->skip_rendering = FALSE;
105    } else {
106        r300->blitter_saved_skip_rendering = 0;
107    }
108}
109
110static void r300_blitter_end(struct r300_context *r300)
111{
112    if (r300->blitter_saved_query) {
113        r300_resume_query(r300, r300->blitter_saved_query);
114        r300->blitter_saved_query = NULL;
115    }
116
117    if (r300->blitter_saved_skip_rendering) {
118        /* Restore the flag. */
119        r300->skip_rendering = r300->blitter_saved_skip_rendering-1;
120    }
121}
122
123static uint32_t r300_depth_clear_cb_value(enum pipe_format format,
124                                          const float* rgba)
125{
126    union util_color uc;
127    util_pack_color(rgba, format, &uc);
128
129    if (util_format_get_blocksizebits(format) == 32)
130        return uc.ui[0];
131    else
132        return uc.us | (uc.us << 16);
133}
134
135static boolean r300_cbzb_clear_allowed(struct r300_context *r300,
136                                       unsigned clear_buffers)
137{
138    struct pipe_framebuffer_state *fb =
139        (struct pipe_framebuffer_state*)r300->fb_state.state;
140
141    /* Only color clear allowed, and only one colorbuffer. */
142    if ((clear_buffers & ~PIPE_CLEAR_COLOR) != 0 || fb->nr_cbufs != 1 || !fb->cbufs[0])
143        return FALSE;
144
145    return r300_surface(fb->cbufs[0])->cbzb_allowed;
146}
147
148static boolean r300_fast_zclear_allowed(struct r300_context *r300,
149                                        unsigned clear_buffers)
150{
151    struct pipe_framebuffer_state *fb =
152        (struct pipe_framebuffer_state*)r300->fb_state.state;
153
154    return r300_resource(fb->zsbuf->texture)->tex.zmask_dwords[fb->zsbuf->u.tex.level] != 0;
155}
156
157static boolean r300_hiz_clear_allowed(struct r300_context *r300)
158{
159    struct pipe_framebuffer_state *fb =
160        (struct pipe_framebuffer_state*)r300->fb_state.state;
161
162    return r300_resource(fb->zsbuf->texture)->tex.hiz_dwords[fb->zsbuf->u.tex.level] != 0;
163}
164
165static uint32_t r300_depth_clear_value(enum pipe_format format,
166                                       double depth, unsigned stencil)
167{
168    switch (format) {
169        case PIPE_FORMAT_Z16_UNORM:
170        case PIPE_FORMAT_X8Z24_UNORM:
171            return util_pack_z(format, depth);
172
173        case PIPE_FORMAT_S8_UINT_Z24_UNORM:
174            return util_pack_z_stencil(format, depth, stencil);
175
176        default:
177            assert(0);
178            return 0;
179    }
180}
181
182static uint32_t r300_hiz_clear_value(double depth)
183{
184    uint32_t r = (uint32_t)(CLAMP(depth, 0, 1) * 255.5);
185    assert(r <= 255);
186    return r | (r << 8) | (r << 16) | (r << 24);
187}
188
189static void r300_set_clear_color(struct r300_context *r300,
190                                 const union pipe_color_union *color)
191{
192    struct pipe_framebuffer_state *fb =
193        (struct pipe_framebuffer_state*)r300->fb_state.state;
194    union util_color uc;
195
196    memset(&uc, 0, sizeof(uc));
197    util_pack_color(color->f, fb->cbufs[0]->format, &uc);
198
199    if (fb->cbufs[0]->format == PIPE_FORMAT_R16G16B16A16_FLOAT ||
200        fb->cbufs[0]->format == PIPE_FORMAT_R16G16B16X16_FLOAT) {
201        /* (0,1,2,3) maps to (B,G,R,A) */
202        r300->color_clear_value_gb = uc.h[0] | ((uint32_t)uc.h[1] << 16);
203        r300->color_clear_value_ar = uc.h[2] | ((uint32_t)uc.h[3] << 16);
204    } else {
205        r300->color_clear_value = uc.ui[0];
206    }
207}
208
209DEBUG_GET_ONCE_BOOL_OPTION(hyperz, "RADEON_HYPERZ", FALSE)
210
211/* Clear currently bound buffers. */
212static void r300_clear(struct pipe_context* pipe,
213                       unsigned buffers,
214                       const struct pipe_scissor_state *scissor_state,
215                       const union pipe_color_union *color,
216                       double depth,
217                       unsigned stencil)
218{
219    /* My notes about Zbuffer compression:
220     *
221     * 1) The zbuffer must be micro-tiled and whole microtiles must be
222     *    written if compression is enabled. If microtiling is disabled,
223     *    it locks up.
224     *
225     * 2) There is ZMASK RAM which contains a compressed zbuffer.
226     *    Each dword of the Z Mask contains compression information
227     *    for 16 4x4 pixel tiles, that is 2 bits for each tile.
228     *    On chips with 2 Z pipes, every other dword maps to a different
229     *    pipe. On newer chipsets, there is a new compression mode
230     *    with 8x8 pixel tiles per 2 bits.
231     *
232     * 3) The FASTFILL bit has nothing to do with filling. It only tells hw
233     *    it should look in the ZMASK RAM first before fetching from a real
234     *    zbuffer.
235     *
236     * 4) If a pixel is in a cleared state, ZB_DEPTHCLEARVALUE is returned
237     *    during zbuffer reads instead of the value that is actually stored
238     *    in the zbuffer memory. A pixel is in a cleared state when its ZMASK
239     *    is equal to 0. Therefore, if you clear ZMASK with zeros, you may
240     *    leave the zbuffer memory uninitialized, but then you must enable
241     *    compression, so that the ZMASK RAM is actually used.
242     *
243     * 5) Each 4x4 (or 8x8) tile is automatically decompressed and recompressed
244     *    during zbuffer updates. A special decompressing operation should be
245     *    used to fully decompress a zbuffer, which basically just stores all
246     *    compressed tiles in ZMASK to the zbuffer memory.
247     *
248     * 6) For a 16-bit zbuffer, compression causes a hung with one or
249     *    two samples and should not be used.
250     *
251     * 7) FORCE_COMPRESSED_STENCIL_VALUE should be enabled for stencil clears
252     *    to avoid needless decompression.
253     *
254     * 8) Fastfill must not be used if reading of compressed Z data is disabled
255     *    and writing of compressed Z data is enabled (RD/WR_COMP_ENABLE),
256     *    i.e. it cannot be used to compress the zbuffer.
257     *
258     * 9) ZB_CB_CLEAR does not interact with zbuffer compression in any way.
259     *
260     * - Marek
261     */
262
263    struct r300_context* r300 = r300_context(pipe);
264    struct pipe_framebuffer_state *fb =
265        (struct pipe_framebuffer_state*)r300->fb_state.state;
266    struct r300_hyperz_state *hyperz =
267        (struct r300_hyperz_state*)r300->hyperz_state.state;
268    uint32_t width = fb->width;
269    uint32_t height = fb->height;
270    uint32_t hyperz_dcv = hyperz->zb_depthclearvalue;
271
272    /* Use fast Z clear.
273     * The zbuffer must be in micro-tiled mode, otherwise it locks up. */
274    if (buffers & PIPE_CLEAR_DEPTHSTENCIL) {
275        boolean zmask_clear, hiz_clear;
276
277        /* If both depth and stencil are present, they must be cleared together. */
278        if (fb->zsbuf->texture->format == PIPE_FORMAT_S8_UINT_Z24_UNORM &&
279            (buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL) {
280            zmask_clear = FALSE;
281            hiz_clear = FALSE;
282        } else {
283            zmask_clear = r300_fast_zclear_allowed(r300, buffers);
284            hiz_clear = r300_hiz_clear_allowed(r300);
285        }
286
287        /* If we need Hyper-Z. */
288        if (zmask_clear || hiz_clear) {
289            /* Try to obtain the access to Hyper-Z buffers if we don't have one. */
290            if (!r300->hyperz_enabled &&
291                (r300->screen->caps.is_r500 || debug_get_option_hyperz())) {
292                r300->hyperz_enabled =
293                    r300->rws->cs_request_feature(&r300->cs,
294                                                RADEON_FID_R300_HYPERZ_ACCESS,
295                                                TRUE);
296                if (r300->hyperz_enabled) {
297                   /* Need to emit HyperZ buffer regs for the first time. */
298                   r300_mark_fb_state_dirty(r300, R300_CHANGED_HYPERZ_FLAG);
299                }
300            }
301
302            /* Setup Hyper-Z clears. */
303            if (r300->hyperz_enabled) {
304                if (zmask_clear) {
305                    hyperz_dcv = hyperz->zb_depthclearvalue =
306                        r300_depth_clear_value(fb->zsbuf->format, depth, stencil);
307
308                    r300_mark_atom_dirty(r300, &r300->zmask_clear);
309                    r300_mark_atom_dirty(r300, &r300->gpu_flush);
310                    buffers &= ~PIPE_CLEAR_DEPTHSTENCIL;
311                }
312
313                if (hiz_clear) {
314                    r300->hiz_clear_value = r300_hiz_clear_value(depth);
315                    r300_mark_atom_dirty(r300, &r300->hiz_clear);
316                    r300_mark_atom_dirty(r300, &r300->gpu_flush);
317                }
318                r300->num_z_clears++;
319            }
320        }
321    }
322
323    /* Use fast color clear for an AA colorbuffer.
324     * The CMASK is shared between all colorbuffers, so we use it
325     * if there is only one colorbuffer bound. */
326    if ((buffers & PIPE_CLEAR_COLOR) && fb->nr_cbufs == 1 && fb->cbufs[0] &&
327        r300_resource(fb->cbufs[0]->texture)->tex.cmask_dwords) {
328        /* Try to obtain the access to the CMASK if we don't have one. */
329        if (!r300->cmask_access) {
330            r300->cmask_access =
331                r300->rws->cs_request_feature(&r300->cs,
332                                              RADEON_FID_R300_CMASK_ACCESS,
333                                              TRUE);
334        }
335
336        /* Setup the clear. */
337        if (r300->cmask_access) {
338            /* Pair the resource with the CMASK to avoid other resources
339             * accessing it. */
340            if (!r300->screen->cmask_resource) {
341                mtx_lock(&r300->screen->cmask_mutex);
342                /* Double checking (first unlocked, then locked). */
343                if (!r300->screen->cmask_resource) {
344                    /* Don't reference this, so that the texture can be
345                     * destroyed while set in cmask_resource.
346                     * Then in texture_destroy, we set cmask_resource to NULL. */
347                    r300->screen->cmask_resource = fb->cbufs[0]->texture;
348                }
349                mtx_unlock(&r300->screen->cmask_mutex);
350            }
351
352            if (r300->screen->cmask_resource == fb->cbufs[0]->texture) {
353                r300_set_clear_color(r300, color);
354                r300_mark_atom_dirty(r300, &r300->cmask_clear);
355                r300_mark_atom_dirty(r300, &r300->gpu_flush);
356                buffers &= ~PIPE_CLEAR_COLOR;
357            }
358        }
359    }
360    /* Enable CBZB clear. */
361    else if (r300_cbzb_clear_allowed(r300, buffers)) {
362        struct r300_surface *surf = r300_surface(fb->cbufs[0]);
363
364        hyperz->zb_depthclearvalue =
365                r300_depth_clear_cb_value(surf->base.format, color->f);
366
367        width = surf->cbzb_width;
368        height = surf->cbzb_height;
369
370        r300->cbzb_clear = TRUE;
371        r300_mark_fb_state_dirty(r300, R300_CHANGED_HYPERZ_FLAG);
372    }
373
374    /* Clear. */
375    if (buffers) {
376        /* Clear using the blitter. */
377        r300_blitter_begin(r300, R300_CLEAR);
378        util_blitter_clear(r300->blitter, width, height, 1,
379                           buffers, color, depth, stencil,
380                           util_framebuffer_get_num_samples(fb) > 1);
381        r300_blitter_end(r300);
382    } else if (r300->zmask_clear.dirty ||
383               r300->hiz_clear.dirty ||
384               r300->cmask_clear.dirty) {
385        /* Just clear zmask and hiz now, this does not use the standard draw
386         * procedure. */
387        /* Calculate zmask_clear and hiz_clear atom sizes. */
388        unsigned dwords =
389            r300->gpu_flush.size +
390            (r300->zmask_clear.dirty ? r300->zmask_clear.size : 0) +
391            (r300->hiz_clear.dirty ? r300->hiz_clear.size : 0) +
392            (r300->cmask_clear.dirty ? r300->cmask_clear.size : 0) +
393            r300_get_num_cs_end_dwords(r300);
394
395        /* Reserve CS space. */
396        if (!r300->rws->cs_check_space(&r300->cs, dwords)) {
397            r300_flush(&r300->context, PIPE_FLUSH_ASYNC, NULL);
398        }
399
400        /* Emit clear packets. */
401        r300_emit_gpu_flush(r300, r300->gpu_flush.size, r300->gpu_flush.state);
402        r300->gpu_flush.dirty = FALSE;
403
404        if (r300->zmask_clear.dirty) {
405            r300_emit_zmask_clear(r300, r300->zmask_clear.size,
406                                  r300->zmask_clear.state);
407            r300->zmask_clear.dirty = FALSE;
408        }
409        if (r300->hiz_clear.dirty) {
410            r300_emit_hiz_clear(r300, r300->hiz_clear.size,
411                                r300->hiz_clear.state);
412            r300->hiz_clear.dirty = FALSE;
413        }
414        if (r300->cmask_clear.dirty) {
415            r300_emit_cmask_clear(r300, r300->cmask_clear.size,
416                                  r300->cmask_clear.state);
417            r300->cmask_clear.dirty = FALSE;
418        }
419    } else {
420        assert(0);
421    }
422
423    /* Disable CBZB clear. */
424    if (r300->cbzb_clear) {
425        r300->cbzb_clear = FALSE;
426        hyperz->zb_depthclearvalue = hyperz_dcv;
427        r300_mark_fb_state_dirty(r300, R300_CHANGED_HYPERZ_FLAG);
428    }
429
430    /* Enable fastfill and/or hiz.
431     *
432     * If we cleared zmask/hiz, it's in use now. The Hyper-Z state update
433     * looks if zmask/hiz is in use and programs hardware accordingly. */
434    if (r300->zmask_in_use || r300->hiz_in_use) {
435        r300_mark_atom_dirty(r300, &r300->hyperz_state);
436    }
437}
438
439/* Clear a region of a color surface to a constant value. */
440static void r300_clear_render_target(struct pipe_context *pipe,
441                                     struct pipe_surface *dst,
442                                     const union pipe_color_union *color,
443                                     unsigned dstx, unsigned dsty,
444                                     unsigned width, unsigned height,
445                                     bool render_condition_enabled)
446{
447    struct r300_context *r300 = r300_context(pipe);
448
449    r300_blitter_begin(r300, R300_CLEAR_SURFACE |
450                       (render_condition_enabled ? 0 : R300_IGNORE_RENDER_COND));
451    util_blitter_clear_render_target(r300->blitter, dst, color,
452                                     dstx, dsty, width, height);
453    r300_blitter_end(r300);
454}
455
456/* Clear a region of a depth stencil surface. */
457static void r300_clear_depth_stencil(struct pipe_context *pipe,
458                                     struct pipe_surface *dst,
459                                     unsigned clear_flags,
460                                     double depth,
461                                     unsigned stencil,
462                                     unsigned dstx, unsigned dsty,
463                                     unsigned width, unsigned height,
464                                     bool render_condition_enabled)
465{
466    struct r300_context *r300 = r300_context(pipe);
467    struct pipe_framebuffer_state *fb =
468        (struct pipe_framebuffer_state*)r300->fb_state.state;
469
470    if (r300->zmask_in_use && !r300->locked_zbuffer) {
471        if (fb->zsbuf->texture == dst->texture) {
472            r300_decompress_zmask(r300);
473        }
474    }
475
476    /* XXX Do not decompress ZMask of the currently-set zbuffer. */
477    r300_blitter_begin(r300, R300_CLEAR_SURFACE |
478                       (render_condition_enabled ? 0 : R300_IGNORE_RENDER_COND));
479    util_blitter_clear_depth_stencil(r300->blitter, dst, clear_flags, depth, stencil,
480                                     dstx, dsty, width, height);
481    r300_blitter_end(r300);
482}
483
484void r300_decompress_zmask(struct r300_context *r300)
485{
486    struct pipe_framebuffer_state *fb =
487        (struct pipe_framebuffer_state*)r300->fb_state.state;
488
489    if (!r300->zmask_in_use || r300->locked_zbuffer)
490        return;
491
492    r300->zmask_decompress = TRUE;
493    r300_mark_atom_dirty(r300, &r300->hyperz_state);
494
495    r300_blitter_begin(r300, R300_DECOMPRESS);
496    util_blitter_custom_clear_depth(r300->blitter, fb->width, fb->height, 0,
497                                    r300->dsa_decompress_zmask);
498    r300_blitter_end(r300);
499
500    r300->zmask_decompress = FALSE;
501    r300->zmask_in_use = FALSE;
502    r300_mark_atom_dirty(r300, &r300->hyperz_state);
503}
504
505void r300_decompress_zmask_locked_unsafe(struct r300_context *r300)
506{
507    struct pipe_framebuffer_state fb;
508
509    memset(&fb, 0, sizeof(fb));
510    fb.width = r300->locked_zbuffer->width;
511    fb.height = r300->locked_zbuffer->height;
512    fb.zsbuf = r300->locked_zbuffer;
513
514    r300->context.set_framebuffer_state(&r300->context, &fb);
515    r300_decompress_zmask(r300);
516}
517
518void r300_decompress_zmask_locked(struct r300_context *r300)
519{
520    struct pipe_framebuffer_state saved_fb;
521
522    memset(&saved_fb, 0, sizeof(saved_fb));
523    util_copy_framebuffer_state(&saved_fb, r300->fb_state.state);
524    r300_decompress_zmask_locked_unsafe(r300);
525    r300->context.set_framebuffer_state(&r300->context, &saved_fb);
526    util_unreference_framebuffer_state(&saved_fb);
527
528    pipe_surface_reference(&r300->locked_zbuffer, NULL);
529}
530
531bool r300_is_blit_supported(enum pipe_format format)
532{
533    const struct util_format_description *desc =
534        util_format_description(format);
535
536    return desc->layout == UTIL_FORMAT_LAYOUT_PLAIN ||
537           desc->layout == UTIL_FORMAT_LAYOUT_S3TC ||
538           desc->layout == UTIL_FORMAT_LAYOUT_RGTC;
539}
540
541/* Copy a block of pixels from one surface to another. */
542static void r300_resource_copy_region(struct pipe_context *pipe,
543                                      struct pipe_resource *dst,
544                                      unsigned dst_level,
545                                      unsigned dstx, unsigned dsty, unsigned dstz,
546                                      struct pipe_resource *src,
547                                      unsigned src_level,
548                                      const struct pipe_box *src_box)
549{
550    struct pipe_screen *screen = pipe->screen;
551    struct r300_context *r300 = r300_context(pipe);
552    struct pipe_framebuffer_state *fb =
553        (struct pipe_framebuffer_state*)r300->fb_state.state;
554    unsigned src_width0 = r300_resource(src)->tex.width0;
555    unsigned src_height0 = r300_resource(src)->tex.height0;
556    unsigned dst_width0 = r300_resource(dst)->tex.width0;
557    unsigned dst_height0 = r300_resource(dst)->tex.height0;
558    unsigned layout;
559    struct pipe_box box, dstbox;
560    struct pipe_sampler_view src_templ, *src_view;
561    struct pipe_surface dst_templ, *dst_view;
562
563    /* Fallback for buffers. */
564    if ((dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) ||
565        !r300_is_blit_supported(dst->format)) {
566        util_resource_copy_region(pipe, dst, dst_level, dstx, dsty, dstz,
567                                  src, src_level, src_box);
568        return;
569    }
570
571    /* Can't read MSAA textures. */
572    if (src->nr_samples > 1 || dst->nr_samples > 1) {
573        return;
574    }
575
576    /* The code below changes the texture format so that the copy can be done
577     * on hardware. E.g. depth-stencil surfaces are copied as RGBA
578     * colorbuffers. */
579
580    util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
581    util_blitter_default_src_texture(r300->blitter, &src_templ, src, src_level);
582
583    layout = util_format_description(dst_templ.format)->layout;
584
585    /* Handle non-renderable plain formats. */
586    if (layout == UTIL_FORMAT_LAYOUT_PLAIN &&
587        (!screen->is_format_supported(screen, src_templ.format, src->target,
588                                      src->nr_samples, src->nr_storage_samples,
589                                      PIPE_BIND_SAMPLER_VIEW) ||
590         !screen->is_format_supported(screen, dst_templ.format, dst->target,
591                                      dst->nr_samples, dst->nr_storage_samples,
592                                      PIPE_BIND_RENDER_TARGET))) {
593        switch (util_format_get_blocksize(dst_templ.format)) {
594            case 1:
595                dst_templ.format = PIPE_FORMAT_I8_UNORM;
596                break;
597            case 2:
598                dst_templ.format = PIPE_FORMAT_B4G4R4A4_UNORM;
599                break;
600            case 4:
601                dst_templ.format = PIPE_FORMAT_B8G8R8A8_UNORM;
602                break;
603            case 8:
604                dst_templ.format = PIPE_FORMAT_R16G16B16A16_UNORM;
605                break;
606            default:
607                debug_printf("r300: copy_region: Unhandled format: %s. Falling back to software.\n"
608                             "r300: copy_region: Software fallback doesn't work for tiled textures.\n",
609                             util_format_short_name(dst_templ.format));
610        }
611        src_templ.format = dst_templ.format;
612    }
613
614    /* Handle compressed formats. */
615    if (layout == UTIL_FORMAT_LAYOUT_S3TC ||
616        layout == UTIL_FORMAT_LAYOUT_RGTC) {
617        assert(src_templ.format == dst_templ.format);
618
619        box = *src_box;
620        src_box = &box;
621
622        dst_width0 = align(dst_width0, 4);
623        dst_height0 = align(dst_height0, 4);
624        src_width0 = align(src_width0, 4);
625        src_height0 = align(src_height0, 4);
626        box.width = align(box.width, 4);
627        box.height = align(box.height, 4);
628
629        switch (util_format_get_blocksize(dst_templ.format)) {
630        case 8:
631            /* one 4x4 pixel block has 8 bytes.
632             * we set 1 pixel = 4 bytes ===> 1 block corrensponds to 2 pixels. */
633            dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
634            dst_width0 = dst_width0 / 2;
635            src_width0 = src_width0 / 2;
636            dstx /= 2;
637            box.x /= 2;
638            box.width /= 2;
639            break;
640        case 16:
641            /* one 4x4 pixel block has 16 bytes.
642             * we set 1 pixel = 4 bytes ===> 1 block corresponds to 4 pixels. */
643            dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
644            break;
645        }
646        src_templ.format = dst_templ.format;
647
648        dst_height0 = dst_height0 / 4;
649        src_height0 = src_height0 / 4;
650        dsty /= 4;
651        box.y /= 4;
652        box.height /= 4;
653    }
654
655    /* Fallback for textures. */
656    if (!screen->is_format_supported(screen, dst_templ.format,
657                                     dst->target, dst->nr_samples,
658                                     dst->nr_storage_samples,
659                                     PIPE_BIND_RENDER_TARGET) ||
660	!screen->is_format_supported(screen, src_templ.format,
661                                     src->target, src->nr_samples,
662                                     src->nr_storage_samples,
663                                     PIPE_BIND_SAMPLER_VIEW)) {
664        assert(0 && "this shouldn't happen, update r300_is_blit_supported");
665        util_resource_copy_region(pipe, dst, dst_level, dstx, dsty, dstz,
666                                  src, src_level, src_box);
667        return;
668    }
669
670    /* Decompress ZMASK. */
671    if (r300->zmask_in_use && !r300->locked_zbuffer) {
672        if (fb->zsbuf->texture == src ||
673            fb->zsbuf->texture == dst) {
674            r300_decompress_zmask(r300);
675        }
676    }
677
678    dst_view = r300_create_surface_custom(pipe, dst, &dst_templ, dst_width0, dst_height0);
679    src_view = r300_create_sampler_view_custom(pipe, src, &src_templ, src_width0, src_height0);
680
681    u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height),
682             abs(src_box->depth), &dstbox);
683
684    r300_blitter_begin(r300, R300_COPY);
685    util_blitter_blit_generic(r300->blitter, dst_view, &dstbox,
686                              src_view, src_box, src_width0, src_height0,
687                              PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
688                              FALSE, FALSE, 0);
689    r300_blitter_end(r300);
690
691    pipe_surface_reference(&dst_view, NULL);
692    pipe_sampler_view_reference(&src_view, NULL);
693}
694
695static boolean r300_is_simple_msaa_resolve(const struct pipe_blit_info *info)
696{
697    unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
698    unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
699
700    return info->src.resource->nr_samples > 1 &&
701           info->dst.resource->nr_samples <= 1 &&
702           info->dst.resource->format == info->src.resource->format &&
703           info->dst.resource->format == info->dst.format &&
704           info->src.resource->format == info->src.format &&
705           !info->scissor_enable &&
706           info->mask == PIPE_MASK_RGBA &&
707           dst_width == info->src.resource->width0 &&
708           dst_height == info->src.resource->height0 &&
709           info->dst.box.x == 0 &&
710           info->dst.box.y == 0 &&
711           info->dst.box.width == dst_width &&
712           info->dst.box.height == dst_height &&
713           info->src.box.x == 0 &&
714           info->src.box.y == 0 &&
715           info->src.box.width == dst_width &&
716           info->src.box.height == dst_height &&
717           (r300_resource(info->dst.resource)->tex.microtile != RADEON_LAYOUT_LINEAR ||
718            r300_resource(info->dst.resource)->tex.macrotile[info->dst.level] != RADEON_LAYOUT_LINEAR);
719}
720
721static void r300_simple_msaa_resolve(struct pipe_context *pipe,
722                                     struct pipe_resource *dst,
723                                     unsigned dst_level,
724                                     unsigned dst_layer,
725                                     struct pipe_resource *src,
726                                     enum pipe_format format)
727{
728    struct r300_context *r300 = r300_context(pipe);
729    struct r300_surface *srcsurf, *dstsurf;
730    struct pipe_surface surf_tmpl;
731    struct r300_aa_state *aa = (struct r300_aa_state*)r300->aa_state.state;
732
733    memset(&surf_tmpl, 0, sizeof(surf_tmpl));
734    surf_tmpl.format = format;
735    srcsurf = r300_surface(pipe->create_surface(pipe, src, &surf_tmpl));
736
737    surf_tmpl.format = format;
738    surf_tmpl.u.tex.level = dst_level;
739    surf_tmpl.u.tex.first_layer =
740    surf_tmpl.u.tex.last_layer = dst_layer;
741    dstsurf = r300_surface(pipe->create_surface(pipe, dst, &surf_tmpl));
742
743    /* COLORPITCH should contain the tiling info of the resolve buffer.
744     * The tiling of the AA buffer isn't programmable anyway. */
745    srcsurf->pitch &= ~(R300_COLOR_TILE(1) | R300_COLOR_MICROTILE(3));
746    srcsurf->pitch |= dstsurf->pitch & (R300_COLOR_TILE(1) | R300_COLOR_MICROTILE(3));
747
748    /* Enable AA resolve. */
749    aa->dest = dstsurf;
750    r300->aa_state.size = 8;
751    r300_mark_atom_dirty(r300, &r300->aa_state);
752
753    /* Resolve the surface. */
754    r300_blitter_begin(r300, R300_CLEAR_SURFACE);
755    util_blitter_custom_color(r300->blitter, &srcsurf->base, NULL);
756    r300_blitter_end(r300);
757
758    /* Disable AA resolve. */
759    aa->dest = NULL;
760    r300->aa_state.size = 4;
761    r300_mark_atom_dirty(r300, &r300->aa_state);
762
763    pipe_surface_reference((struct pipe_surface**)&srcsurf, NULL);
764    pipe_surface_reference((struct pipe_surface**)&dstsurf, NULL);
765}
766
767static void r300_msaa_resolve(struct pipe_context *pipe,
768                              const struct pipe_blit_info *info)
769{
770    struct r300_context *r300 = r300_context(pipe);
771    struct pipe_screen *screen = pipe->screen;
772    struct pipe_resource *tmp, templ;
773    struct pipe_blit_info blit;
774
775    assert(info->src.level == 0);
776    assert(info->src.box.z == 0);
777    assert(info->src.box.depth == 1);
778    assert(info->dst.box.depth == 1);
779
780    if (r300_is_simple_msaa_resolve(info)) {
781        r300_simple_msaa_resolve(pipe, info->dst.resource, info->dst.level,
782                                 info->dst.box.z, info->src.resource,
783                                 info->src.format);
784        return;
785    }
786
787    /* resolve into a temporary texture, then blit */
788    memset(&templ, 0, sizeof(templ));
789    templ.target = PIPE_TEXTURE_2D;
790    templ.format = info->src.resource->format;
791    templ.width0 = info->src.resource->width0;
792    templ.height0 = info->src.resource->height0;
793    templ.depth0 = 1;
794    templ.array_size = 1;
795    templ.usage = PIPE_USAGE_DEFAULT;
796    templ.flags = R300_RESOURCE_FORCE_MICROTILING;
797
798    tmp = screen->resource_create(screen, &templ);
799
800    /* resolve */
801    r300_simple_msaa_resolve(pipe, tmp, 0, 0, info->src.resource,
802                             info->src.format);
803
804    /* blit */
805    blit = *info;
806    blit.src.resource = tmp;
807    blit.src.box.z = 0;
808
809    r300_blitter_begin(r300, R300_BLIT | R300_IGNORE_RENDER_COND);
810    util_blitter_blit(r300->blitter, &blit);
811    r300_blitter_end(r300);
812
813    pipe_resource_reference(&tmp, NULL);
814}
815
816static void r300_blit(struct pipe_context *pipe,
817                      const struct pipe_blit_info *blit)
818{
819    struct r300_context *r300 = r300_context(pipe);
820    struct pipe_framebuffer_state *fb =
821        (struct pipe_framebuffer_state*)r300->fb_state.state;
822    struct pipe_blit_info info = *blit;
823
824    /* The driver supports sRGB textures but not framebuffers. Blitting
825     * from sRGB to sRGB should be the same as blitting from linear
826     * to linear, so use that, This avoids incorrect linearization.
827     */
828    if (util_format_is_srgb(info.src.format)) {
829      info.src.format = util_format_linear(info.src.format);
830      info.dst.format = util_format_linear(info.dst.format);
831    }
832
833    /* MSAA resolve. */
834    if (info.src.resource->nr_samples > 1 &&
835        !util_format_is_depth_or_stencil(info.src.resource->format)) {
836        r300_msaa_resolve(pipe, &info);
837        return;
838    }
839
840    /* Can't read MSAA textures. */
841    if (info.src.resource->nr_samples > 1) {
842        return;
843    }
844
845    /* Blit a combined depth-stencil resource as color.
846     * S8Z24 is the only supported stencil format. */
847    if ((info.mask & PIPE_MASK_S) &&
848        info.src.format == PIPE_FORMAT_S8_UINT_Z24_UNORM &&
849        info.dst.format == PIPE_FORMAT_S8_UINT_Z24_UNORM) {
850        if (info.dst.resource->nr_samples > 1) {
851            /* Cannot do that with MSAA buffers. */
852            info.mask &= ~PIPE_MASK_S;
853            if (!(info.mask & PIPE_MASK_Z)) {
854                return;
855            }
856        } else {
857            /* Single-sample buffer. */
858            info.src.format = PIPE_FORMAT_B8G8R8A8_UNORM;
859            info.dst.format = PIPE_FORMAT_B8G8R8A8_UNORM;
860            if (info.mask & PIPE_MASK_Z) {
861                info.mask = PIPE_MASK_RGBA; /* depth+stencil */
862            } else {
863                info.mask = PIPE_MASK_B; /* stencil only */
864            }
865        }
866    }
867
868    /* Decompress ZMASK. */
869    if (r300->zmask_in_use && !r300->locked_zbuffer) {
870        if (fb->zsbuf->texture == info.src.resource ||
871            fb->zsbuf->texture == info.dst.resource) {
872            r300_decompress_zmask(r300);
873        }
874    }
875
876    r300_blitter_begin(r300, R300_BLIT |
877		       (info.render_condition_enable ? 0 : R300_IGNORE_RENDER_COND));
878    util_blitter_blit(r300->blitter, &info);
879    r300_blitter_end(r300);
880}
881
882static void r300_flush_resource(struct pipe_context *ctx,
883				struct pipe_resource *resource)
884{
885}
886
887void r300_init_blit_functions(struct r300_context *r300)
888{
889    r300->context.clear = r300_clear;
890    r300->context.clear_render_target = r300_clear_render_target;
891    r300->context.clear_depth_stencil = r300_clear_depth_stencil;
892    r300->context.resource_copy_region = r300_resource_copy_region;
893    r300->context.blit = r300_blit;
894    r300->context.flush_resource = r300_flush_resource;
895}
896