1/*
2 * Videotoolbox hardware acceleration
3 *
4 * copyright (c) 2012 Sebastien Zwickert
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include "config.h"
24#include "config_components.h"
25#include "videotoolbox.h"
26#include "libavutil/hwcontext_videotoolbox.h"
27#include "vt_internal.h"
28#include "libavutil/avutil.h"
29#include "libavutil/hwcontext.h"
30#include "libavutil/pixdesc.h"
31#include "bytestream.h"
32#include "decode.h"
33#include "internal.h"
34#include "h264dec.h"
35#include "hevcdec.h"
36#include "mpegvideo.h"
37#include "proresdec.h"
38#include <Availability.h>
39#include <AvailabilityMacros.h>
40#include <TargetConditionals.h>
41
42#ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
43#  define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
44#endif
45#ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
46#  define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
47#endif
48
49#if !HAVE_KCMVIDEOCODECTYPE_HEVC
50enum { kCMVideoCodecType_HEVC = 'hvc1' };
51#endif
52
53#if !HAVE_KCMVIDEOCODECTYPE_VP9
54enum { kCMVideoCodecType_VP9 = 'vp09' };
55#endif
56
57#define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING  12
58
59typedef struct VTHWFrame {
60    CVPixelBufferRef pixbuf;
61    AVBufferRef *hw_frames_ctx;
62} VTHWFrame;
63
64static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
65{
66    VTHWFrame *ref = (VTHWFrame *)data;
67    av_buffer_unref(&ref->hw_frames_ctx);
68    CVPixelBufferRelease(ref->pixbuf);
69
70    av_free(data);
71}
72
73int ff_videotoolbox_buffer_copy(VTContext *vtctx,
74                                const uint8_t *buffer,
75                                uint32_t size)
76{
77    void *tmp;
78
79    tmp = av_fast_realloc(vtctx->bitstream,
80                         &vtctx->allocated_size,
81                         size);
82
83    if (!tmp)
84        return AVERROR(ENOMEM);
85
86    vtctx->bitstream = tmp;
87    memcpy(vtctx->bitstream, buffer, size);
88    vtctx->bitstream_size = size;
89
90    return 0;
91}
92
93static int videotoolbox_postproc_frame(void *avctx, AVFrame *frame)
94{
95    int ret;
96    VTHWFrame *ref = (VTHWFrame *)frame->buf[0]->data;
97
98    if (!ref->pixbuf) {
99        av_log(avctx, AV_LOG_ERROR, "No frame decoded?\n");
100        av_frame_unref(frame);
101        return AVERROR_EXTERNAL;
102    }
103
104    frame->crop_right = 0;
105    frame->crop_left = 0;
106    frame->crop_top = 0;
107    frame->crop_bottom = 0;
108
109    if ((ret = av_vt_pixbuf_set_attachments(avctx, ref->pixbuf, frame)) < 0)
110        return ret;
111
112    frame->data[3] = (uint8_t*)ref->pixbuf;
113
114    if (ref->hw_frames_ctx) {
115        av_buffer_unref(&frame->hw_frames_ctx);
116        frame->hw_frames_ctx = av_buffer_ref(ref->hw_frames_ctx);
117        if (!frame->hw_frames_ctx)
118            return AVERROR(ENOMEM);
119    }
120
121    return 0;
122}
123
124int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
125{
126    size_t      size = sizeof(VTHWFrame);
127    uint8_t    *data = NULL;
128    AVBufferRef *buf = NULL;
129    int ret = ff_attach_decode_data(frame);
130    FrameDecodeData *fdd;
131    if (ret < 0)
132        return ret;
133
134    data = av_mallocz(size);
135    if (!data)
136        return AVERROR(ENOMEM);
137    buf = av_buffer_create(data, size, videotoolbox_buffer_release, NULL, 0);
138    if (!buf) {
139        av_freep(&data);
140        return AVERROR(ENOMEM);
141    }
142    frame->buf[0] = buf;
143
144    fdd = (FrameDecodeData*)frame->private_ref->data;
145    fdd->post_process = videotoolbox_postproc_frame;
146
147    frame->width  = avctx->width;
148    frame->height = avctx->height;
149    frame->format = avctx->pix_fmt;
150
151    return 0;
152}
153
154#define AV_W8(p, v) *(p) = (v)
155
156static int escape_ps(uint8_t* dst, const uint8_t* src, int src_size)
157{
158    int i;
159    int size = src_size;
160    uint8_t* p = dst;
161
162    for (i = 0; i < src_size; i++) {
163        if (i + 2 < src_size &&
164            src[i]     == 0x00 &&
165            src[i + 1] == 0x00 &&
166            src[i + 2] <= 0x03) {
167            if (dst) {
168                *p++ = src[i++];
169                *p++ = src[i];
170                *p++ = 0x03;
171            } else {
172                i++;
173            }
174            size++;
175        } else if (dst)
176            *p++ = src[i];
177    }
178
179    if (dst)
180        av_assert0((p - dst) == size);
181
182    return size;
183}
184
185CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
186{
187    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
188    H264Context *h = avctx->priv_data;
189    CFDataRef data = NULL;
190    uint8_t *p;
191    int sps_size = escape_ps(NULL, h->ps.sps->data, h->ps.sps->data_size);
192    int pps_size = escape_ps(NULL, h->ps.pps->data, h->ps.pps->data_size);
193    int vt_extradata_size;
194    uint8_t *vt_extradata;
195
196    vt_extradata_size = 6 + 2 + sps_size + 3 + pps_size;
197    vt_extradata = av_malloc(vt_extradata_size);
198
199    if (!vt_extradata)
200        return NULL;
201
202    p = vt_extradata;
203
204    AV_W8(p + 0, 1); /* version */
205    AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
206    AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
207    AV_W8(p + 3, h->ps.sps->data[3]); /* level */
208    AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
209    AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
210    AV_WB16(p + 6, sps_size);
211    p += 8;
212    p += escape_ps(p, h->ps.sps->data, h->ps.sps->data_size);
213    AV_W8(p + 0, 1); /* number of pps */
214    AV_WB16(p + 1, pps_size);
215    p += 3;
216    p += escape_ps(p, h->ps.pps->data, h->ps.pps->data_size);
217
218    av_assert0(p - vt_extradata == vt_extradata_size);
219
220    // save sps header (profile/level) used to create decoder session,
221    // so we can detect changes and recreate it.
222    if (vtctx)
223        memcpy(vtctx->sps, h->ps.sps->data + 1, 3);
224
225    data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
226    av_free(vt_extradata);
227    return data;
228}
229
230CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
231{
232    HEVCContext *h = avctx->priv_data;
233    int i, num_vps = 0, num_sps = 0, num_pps = 0;
234    const HEVCVPS *vps = h->ps.vps;
235    const HEVCSPS *sps = h->ps.sps;
236    const HEVCPPS *pps = h->ps.pps;
237    PTLCommon ptlc = vps->ptl.general_ptl;
238    VUI vui = sps->vui;
239    uint8_t parallelismType;
240    CFDataRef data = NULL;
241    uint8_t *p;
242    int vt_extradata_size = 23 + 3 + 3 + 3;
243    uint8_t *vt_extradata;
244
245#define COUNT_SIZE_PS(T, t) \
246    for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
247        if (h->ps.t##ps_list[i]) { \
248            const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
249            vt_extradata_size += 2 + escape_ps(NULL, lps->data, lps->data_size); \
250            num_##t##ps++; \
251        } \
252    }
253
254    COUNT_SIZE_PS(V, v)
255    COUNT_SIZE_PS(S, s)
256    COUNT_SIZE_PS(P, p)
257
258    vt_extradata = av_malloc(vt_extradata_size);
259    if (!vt_extradata)
260        return NULL;
261    p = vt_extradata;
262
263    /* unsigned int(8) configurationVersion = 1; */
264    AV_W8(p + 0, 1);
265
266    /*
267     * unsigned int(2) general_profile_space;
268     * unsigned int(1) general_tier_flag;
269     * unsigned int(5) general_profile_idc;
270     */
271    AV_W8(p + 1, ptlc.profile_space << 6 |
272                 ptlc.tier_flag     << 5 |
273                 ptlc.profile_idc);
274
275    /* unsigned int(32) general_profile_compatibility_flags; */
276    for (i = 0; i < 4; i++) {
277        AV_W8(p + 2 + i, ptlc.profile_compatibility_flag[i * 8] << 7 |
278                         ptlc.profile_compatibility_flag[i * 8 + 1] << 6 |
279                         ptlc.profile_compatibility_flag[i * 8 + 2] << 5 |
280                         ptlc.profile_compatibility_flag[i * 8 + 3] << 4 |
281                         ptlc.profile_compatibility_flag[i * 8 + 4] << 3 |
282                         ptlc.profile_compatibility_flag[i * 8 + 5] << 2 |
283                         ptlc.profile_compatibility_flag[i * 8 + 6] << 1 |
284                         ptlc.profile_compatibility_flag[i * 8 + 7]);
285    }
286
287    /* unsigned int(48) general_constraint_indicator_flags; */
288    AV_W8(p + 6, ptlc.progressive_source_flag    << 7 |
289                 ptlc.interlaced_source_flag     << 6 |
290                 ptlc.non_packed_constraint_flag << 5 |
291                 ptlc.frame_only_constraint_flag << 4);
292    AV_W8(p + 7, 0);
293    AV_WN32(p + 8, 0);
294
295    /* unsigned int(8) general_level_idc; */
296    AV_W8(p + 12, ptlc.level_idc);
297
298    /*
299     * bit(4) reserved = ‘1111’b;
300     * unsigned int(12) min_spatial_segmentation_idc;
301     */
302    AV_W8(p + 13, 0xf0 | (vui.min_spatial_segmentation_idc >> 4));
303    AV_W8(p + 14, vui.min_spatial_segmentation_idc & 0xff);
304
305    /*
306     * bit(6) reserved = ‘111111’b;
307     * unsigned int(2) parallelismType;
308     */
309    if (!vui.min_spatial_segmentation_idc)
310        parallelismType = 0;
311    else if (pps->entropy_coding_sync_enabled_flag && pps->tiles_enabled_flag)
312        parallelismType = 0;
313    else if (pps->entropy_coding_sync_enabled_flag)
314        parallelismType = 3;
315    else if (pps->tiles_enabled_flag)
316        parallelismType = 2;
317    else
318        parallelismType = 1;
319    AV_W8(p + 15, 0xfc | parallelismType);
320
321    /*
322     * bit(6) reserved = ‘111111’b;
323     * unsigned int(2) chromaFormat;
324     */
325    AV_W8(p + 16, sps->chroma_format_idc | 0xfc);
326
327    /*
328     * bit(5) reserved = ‘11111’b;
329     * unsigned int(3) bitDepthLumaMinus8;
330     */
331    AV_W8(p + 17, (sps->bit_depth - 8) | 0xf8);
332
333    /*
334     * bit(5) reserved = ‘11111’b;
335     * unsigned int(3) bitDepthChromaMinus8;
336     */
337    AV_W8(p + 18, (sps->bit_depth_chroma - 8) | 0xf8);
338
339    /* bit(16) avgFrameRate; */
340    AV_WB16(p + 19, 0);
341
342    /*
343     * bit(2) constantFrameRate;
344     * bit(3) numTemporalLayers;
345     * bit(1) temporalIdNested;
346     * unsigned int(2) lengthSizeMinusOne;
347     */
348    AV_W8(p + 21, 0                             << 6 |
349                  sps->max_sub_layers           << 3 |
350                  sps->temporal_id_nesting_flag << 2 |
351                  3);
352
353    /* unsigned int(8) numOfArrays; */
354    AV_W8(p + 22, 3);
355
356    p += 23;
357
358#define APPEND_PS(T, t) \
359    /* \
360     * bit(1) array_completeness; \
361     * unsigned int(1) reserved = 0; \
362     * unsigned int(6) NAL_unit_type; \
363     */ \
364    AV_W8(p, 1 << 7 | \
365             HEVC_NAL_##T##PS & 0x3f); \
366    /* unsigned int(16) numNalus; */ \
367    AV_WB16(p + 1, num_##t##ps); \
368    p += 3; \
369    for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
370        if (h->ps.t##ps_list[i]) { \
371            const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
372            int size = escape_ps(p + 2, lps->data, lps->data_size); \
373            /* unsigned int(16) nalUnitLength; */ \
374            AV_WB16(p, size); \
375            /* bit(8*nalUnitLength) nalUnit; */ \
376            p += 2 + size; \
377        } \
378    }
379
380    APPEND_PS(V, v)
381    APPEND_PS(S, s)
382    APPEND_PS(P, p)
383
384    av_assert0(p - vt_extradata == vt_extradata_size);
385
386    data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
387    av_free(vt_extradata);
388    return data;
389}
390
391int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
392                                     const uint8_t *buffer,
393                                     uint32_t size)
394{
395    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
396    H264Context *h = avctx->priv_data;
397
398    if (h->is_avc == 1) {
399        return ff_videotoolbox_buffer_copy(vtctx, buffer, size);
400    }
401
402    return 0;
403}
404
405static int videotoolbox_h264_decode_params(AVCodecContext *avctx,
406                                           int type,
407                                           const uint8_t *buffer,
408                                           uint32_t size)
409{
410    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
411    H264Context *h = avctx->priv_data;
412
413    // save sps header (profile/level) used to create decoder session
414    if (!vtctx->sps[0])
415        memcpy(vtctx->sps, h->ps.sps->data + 1, 3);
416
417    if (type == H264_NAL_SPS) {
418        if (size > 4 && memcmp(vtctx->sps, buffer + 1, 3) != 0) {
419            vtctx->reconfig_needed = true;
420            memcpy(vtctx->sps, buffer + 1, 3);
421        }
422    }
423
424    // pass-through SPS/PPS changes to the decoder
425    return ff_videotoolbox_h264_decode_slice(avctx, buffer, size);
426}
427
428static int videotoolbox_common_decode_slice(AVCodecContext *avctx,
429                                            const uint8_t *buffer,
430                                            uint32_t size)
431{
432    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
433    void *tmp;
434
435    tmp = av_fast_realloc(vtctx->bitstream,
436                          &vtctx->allocated_size,
437                          vtctx->bitstream_size+size+4);
438    if (!tmp)
439        return AVERROR(ENOMEM);
440
441    vtctx->bitstream = tmp;
442
443    AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
444    memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
445
446    vtctx->bitstream_size += size + 4;
447
448    return 0;
449}
450
451int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
452                                      const uint8_t *buffer,
453                                      uint32_t size)
454{
455    H264Context *h = avctx->priv_data;
456
457    if (h->is_avc == 1)
458        return 0;
459
460    return videotoolbox_common_decode_slice(avctx, buffer, size);
461}
462
463#if CONFIG_VIDEOTOOLBOX
464// Return the AVVideotoolboxContext that matters currently. Where it comes from
465// depends on the API used.
466static AVVideotoolboxContext *videotoolbox_get_context(AVCodecContext *avctx)
467{
468    // Somewhat tricky because the user can call av_videotoolbox_default_free()
469    // at any time, even when the codec is closed.
470    if (avctx->internal && avctx->internal->hwaccel_priv_data) {
471        VTContext *vtctx = avctx->internal->hwaccel_priv_data;
472        if (vtctx->vt_ctx)
473            return vtctx->vt_ctx;
474    }
475    return avctx->hwaccel_context;
476}
477
478static void videotoolbox_stop(AVCodecContext *avctx)
479{
480    AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
481    if (!videotoolbox)
482        return;
483
484    if (videotoolbox->cm_fmt_desc) {
485        CFRelease(videotoolbox->cm_fmt_desc);
486        videotoolbox->cm_fmt_desc = NULL;
487    }
488
489    if (videotoolbox->session) {
490        VTDecompressionSessionInvalidate(videotoolbox->session);
491        CFRelease(videotoolbox->session);
492        videotoolbox->session = NULL;
493    }
494}
495
496int ff_videotoolbox_uninit(AVCodecContext *avctx)
497{
498    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
499    if (!vtctx)
500        return 0;
501
502    av_freep(&vtctx->bitstream);
503    if (vtctx->frame)
504        CVPixelBufferRelease(vtctx->frame);
505
506    if (vtctx->vt_ctx)
507        videotoolbox_stop(avctx);
508
509    av_buffer_unref(&vtctx->cached_hw_frames_ctx);
510    av_freep(&vtctx->vt_ctx);
511
512    return 0;
513}
514
515static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
516{
517    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
518    CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->frame;
519    OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
520    enum AVPixelFormat sw_format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
521    int width = CVPixelBufferGetWidth(pixbuf);
522    int height = CVPixelBufferGetHeight(pixbuf);
523    AVHWFramesContext *cached_frames;
524    VTHWFrame *ref;
525    int ret;
526
527    if (!frame->buf[0] || frame->data[3]) {
528        av_log(avctx, AV_LOG_ERROR, "videotoolbox: invalid state\n");
529        av_frame_unref(frame);
530        return AVERROR_EXTERNAL;
531    }
532
533    ref = (VTHWFrame *)frame->buf[0]->data;
534
535    if (ref->pixbuf)
536        CVPixelBufferRelease(ref->pixbuf);
537    ref->pixbuf = vtctx->frame;
538    vtctx->frame = NULL;
539
540    // Old API code path.
541    if (!vtctx->cached_hw_frames_ctx)
542        return 0;
543
544    cached_frames = (AVHWFramesContext*)vtctx->cached_hw_frames_ctx->data;
545
546    if (cached_frames->sw_format != sw_format ||
547        cached_frames->width != width ||
548        cached_frames->height != height) {
549        AVBufferRef *hw_frames_ctx = av_hwframe_ctx_alloc(cached_frames->device_ref);
550        AVHWFramesContext *hw_frames;
551        if (!hw_frames_ctx)
552            return AVERROR(ENOMEM);
553
554        hw_frames = (AVHWFramesContext*)hw_frames_ctx->data;
555        hw_frames->format = cached_frames->format;
556        hw_frames->sw_format = sw_format;
557        hw_frames->width = width;
558        hw_frames->height = height;
559
560        ret = av_hwframe_ctx_init(hw_frames_ctx);
561        if (ret < 0) {
562            av_buffer_unref(&hw_frames_ctx);
563            return ret;
564        }
565
566        av_buffer_unref(&vtctx->cached_hw_frames_ctx);
567        vtctx->cached_hw_frames_ctx = hw_frames_ctx;
568    }
569
570    av_buffer_unref(&ref->hw_frames_ctx);
571    ref->hw_frames_ctx = av_buffer_ref(vtctx->cached_hw_frames_ctx);
572    if (!ref->hw_frames_ctx)
573        return AVERROR(ENOMEM);
574
575    return 0;
576}
577
578static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
579{
580    int i;
581    uint8_t b;
582
583    for (i = 3; i >= 0; i--) {
584        b = (length >> (i * 7)) & 0x7F;
585        if (i != 0)
586            b |= 0x80;
587
588        bytestream2_put_byteu(pb, b);
589    }
590}
591
592static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
593{
594    CFDataRef data;
595    uint8_t *rw_extradata;
596    PutByteContext pb;
597    int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
598    // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
599    int config_size = 13 + 5 + avctx->extradata_size;
600    int s;
601
602    if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
603        return NULL;
604
605    bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
606    bytestream2_put_byteu(&pb, 0);        // version
607    bytestream2_put_ne24(&pb, 0);         // flags
608
609    // elementary stream descriptor
610    bytestream2_put_byteu(&pb, 0x03);     // ES_DescrTag
611    videotoolbox_write_mp4_descr_length(&pb, full_size);
612    bytestream2_put_ne16(&pb, 0);         // esid
613    bytestream2_put_byteu(&pb, 0);        // stream priority (0-32)
614
615    // decoder configuration descriptor
616    bytestream2_put_byteu(&pb, 0x04);     // DecoderConfigDescrTag
617    videotoolbox_write_mp4_descr_length(&pb, config_size);
618    bytestream2_put_byteu(&pb, 32);       // object type indication. 32 = AV_CODEC_ID_MPEG4
619    bytestream2_put_byteu(&pb, 0x11);     // stream type
620    bytestream2_put_ne24(&pb, 0);         // buffer size
621    bytestream2_put_ne32(&pb, 0);         // max bitrate
622    bytestream2_put_ne32(&pb, 0);         // avg bitrate
623
624    // decoder specific descriptor
625    bytestream2_put_byteu(&pb, 0x05);     ///< DecSpecificInfoTag
626    videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
627
628    bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
629
630    // SLConfigDescriptor
631    bytestream2_put_byteu(&pb, 0x06);     // SLConfigDescrTag
632    bytestream2_put_byteu(&pb, 0x01);     // length
633    bytestream2_put_byteu(&pb, 0x02);     //
634
635    s = bytestream2_size_p(&pb);
636
637    data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
638
639    av_freep(&rw_extradata);
640    return data;
641}
642
643static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
644                                                           void *buffer,
645                                                           int size)
646{
647    OSStatus status;
648    CMBlockBufferRef  block_buf;
649    CMSampleBufferRef sample_buf;
650
651    block_buf  = NULL;
652    sample_buf = NULL;
653
654    status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
655                                                buffer,             // memoryBlock
656                                                size,               // blockLength
657                                                kCFAllocatorNull,   // blockAllocator
658                                                NULL,               // customBlockSource
659                                                0,                  // offsetToData
660                                                size,               // dataLength
661                                                0,                  // flags
662                                                &block_buf);
663
664    if (!status) {
665        status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
666                                      block_buf,            // dataBuffer
667                                      TRUE,                 // dataReady
668                                      0,                    // makeDataReadyCallback
669                                      0,                    // makeDataReadyRefcon
670                                      fmt_desc,             // formatDescription
671                                      1,                    // numSamples
672                                      0,                    // numSampleTimingEntries
673                                      NULL,                 // sampleTimingArray
674                                      0,                    // numSampleSizeEntries
675                                      NULL,                 // sampleSizeArray
676                                      &sample_buf);
677    }
678
679    if (block_buf)
680        CFRelease(block_buf);
681
682    return sample_buf;
683}
684
685static void videotoolbox_decoder_callback(void *opaque,
686                                          void *sourceFrameRefCon,
687                                          OSStatus status,
688                                          VTDecodeInfoFlags flags,
689                                          CVImageBufferRef image_buffer,
690                                          CMTime pts,
691                                          CMTime duration)
692{
693    VTContext *vtctx = opaque;
694
695    if (vtctx->frame) {
696        CVPixelBufferRelease(vtctx->frame);
697        vtctx->frame = NULL;
698    }
699
700    if (!image_buffer) {
701        av_log(vtctx->logctx, status ? AV_LOG_WARNING : AV_LOG_DEBUG,
702               "vt decoder cb: output image buffer is null: %i\n", status);
703        return;
704    }
705
706    vtctx->frame = CVPixelBufferRetain(image_buffer);
707}
708
709static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
710{
711    OSStatus status;
712    CMSampleBufferRef sample_buf;
713    AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
714    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
715
716    sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
717                                                   vtctx->bitstream,
718                                                   vtctx->bitstream_size);
719
720    if (!sample_buf)
721        return -1;
722
723    status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
724                                               sample_buf,
725                                               0,       // decodeFlags
726                                               NULL,    // sourceFrameRefCon
727                                               0);      // infoFlagsOut
728    if (status == noErr)
729        status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
730
731    CFRelease(sample_buf);
732
733    return status;
734}
735
736static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
737                                                                   CFDictionaryRef decoder_spec,
738                                                                   int width,
739                                                                   int height)
740{
741    CMFormatDescriptionRef cm_fmt_desc;
742    OSStatus status;
743
744    status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
745                                            codec_type,
746                                            width,
747                                            height,
748                                            decoder_spec, // Dictionary of extension
749                                            &cm_fmt_desc);
750
751    if (status)
752        return NULL;
753
754    return cm_fmt_desc;
755}
756
757static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
758                                                             int height,
759                                                             OSType pix_fmt)
760{
761    CFMutableDictionaryRef buffer_attributes;
762    CFMutableDictionaryRef io_surface_properties;
763    CFNumberRef cv_pix_fmt;
764    CFNumberRef w;
765    CFNumberRef h;
766
767    w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
768    h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
769    cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
770
771    buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
772                                                  4,
773                                                  &kCFTypeDictionaryKeyCallBacks,
774                                                  &kCFTypeDictionaryValueCallBacks);
775    io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
776                                                      0,
777                                                      &kCFTypeDictionaryKeyCallBacks,
778                                                      &kCFTypeDictionaryValueCallBacks);
779
780    if (pix_fmt)
781        CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
782    CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
783    CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
784    CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
785#if TARGET_OS_IPHONE
786    CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
787#else
788    CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
789#endif
790
791    CFRelease(io_surface_properties);
792    CFRelease(cv_pix_fmt);
793    CFRelease(w);
794    CFRelease(h);
795
796    return buffer_attributes;
797}
798
799static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
800                                                          AVCodecContext *avctx)
801{
802    CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
803                                                                   0,
804                                                                   &kCFTypeDictionaryKeyCallBacks,
805                                                                   &kCFTypeDictionaryValueCallBacks);
806
807    CFDictionarySetValue(config_info,
808                         codec_type == kCMVideoCodecType_HEVC ?
809                            kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder :
810                            kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
811                         kCFBooleanTrue);
812
813    CFMutableDictionaryRef avc_info;
814    CFDataRef data = NULL;
815
816    avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
817                                         1,
818                                         &kCFTypeDictionaryKeyCallBacks,
819                                         &kCFTypeDictionaryValueCallBacks);
820
821    switch (codec_type) {
822    case kCMVideoCodecType_MPEG4Video :
823        if (avctx->extradata_size)
824            data = videotoolbox_esds_extradata_create(avctx);
825        if (data)
826            CFDictionarySetValue(avc_info, CFSTR("esds"), data);
827        break;
828    case kCMVideoCodecType_H264 :
829        data = ff_videotoolbox_avcc_extradata_create(avctx);
830        if (data)
831            CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
832        break;
833    case kCMVideoCodecType_HEVC :
834        data = ff_videotoolbox_hvcc_extradata_create(avctx);
835        if (data)
836            CFDictionarySetValue(avc_info, CFSTR("hvcC"), data);
837        break;
838#if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
839    case kCMVideoCodecType_VP9 :
840        data = ff_videotoolbox_vpcc_extradata_create(avctx);
841        if (data)
842            CFDictionarySetValue(avc_info, CFSTR("vpcC"), data);
843        break;
844#endif
845    default:
846        break;
847    }
848
849    CFDictionarySetValue(config_info,
850            kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
851            avc_info);
852
853    if (data)
854        CFRelease(data);
855
856    CFRelease(avc_info);
857    return config_info;
858}
859
860static int videotoolbox_start(AVCodecContext *avctx)
861{
862    AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
863    OSStatus status;
864    VTDecompressionOutputCallbackRecord decoder_cb;
865    CFDictionaryRef decoder_spec;
866    CFDictionaryRef buf_attr;
867
868    if (!videotoolbox) {
869        av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
870        return -1;
871    }
872
873    switch( avctx->codec_id ) {
874    case AV_CODEC_ID_H263 :
875        videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
876        break;
877    case AV_CODEC_ID_H264 :
878        videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
879        break;
880    case AV_CODEC_ID_HEVC :
881        videotoolbox->cm_codec_type = kCMVideoCodecType_HEVC;
882        break;
883    case AV_CODEC_ID_MPEG1VIDEO :
884        videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
885        break;
886    case AV_CODEC_ID_MPEG2VIDEO :
887        videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
888        break;
889    case AV_CODEC_ID_MPEG4 :
890        videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
891        break;
892    case AV_CODEC_ID_PRORES :
893        switch (avctx->codec_tag) {
894        default:
895            av_log(avctx, AV_LOG_WARNING, "Unknown prores profile %d\n", avctx->codec_tag);
896        // fall-through
897        case MKTAG('a','p','c','o'): // kCMVideoCodecType_AppleProRes422Proxy
898        case MKTAG('a','p','c','s'): // kCMVideoCodecType_AppleProRes422LT
899        case MKTAG('a','p','c','n'): // kCMVideoCodecType_AppleProRes422
900        case MKTAG('a','p','c','h'): // kCMVideoCodecType_AppleProRes422HQ
901        case MKTAG('a','p','4','h'): // kCMVideoCodecType_AppleProRes4444
902        case MKTAG('a','p','4','x'): // kCMVideoCodecType_AppleProRes4444XQ
903            videotoolbox->cm_codec_type = av_bswap32(avctx->codec_tag);
904            break;
905        }
906        break;
907    case AV_CODEC_ID_VP9 :
908        videotoolbox->cm_codec_type = kCMVideoCodecType_VP9;
909        break;
910    default :
911        break;
912    }
913
914#if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) && AV_HAS_BUILTIN(__builtin_available)
915    if (avctx->codec_id == AV_CODEC_ID_PRORES) {
916        if (__builtin_available(macOS 10.9, *)) {
917            VTRegisterProfessionalVideoWorkflowVideoDecoders();
918        }
919    }
920#endif
921
922#if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) && AV_HAS_BUILTIN(__builtin_available)
923    if (__builtin_available(macOS 11.0, *)) {
924        VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->cm_codec_type);
925    }
926#endif
927
928    decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
929
930    if (!decoder_spec) {
931        av_log(avctx, AV_LOG_ERROR, "decoder specification creation failed\n");
932        return -1;
933    }
934
935    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
936                                                                decoder_spec,
937                                                                avctx->width,
938                                                                avctx->height);
939    if (!videotoolbox->cm_fmt_desc) {
940        if (decoder_spec)
941            CFRelease(decoder_spec);
942
943        av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
944        return -1;
945    }
946
947    buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
948                                                     avctx->height,
949                                                     videotoolbox->cv_pix_fmt_type);
950
951    decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
952    decoder_cb.decompressionOutputRefCon   = avctx->internal->hwaccel_priv_data;
953
954    status = VTDecompressionSessionCreate(NULL,                      // allocator
955                                          videotoolbox->cm_fmt_desc, // videoFormatDescription
956                                          decoder_spec,              // videoDecoderSpecification
957                                          buf_attr,                  // destinationImageBufferAttributes
958                                          &decoder_cb,               // outputCallback
959                                          &videotoolbox->session);   // decompressionSessionOut
960
961    if (decoder_spec)
962        CFRelease(decoder_spec);
963    if (buf_attr)
964        CFRelease(buf_attr);
965
966    switch (status) {
967    case kVTVideoDecoderNotAvailableNowErr:
968        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox session not available.\n");
969        return AVERROR(ENOSYS);
970    case kVTVideoDecoderUnsupportedDataFormatErr:
971        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox does not support this format.\n");
972        return AVERROR(ENOSYS);
973    case kVTCouldNotFindVideoDecoderErr:
974        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox decoder for this format not found.\n");
975        return AVERROR(ENOSYS);
976    case kVTVideoDecoderMalfunctionErr:
977        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n");
978        return AVERROR(EINVAL);
979    case kVTVideoDecoderBadDataErr:
980        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n");
981        return AVERROR_INVALIDDATA;
982    case 0:
983        return 0;
984    default:
985        av_log(avctx, AV_LOG_VERBOSE, "Unknown VideoToolbox session creation error %d\n", (int)status);
986        return AVERROR_UNKNOWN;
987    }
988}
989
990static const char *videotoolbox_error_string(OSStatus status)
991{
992    switch (status) {
993        case kVTVideoDecoderBadDataErr:
994            return "bad data";
995        case kVTVideoDecoderMalfunctionErr:
996            return "decoder malfunction";
997        case kVTInvalidSessionErr:
998            return "invalid session";
999    }
1000    return "unknown";
1001}
1002
1003int ff_videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
1004{
1005    OSStatus status;
1006    AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
1007    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1008
1009    if (vtctx->reconfig_needed == true) {
1010        vtctx->reconfig_needed = false;
1011        av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox decoder needs reconfig, restarting..\n");
1012        videotoolbox_stop(avctx);
1013        if (videotoolbox_start(avctx) != 0) {
1014            return AVERROR_EXTERNAL;
1015        }
1016    }
1017
1018    if (!videotoolbox->session || !vtctx->bitstream || !vtctx->bitstream_size)
1019        return AVERROR_INVALIDDATA;
1020
1021    status = videotoolbox_session_decode_frame(avctx);
1022    if (status != noErr) {
1023        if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr)
1024            vtctx->reconfig_needed = true;
1025        av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (int)status);
1026        return AVERROR_UNKNOWN;
1027    }
1028
1029    if (!vtctx->frame) {
1030        vtctx->reconfig_needed = true;
1031        return AVERROR_UNKNOWN;
1032    }
1033
1034    return videotoolbox_buffer_create(avctx, frame);
1035}
1036
1037static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
1038{
1039    H264Context *h = avctx->priv_data;
1040    AVFrame *frame = h->cur_pic_ptr->f;
1041    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1042    int ret = ff_videotoolbox_common_end_frame(avctx, frame);
1043    vtctx->bitstream_size = 0;
1044    return ret;
1045}
1046
1047static int videotoolbox_hevc_start_frame(AVCodecContext *avctx,
1048                                         const uint8_t *buffer,
1049                                         uint32_t size)
1050{
1051    return 0;
1052}
1053
1054static int videotoolbox_hevc_decode_slice(AVCodecContext *avctx,
1055                                          const uint8_t *buffer,
1056                                          uint32_t size)
1057{
1058    return videotoolbox_common_decode_slice(avctx, buffer, size);
1059}
1060
1061
1062static int videotoolbox_hevc_decode_params(AVCodecContext *avctx,
1063                                           int type,
1064                                           const uint8_t *buffer,
1065                                           uint32_t size)
1066{
1067    return videotoolbox_common_decode_slice(avctx, buffer, size);
1068}
1069
1070static int videotoolbox_hevc_end_frame(AVCodecContext *avctx)
1071{
1072    HEVCContext *h = avctx->priv_data;
1073    AVFrame *frame = h->ref->frame;
1074    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1075
1076    h->output_frame->crop_right = 0;
1077    h->output_frame->crop_left = 0;
1078    h->output_frame->crop_top = 0;
1079    h->output_frame->crop_bottom = 0;
1080
1081    int ret = ff_videotoolbox_common_end_frame(avctx, frame);
1082    vtctx->bitstream_size = 0;
1083    return ret;
1084}
1085
1086static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
1087                                         const uint8_t *buffer,
1088                                         uint32_t size)
1089{
1090    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1091
1092    return ff_videotoolbox_buffer_copy(vtctx, buffer, size);
1093}
1094
1095static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
1096                                          const uint8_t *buffer,
1097                                          uint32_t size)
1098{
1099    return 0;
1100}
1101
1102static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
1103{
1104    MpegEncContext *s = avctx->priv_data;
1105    AVFrame *frame = s->current_picture_ptr->f;
1106
1107    return ff_videotoolbox_common_end_frame(avctx, frame);
1108}
1109
1110static int videotoolbox_prores_start_frame(AVCodecContext *avctx,
1111                                         const uint8_t *buffer,
1112                                         uint32_t size)
1113{
1114    return 0;
1115}
1116
1117static int videotoolbox_prores_decode_slice(AVCodecContext *avctx,
1118                                          const uint8_t *buffer,
1119                                          uint32_t size)
1120{
1121    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1122
1123    return ff_videotoolbox_buffer_copy(vtctx, buffer, size);
1124}
1125
1126static int videotoolbox_prores_end_frame(AVCodecContext *avctx)
1127{
1128    ProresContext *ctx = avctx->priv_data;
1129    AVFrame *frame = ctx->frame;
1130
1131    return ff_videotoolbox_common_end_frame(avctx, frame);
1132}
1133
1134static enum AVPixelFormat videotoolbox_best_pixel_format(AVCodecContext *avctx) {
1135    const AVPixFmtDescriptor *descriptor = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
1136    if (!descriptor)
1137        return AV_PIX_FMT_NV12; // same as av_videotoolbox_alloc_context()
1138
1139    int depth = descriptor->comp[0].depth;
1140
1141    if (descriptor->flags & AV_PIX_FMT_FLAG_ALPHA)
1142        return AV_PIX_FMT_AYUV64;
1143
1144#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
1145    if (depth > 10)
1146        return descriptor->log2_chroma_w == 0 ? AV_PIX_FMT_P416 : AV_PIX_FMT_P216;
1147#endif
1148
1149#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
1150    if (descriptor->log2_chroma_w == 0) {
1151#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
1152        if (depth <= 8)
1153            return AV_PIX_FMT_NV24;
1154#endif
1155        return AV_PIX_FMT_P410;
1156    }
1157#endif
1158#if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
1159    if (descriptor->log2_chroma_h == 0) {
1160#if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
1161        if (depth <= 8)
1162            return AV_PIX_FMT_NV16;
1163#endif
1164        return AV_PIX_FMT_P210;
1165    }
1166#endif
1167#if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
1168    if (depth > 8) {
1169        return AV_PIX_FMT_P010;
1170    }
1171#endif
1172
1173    return AV_PIX_FMT_NV12;
1174}
1175
1176int ff_videotoolbox_common_init(AVCodecContext *avctx)
1177{
1178    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
1179    AVHWFramesContext *hw_frames;
1180    int err;
1181
1182    vtctx->logctx = avctx;
1183
1184    // Old API - do nothing.
1185    if (avctx->hwaccel_context)
1186        return 0;
1187
1188    if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
1189        av_log(avctx, AV_LOG_ERROR,
1190               "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1191        return AVERROR(EINVAL);
1192    }
1193
1194    vtctx->vt_ctx = av_videotoolbox_alloc_context();
1195    if (!vtctx->vt_ctx) {
1196        err = AVERROR(ENOMEM);
1197        goto fail;
1198    }
1199
1200    if (avctx->hw_frames_ctx) {
1201        hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
1202    } else {
1203        avctx->hw_frames_ctx = av_hwframe_ctx_alloc(avctx->hw_device_ctx);
1204        if (!avctx->hw_frames_ctx) {
1205            err = AVERROR(ENOMEM);
1206            goto fail;
1207        }
1208
1209        hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
1210        hw_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
1211        hw_frames->sw_format = videotoolbox_best_pixel_format(avctx);
1212        hw_frames->width = avctx->width;
1213        hw_frames->height = avctx->height;
1214
1215        err = av_hwframe_ctx_init(avctx->hw_frames_ctx);
1216        if (err < 0) {
1217            av_buffer_unref(&avctx->hw_frames_ctx);
1218            goto fail;
1219        }
1220    }
1221
1222    vtctx->cached_hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
1223    if (!vtctx->cached_hw_frames_ctx) {
1224        err = AVERROR(ENOMEM);
1225        goto fail;
1226    }
1227
1228    bool full_range = avctx->color_range == AVCOL_RANGE_JPEG;
1229    vtctx->vt_ctx->cv_pix_fmt_type =
1230        av_map_videotoolbox_format_from_pixfmt2(hw_frames->sw_format, full_range);
1231    if (!vtctx->vt_ctx->cv_pix_fmt_type) {
1232        const AVPixFmtDescriptor *attempted_format =
1233            av_pix_fmt_desc_get(hw_frames->sw_format);
1234        av_log(avctx, AV_LOG_ERROR,
1235               "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1236               "a VideoToolbox format!\n",
1237               attempted_format ? attempted_format->name : "<unknown>",
1238               av_color_range_name(avctx->color_range));
1239        err = AVERROR(EINVAL);
1240        goto fail;
1241    }
1242
1243    err = videotoolbox_start(avctx);
1244    if (err < 0)
1245        goto fail;
1246
1247    return 0;
1248
1249fail:
1250    ff_videotoolbox_uninit(avctx);
1251    return err;
1252}
1253
1254int ff_videotoolbox_frame_params(AVCodecContext *avctx,
1255                                 AVBufferRef *hw_frames_ctx)
1256{
1257    AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
1258
1259    frames_ctx->format            = AV_PIX_FMT_VIDEOTOOLBOX;
1260    frames_ctx->width             = avctx->coded_width;
1261    frames_ctx->height            = avctx->coded_height;
1262    frames_ctx->sw_format         = videotoolbox_best_pixel_format(avctx);
1263
1264    return 0;
1265}
1266
1267const AVHWAccel ff_h263_videotoolbox_hwaccel = {
1268    .name           = "h263_videotoolbox",
1269    .type           = AVMEDIA_TYPE_VIDEO,
1270    .id             = AV_CODEC_ID_H263,
1271    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1272    .alloc_frame    = ff_videotoolbox_alloc_frame,
1273    .start_frame    = videotoolbox_mpeg_start_frame,
1274    .decode_slice   = videotoolbox_mpeg_decode_slice,
1275    .end_frame      = videotoolbox_mpeg_end_frame,
1276    .frame_params   = ff_videotoolbox_frame_params,
1277    .init           = ff_videotoolbox_common_init,
1278    .uninit         = ff_videotoolbox_uninit,
1279    .priv_data_size = sizeof(VTContext),
1280};
1281
1282const AVHWAccel ff_hevc_videotoolbox_hwaccel = {
1283    .name           = "hevc_videotoolbox",
1284    .type           = AVMEDIA_TYPE_VIDEO,
1285    .id             = AV_CODEC_ID_HEVC,
1286    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1287    .alloc_frame    = ff_videotoolbox_alloc_frame,
1288    .start_frame    = videotoolbox_hevc_start_frame,
1289    .decode_slice   = videotoolbox_hevc_decode_slice,
1290    .decode_params  = videotoolbox_hevc_decode_params,
1291    .end_frame      = videotoolbox_hevc_end_frame,
1292    .frame_params   = ff_videotoolbox_frame_params,
1293    .init           = ff_videotoolbox_common_init,
1294    .uninit         = ff_videotoolbox_uninit,
1295    .priv_data_size = sizeof(VTContext),
1296};
1297
1298const AVHWAccel ff_h264_videotoolbox_hwaccel = {
1299    .name           = "h264_videotoolbox",
1300    .type           = AVMEDIA_TYPE_VIDEO,
1301    .id             = AV_CODEC_ID_H264,
1302    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1303    .alloc_frame    = ff_videotoolbox_alloc_frame,
1304    .start_frame    = ff_videotoolbox_h264_start_frame,
1305    .decode_slice   = ff_videotoolbox_h264_decode_slice,
1306    .decode_params  = videotoolbox_h264_decode_params,
1307    .end_frame      = videotoolbox_h264_end_frame,
1308    .frame_params   = ff_videotoolbox_frame_params,
1309    .init           = ff_videotoolbox_common_init,
1310    .uninit         = ff_videotoolbox_uninit,
1311    .priv_data_size = sizeof(VTContext),
1312};
1313
1314const AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
1315    .name           = "mpeg1_videotoolbox",
1316    .type           = AVMEDIA_TYPE_VIDEO,
1317    .id             = AV_CODEC_ID_MPEG1VIDEO,
1318    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1319    .alloc_frame    = ff_videotoolbox_alloc_frame,
1320    .start_frame    = videotoolbox_mpeg_start_frame,
1321    .decode_slice   = videotoolbox_mpeg_decode_slice,
1322    .end_frame      = videotoolbox_mpeg_end_frame,
1323    .frame_params   = ff_videotoolbox_frame_params,
1324    .init           = ff_videotoolbox_common_init,
1325    .uninit         = ff_videotoolbox_uninit,
1326    .priv_data_size = sizeof(VTContext),
1327};
1328
1329const AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
1330    .name           = "mpeg2_videotoolbox",
1331    .type           = AVMEDIA_TYPE_VIDEO,
1332    .id             = AV_CODEC_ID_MPEG2VIDEO,
1333    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1334    .alloc_frame    = ff_videotoolbox_alloc_frame,
1335    .start_frame    = videotoolbox_mpeg_start_frame,
1336    .decode_slice   = videotoolbox_mpeg_decode_slice,
1337    .end_frame      = videotoolbox_mpeg_end_frame,
1338    .frame_params   = ff_videotoolbox_frame_params,
1339    .init           = ff_videotoolbox_common_init,
1340    .uninit         = ff_videotoolbox_uninit,
1341    .priv_data_size = sizeof(VTContext),
1342};
1343
1344const AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
1345    .name           = "mpeg4_videotoolbox",
1346    .type           = AVMEDIA_TYPE_VIDEO,
1347    .id             = AV_CODEC_ID_MPEG4,
1348    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1349    .alloc_frame    = ff_videotoolbox_alloc_frame,
1350    .start_frame    = videotoolbox_mpeg_start_frame,
1351    .decode_slice   = videotoolbox_mpeg_decode_slice,
1352    .end_frame      = videotoolbox_mpeg_end_frame,
1353    .frame_params   = ff_videotoolbox_frame_params,
1354    .init           = ff_videotoolbox_common_init,
1355    .uninit         = ff_videotoolbox_uninit,
1356    .priv_data_size = sizeof(VTContext),
1357};
1358
1359const AVHWAccel ff_prores_videotoolbox_hwaccel = {
1360    .name           = "prores_videotoolbox",
1361    .type           = AVMEDIA_TYPE_VIDEO,
1362    .id             = AV_CODEC_ID_PRORES,
1363    .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1364    .alloc_frame    = ff_videotoolbox_alloc_frame,
1365    .start_frame    = videotoolbox_prores_start_frame,
1366    .decode_slice   = videotoolbox_prores_decode_slice,
1367    .end_frame      = videotoolbox_prores_end_frame,
1368    .frame_params   = ff_videotoolbox_frame_params,
1369    .init           = ff_videotoolbox_common_init,
1370    .uninit         = ff_videotoolbox_uninit,
1371    .priv_data_size = sizeof(VTContext),
1372};
1373
1374static AVVideotoolboxContext *av_videotoolbox_alloc_context_with_pix_fmt(enum AVPixelFormat pix_fmt,
1375                                                                         bool full_range)
1376{
1377    AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
1378
1379    if (ret) {
1380        ret->output_callback = videotoolbox_decoder_callback;
1381
1382        OSType cv_pix_fmt_type = av_map_videotoolbox_format_from_pixfmt2(pix_fmt, full_range);
1383        if (cv_pix_fmt_type == 0) {
1384            cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1385        }
1386        ret->cv_pix_fmt_type = cv_pix_fmt_type;
1387    }
1388
1389    return ret;
1390}
1391
1392AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
1393{
1394    return av_videotoolbox_alloc_context_with_pix_fmt(AV_PIX_FMT_NONE, false);
1395}
1396
1397int av_videotoolbox_default_init(AVCodecContext *avctx)
1398{
1399    return av_videotoolbox_default_init2(avctx, NULL);
1400}
1401
1402int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
1403{
1404    enum AVPixelFormat pix_fmt = videotoolbox_best_pixel_format(avctx);
1405    bool full_range = avctx->color_range == AVCOL_RANGE_JPEG;
1406    avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context_with_pix_fmt(pix_fmt, full_range);
1407    if (!avctx->hwaccel_context)
1408        return AVERROR(ENOMEM);
1409    return videotoolbox_start(avctx);
1410}
1411
1412void av_videotoolbox_default_free(AVCodecContext *avctx)
1413{
1414
1415    videotoolbox_stop(avctx);
1416    av_freep(&avctx->hwaccel_context);
1417}
1418#endif /* CONFIG_VIDEOTOOLBOX */
1419