1/*
2 * Android MediaCodec MPEG-2 / H.264 / H.265 / MPEG-4 / VP8 / VP9 decoders
3 *
4 * Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include "config_components.h"
24
25#include <stdint.h>
26#include <string.h>
27
28#include "libavutil/avassert.h"
29#include "libavutil/common.h"
30#include "libavutil/opt.h"
31#include "libavutil/intreadwrite.h"
32#include "libavutil/pixfmt.h"
33#include "libavutil/internal.h"
34
35#include "avcodec.h"
36#include "codec_internal.h"
37#include "decode.h"
38#include "h264_parse.h"
39#include "h264_ps.h"
40#include "hevc_parse.h"
41#include "hwconfig.h"
42#include "internal.h"
43#include "mediacodec_wrapper.h"
44#include "mediacodecdec_common.h"
45
46typedef struct MediaCodecH264DecContext {
47
48    AVClass *avclass;
49
50    MediaCodecDecContext *ctx;
51
52    AVPacket buffered_pkt;
53
54    int delay_flush;
55    int amlogic_mpeg2_api23_workaround;
56
57} MediaCodecH264DecContext;
58
59static av_cold int mediacodec_decode_close(AVCodecContext *avctx)
60{
61    MediaCodecH264DecContext *s = avctx->priv_data;
62
63    ff_mediacodec_dec_close(avctx, s->ctx);
64    s->ctx = NULL;
65
66    av_packet_unref(&s->buffered_pkt);
67
68    return 0;
69}
70
71#if CONFIG_H264_MEDIACODEC_DECODER || CONFIG_HEVC_MEDIACODEC_DECODER
72static int h2645_ps_to_nalu(const uint8_t *src, int src_size, uint8_t **out, int *out_size)
73{
74    int i;
75    int ret = 0;
76    uint8_t *p = NULL;
77    static const uint8_t nalu_header[] = { 0x00, 0x00, 0x00, 0x01 };
78
79    if (!out || !out_size) {
80        return AVERROR(EINVAL);
81    }
82
83    p = av_malloc(sizeof(nalu_header) + src_size);
84    if (!p) {
85        return AVERROR(ENOMEM);
86    }
87
88    *out = p;
89    *out_size = sizeof(nalu_header) + src_size;
90
91    memcpy(p, nalu_header, sizeof(nalu_header));
92    memcpy(p + sizeof(nalu_header), src, src_size);
93
94    /* Escape 0x00, 0x00, 0x0{0-3} pattern */
95    for (i = 4; i < *out_size; i++) {
96        if (i < *out_size - 3 &&
97            p[i + 0] == 0 &&
98            p[i + 1] == 0 &&
99            p[i + 2] <= 3) {
100            uint8_t *new;
101
102            *out_size += 1;
103            new = av_realloc(*out, *out_size);
104            if (!new) {
105                ret = AVERROR(ENOMEM);
106                goto done;
107            }
108            *out = p = new;
109
110            i = i + 2;
111            memmove(p + i + 1, p + i, *out_size - (i + 1));
112            p[i] = 0x03;
113        }
114    }
115done:
116    if (ret < 0) {
117        av_freep(out);
118        *out_size = 0;
119    }
120
121    return ret;
122}
123#endif
124
125#if CONFIG_H264_MEDIACODEC_DECODER
126static int h264_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
127{
128    int i;
129    int ret;
130
131    H264ParamSets ps;
132    const PPS *pps = NULL;
133    const SPS *sps = NULL;
134    int is_avc = 0;
135    int nal_length_size = 0;
136
137    memset(&ps, 0, sizeof(ps));
138
139    ret = ff_h264_decode_extradata(avctx->extradata, avctx->extradata_size,
140                                   &ps, &is_avc, &nal_length_size, 0, avctx);
141    if (ret < 0) {
142        goto done;
143    }
144
145    for (i = 0; i < MAX_PPS_COUNT; i++) {
146        if (ps.pps_list[i]) {
147            pps = (const PPS*)ps.pps_list[i]->data;
148            break;
149        }
150    }
151
152    if (pps) {
153        if (ps.sps_list[pps->sps_id]) {
154            sps = (const SPS*)ps.sps_list[pps->sps_id]->data;
155        }
156    }
157
158    if (pps && sps) {
159        uint8_t *data = NULL;
160        int data_size = 0;
161
162        avctx->profile = ff_h264_get_profile(sps);
163        avctx->level = sps->level_idc;
164
165        if ((ret = h2645_ps_to_nalu(sps->data, sps->data_size, &data, &data_size)) < 0) {
166            goto done;
167        }
168        ff_AMediaFormat_setBuffer(format, "csd-0", (void*)data, data_size);
169        av_freep(&data);
170
171        if ((ret = h2645_ps_to_nalu(pps->data, pps->data_size, &data, &data_size)) < 0) {
172            goto done;
173        }
174        ff_AMediaFormat_setBuffer(format, "csd-1", (void*)data, data_size);
175        av_freep(&data);
176    } else {
177        const int warn = is_avc && (avctx->codec_tag == MKTAG('a','v','c','1') ||
178                                    avctx->codec_tag == MKTAG('a','v','c','2'));
179        av_log(avctx, warn ? AV_LOG_WARNING : AV_LOG_DEBUG,
180               "Could not extract PPS/SPS from extradata\n");
181        ret = 0;
182    }
183
184done:
185    ff_h264_ps_uninit(&ps);
186
187    return ret;
188}
189#endif
190
191#if CONFIG_HEVC_MEDIACODEC_DECODER
192static int hevc_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
193{
194    int i;
195    int ret;
196
197    HEVCParamSets ps;
198    HEVCSEI sei;
199
200    const HEVCVPS *vps = NULL;
201    const HEVCPPS *pps = NULL;
202    const HEVCSPS *sps = NULL;
203    int is_nalff = 0;
204    int nal_length_size = 0;
205
206    uint8_t *vps_data = NULL;
207    uint8_t *sps_data = NULL;
208    uint8_t *pps_data = NULL;
209    int vps_data_size = 0;
210    int sps_data_size = 0;
211    int pps_data_size = 0;
212
213    memset(&ps, 0, sizeof(ps));
214    memset(&sei, 0, sizeof(sei));
215
216    ret = ff_hevc_decode_extradata(avctx->extradata, avctx->extradata_size,
217                                   &ps, &sei, &is_nalff, &nal_length_size, 0, 1, avctx);
218    if (ret < 0) {
219        goto done;
220    }
221
222    for (i = 0; i < HEVC_MAX_VPS_COUNT; i++) {
223        if (ps.vps_list[i]) {
224            vps = (const HEVCVPS*)ps.vps_list[i]->data;
225            break;
226        }
227    }
228
229    for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
230        if (ps.pps_list[i]) {
231            pps = (const HEVCPPS*)ps.pps_list[i]->data;
232            break;
233        }
234    }
235
236    if (pps) {
237        if (ps.sps_list[pps->sps_id]) {
238            sps = (const HEVCSPS*)ps.sps_list[pps->sps_id]->data;
239        }
240    }
241
242    if (vps && pps && sps) {
243        uint8_t *data;
244        int data_size;
245
246        avctx->profile = sps->ptl.general_ptl.profile_idc;
247        avctx->level   = sps->ptl.general_ptl.level_idc;
248
249        if ((ret = h2645_ps_to_nalu(vps->data, vps->data_size, &vps_data, &vps_data_size)) < 0 ||
250            (ret = h2645_ps_to_nalu(sps->data, sps->data_size, &sps_data, &sps_data_size)) < 0 ||
251            (ret = h2645_ps_to_nalu(pps->data, pps->data_size, &pps_data, &pps_data_size)) < 0) {
252            goto done;
253        }
254
255        data_size = vps_data_size + sps_data_size + pps_data_size;
256        data = av_mallocz(data_size);
257        if (!data) {
258            ret = AVERROR(ENOMEM);
259            goto done;
260        }
261
262        memcpy(data                                , vps_data, vps_data_size);
263        memcpy(data + vps_data_size                , sps_data, sps_data_size);
264        memcpy(data + vps_data_size + sps_data_size, pps_data, pps_data_size);
265
266        ff_AMediaFormat_setBuffer(format, "csd-0", data, data_size);
267
268        av_freep(&data);
269    } else {
270        const int warn = is_nalff && avctx->codec_tag == MKTAG('h','v','c','1');
271        av_log(avctx, warn ? AV_LOG_WARNING : AV_LOG_DEBUG,
272               "Could not extract VPS/PPS/SPS from extradata\n");
273        ret = 0;
274    }
275
276done:
277    ff_hevc_ps_uninit(&ps);
278
279    av_freep(&vps_data);
280    av_freep(&sps_data);
281    av_freep(&pps_data);
282
283    return ret;
284}
285#endif
286
287#if CONFIG_MPEG2_MEDIACODEC_DECODER || \
288    CONFIG_MPEG4_MEDIACODEC_DECODER || \
289    CONFIG_VP8_MEDIACODEC_DECODER   || \
290    CONFIG_VP9_MEDIACODEC_DECODER
291static int common_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
292{
293    int ret = 0;
294
295    if (avctx->extradata) {
296        ff_AMediaFormat_setBuffer(format, "csd-0", avctx->extradata, avctx->extradata_size);
297    }
298
299    return ret;
300}
301#endif
302
303static av_cold int mediacodec_decode_init(AVCodecContext *avctx)
304{
305    int ret;
306    int sdk_int;
307
308    const char *codec_mime = NULL;
309
310    FFAMediaFormat *format = NULL;
311    MediaCodecH264DecContext *s = avctx->priv_data;
312
313    format = ff_AMediaFormat_new();
314    if (!format) {
315        av_log(avctx, AV_LOG_ERROR, "Failed to create media format\n");
316        ret = AVERROR_EXTERNAL;
317        goto done;
318    }
319
320    switch (avctx->codec_id) {
321#if CONFIG_H264_MEDIACODEC_DECODER
322    case AV_CODEC_ID_H264:
323        codec_mime = "video/avc";
324
325        ret = h264_set_extradata(avctx, format);
326        if (ret < 0)
327            goto done;
328        break;
329#endif
330#if CONFIG_HEVC_MEDIACODEC_DECODER
331    case AV_CODEC_ID_HEVC:
332        codec_mime = "video/hevc";
333
334        ret = hevc_set_extradata(avctx, format);
335        if (ret < 0)
336            goto done;
337        break;
338#endif
339#if CONFIG_MPEG2_MEDIACODEC_DECODER
340    case AV_CODEC_ID_MPEG2VIDEO:
341        codec_mime = "video/mpeg2";
342
343        ret = common_set_extradata(avctx, format);
344        if (ret < 0)
345            goto done;
346        break;
347#endif
348#if CONFIG_MPEG4_MEDIACODEC_DECODER
349    case AV_CODEC_ID_MPEG4:
350        codec_mime = "video/mp4v-es",
351
352        ret = common_set_extradata(avctx, format);
353        if (ret < 0)
354            goto done;
355        break;
356#endif
357#if CONFIG_VP8_MEDIACODEC_DECODER
358    case AV_CODEC_ID_VP8:
359        codec_mime = "video/x-vnd.on2.vp8";
360
361        ret = common_set_extradata(avctx, format);
362        if (ret < 0)
363            goto done;
364        break;
365#endif
366#if CONFIG_VP9_MEDIACODEC_DECODER
367    case AV_CODEC_ID_VP9:
368        codec_mime = "video/x-vnd.on2.vp9";
369
370        ret = common_set_extradata(avctx, format);
371        if (ret < 0)
372            goto done;
373        break;
374#endif
375    default:
376        av_assert0(0);
377    }
378
379    ff_AMediaFormat_setString(format, "mime", codec_mime);
380    ff_AMediaFormat_setInt32(format, "width", avctx->width);
381    ff_AMediaFormat_setInt32(format, "height", avctx->height);
382
383    s->ctx = av_mallocz(sizeof(*s->ctx));
384    if (!s->ctx) {
385        av_log(avctx, AV_LOG_ERROR, "Failed to allocate MediaCodecDecContext\n");
386        ret = AVERROR(ENOMEM);
387        goto done;
388    }
389
390    s->ctx->delay_flush = s->delay_flush;
391
392    if ((ret = ff_mediacodec_dec_init(avctx, s->ctx, codec_mime, format)) < 0) {
393        s->ctx = NULL;
394        goto done;
395    }
396
397    av_log(avctx, AV_LOG_INFO,
398           "MediaCodec started successfully: codec = %s, ret = %d\n",
399           s->ctx->codec_name, ret);
400
401    sdk_int = ff_Build_SDK_INT(avctx);
402    if (sdk_int <= 23 &&
403        strcmp(s->ctx->codec_name, "OMX.amlogic.mpeg2.decoder.awesome") == 0) {
404        av_log(avctx, AV_LOG_INFO, "Enabling workaround for %s on API=%d\n",
405               s->ctx->codec_name, sdk_int);
406        s->amlogic_mpeg2_api23_workaround = 1;
407    }
408
409done:
410    if (format) {
411        ff_AMediaFormat_delete(format);
412    }
413
414    if (ret < 0) {
415        mediacodec_decode_close(avctx);
416    }
417
418    return ret;
419}
420
421static int mediacodec_receive_frame(AVCodecContext *avctx, AVFrame *frame)
422{
423    MediaCodecH264DecContext *s = avctx->priv_data;
424    int ret;
425    ssize_t index;
426
427    /* In delay_flush mode, wait until the user has released or rendered
428       all retained frames. */
429    if (s->delay_flush && ff_mediacodec_dec_is_flushing(avctx, s->ctx)) {
430        if (!ff_mediacodec_dec_flush(avctx, s->ctx)) {
431            return AVERROR(EAGAIN);
432        }
433    }
434
435    /* poll for new frame */
436    ret = ff_mediacodec_dec_receive(avctx, s->ctx, frame, false);
437    if (ret != AVERROR(EAGAIN))
438        return ret;
439
440    /* feed decoder */
441    while (1) {
442        if (s->ctx->current_input_buffer < 0) {
443            /* poll for input space */
444            index = ff_AMediaCodec_dequeueInputBuffer(s->ctx->codec, 0);
445            if (index < 0) {
446                /* no space, block for an output frame to appear */
447                return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
448            }
449            s->ctx->current_input_buffer = index;
450        }
451
452        /* try to flush any buffered packet data */
453        if (s->buffered_pkt.size > 0) {
454            ret = ff_mediacodec_dec_send(avctx, s->ctx, &s->buffered_pkt, false);
455            if (ret >= 0) {
456                s->buffered_pkt.size -= ret;
457                s->buffered_pkt.data += ret;
458                if (s->buffered_pkt.size <= 0) {
459                    av_packet_unref(&s->buffered_pkt);
460                } else {
461                    av_log(avctx, AV_LOG_WARNING,
462                           "could not send entire packet in single input buffer (%d < %d)\n",
463                           ret, s->buffered_pkt.size+ret);
464                }
465            } else if (ret < 0 && ret != AVERROR(EAGAIN)) {
466                return ret;
467            }
468
469            if (s->amlogic_mpeg2_api23_workaround && s->buffered_pkt.size <= 0) {
470                /* fallthrough to fetch next packet regardless of input buffer space */
471            } else {
472                /* poll for space again */
473                continue;
474            }
475        }
476
477        /* fetch new packet or eof */
478        ret = ff_decode_get_packet(avctx, &s->buffered_pkt);
479        if (ret == AVERROR_EOF) {
480            AVPacket null_pkt = { 0 };
481            ret = ff_mediacodec_dec_send(avctx, s->ctx, &null_pkt, true);
482            if (ret < 0)
483                return ret;
484            return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
485        } else if (ret == AVERROR(EAGAIN) && s->ctx->current_input_buffer < 0) {
486            return ff_mediacodec_dec_receive(avctx, s->ctx, frame, true);
487        } else if (ret < 0) {
488            return ret;
489        }
490    }
491
492    return AVERROR(EAGAIN);
493}
494
495static void mediacodec_decode_flush(AVCodecContext *avctx)
496{
497    MediaCodecH264DecContext *s = avctx->priv_data;
498
499    av_packet_unref(&s->buffered_pkt);
500
501    ff_mediacodec_dec_flush(avctx, s->ctx);
502}
503
504static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = {
505    &(const AVCodecHWConfigInternal) {
506        .public          = {
507            .pix_fmt     = AV_PIX_FMT_MEDIACODEC,
508            .methods     = AV_CODEC_HW_CONFIG_METHOD_AD_HOC |
509                           AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX,
510            .device_type = AV_HWDEVICE_TYPE_MEDIACODEC,
511        },
512        .hwaccel         = NULL,
513    },
514    NULL
515};
516
517#define OFFSET(x) offsetof(MediaCodecH264DecContext, x)
518#define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
519static const AVOption ff_mediacodec_vdec_options[] = {
520    { "delay_flush", "Delay flush until hw output buffers are returned to the decoder",
521                     OFFSET(delay_flush), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VD },
522    { NULL }
523};
524
525#define DECLARE_MEDIACODEC_VCLASS(short_name)                   \
526static const AVClass ff_##short_name##_mediacodec_dec_class = { \
527    .class_name = #short_name "_mediacodec",                    \
528    .item_name  = av_default_item_name,                         \
529    .option     = ff_mediacodec_vdec_options,                   \
530    .version    = LIBAVUTIL_VERSION_INT,                        \
531};
532
533#define DECLARE_MEDIACODEC_VDEC(short_name, full_name, codec_id, bsf)                          \
534DECLARE_MEDIACODEC_VCLASS(short_name)                                                          \
535const FFCodec ff_ ## short_name ## _mediacodec_decoder = {                                     \
536    .p.name         = #short_name "_mediacodec",                                               \
537    .p.long_name    = NULL_IF_CONFIG_SMALL(full_name " Android MediaCodec decoder"),           \
538    .p.type         = AVMEDIA_TYPE_VIDEO,                                                      \
539    .p.id           = codec_id,                                                                \
540    .p.priv_class   = &ff_##short_name##_mediacodec_dec_class,                                 \
541    .priv_data_size = sizeof(MediaCodecH264DecContext),                                        \
542    .init           = mediacodec_decode_init,                                                  \
543    FF_CODEC_RECEIVE_FRAME_CB(mediacodec_receive_frame),                                       \
544    .flush          = mediacodec_decode_flush,                                                 \
545    .close          = mediacodec_decode_close,                                                 \
546    .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \
547    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,                                               \
548    .bsfs           = bsf,                                                                     \
549    .hw_configs     = mediacodec_hw_configs,                                                   \
550    .p.wrapper_name = "mediacodec",                                                            \
551};                                                                                             \
552
553#if CONFIG_H264_MEDIACODEC_DECODER
554DECLARE_MEDIACODEC_VDEC(h264, "H.264", AV_CODEC_ID_H264, "h264_mp4toannexb")
555#endif
556
557#if CONFIG_HEVC_MEDIACODEC_DECODER
558DECLARE_MEDIACODEC_VDEC(hevc, "H.265", AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
559#endif
560
561#if CONFIG_MPEG2_MEDIACODEC_DECODER
562DECLARE_MEDIACODEC_VDEC(mpeg2, "MPEG-2", AV_CODEC_ID_MPEG2VIDEO, NULL)
563#endif
564
565#if CONFIG_MPEG4_MEDIACODEC_DECODER
566DECLARE_MEDIACODEC_VDEC(mpeg4, "MPEG-4", AV_CODEC_ID_MPEG4, NULL)
567#endif
568
569#if CONFIG_VP8_MEDIACODEC_DECODER
570DECLARE_MEDIACODEC_VDEC(vp8, "VP8", AV_CODEC_ID_VP8, NULL)
571#endif
572
573#if CONFIG_VP9_MEDIACODEC_DECODER
574DECLARE_MEDIACODEC_VDEC(vp9, "VP9", AV_CODEC_ID_VP9, NULL)
575#endif
576