1/* 2 * RockChip MPP Video Decoder 3 * Copyright (c) 2017 Lionel CHAZALLON 4 * 5 * This file is part of FFmpeg. 6 * 7 * FFmpeg is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * FFmpeg is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with FFmpeg; if not, write to the Free Software 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 */ 21 22#include <drm_fourcc.h> 23#include <pthread.h> 24#include <rockchip/mpp_buffer.h> 25#include <rockchip/rk_mpi.h> 26#include <time.h> 27#include <unistd.h> 28 29#include "avcodec.h" 30#include "codec_internal.h" 31#include "decode.h" 32#include "hwconfig.h" 33#include "libavutil/buffer.h" 34#include "libavutil/common.h" 35#include "libavutil/frame.h" 36#include "libavutil/hwcontext.h" 37#include "libavutil/hwcontext_drm.h" 38#include "libavutil/imgutils.h" 39#include "libavutil/log.h" 40 41#define RECEIVE_FRAME_TIMEOUT 100 42#define FRAMEGROUP_MAX_FRAMES 16 43#define INPUT_MAX_PACKETS 4 44 45typedef struct { 46 MppCtx ctx; 47 MppApi *mpi; 48 MppBufferGroup frame_group; 49 50 char first_packet; 51 char eos_reached; 52 53 AVBufferRef *frames_ref; 54 AVBufferRef *device_ref; 55} RKMPPDecoder; 56 57typedef struct { 58 AVClass *av_class; 59 AVBufferRef *decoder_ref; 60} RKMPPDecodeContext; 61 62typedef struct { 63 MppFrame frame; 64 AVBufferRef *decoder_ref; 65} RKMPPFrameContext; 66 67static MppCodingType rkmpp_get_codingtype(AVCodecContext *avctx) 68{ 69 switch (avctx->codec_id) { 70 case AV_CODEC_ID_H264: return MPP_VIDEO_CodingAVC; 71 case AV_CODEC_ID_HEVC: return MPP_VIDEO_CodingHEVC; 72 case AV_CODEC_ID_VP8: return MPP_VIDEO_CodingVP8; 73 case AV_CODEC_ID_VP9: return MPP_VIDEO_CodingVP9; 74 default: return MPP_VIDEO_CodingUnused; 75 } 76} 77 78static uint32_t rkmpp_get_frameformat(MppFrameFormat mppformat) 79{ 80 switch (mppformat) { 81 case MPP_FMT_YUV420SP: return DRM_FORMAT_NV12; 82#ifdef DRM_FORMAT_NV12_10 83 case MPP_FMT_YUV420SP_10BIT: return DRM_FORMAT_NV12_10; 84#endif 85 default: return 0; 86 } 87} 88 89static int rkmpp_write_data(AVCodecContext *avctx, uint8_t *buffer, int size, int64_t pts) 90{ 91 RKMPPDecodeContext *rk_context = avctx->priv_data; 92 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data; 93 int ret; 94 MppPacket packet; 95 96 // create the MPP packet 97 ret = mpp_packet_init(&packet, buffer, size); 98 if (ret != MPP_OK) { 99 av_log(avctx, AV_LOG_ERROR, "Failed to init MPP packet (code = %d)\n", ret); 100 return AVERROR_UNKNOWN; 101 } 102 103 mpp_packet_set_pts(packet, pts); 104 105 if (!buffer) 106 mpp_packet_set_eos(packet); 107 108 ret = decoder->mpi->decode_put_packet(decoder->ctx, packet); 109 if (ret != MPP_OK) { 110 if (ret == MPP_ERR_BUFFER_FULL) { 111 av_log(avctx, AV_LOG_DEBUG, "Buffer full writing %d bytes to decoder\n", size); 112 ret = AVERROR(EAGAIN); 113 } else 114 ret = AVERROR_UNKNOWN; 115 } 116 else 117 av_log(avctx, AV_LOG_DEBUG, "Wrote %d bytes to decoder\n", size); 118 119 mpp_packet_deinit(&packet); 120 121 return ret; 122} 123 124static int rkmpp_close_decoder(AVCodecContext *avctx) 125{ 126 RKMPPDecodeContext *rk_context = avctx->priv_data; 127 av_buffer_unref(&rk_context->decoder_ref); 128 return 0; 129} 130 131static void rkmpp_release_decoder(void *opaque, uint8_t *data) 132{ 133 RKMPPDecoder *decoder = (RKMPPDecoder *)data; 134 135 if (decoder->mpi) { 136 decoder->mpi->reset(decoder->ctx); 137 mpp_destroy(decoder->ctx); 138 decoder->ctx = NULL; 139 } 140 141 if (decoder->frame_group) { 142 mpp_buffer_group_put(decoder->frame_group); 143 decoder->frame_group = NULL; 144 } 145 146 av_buffer_unref(&decoder->frames_ref); 147 av_buffer_unref(&decoder->device_ref); 148 149 av_free(decoder); 150} 151 152static int rkmpp_init_decoder(AVCodecContext *avctx) 153{ 154 RKMPPDecodeContext *rk_context = avctx->priv_data; 155 RKMPPDecoder *decoder = NULL; 156 MppCodingType codectype = MPP_VIDEO_CodingUnused; 157 int ret; 158 RK_S64 paramS64; 159 RK_S32 paramS32; 160 161 avctx->pix_fmt = AV_PIX_FMT_DRM_PRIME; 162 163 // create a decoder and a ref to it 164 decoder = av_mallocz(sizeof(RKMPPDecoder)); 165 if (!decoder) { 166 ret = AVERROR(ENOMEM); 167 goto fail; 168 } 169 170 rk_context->decoder_ref = av_buffer_create((uint8_t *)decoder, sizeof(*decoder), rkmpp_release_decoder, 171 NULL, AV_BUFFER_FLAG_READONLY); 172 if (!rk_context->decoder_ref) { 173 av_free(decoder); 174 ret = AVERROR(ENOMEM); 175 goto fail; 176 } 177 178 av_log(avctx, AV_LOG_DEBUG, "Initializing RKMPP decoder.\n"); 179 180 codectype = rkmpp_get_codingtype(avctx); 181 if (codectype == MPP_VIDEO_CodingUnused) { 182 av_log(avctx, AV_LOG_ERROR, "Unknown codec type (%d).\n", avctx->codec_id); 183 ret = AVERROR_UNKNOWN; 184 goto fail; 185 } 186 187 ret = mpp_check_support_format(MPP_CTX_DEC, codectype); 188 if (ret != MPP_OK) { 189 av_log(avctx, AV_LOG_ERROR, "Codec type (%d) unsupported by MPP\n", avctx->codec_id); 190 ret = AVERROR_UNKNOWN; 191 goto fail; 192 } 193 194 // Create the MPP context 195 ret = mpp_create(&decoder->ctx, &decoder->mpi); 196 if (ret != MPP_OK) { 197 av_log(avctx, AV_LOG_ERROR, "Failed to create MPP context (code = %d).\n", ret); 198 ret = AVERROR_UNKNOWN; 199 goto fail; 200 } 201 202 // initialize mpp 203 ret = mpp_init(decoder->ctx, MPP_CTX_DEC, codectype); 204 if (ret != MPP_OK) { 205 av_log(avctx, AV_LOG_ERROR, "Failed to initialize MPP context (code = %d).\n", ret); 206 ret = AVERROR_UNKNOWN; 207 goto fail; 208 } 209 210 // make decode calls blocking with a timeout 211 paramS32 = MPP_POLL_BLOCK; 212 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK, ¶mS32); 213 if (ret != MPP_OK) { 214 av_log(avctx, AV_LOG_ERROR, "Failed to set blocking mode on MPI (code = %d).\n", ret); 215 ret = AVERROR_UNKNOWN; 216 goto fail; 217 } 218 219 paramS64 = RECEIVE_FRAME_TIMEOUT; 220 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK_TIMEOUT, ¶mS64); 221 if (ret != MPP_OK) { 222 av_log(avctx, AV_LOG_ERROR, "Failed to set block timeout on MPI (code = %d).\n", ret); 223 ret = AVERROR_UNKNOWN; 224 goto fail; 225 } 226 227 ret = mpp_buffer_group_get_internal(&decoder->frame_group, MPP_BUFFER_TYPE_ION); 228 if (ret) { 229 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve buffer group (code = %d)\n", ret); 230 ret = AVERROR_UNKNOWN; 231 goto fail; 232 } 233 234 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_SET_EXT_BUF_GROUP, decoder->frame_group); 235 if (ret) { 236 av_log(avctx, AV_LOG_ERROR, "Failed to assign buffer group (code = %d)\n", ret); 237 ret = AVERROR_UNKNOWN; 238 goto fail; 239 } 240 241 ret = mpp_buffer_group_limit_config(decoder->frame_group, 0, FRAMEGROUP_MAX_FRAMES); 242 if (ret) { 243 av_log(avctx, AV_LOG_ERROR, "Failed to set buffer group limit (code = %d)\n", ret); 244 ret = AVERROR_UNKNOWN; 245 goto fail; 246 } 247 248 decoder->first_packet = 1; 249 250 av_log(avctx, AV_LOG_DEBUG, "RKMPP decoder initialized successfully.\n"); 251 252 decoder->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM); 253 if (!decoder->device_ref) { 254 ret = AVERROR(ENOMEM); 255 goto fail; 256 } 257 ret = av_hwdevice_ctx_init(decoder->device_ref); 258 if (ret < 0) 259 goto fail; 260 261 return 0; 262 263fail: 264 av_log(avctx, AV_LOG_ERROR, "Failed to initialize RKMPP decoder.\n"); 265 rkmpp_close_decoder(avctx); 266 return ret; 267} 268 269static int rkmpp_send_packet(AVCodecContext *avctx, const AVPacket *avpkt) 270{ 271 RKMPPDecodeContext *rk_context = avctx->priv_data; 272 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data; 273 int ret; 274 275 // handle EOF 276 if (!avpkt->size) { 277 av_log(avctx, AV_LOG_DEBUG, "End of stream.\n"); 278 decoder->eos_reached = 1; 279 ret = rkmpp_write_data(avctx, NULL, 0, 0); 280 if (ret) 281 av_log(avctx, AV_LOG_ERROR, "Failed to send EOS to decoder (code = %d)\n", ret); 282 return ret; 283 } 284 285 // on first packet, send extradata 286 if (decoder->first_packet) { 287 if (avctx->extradata_size) { 288 ret = rkmpp_write_data(avctx, avctx->extradata, 289 avctx->extradata_size, 290 avpkt->pts); 291 if (ret) { 292 av_log(avctx, AV_LOG_ERROR, "Failed to write extradata to decoder (code = %d)\n", ret); 293 return ret; 294 } 295 } 296 decoder->first_packet = 0; 297 } 298 299 // now send packet 300 ret = rkmpp_write_data(avctx, avpkt->data, avpkt->size, avpkt->pts); 301 if (ret && ret!=AVERROR(EAGAIN)) 302 av_log(avctx, AV_LOG_ERROR, "Failed to write data to decoder (code = %d)\n", ret); 303 304 return ret; 305} 306 307static void rkmpp_release_frame(void *opaque, uint8_t *data) 308{ 309 AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)data; 310 AVBufferRef *framecontextref = (AVBufferRef *)opaque; 311 RKMPPFrameContext *framecontext = (RKMPPFrameContext *)framecontextref->data; 312 313 mpp_frame_deinit(&framecontext->frame); 314 av_buffer_unref(&framecontext->decoder_ref); 315 av_buffer_unref(&framecontextref); 316 317 av_free(desc); 318} 319 320static int rkmpp_retrieve_frame(AVCodecContext *avctx, AVFrame *frame) 321{ 322 RKMPPDecodeContext *rk_context = avctx->priv_data; 323 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data; 324 RKMPPFrameContext *framecontext = NULL; 325 AVBufferRef *framecontextref = NULL; 326 int ret; 327 MppFrame mppframe = NULL; 328 MppBuffer buffer = NULL; 329 AVDRMFrameDescriptor *desc = NULL; 330 AVDRMLayerDescriptor *layer = NULL; 331 int mode; 332 MppFrameFormat mppformat; 333 uint32_t drmformat; 334 335 ret = decoder->mpi->decode_get_frame(decoder->ctx, &mppframe); 336 if (ret != MPP_OK && ret != MPP_ERR_TIMEOUT) { 337 av_log(avctx, AV_LOG_ERROR, "Failed to get a frame from MPP (code = %d)\n", ret); 338 goto fail; 339 } 340 341 if (mppframe) { 342 // Check whether we have a special frame or not 343 if (mpp_frame_get_info_change(mppframe)) { 344 AVHWFramesContext *hwframes; 345 346 av_log(avctx, AV_LOG_INFO, "Decoder noticed an info change (%dx%d), format=%d\n", 347 (int)mpp_frame_get_width(mppframe), (int)mpp_frame_get_height(mppframe), 348 (int)mpp_frame_get_fmt(mppframe)); 349 350 avctx->width = mpp_frame_get_width(mppframe); 351 avctx->height = mpp_frame_get_height(mppframe); 352 353 decoder->mpi->control(decoder->ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL); 354 355 av_buffer_unref(&decoder->frames_ref); 356 357 decoder->frames_ref = av_hwframe_ctx_alloc(decoder->device_ref); 358 if (!decoder->frames_ref) { 359 ret = AVERROR(ENOMEM); 360 goto fail; 361 } 362 363 mppformat = mpp_frame_get_fmt(mppframe); 364 drmformat = rkmpp_get_frameformat(mppformat); 365 366 hwframes = (AVHWFramesContext*)decoder->frames_ref->data; 367 hwframes->format = AV_PIX_FMT_DRM_PRIME; 368 hwframes->sw_format = drmformat == DRM_FORMAT_NV12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NONE; 369 hwframes->width = avctx->width; 370 hwframes->height = avctx->height; 371 ret = av_hwframe_ctx_init(decoder->frames_ref); 372 if (ret < 0) 373 goto fail; 374 375 // here decoder is fully initialized, we need to feed it again with data 376 ret = AVERROR(EAGAIN); 377 goto fail; 378 } else if (mpp_frame_get_eos(mppframe)) { 379 av_log(avctx, AV_LOG_DEBUG, "Received a EOS frame.\n"); 380 decoder->eos_reached = 1; 381 ret = AVERROR_EOF; 382 goto fail; 383 } else if (mpp_frame_get_discard(mppframe)) { 384 av_log(avctx, AV_LOG_DEBUG, "Received a discard frame.\n"); 385 ret = AVERROR(EAGAIN); 386 goto fail; 387 } else if (mpp_frame_get_errinfo(mppframe)) { 388 av_log(avctx, AV_LOG_ERROR, "Received a errinfo frame.\n"); 389 ret = AVERROR_UNKNOWN; 390 goto fail; 391 } 392 393 // here we should have a valid frame 394 av_log(avctx, AV_LOG_DEBUG, "Received a frame.\n"); 395 396 // setup general frame fields 397 frame->format = AV_PIX_FMT_DRM_PRIME; 398 frame->width = mpp_frame_get_width(mppframe); 399 frame->height = mpp_frame_get_height(mppframe); 400 frame->pts = mpp_frame_get_pts(mppframe); 401 frame->color_range = mpp_frame_get_color_range(mppframe); 402 frame->color_primaries = mpp_frame_get_color_primaries(mppframe); 403 frame->color_trc = mpp_frame_get_color_trc(mppframe); 404 frame->colorspace = mpp_frame_get_colorspace(mppframe); 405 406 mode = mpp_frame_get_mode(mppframe); 407 frame->interlaced_frame = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_DEINTERLACED); 408 frame->top_field_first = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_TOP_FIRST); 409 410 mppformat = mpp_frame_get_fmt(mppframe); 411 drmformat = rkmpp_get_frameformat(mppformat); 412 413 // now setup the frame buffer info 414 buffer = mpp_frame_get_buffer(mppframe); 415 if (buffer) { 416 desc = av_mallocz(sizeof(AVDRMFrameDescriptor)); 417 if (!desc) { 418 ret = AVERROR(ENOMEM); 419 goto fail; 420 } 421 422 desc->nb_objects = 1; 423 desc->objects[0].fd = mpp_buffer_get_fd(buffer); 424 desc->objects[0].size = mpp_buffer_get_size(buffer); 425 426 desc->nb_layers = 1; 427 layer = &desc->layers[0]; 428 layer->format = drmformat; 429 layer->nb_planes = 2; 430 431 layer->planes[0].object_index = 0; 432 layer->planes[0].offset = 0; 433 layer->planes[0].pitch = mpp_frame_get_hor_stride(mppframe); 434 435 layer->planes[1].object_index = 0; 436 layer->planes[1].offset = layer->planes[0].pitch * mpp_frame_get_ver_stride(mppframe); 437 layer->planes[1].pitch = layer->planes[0].pitch; 438 439 // we also allocate a struct in buf[0] that will allow to hold additionnal information 440 // for releasing properly MPP frames and decoder 441 framecontextref = av_buffer_allocz(sizeof(*framecontext)); 442 if (!framecontextref) { 443 ret = AVERROR(ENOMEM); 444 goto fail; 445 } 446 447 // MPP decoder needs to be closed only when all frames have been released. 448 framecontext = (RKMPPFrameContext *)framecontextref->data; 449 framecontext->decoder_ref = av_buffer_ref(rk_context->decoder_ref); 450 framecontext->frame = mppframe; 451 452 frame->data[0] = (uint8_t *)desc; 453 frame->buf[0] = av_buffer_create((uint8_t *)desc, sizeof(*desc), rkmpp_release_frame, 454 framecontextref, AV_BUFFER_FLAG_READONLY); 455 456 if (!frame->buf[0]) { 457 ret = AVERROR(ENOMEM); 458 goto fail; 459 } 460 461 frame->hw_frames_ctx = av_buffer_ref(decoder->frames_ref); 462 if (!frame->hw_frames_ctx) { 463 ret = AVERROR(ENOMEM); 464 goto fail; 465 } 466 467 return 0; 468 } else { 469 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret); 470 mpp_frame_deinit(&mppframe); 471 } 472 } else if (decoder->eos_reached) { 473 return AVERROR_EOF; 474 } else if (ret == MPP_ERR_TIMEOUT) { 475 av_log(avctx, AV_LOG_DEBUG, "Timeout when trying to get a frame from MPP\n"); 476 } 477 478 return AVERROR(EAGAIN); 479 480fail: 481 if (mppframe) 482 mpp_frame_deinit(&mppframe); 483 484 if (framecontext) 485 av_buffer_unref(&framecontext->decoder_ref); 486 487 if (framecontextref) 488 av_buffer_unref(&framecontextref); 489 490 if (desc) 491 av_free(desc); 492 493 return ret; 494} 495 496static int rkmpp_receive_frame(AVCodecContext *avctx, AVFrame *frame) 497{ 498 RKMPPDecodeContext *rk_context = avctx->priv_data; 499 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data; 500 int ret = MPP_NOK; 501 AVPacket pkt = {0}; 502 RK_S32 usedslots, freeslots; 503 504 if (!decoder->eos_reached) { 505 // we get the available slots in decoder 506 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_GET_STREAM_COUNT, &usedslots); 507 if (ret != MPP_OK) { 508 av_log(avctx, AV_LOG_ERROR, "Failed to get decoder used slots (code = %d).\n", ret); 509 return ret; 510 } 511 512 freeslots = INPUT_MAX_PACKETS - usedslots; 513 if (freeslots > 0) { 514 ret = ff_decode_get_packet(avctx, &pkt); 515 if (ret < 0 && ret != AVERROR_EOF) { 516 return ret; 517 } 518 519 ret = rkmpp_send_packet(avctx, &pkt); 520 av_packet_unref(&pkt); 521 522 if (ret < 0) { 523 av_log(avctx, AV_LOG_ERROR, "Failed to send packet to decoder (code = %d)\n", ret); 524 return ret; 525 } 526 } 527 528 // make sure we keep decoder full 529 if (freeslots > 1) 530 return AVERROR(EAGAIN); 531 } 532 533 return rkmpp_retrieve_frame(avctx, frame); 534} 535 536static void rkmpp_flush(AVCodecContext *avctx) 537{ 538 RKMPPDecodeContext *rk_context = avctx->priv_data; 539 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data; 540 int ret = MPP_NOK; 541 542 av_log(avctx, AV_LOG_DEBUG, "Flush.\n"); 543 544 ret = decoder->mpi->reset(decoder->ctx); 545 if (ret == MPP_OK) { 546 decoder->first_packet = 1; 547 } else 548 av_log(avctx, AV_LOG_ERROR, "Failed to reset MPI (code = %d)\n", ret); 549} 550 551static const AVCodecHWConfigInternal *const rkmpp_hw_configs[] = { 552 HW_CONFIG_INTERNAL(DRM_PRIME), 553 NULL 554}; 555 556#define RKMPP_DEC_CLASS(NAME) \ 557 static const AVClass rkmpp_##NAME##_dec_class = { \ 558 .class_name = "rkmpp_" #NAME "_dec", \ 559 .version = LIBAVUTIL_VERSION_INT, \ 560 }; 561 562#define RKMPP_DEC(NAME, ID, BSFS) \ 563 RKMPP_DEC_CLASS(NAME) \ 564 const FFCodec ff_##NAME##_rkmpp_decoder = { \ 565 .p.name = #NAME "_rkmpp", \ 566 .p.long_name = NULL_IF_CONFIG_SMALL(#NAME " (rkmpp)"), \ 567 .p.type = AVMEDIA_TYPE_VIDEO, \ 568 .p.id = ID, \ 569 .priv_data_size = sizeof(RKMPPDecodeContext), \ 570 .init = rkmpp_init_decoder, \ 571 .close = rkmpp_close_decoder, \ 572 FF_CODEC_RECEIVE_FRAME_CB(rkmpp_receive_frame), \ 573 .flush = rkmpp_flush, \ 574 .p.priv_class = &rkmpp_##NAME##_dec_class, \ 575 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \ 576 .p.pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ 577 AV_PIX_FMT_NONE}, \ 578 .hw_configs = rkmpp_hw_configs, \ 579 .bsfs = BSFS, \ 580 .p.wrapper_name = "rkmpp", \ 581 }; 582 583RKMPP_DEC(h264, AV_CODEC_ID_H264, "h264_mp4toannexb") 584RKMPP_DEC(hevc, AV_CODEC_ID_HEVC, "hevc_mp4toannexb") 585RKMPP_DEC(vp8, AV_CODEC_ID_VP8, NULL) 586RKMPP_DEC(vp9, AV_CODEC_ID_VP9, NULL) 587