1/* 2 * copyright (c) 2015 Rick Kern <kernrj@gmail.com> 3 * 4 * This file is part of FFmpeg. 5 * 6 * FFmpeg is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * FFmpeg is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with FFmpeg; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 */ 20 21#include <VideoToolbox/VideoToolbox.h> 22#include <CoreVideo/CoreVideo.h> 23#include <CoreMedia/CoreMedia.h> 24#include <TargetConditionals.h> 25#include <Availability.h> 26#include "avcodec.h" 27#include "libavutil/opt.h" 28#include "libavutil/avassert.h" 29#include "libavutil/avstring.h" 30#include "libavcodec/avcodec.h" 31#include "libavutil/pixdesc.h" 32#include "libavutil/hwcontext_videotoolbox.h" 33#include "codec_internal.h" 34#include "internal.h" 35#include <pthread.h> 36#include "atsc_a53.h" 37#include "encode.h" 38#include "h264.h" 39#include "h264_sei.h" 40#include <dlfcn.h> 41 42#if !HAVE_KCMVIDEOCODECTYPE_HEVC 43enum { kCMVideoCodecType_HEVC = 'hvc1' }; 44#endif 45 46#if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA 47enum { kCMVideoCodecType_HEVCWithAlpha = 'muxa' }; 48#endif 49 50#if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE 51enum { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20' }; 52enum { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420' }; 53#endif 54 55#ifndef TARGET_CPU_ARM64 56# define TARGET_CPU_ARM64 0 57#endif 58 59typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc, 60 size_t parameterSetIndex, 61 const uint8_t **parameterSetPointerOut, 62 size_t *parameterSetSizeOut, 63 size_t *parameterSetCountOut, 64 int *NALUnitHeaderLengthOut); 65 66//These symbols may not be present 67static struct{ 68 CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020; 69 CFStringRef kCVImageBufferTransferFunction_ITU_R_2020; 70 CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020; 71 72 CFStringRef kVTCompressionPropertyKey_H264EntropyMode; 73 CFStringRef kVTH264EntropyMode_CAVLC; 74 CFStringRef kVTH264EntropyMode_CABAC; 75 76 CFStringRef kVTProfileLevel_H264_Baseline_4_0; 77 CFStringRef kVTProfileLevel_H264_Baseline_4_2; 78 CFStringRef kVTProfileLevel_H264_Baseline_5_0; 79 CFStringRef kVTProfileLevel_H264_Baseline_5_1; 80 CFStringRef kVTProfileLevel_H264_Baseline_5_2; 81 CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel; 82 CFStringRef kVTProfileLevel_H264_Main_4_2; 83 CFStringRef kVTProfileLevel_H264_Main_5_1; 84 CFStringRef kVTProfileLevel_H264_Main_5_2; 85 CFStringRef kVTProfileLevel_H264_Main_AutoLevel; 86 CFStringRef kVTProfileLevel_H264_High_3_0; 87 CFStringRef kVTProfileLevel_H264_High_3_1; 88 CFStringRef kVTProfileLevel_H264_High_3_2; 89 CFStringRef kVTProfileLevel_H264_High_4_0; 90 CFStringRef kVTProfileLevel_H264_High_4_1; 91 CFStringRef kVTProfileLevel_H264_High_4_2; 92 CFStringRef kVTProfileLevel_H264_High_5_1; 93 CFStringRef kVTProfileLevel_H264_High_5_2; 94 CFStringRef kVTProfileLevel_H264_High_AutoLevel; 95 CFStringRef kVTProfileLevel_H264_Extended_5_0; 96 CFStringRef kVTProfileLevel_H264_Extended_AutoLevel; 97 98 CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel; 99 CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel; 100 101 CFStringRef kVTCompressionPropertyKey_RealTime; 102 CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha; 103 CFStringRef kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality; 104 105 CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder; 106 CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder; 107 108 getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex; 109} compat_keys; 110 111#define GET_SYM(symbol, defaultVal) \ 112do{ \ 113 CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol); \ 114 if(!handle) \ 115 compat_keys.symbol = CFSTR(defaultVal); \ 116 else \ 117 compat_keys.symbol = *handle; \ 118}while(0) 119 120static pthread_once_t once_ctrl = PTHREAD_ONCE_INIT; 121 122static void loadVTEncSymbols(){ 123 compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex = 124 (getParameterSetAtIndex)dlsym( 125 RTLD_DEFAULT, 126 "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex" 127 ); 128 129 GET_SYM(kCVImageBufferColorPrimaries_ITU_R_2020, "ITU_R_2020"); 130 GET_SYM(kCVImageBufferTransferFunction_ITU_R_2020, "ITU_R_2020"); 131 GET_SYM(kCVImageBufferYCbCrMatrix_ITU_R_2020, "ITU_R_2020"); 132 133 GET_SYM(kVTCompressionPropertyKey_H264EntropyMode, "H264EntropyMode"); 134 GET_SYM(kVTH264EntropyMode_CAVLC, "CAVLC"); 135 GET_SYM(kVTH264EntropyMode_CABAC, "CABAC"); 136 137 GET_SYM(kVTProfileLevel_H264_Baseline_4_0, "H264_Baseline_4_0"); 138 GET_SYM(kVTProfileLevel_H264_Baseline_4_2, "H264_Baseline_4_2"); 139 GET_SYM(kVTProfileLevel_H264_Baseline_5_0, "H264_Baseline_5_0"); 140 GET_SYM(kVTProfileLevel_H264_Baseline_5_1, "H264_Baseline_5_1"); 141 GET_SYM(kVTProfileLevel_H264_Baseline_5_2, "H264_Baseline_5_2"); 142 GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel"); 143 GET_SYM(kVTProfileLevel_H264_Main_4_2, "H264_Main_4_2"); 144 GET_SYM(kVTProfileLevel_H264_Main_5_1, "H264_Main_5_1"); 145 GET_SYM(kVTProfileLevel_H264_Main_5_2, "H264_Main_5_2"); 146 GET_SYM(kVTProfileLevel_H264_Main_AutoLevel, "H264_Main_AutoLevel"); 147 GET_SYM(kVTProfileLevel_H264_High_3_0, "H264_High_3_0"); 148 GET_SYM(kVTProfileLevel_H264_High_3_1, "H264_High_3_1"); 149 GET_SYM(kVTProfileLevel_H264_High_3_2, "H264_High_3_2"); 150 GET_SYM(kVTProfileLevel_H264_High_4_0, "H264_High_4_0"); 151 GET_SYM(kVTProfileLevel_H264_High_4_1, "H264_High_4_1"); 152 GET_SYM(kVTProfileLevel_H264_High_4_2, "H264_High_4_2"); 153 GET_SYM(kVTProfileLevel_H264_High_5_1, "H264_High_5_1"); 154 GET_SYM(kVTProfileLevel_H264_High_5_2, "H264_High_5_2"); 155 GET_SYM(kVTProfileLevel_H264_High_AutoLevel, "H264_High_AutoLevel"); 156 GET_SYM(kVTProfileLevel_H264_Extended_5_0, "H264_Extended_5_0"); 157 GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel"); 158 159 GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel, "HEVC_Main_AutoLevel"); 160 GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel, "HEVC_Main10_AutoLevel"); 161 162 GET_SYM(kVTCompressionPropertyKey_RealTime, "RealTime"); 163 GET_SYM(kVTCompressionPropertyKey_TargetQualityForAlpha, 164 "TargetQualityForAlpha"); 165 GET_SYM(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, 166 "PrioritizeEncodingSpeedOverQuality"); 167 168 GET_SYM(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, 169 "EnableHardwareAcceleratedVideoEncoder"); 170 GET_SYM(kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder, 171 "RequireHardwareAcceleratedVideoEncoder"); 172} 173 174typedef enum VT_H264Profile { 175 H264_PROF_AUTO, 176 H264_PROF_BASELINE, 177 H264_PROF_MAIN, 178 H264_PROF_HIGH, 179 H264_PROF_EXTENDED, 180 H264_PROF_COUNT 181} VT_H264Profile; 182 183typedef enum VTH264Entropy{ 184 VT_ENTROPY_NOT_SET, 185 VT_CAVLC, 186 VT_CABAC 187} VTH264Entropy; 188 189typedef enum VT_HEVCProfile { 190 HEVC_PROF_AUTO, 191 HEVC_PROF_MAIN, 192 HEVC_PROF_MAIN10, 193 HEVC_PROF_COUNT 194} VT_HEVCProfile; 195 196static const uint8_t start_code[] = { 0, 0, 0, 1 }; 197 198typedef struct ExtraSEI { 199 void *data; 200 size_t size; 201} ExtraSEI; 202 203typedef struct BufNode { 204 CMSampleBufferRef cm_buffer; 205 ExtraSEI *sei; 206 struct BufNode* next; 207 int error; 208} BufNode; 209 210typedef struct VTEncContext { 211 AVClass *class; 212 enum AVCodecID codec_id; 213 VTCompressionSessionRef session; 214 CFStringRef ycbcr_matrix; 215 CFStringRef color_primaries; 216 CFStringRef transfer_function; 217 getParameterSetAtIndex get_param_set_func; 218 219 pthread_mutex_t lock; 220 pthread_cond_t cv_sample_sent; 221 222 int async_error; 223 224 BufNode *q_head; 225 BufNode *q_tail; 226 227 int64_t frame_ct_out; 228 int64_t frame_ct_in; 229 230 int64_t first_pts; 231 int64_t dts_delta; 232 233 int64_t profile; 234 int level; 235 int entropy; 236 int realtime; 237 int frames_before; 238 int frames_after; 239 240 int allow_sw; 241 int require_sw; 242 double alpha_quality; 243 int prio_speed; 244 245 bool flushing; 246 int has_b_frames; 247 bool warned_color_range; 248 249 /* can't be bool type since AVOption will access it as int */ 250 int a53_cc; 251} VTEncContext; 252 253static int vtenc_populate_extradata(AVCodecContext *avctx, 254 CMVideoCodecType codec_type, 255 CFStringRef profile_level, 256 CFNumberRef gamma_level, 257 CFDictionaryRef enc_info, 258 CFDictionaryRef pixel_buffer_info); 259 260/** 261 * NULL-safe release of *refPtr, and sets value to NULL. 262 */ 263static void vt_release_num(CFNumberRef* refPtr){ 264 if (!*refPtr) { 265 return; 266 } 267 268 CFRelease(*refPtr); 269 *refPtr = NULL; 270} 271 272static void set_async_error(VTEncContext *vtctx, int err) 273{ 274 BufNode *info; 275 276 pthread_mutex_lock(&vtctx->lock); 277 278 vtctx->async_error = err; 279 280 info = vtctx->q_head; 281 vtctx->q_head = vtctx->q_tail = NULL; 282 283 while (info) { 284 BufNode *next = info->next; 285 CFRelease(info->cm_buffer); 286 av_free(info); 287 info = next; 288 } 289 290 pthread_mutex_unlock(&vtctx->lock); 291} 292 293static void clear_frame_queue(VTEncContext *vtctx) 294{ 295 set_async_error(vtctx, 0); 296} 297 298static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei) 299{ 300 BufNode *info; 301 302 pthread_mutex_lock(&vtctx->lock); 303 304 if (vtctx->async_error) { 305 pthread_mutex_unlock(&vtctx->lock); 306 return vtctx->async_error; 307 } 308 309 if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) { 310 *buf = NULL; 311 312 pthread_mutex_unlock(&vtctx->lock); 313 return 0; 314 } 315 316 while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) { 317 pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock); 318 } 319 320 if (!vtctx->q_head) { 321 pthread_mutex_unlock(&vtctx->lock); 322 *buf = NULL; 323 return 0; 324 } 325 326 info = vtctx->q_head; 327 vtctx->q_head = vtctx->q_head->next; 328 if (!vtctx->q_head) { 329 vtctx->q_tail = NULL; 330 } 331 332 vtctx->frame_ct_out++; 333 pthread_mutex_unlock(&vtctx->lock); 334 335 *buf = info->cm_buffer; 336 if (sei && *buf) { 337 *sei = info->sei; 338 } else if (info->sei) { 339 if (info->sei->data) av_free(info->sei->data); 340 av_free(info->sei); 341 } 342 av_free(info); 343 344 345 return 0; 346} 347 348static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei) 349{ 350 BufNode *info = av_malloc(sizeof(BufNode)); 351 if (!info) { 352 set_async_error(vtctx, AVERROR(ENOMEM)); 353 return; 354 } 355 356 CFRetain(buffer); 357 info->cm_buffer = buffer; 358 info->sei = sei; 359 info->next = NULL; 360 361 pthread_mutex_lock(&vtctx->lock); 362 363 if (!vtctx->q_head) { 364 vtctx->q_head = info; 365 } else { 366 vtctx->q_tail->next = info; 367 } 368 369 vtctx->q_tail = info; 370 371 pthread_cond_signal(&vtctx->cv_sample_sent); 372 pthread_mutex_unlock(&vtctx->lock); 373} 374 375static int count_nalus(size_t length_code_size, 376 CMSampleBufferRef sample_buffer, 377 int *count) 378{ 379 size_t offset = 0; 380 int status; 381 int nalu_ct = 0; 382 uint8_t size_buf[4]; 383 size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer); 384 CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer); 385 386 if (length_code_size > 4) 387 return AVERROR_INVALIDDATA; 388 389 while (offset < src_size) { 390 size_t curr_src_len; 391 size_t box_len = 0; 392 size_t i; 393 394 status = CMBlockBufferCopyDataBytes(block, 395 offset, 396 length_code_size, 397 size_buf); 398 399 for (i = 0; i < length_code_size; i++) { 400 box_len <<= 8; 401 box_len |= size_buf[i]; 402 } 403 404 curr_src_len = box_len + length_code_size; 405 offset += curr_src_len; 406 407 nalu_ct++; 408 } 409 410 *count = nalu_ct; 411 return 0; 412} 413 414static CMVideoCodecType get_cm_codec_type(AVCodecContext *avctx, 415 int64_t profile, 416 double alpha_quality) 417{ 418 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX ? avctx->sw_pix_fmt : avctx->pix_fmt); 419 switch (avctx->codec_id) { 420 case AV_CODEC_ID_H264: return kCMVideoCodecType_H264; 421 case AV_CODEC_ID_HEVC: 422 if (desc && (desc->flags & AV_PIX_FMT_FLAG_ALPHA) && alpha_quality > 0.0) { 423 return kCMVideoCodecType_HEVCWithAlpha; 424 } 425 return kCMVideoCodecType_HEVC; 426 case AV_CODEC_ID_PRORES: 427 switch (profile) { 428 case FF_PROFILE_PRORES_PROXY: 429 return MKBETAG('a','p','c','o'); // kCMVideoCodecType_AppleProRes422Proxy 430 case FF_PROFILE_PRORES_LT: 431 return MKBETAG('a','p','c','s'); // kCMVideoCodecType_AppleProRes422LT 432 case FF_PROFILE_PRORES_STANDARD: 433 return MKBETAG('a','p','c','n'); // kCMVideoCodecType_AppleProRes422 434 case FF_PROFILE_PRORES_HQ: 435 return MKBETAG('a','p','c','h'); // kCMVideoCodecType_AppleProRes422HQ 436 case FF_PROFILE_PRORES_4444: 437 return MKBETAG('a','p','4','h'); // kCMVideoCodecType_AppleProRes4444 438 case FF_PROFILE_PRORES_XQ: 439 return MKBETAG('a','p','4','x'); // kCMVideoCodecType_AppleProRes4444XQ 440 441 default: 442 av_log(avctx, AV_LOG_ERROR, "Unknown profile ID: %"PRId64", using auto\n", profile); 443 case FF_PROFILE_UNKNOWN: 444 if (desc && 445 ((desc->flags & AV_PIX_FMT_FLAG_ALPHA) || 446 desc->log2_chroma_w == 0)) 447 return MKBETAG('a','p','4','h'); // kCMVideoCodecType_AppleProRes4444 448 else 449 return MKBETAG('a','p','c','n'); // kCMVideoCodecType_AppleProRes422 450 } 451 default: return 0; 452 } 453} 454 455/** 456 * Get the parameter sets from a CMSampleBufferRef. 457 * @param dst If *dst isn't NULL, the parameters are copied into existing 458 * memory. *dst_size must be set accordingly when *dst != NULL. 459 * If *dst is NULL, it will be allocated. 460 * In all cases, *dst_size is set to the number of bytes used starting 461 * at *dst. 462 */ 463static int get_params_size( 464 AVCodecContext *avctx, 465 CMVideoFormatDescriptionRef vid_fmt, 466 size_t *size) 467{ 468 VTEncContext *vtctx = avctx->priv_data; 469 size_t total_size = 0; 470 size_t ps_count; 471 int is_count_bad = 0; 472 size_t i; 473 int status; 474 status = vtctx->get_param_set_func(vid_fmt, 475 0, 476 NULL, 477 NULL, 478 &ps_count, 479 NULL); 480 if (status) { 481 is_count_bad = 1; 482 ps_count = 0; 483 status = 0; 484 } 485 486 for (i = 0; i < ps_count || is_count_bad; i++) { 487 const uint8_t *ps; 488 size_t ps_size; 489 status = vtctx->get_param_set_func(vid_fmt, 490 i, 491 &ps, 492 &ps_size, 493 NULL, 494 NULL); 495 if (status) { 496 /* 497 * When ps_count is invalid, status != 0 ends the loop normally 498 * unless we didn't get any parameter sets. 499 */ 500 if (i > 0 && is_count_bad) status = 0; 501 502 break; 503 } 504 505 total_size += ps_size + sizeof(start_code); 506 } 507 508 if (status) { 509 av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status); 510 return AVERROR_EXTERNAL; 511 } 512 513 *size = total_size; 514 return 0; 515} 516 517static int copy_param_sets( 518 AVCodecContext *avctx, 519 CMVideoFormatDescriptionRef vid_fmt, 520 uint8_t *dst, 521 size_t dst_size) 522{ 523 VTEncContext *vtctx = avctx->priv_data; 524 size_t ps_count; 525 int is_count_bad = 0; 526 int status; 527 size_t offset = 0; 528 size_t i; 529 530 status = vtctx->get_param_set_func(vid_fmt, 531 0, 532 NULL, 533 NULL, 534 &ps_count, 535 NULL); 536 if (status) { 537 is_count_bad = 1; 538 ps_count = 0; 539 status = 0; 540 } 541 542 543 for (i = 0; i < ps_count || is_count_bad; i++) { 544 const uint8_t *ps; 545 size_t ps_size; 546 size_t next_offset; 547 548 status = vtctx->get_param_set_func(vid_fmt, 549 i, 550 &ps, 551 &ps_size, 552 NULL, 553 NULL); 554 if (status) { 555 if (i > 0 && is_count_bad) status = 0; 556 557 break; 558 } 559 560 next_offset = offset + sizeof(start_code) + ps_size; 561 if (dst_size < next_offset) { 562 av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n"); 563 return AVERROR_BUFFER_TOO_SMALL; 564 } 565 566 memcpy(dst + offset, start_code, sizeof(start_code)); 567 offset += sizeof(start_code); 568 569 memcpy(dst + offset, ps, ps_size); 570 offset = next_offset; 571 } 572 573 if (status) { 574 av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status); 575 return AVERROR_EXTERNAL; 576 } 577 578 return 0; 579} 580 581static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer) 582{ 583 VTEncContext *vtctx = avctx->priv_data; 584 CMVideoFormatDescriptionRef vid_fmt; 585 size_t total_size; 586 int status; 587 588 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer); 589 if (!vid_fmt) { 590 av_log(avctx, AV_LOG_ERROR, "No video format.\n"); 591 return AVERROR_EXTERNAL; 592 } 593 594 if (vtctx->get_param_set_func) { 595 status = get_params_size(avctx, vid_fmt, &total_size); 596 if (status) { 597 av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n"); 598 return status; 599 } 600 601 avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE); 602 if (!avctx->extradata) { 603 return AVERROR(ENOMEM); 604 } 605 avctx->extradata_size = total_size; 606 607 status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size); 608 609 if (status) { 610 av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n"); 611 return status; 612 } 613 } else { 614 CFDataRef data = CMFormatDescriptionGetExtension(vid_fmt, kCMFormatDescriptionExtension_VerbatimSampleDescription); 615 if (data && CFGetTypeID(data) == CFDataGetTypeID()) { 616 CFIndex size = CFDataGetLength(data); 617 618 avctx->extradata = av_mallocz(size + AV_INPUT_BUFFER_PADDING_SIZE); 619 if (!avctx->extradata) 620 return AVERROR(ENOMEM); 621 avctx->extradata_size = size; 622 623 CFDataGetBytes(data, CFRangeMake(0, size), avctx->extradata); 624 } 625 } 626 627 return 0; 628} 629 630static void vtenc_output_callback( 631 void *ctx, 632 void *sourceFrameCtx, 633 OSStatus status, 634 VTEncodeInfoFlags flags, 635 CMSampleBufferRef sample_buffer) 636{ 637 AVCodecContext *avctx = ctx; 638 VTEncContext *vtctx = avctx->priv_data; 639 ExtraSEI *sei = sourceFrameCtx; 640 641 if (vtctx->async_error) { 642 return; 643 } 644 645 if (status) { 646 av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status); 647 set_async_error(vtctx, AVERROR_EXTERNAL); 648 return; 649 } 650 651 if (!sample_buffer) { 652 return; 653 } 654 655 if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) { 656 int set_status = set_extradata(avctx, sample_buffer); 657 if (set_status) { 658 set_async_error(vtctx, set_status); 659 return; 660 } 661 } 662 663 vtenc_q_push(vtctx, sample_buffer, sei); 664} 665 666static int get_length_code_size( 667 AVCodecContext *avctx, 668 CMSampleBufferRef sample_buffer, 669 size_t *size) 670{ 671 VTEncContext *vtctx = avctx->priv_data; 672 CMVideoFormatDescriptionRef vid_fmt; 673 int isize; 674 int status; 675 676 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer); 677 if (!vid_fmt) { 678 av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n"); 679 return AVERROR_EXTERNAL; 680 } 681 682 status = vtctx->get_param_set_func(vid_fmt, 683 0, 684 NULL, 685 NULL, 686 NULL, 687 &isize); 688 if (status) { 689 av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status); 690 return AVERROR_EXTERNAL; 691 } 692 693 *size = isize; 694 return 0; 695} 696 697/* 698 * Returns true on success. 699 * 700 * If profile_level_val is NULL and this method returns true, don't specify the 701 * profile/level to the encoder. 702 */ 703static bool get_vt_h264_profile_level(AVCodecContext *avctx, 704 CFStringRef *profile_level_val) 705{ 706 VTEncContext *vtctx = avctx->priv_data; 707 int64_t profile = vtctx->profile; 708 709 if (profile == H264_PROF_AUTO && vtctx->level) { 710 //Need to pick a profile if level is not auto-selected. 711 profile = vtctx->has_b_frames ? H264_PROF_MAIN : H264_PROF_BASELINE; 712 } 713 714 *profile_level_val = NULL; 715 716 switch (profile) { 717 case H264_PROF_AUTO: 718 return true; 719 720 case H264_PROF_BASELINE: 721 switch (vtctx->level) { 722 case 0: *profile_level_val = 723 compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break; 724 case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break; 725 case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break; 726 case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break; 727 case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break; 728 case 40: *profile_level_val = 729 compat_keys.kVTProfileLevel_H264_Baseline_4_0; break; 730 case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break; 731 case 42: *profile_level_val = 732 compat_keys.kVTProfileLevel_H264_Baseline_4_2; break; 733 case 50: *profile_level_val = 734 compat_keys.kVTProfileLevel_H264_Baseline_5_0; break; 735 case 51: *profile_level_val = 736 compat_keys.kVTProfileLevel_H264_Baseline_5_1; break; 737 case 52: *profile_level_val = 738 compat_keys.kVTProfileLevel_H264_Baseline_5_2; break; 739 } 740 break; 741 742 case H264_PROF_MAIN: 743 switch (vtctx->level) { 744 case 0: *profile_level_val = 745 compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break; 746 case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break; 747 case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break; 748 case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break; 749 case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break; 750 case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break; 751 case 42: *profile_level_val = 752 compat_keys.kVTProfileLevel_H264_Main_4_2; break; 753 case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break; 754 case 51: *profile_level_val = 755 compat_keys.kVTProfileLevel_H264_Main_5_1; break; 756 case 52: *profile_level_val = 757 compat_keys.kVTProfileLevel_H264_Main_5_2; break; 758 } 759 break; 760 761 case H264_PROF_HIGH: 762 switch (vtctx->level) { 763 case 0: *profile_level_val = 764 compat_keys.kVTProfileLevel_H264_High_AutoLevel; break; 765 case 30: *profile_level_val = 766 compat_keys.kVTProfileLevel_H264_High_3_0; break; 767 case 31: *profile_level_val = 768 compat_keys.kVTProfileLevel_H264_High_3_1; break; 769 case 32: *profile_level_val = 770 compat_keys.kVTProfileLevel_H264_High_3_2; break; 771 case 40: *profile_level_val = 772 compat_keys.kVTProfileLevel_H264_High_4_0; break; 773 case 41: *profile_level_val = 774 compat_keys.kVTProfileLevel_H264_High_4_1; break; 775 case 42: *profile_level_val = 776 compat_keys.kVTProfileLevel_H264_High_4_2; break; 777 case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break; 778 case 51: *profile_level_val = 779 compat_keys.kVTProfileLevel_H264_High_5_1; break; 780 case 52: *profile_level_val = 781 compat_keys.kVTProfileLevel_H264_High_5_2; break; 782 } 783 break; 784 case H264_PROF_EXTENDED: 785 switch (vtctx->level) { 786 case 0: *profile_level_val = 787 compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break; 788 case 50: *profile_level_val = 789 compat_keys.kVTProfileLevel_H264_Extended_5_0; break; 790 } 791 break; 792 } 793 794 if (!*profile_level_val) { 795 av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n"); 796 return false; 797 } 798 799 return true; 800} 801 802/* 803 * Returns true on success. 804 * 805 * If profile_level_val is NULL and this method returns true, don't specify the 806 * profile/level to the encoder. 807 */ 808static bool get_vt_hevc_profile_level(AVCodecContext *avctx, 809 CFStringRef *profile_level_val) 810{ 811 VTEncContext *vtctx = avctx->priv_data; 812 int64_t profile = vtctx->profile; 813 814 *profile_level_val = NULL; 815 816 switch (profile) { 817 case HEVC_PROF_AUTO: 818 return true; 819 case HEVC_PROF_MAIN: 820 *profile_level_val = 821 compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel; 822 break; 823 case HEVC_PROF_MAIN10: 824 *profile_level_val = 825 compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel; 826 break; 827 } 828 829 if (!*profile_level_val) { 830 av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n"); 831 return false; 832 } 833 834 return true; 835} 836 837static int get_cv_pixel_format(AVCodecContext* avctx, 838 enum AVPixelFormat fmt, 839 enum AVColorRange range, 840 int* av_pixel_format, 841 int* range_guessed) 842{ 843 if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG && 844 range != AVCOL_RANGE_JPEG; 845 846 //MPEG range is used when no range is set 847 *av_pixel_format = av_map_videotoolbox_format_from_pixfmt2(fmt, range == AVCOL_RANGE_JPEG); 848 849 return *av_pixel_format ? 0 : AVERROR(EINVAL); 850} 851 852static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) { 853 VTEncContext *vtctx = avctx->priv_data; 854 855 if (vtctx->color_primaries) { 856 CFDictionarySetValue(dict, 857 kCVImageBufferColorPrimariesKey, 858 vtctx->color_primaries); 859 } 860 861 if (vtctx->transfer_function) { 862 CFDictionarySetValue(dict, 863 kCVImageBufferTransferFunctionKey, 864 vtctx->transfer_function); 865 } 866 867 if (vtctx->ycbcr_matrix) { 868 CFDictionarySetValue(dict, 869 kCVImageBufferYCbCrMatrixKey, 870 vtctx->ycbcr_matrix); 871 } 872} 873 874static int create_cv_pixel_buffer_info(AVCodecContext* avctx, 875 CFMutableDictionaryRef* dict) 876{ 877 CFNumberRef cv_color_format_num = NULL; 878 CFNumberRef width_num = NULL; 879 CFNumberRef height_num = NULL; 880 CFMutableDictionaryRef pixel_buffer_info = NULL; 881 int cv_color_format; 882 int status = get_cv_pixel_format(avctx, 883 avctx->pix_fmt, 884 avctx->color_range, 885 &cv_color_format, 886 NULL); 887 if (status) return status; 888 889 pixel_buffer_info = CFDictionaryCreateMutable( 890 kCFAllocatorDefault, 891 20, 892 &kCFCopyStringDictionaryKeyCallBacks, 893 &kCFTypeDictionaryValueCallBacks); 894 895 if (!pixel_buffer_info) goto pbinfo_nomem; 896 897 cv_color_format_num = CFNumberCreate(kCFAllocatorDefault, 898 kCFNumberSInt32Type, 899 &cv_color_format); 900 if (!cv_color_format_num) goto pbinfo_nomem; 901 902 CFDictionarySetValue(pixel_buffer_info, 903 kCVPixelBufferPixelFormatTypeKey, 904 cv_color_format_num); 905 vt_release_num(&cv_color_format_num); 906 907 width_num = CFNumberCreate(kCFAllocatorDefault, 908 kCFNumberSInt32Type, 909 &avctx->width); 910 if (!width_num) goto pbinfo_nomem; 911 912 CFDictionarySetValue(pixel_buffer_info, 913 kCVPixelBufferWidthKey, 914 width_num); 915 vt_release_num(&width_num); 916 917 height_num = CFNumberCreate(kCFAllocatorDefault, 918 kCFNumberSInt32Type, 919 &avctx->height); 920 if (!height_num) goto pbinfo_nomem; 921 922 CFDictionarySetValue(pixel_buffer_info, 923 kCVPixelBufferHeightKey, 924 height_num); 925 vt_release_num(&height_num); 926 927 add_color_attr(avctx, pixel_buffer_info); 928 929 *dict = pixel_buffer_info; 930 return 0; 931 932pbinfo_nomem: 933 vt_release_num(&cv_color_format_num); 934 vt_release_num(&width_num); 935 vt_release_num(&height_num); 936 if (pixel_buffer_info) CFRelease(pixel_buffer_info); 937 938 return AVERROR(ENOMEM); 939} 940 941static int get_cv_color_primaries(AVCodecContext *avctx, 942 CFStringRef *primaries) 943{ 944 enum AVColorPrimaries pri = avctx->color_primaries; 945 switch (pri) { 946 case AVCOL_PRI_UNSPECIFIED: 947 *primaries = NULL; 948 break; 949 950 case AVCOL_PRI_BT470BG: 951 *primaries = kCVImageBufferColorPrimaries_EBU_3213; 952 break; 953 954 case AVCOL_PRI_SMPTE170M: 955 *primaries = kCVImageBufferColorPrimaries_SMPTE_C; 956 break; 957 958 case AVCOL_PRI_BT709: 959 *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2; 960 break; 961 962 case AVCOL_PRI_BT2020: 963 *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020; 964 break; 965 966 default: 967 av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri)); 968 *primaries = NULL; 969 return -1; 970 } 971 972 return 0; 973} 974 975static int get_cv_transfer_function(AVCodecContext *avctx, 976 CFStringRef *transfer_fnc, 977 CFNumberRef *gamma_level) 978{ 979 enum AVColorTransferCharacteristic trc = avctx->color_trc; 980 Float32 gamma; 981 *gamma_level = NULL; 982 983 switch (trc) { 984 case AVCOL_TRC_UNSPECIFIED: 985 *transfer_fnc = NULL; 986 break; 987 988 case AVCOL_TRC_BT709: 989 *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2; 990 break; 991 992 case AVCOL_TRC_SMPTE240M: 993 *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995; 994 break; 995 996#if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ 997 case AVCOL_TRC_SMPTE2084: 998 *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; 999 break; 1000#endif 1001#if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR 1002 case AVCOL_TRC_LINEAR: 1003 *transfer_fnc = kCVImageBufferTransferFunction_Linear; 1004 break; 1005#endif 1006#if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG 1007 case AVCOL_TRC_ARIB_STD_B67: 1008 *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG; 1009 break; 1010#endif 1011 1012 case AVCOL_TRC_GAMMA22: 1013 gamma = 2.2; 1014 *transfer_fnc = kCVImageBufferTransferFunction_UseGamma; 1015 *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); 1016 break; 1017 1018 case AVCOL_TRC_GAMMA28: 1019 gamma = 2.8; 1020 *transfer_fnc = kCVImageBufferTransferFunction_UseGamma; 1021 *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); 1022 break; 1023 1024 case AVCOL_TRC_BT2020_10: 1025 case AVCOL_TRC_BT2020_12: 1026 *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020; 1027 break; 1028 1029 default: 1030 *transfer_fnc = NULL; 1031 av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc)); 1032 return -1; 1033 } 1034 1035 return 0; 1036} 1037 1038static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) { 1039 switch(avctx->colorspace) { 1040 case AVCOL_SPC_BT709: 1041 *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; 1042 break; 1043 1044 case AVCOL_SPC_UNSPECIFIED: 1045 case AVCOL_SPC_RGB: 1046 *matrix = NULL; 1047 break; 1048 1049 case AVCOL_SPC_BT470BG: 1050 case AVCOL_SPC_SMPTE170M: 1051 *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; 1052 break; 1053 1054 case AVCOL_SPC_SMPTE240M: 1055 *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; 1056 break; 1057 1058 case AVCOL_SPC_BT2020_NCL: 1059 *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020; 1060 break; 1061 1062 default: 1063 av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace)); 1064 return -1; 1065 } 1066 1067 return 0; 1068} 1069 1070// constant quality only on Macs with Apple Silicon 1071static bool vtenc_qscale_enabled(void) 1072{ 1073 return !TARGET_OS_IPHONE && TARGET_CPU_ARM64; 1074} 1075 1076static int vtenc_create_encoder(AVCodecContext *avctx, 1077 CMVideoCodecType codec_type, 1078 CFStringRef profile_level, 1079 CFNumberRef gamma_level, 1080 CFDictionaryRef enc_info, 1081 CFDictionaryRef pixel_buffer_info, 1082 VTCompressionSessionRef *session) 1083{ 1084 VTEncContext *vtctx = avctx->priv_data; 1085 SInt32 bit_rate = avctx->bit_rate; 1086 SInt32 max_rate = avctx->rc_max_rate; 1087 Float32 quality = avctx->global_quality / FF_QP2LAMBDA; 1088 CFNumberRef bit_rate_num; 1089 CFNumberRef quality_num; 1090 CFNumberRef bytes_per_second; 1091 CFNumberRef one_second; 1092 CFArrayRef data_rate_limits; 1093 int64_t bytes_per_second_value = 0; 1094 int64_t one_second_value = 0; 1095 void *nums[2]; 1096 1097 int status = VTCompressionSessionCreate(kCFAllocatorDefault, 1098 avctx->width, 1099 avctx->height, 1100 codec_type, 1101 enc_info, 1102 pixel_buffer_info, 1103 kCFAllocatorDefault, 1104 vtenc_output_callback, 1105 avctx, 1106 session); 1107 1108 if (status || !vtctx->session) { 1109 av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status); 1110 1111#if !TARGET_OS_IPHONE 1112 if (!vtctx->allow_sw) { 1113 av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n"); 1114 } 1115#endif 1116 1117 return AVERROR_EXTERNAL; 1118 } 1119 1120 if (avctx->flags & AV_CODEC_FLAG_QSCALE && !vtenc_qscale_enabled()) { 1121 av_log(avctx, AV_LOG_ERROR, "Error: -q:v qscale not available for encoder. Use -b:v bitrate instead.\n"); 1122 return AVERROR_EXTERNAL; 1123 } 1124 1125 if (avctx->flags & AV_CODEC_FLAG_QSCALE) { 1126 quality = quality >= 100 ? 1.0 : quality / 100; 1127 quality_num = CFNumberCreate(kCFAllocatorDefault, 1128 kCFNumberFloat32Type, 1129 &quality); 1130 if (!quality_num) return AVERROR(ENOMEM); 1131 1132 status = VTSessionSetProperty(vtctx->session, 1133 kVTCompressionPropertyKey_Quality, 1134 quality_num); 1135 CFRelease(quality_num); 1136 } else if (avctx->codec_id != AV_CODEC_ID_PRORES) { 1137 bit_rate_num = CFNumberCreate(kCFAllocatorDefault, 1138 kCFNumberSInt32Type, 1139 &bit_rate); 1140 if (!bit_rate_num) return AVERROR(ENOMEM); 1141 1142 status = VTSessionSetProperty(vtctx->session, 1143 kVTCompressionPropertyKey_AverageBitRate, 1144 bit_rate_num); 1145 CFRelease(bit_rate_num); 1146 } 1147 1148 if (status) { 1149 av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status); 1150 return AVERROR_EXTERNAL; 1151 } 1152 1153 if (vtctx->prio_speed >= 0) { 1154 status = VTSessionSetProperty(vtctx->session, 1155 compat_keys.kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, 1156 vtctx->prio_speed ? kCFBooleanTrue : kCFBooleanFalse); 1157 if (status) { 1158 av_log(avctx, AV_LOG_WARNING, "PrioritizeEncodingSpeedOverQuality property is not supported on this device. Ignoring.\n"); 1159 } 1160 } 1161 1162 if ((vtctx->codec_id == AV_CODEC_ID_H264 || vtctx->codec_id == AV_CODEC_ID_HEVC) 1163 && max_rate > 0) { 1164 bytes_per_second_value = max_rate >> 3; 1165 bytes_per_second = CFNumberCreate(kCFAllocatorDefault, 1166 kCFNumberSInt64Type, 1167 &bytes_per_second_value); 1168 if (!bytes_per_second) { 1169 return AVERROR(ENOMEM); 1170 } 1171 one_second_value = 1; 1172 one_second = CFNumberCreate(kCFAllocatorDefault, 1173 kCFNumberSInt64Type, 1174 &one_second_value); 1175 if (!one_second) { 1176 CFRelease(bytes_per_second); 1177 return AVERROR(ENOMEM); 1178 } 1179 nums[0] = (void *)bytes_per_second; 1180 nums[1] = (void *)one_second; 1181 data_rate_limits = CFArrayCreate(kCFAllocatorDefault, 1182 (const void **)nums, 1183 2, 1184 &kCFTypeArrayCallBacks); 1185 1186 if (!data_rate_limits) { 1187 CFRelease(bytes_per_second); 1188 CFRelease(one_second); 1189 return AVERROR(ENOMEM); 1190 } 1191 status = VTSessionSetProperty(vtctx->session, 1192 kVTCompressionPropertyKey_DataRateLimits, 1193 data_rate_limits); 1194 1195 CFRelease(bytes_per_second); 1196 CFRelease(one_second); 1197 CFRelease(data_rate_limits); 1198 1199 if (status) { 1200 av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status); 1201 // kVTCompressionPropertyKey_DataRateLimits is available for HEVC 1202 // now but not on old release. There is no document about since 1203 // when. So ignore the error if it failed for hevc. 1204 if (vtctx->codec_id != AV_CODEC_ID_HEVC) 1205 return AVERROR_EXTERNAL; 1206 } 1207 } 1208 1209 if (vtctx->codec_id == AV_CODEC_ID_HEVC) { 1210 if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) { 1211 CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault, 1212 kCFNumberDoubleType, 1213 &vtctx->alpha_quality); 1214 if (!alpha_quality_num) return AVERROR(ENOMEM); 1215 1216 status = VTSessionSetProperty(vtctx->session, 1217 compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha, 1218 alpha_quality_num); 1219 CFRelease(alpha_quality_num); 1220 } 1221 } 1222 1223 if (profile_level) { 1224 status = VTSessionSetProperty(vtctx->session, 1225 kVTCompressionPropertyKey_ProfileLevel, 1226 profile_level); 1227 if (status) { 1228 av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status); 1229 } 1230 } 1231 1232 if (avctx->gop_size > 0 && avctx->codec_id != AV_CODEC_ID_PRORES) { 1233 CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault, 1234 kCFNumberIntType, 1235 &avctx->gop_size); 1236 if (!interval) { 1237 return AVERROR(ENOMEM); 1238 } 1239 1240 status = VTSessionSetProperty(vtctx->session, 1241 kVTCompressionPropertyKey_MaxKeyFrameInterval, 1242 interval); 1243 CFRelease(interval); 1244 1245 if (status) { 1246 av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status); 1247 return AVERROR_EXTERNAL; 1248 } 1249 } 1250 1251 if (vtctx->frames_before) { 1252 status = VTSessionSetProperty(vtctx->session, 1253 kVTCompressionPropertyKey_MoreFramesBeforeStart, 1254 kCFBooleanTrue); 1255 1256 if (status == kVTPropertyNotSupportedErr) { 1257 av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n"); 1258 } else if (status) { 1259 av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status); 1260 } 1261 } 1262 1263 if (vtctx->frames_after) { 1264 status = VTSessionSetProperty(vtctx->session, 1265 kVTCompressionPropertyKey_MoreFramesAfterEnd, 1266 kCFBooleanTrue); 1267 1268 if (status == kVTPropertyNotSupportedErr) { 1269 av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n"); 1270 } else if (status) { 1271 av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status); 1272 } 1273 } 1274 1275 if (avctx->sample_aspect_ratio.num != 0) { 1276 CFNumberRef num; 1277 CFNumberRef den; 1278 CFMutableDictionaryRef par; 1279 AVRational *avpar = &avctx->sample_aspect_ratio; 1280 1281 av_reduce(&avpar->num, &avpar->den, 1282 avpar->num, avpar->den, 1283 0xFFFFFFFF); 1284 1285 num = CFNumberCreate(kCFAllocatorDefault, 1286 kCFNumberIntType, 1287 &avpar->num); 1288 1289 den = CFNumberCreate(kCFAllocatorDefault, 1290 kCFNumberIntType, 1291 &avpar->den); 1292 1293 1294 1295 par = CFDictionaryCreateMutable(kCFAllocatorDefault, 1296 2, 1297 &kCFCopyStringDictionaryKeyCallBacks, 1298 &kCFTypeDictionaryValueCallBacks); 1299 1300 if (!par || !num || !den) { 1301 if (par) CFRelease(par); 1302 if (num) CFRelease(num); 1303 if (den) CFRelease(den); 1304 1305 return AVERROR(ENOMEM); 1306 } 1307 1308 CFDictionarySetValue( 1309 par, 1310 kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing, 1311 num); 1312 1313 CFDictionarySetValue( 1314 par, 1315 kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing, 1316 den); 1317 1318 status = VTSessionSetProperty(vtctx->session, 1319 kVTCompressionPropertyKey_PixelAspectRatio, 1320 par); 1321 1322 CFRelease(par); 1323 CFRelease(num); 1324 CFRelease(den); 1325 1326 if (status) { 1327 av_log(avctx, 1328 AV_LOG_ERROR, 1329 "Error setting pixel aspect ratio to %d:%d: %d.\n", 1330 avctx->sample_aspect_ratio.num, 1331 avctx->sample_aspect_ratio.den, 1332 status); 1333 1334 return AVERROR_EXTERNAL; 1335 } 1336 } 1337 1338 1339 if (vtctx->transfer_function) { 1340 status = VTSessionSetProperty(vtctx->session, 1341 kVTCompressionPropertyKey_TransferFunction, 1342 vtctx->transfer_function); 1343 1344 if (status) { 1345 av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status); 1346 } 1347 } 1348 1349 1350 if (vtctx->ycbcr_matrix) { 1351 status = VTSessionSetProperty(vtctx->session, 1352 kVTCompressionPropertyKey_YCbCrMatrix, 1353 vtctx->ycbcr_matrix); 1354 1355 if (status) { 1356 av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status); 1357 } 1358 } 1359 1360 1361 if (vtctx->color_primaries) { 1362 status = VTSessionSetProperty(vtctx->session, 1363 kVTCompressionPropertyKey_ColorPrimaries, 1364 vtctx->color_primaries); 1365 1366 if (status) { 1367 av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status); 1368 } 1369 } 1370 1371 if (gamma_level) { 1372 status = VTSessionSetProperty(vtctx->session, 1373 kCVImageBufferGammaLevelKey, 1374 gamma_level); 1375 1376 if (status) { 1377 av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status); 1378 } 1379 } 1380 1381 if (!vtctx->has_b_frames && avctx->codec_id != AV_CODEC_ID_PRORES) { 1382 status = VTSessionSetProperty(vtctx->session, 1383 kVTCompressionPropertyKey_AllowFrameReordering, 1384 kCFBooleanFalse); 1385 1386 if (status) { 1387 av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status); 1388 return AVERROR_EXTERNAL; 1389 } 1390 } 1391 1392 if (vtctx->entropy != VT_ENTROPY_NOT_SET) { 1393 CFStringRef entropy = vtctx->entropy == VT_CABAC ? 1394 compat_keys.kVTH264EntropyMode_CABAC: 1395 compat_keys.kVTH264EntropyMode_CAVLC; 1396 1397 status = VTSessionSetProperty(vtctx->session, 1398 compat_keys.kVTCompressionPropertyKey_H264EntropyMode, 1399 entropy); 1400 1401 if (status) { 1402 av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status); 1403 } 1404 } 1405 1406 if (vtctx->realtime >= 0) { 1407 status = VTSessionSetProperty(vtctx->session, 1408 compat_keys.kVTCompressionPropertyKey_RealTime, 1409 vtctx->realtime ? kCFBooleanTrue : kCFBooleanFalse); 1410 1411 if (status) { 1412 av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status); 1413 } 1414 } 1415 1416 status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session); 1417 if (status) { 1418 av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status); 1419 return AVERROR_EXTERNAL; 1420 } 1421 1422 return 0; 1423} 1424 1425static int vtenc_configure_encoder(AVCodecContext *avctx) 1426{ 1427 CFMutableDictionaryRef enc_info; 1428 CFMutableDictionaryRef pixel_buffer_info; 1429 CMVideoCodecType codec_type; 1430 VTEncContext *vtctx = avctx->priv_data; 1431 CFStringRef profile_level = NULL; 1432 CFNumberRef gamma_level = NULL; 1433 int status; 1434 1435 codec_type = get_cm_codec_type(avctx, vtctx->profile, vtctx->alpha_quality); 1436 if (!codec_type) { 1437 av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id); 1438 return AVERROR(EINVAL); 1439 } 1440 1441#if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) 1442 if (avctx->codec_id == AV_CODEC_ID_PRORES) { 1443 if (__builtin_available(macOS 10.10, *)) { 1444 VTRegisterProfessionalVideoWorkflowVideoEncoders(); 1445 } 1446 } 1447#endif 1448 1449 vtctx->codec_id = avctx->codec_id; 1450 1451 if (vtctx->codec_id == AV_CODEC_ID_H264) { 1452 vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex; 1453 1454 vtctx->has_b_frames = avctx->max_b_frames > 0; 1455 if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){ 1456 av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n"); 1457 vtctx->has_b_frames = 0; 1458 } 1459 1460 if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) { 1461 av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n"); 1462 vtctx->entropy = VT_ENTROPY_NOT_SET; 1463 } 1464 1465 if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL); 1466 } else if (vtctx->codec_id == AV_CODEC_ID_HEVC) { 1467 vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex; 1468 if (!vtctx->get_param_set_func) return AVERROR(EINVAL); 1469 if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL); 1470 // HEVC has b-byramid 1471 vtctx->has_b_frames = avctx->max_b_frames > 0 ? 2 : 0; 1472 } else if (vtctx->codec_id == AV_CODEC_ID_PRORES) { 1473 avctx->codec_tag = av_bswap32(codec_type); 1474 } 1475 1476 enc_info = CFDictionaryCreateMutable( 1477 kCFAllocatorDefault, 1478 20, 1479 &kCFCopyStringDictionaryKeyCallBacks, 1480 &kCFTypeDictionaryValueCallBacks 1481 ); 1482 1483 if (!enc_info) return AVERROR(ENOMEM); 1484 1485#if !TARGET_OS_IPHONE 1486 if(vtctx->require_sw) { 1487 CFDictionarySetValue(enc_info, 1488 compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, 1489 kCFBooleanFalse); 1490 } else if (!vtctx->allow_sw) { 1491 CFDictionarySetValue(enc_info, 1492 compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder, 1493 kCFBooleanTrue); 1494 } else { 1495 CFDictionarySetValue(enc_info, 1496 compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, 1497 kCFBooleanTrue); 1498 } 1499#endif 1500 1501 if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) { 1502 status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info); 1503 if (status) 1504 goto init_cleanup; 1505 } else { 1506 pixel_buffer_info = NULL; 1507 } 1508 1509 vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0; 1510 1511 get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level); 1512 get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix); 1513 get_cv_color_primaries(avctx, &vtctx->color_primaries); 1514 1515 1516 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) { 1517 status = vtenc_populate_extradata(avctx, 1518 codec_type, 1519 profile_level, 1520 gamma_level, 1521 enc_info, 1522 pixel_buffer_info); 1523 if (status) 1524 goto init_cleanup; 1525 } 1526 1527 status = vtenc_create_encoder(avctx, 1528 codec_type, 1529 profile_level, 1530 gamma_level, 1531 enc_info, 1532 pixel_buffer_info, 1533 &vtctx->session); 1534 1535init_cleanup: 1536 if (gamma_level) 1537 CFRelease(gamma_level); 1538 1539 if (pixel_buffer_info) 1540 CFRelease(pixel_buffer_info); 1541 1542 CFRelease(enc_info); 1543 1544 return status; 1545} 1546 1547static av_cold int vtenc_init(AVCodecContext *avctx) 1548{ 1549 VTEncContext *vtctx = avctx->priv_data; 1550 CFBooleanRef has_b_frames_cfbool; 1551 int status; 1552 1553 pthread_once(&once_ctrl, loadVTEncSymbols); 1554 1555 pthread_mutex_init(&vtctx->lock, NULL); 1556 pthread_cond_init(&vtctx->cv_sample_sent, NULL); 1557 1558 vtctx->session = NULL; 1559 status = vtenc_configure_encoder(avctx); 1560 if (status) return status; 1561 1562 status = VTSessionCopyProperty(vtctx->session, 1563 kVTCompressionPropertyKey_AllowFrameReordering, 1564 kCFAllocatorDefault, 1565 &has_b_frames_cfbool); 1566 1567 if (!status && has_b_frames_cfbool) { 1568 //Some devices don't output B-frames for main profile, even if requested. 1569 // HEVC has b-pyramid 1570 if (CFBooleanGetValue(has_b_frames_cfbool)) 1571 vtctx->has_b_frames = avctx->codec_id == AV_CODEC_ID_HEVC ? 2 : 1; 1572 else 1573 vtctx->has_b_frames = 0; 1574 CFRelease(has_b_frames_cfbool); 1575 } 1576 avctx->has_b_frames = vtctx->has_b_frames; 1577 1578 return 0; 1579} 1580 1581static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame) 1582{ 1583 CFArrayRef attachments; 1584 CFDictionaryRef attachment; 1585 CFBooleanRef not_sync; 1586 CFIndex len; 1587 1588 attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false); 1589 len = !attachments ? 0 : CFArrayGetCount(attachments); 1590 1591 if (!len) { 1592 *is_key_frame = true; 1593 return; 1594 } 1595 1596 attachment = CFArrayGetValueAtIndex(attachments, 0); 1597 1598 if (CFDictionaryGetValueIfPresent(attachment, 1599 kCMSampleAttachmentKey_NotSync, 1600 (const void **)¬_sync)) 1601 { 1602 *is_key_frame = !CFBooleanGetValue(not_sync); 1603 } else { 1604 *is_key_frame = true; 1605 } 1606} 1607 1608static int is_post_sei_nal_type(int nal_type){ 1609 return nal_type != H264_NAL_SEI && 1610 nal_type != H264_NAL_SPS && 1611 nal_type != H264_NAL_PPS && 1612 nal_type != H264_NAL_AUD; 1613} 1614 1615/* 1616 * Finds the sei message start/size of type find_sei_type. 1617 * If more than one of that type exists, the last one is returned. 1618 */ 1619static int find_sei_end(AVCodecContext *avctx, 1620 uint8_t *nal_data, 1621 size_t nal_size, 1622 uint8_t **sei_end) 1623{ 1624 int nal_type; 1625 size_t sei_payload_size = 0; 1626 int sei_payload_type = 0; 1627 *sei_end = NULL; 1628 uint8_t *nal_start = nal_data; 1629 1630 if (!nal_size) 1631 return 0; 1632 1633 nal_type = *nal_data & 0x1F; 1634 if (nal_type != H264_NAL_SEI) 1635 return 0; 1636 1637 nal_data++; 1638 nal_size--; 1639 1640 if (nal_data[nal_size - 1] == 0x80) 1641 nal_size--; 1642 1643 while (nal_size > 0 && *nal_data > 0) { 1644 do{ 1645 sei_payload_type += *nal_data; 1646 nal_data++; 1647 nal_size--; 1648 } while (nal_size > 0 && *nal_data == 0xFF); 1649 1650 if (!nal_size) { 1651 av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n"); 1652 return AVERROR_INVALIDDATA; 1653 } 1654 1655 do{ 1656 sei_payload_size += *nal_data; 1657 nal_data++; 1658 nal_size--; 1659 } while (nal_size > 0 && *nal_data == 0xFF); 1660 1661 if (nal_size < sei_payload_size) { 1662 av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n"); 1663 return AVERROR_INVALIDDATA; 1664 } 1665 1666 nal_data += sei_payload_size; 1667 nal_size -= sei_payload_size; 1668 } 1669 1670 *sei_end = nal_data; 1671 1672 return nal_data - nal_start + 1; 1673} 1674 1675/** 1676 * Copies the data inserting emulation prevention bytes as needed. 1677 * Existing data in the destination can be taken into account by providing 1678 * dst with a dst_offset > 0. 1679 * 1680 * @return The number of bytes copied on success. On failure, the negative of 1681 * the number of bytes needed to copy src is returned. 1682 */ 1683static int copy_emulation_prev(const uint8_t *src, 1684 size_t src_size, 1685 uint8_t *dst, 1686 ssize_t dst_offset, 1687 size_t dst_size) 1688{ 1689 int zeros = 0; 1690 int wrote_bytes; 1691 uint8_t* dst_start; 1692 uint8_t* dst_end = dst + dst_size; 1693 const uint8_t* src_end = src + src_size; 1694 int start_at = dst_offset > 2 ? dst_offset - 2 : 0; 1695 int i; 1696 for (i = start_at; i < dst_offset && i < dst_size; i++) { 1697 if (!dst[i]) 1698 zeros++; 1699 else 1700 zeros = 0; 1701 } 1702 1703 dst += dst_offset; 1704 dst_start = dst; 1705 for (; src < src_end; src++, dst++) { 1706 if (zeros == 2) { 1707 int insert_ep3_byte = *src <= 3; 1708 if (insert_ep3_byte) { 1709 if (dst < dst_end) 1710 *dst = 3; 1711 dst++; 1712 } 1713 1714 zeros = 0; 1715 } 1716 1717 if (dst < dst_end) 1718 *dst = *src; 1719 1720 if (!*src) 1721 zeros++; 1722 else 1723 zeros = 0; 1724 } 1725 1726 wrote_bytes = dst - dst_start; 1727 1728 if (dst > dst_end) 1729 return -wrote_bytes; 1730 1731 return wrote_bytes; 1732} 1733 1734static int write_sei(const ExtraSEI *sei, 1735 int sei_type, 1736 uint8_t *dst, 1737 size_t dst_size) 1738{ 1739 uint8_t *sei_start = dst; 1740 size_t remaining_sei_size = sei->size; 1741 size_t remaining_dst_size = dst_size; 1742 int header_bytes; 1743 int bytes_written; 1744 ssize_t offset; 1745 1746 if (!remaining_dst_size) 1747 return AVERROR_BUFFER_TOO_SMALL; 1748 1749 while (sei_type && remaining_dst_size != 0) { 1750 int sei_byte = sei_type > 255 ? 255 : sei_type; 1751 *dst = sei_byte; 1752 1753 sei_type -= sei_byte; 1754 dst++; 1755 remaining_dst_size--; 1756 } 1757 1758 if (!dst_size) 1759 return AVERROR_BUFFER_TOO_SMALL; 1760 1761 while (remaining_sei_size && remaining_dst_size != 0) { 1762 int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size; 1763 *dst = size_byte; 1764 1765 remaining_sei_size -= size_byte; 1766 dst++; 1767 remaining_dst_size--; 1768 } 1769 1770 if (remaining_dst_size < sei->size) 1771 return AVERROR_BUFFER_TOO_SMALL; 1772 1773 header_bytes = dst - sei_start; 1774 1775 offset = header_bytes; 1776 bytes_written = copy_emulation_prev(sei->data, 1777 sei->size, 1778 sei_start, 1779 offset, 1780 dst_size); 1781 if (bytes_written < 0) 1782 return AVERROR_BUFFER_TOO_SMALL; 1783 1784 bytes_written += header_bytes; 1785 return bytes_written; 1786} 1787 1788/** 1789 * Copies NAL units and replaces length codes with 1790 * H.264 Annex B start codes. On failure, the contents of 1791 * dst_data may have been modified. 1792 * 1793 * @param length_code_size Byte length of each length code 1794 * @param sample_buffer NAL units prefixed with length codes. 1795 * @param sei Optional A53 closed captions SEI data. 1796 * @param dst_data Must be zeroed before calling this function. 1797 * Contains the copied NAL units prefixed with 1798 * start codes when the function returns 1799 * successfully. 1800 * @param dst_size Length of dst_data 1801 * @return 0 on success 1802 * AVERROR_INVALIDDATA if length_code_size is invalid 1803 * AVERROR_BUFFER_TOO_SMALL if dst_data is too small 1804 * or if a length_code in src_data specifies data beyond 1805 * the end of its buffer. 1806 */ 1807static int copy_replace_length_codes( 1808 AVCodecContext *avctx, 1809 size_t length_code_size, 1810 CMSampleBufferRef sample_buffer, 1811 ExtraSEI *sei, 1812 uint8_t *dst_data, 1813 size_t dst_size) 1814{ 1815 size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer); 1816 size_t remaining_src_size = src_size; 1817 size_t remaining_dst_size = dst_size; 1818 size_t src_offset = 0; 1819 int wrote_sei = 0; 1820 int status; 1821 uint8_t size_buf[4]; 1822 uint8_t nal_type; 1823 CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer); 1824 1825 if (length_code_size > 4) { 1826 return AVERROR_INVALIDDATA; 1827 } 1828 1829 while (remaining_src_size > 0) { 1830 size_t curr_src_len; 1831 size_t curr_dst_len; 1832 size_t box_len = 0; 1833 size_t i; 1834 1835 uint8_t *dst_box; 1836 1837 status = CMBlockBufferCopyDataBytes(block, 1838 src_offset, 1839 length_code_size, 1840 size_buf); 1841 if (status) { 1842 av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status); 1843 return AVERROR_EXTERNAL; 1844 } 1845 1846 status = CMBlockBufferCopyDataBytes(block, 1847 src_offset + length_code_size, 1848 1, 1849 &nal_type); 1850 1851 if (status) { 1852 av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status); 1853 return AVERROR_EXTERNAL; 1854 } 1855 1856 nal_type &= 0x1F; 1857 1858 for (i = 0; i < length_code_size; i++) { 1859 box_len <<= 8; 1860 box_len |= size_buf[i]; 1861 } 1862 1863 if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) { 1864 //No SEI NAL unit - insert. 1865 int wrote_bytes; 1866 1867 memcpy(dst_data, start_code, sizeof(start_code)); 1868 dst_data += sizeof(start_code); 1869 remaining_dst_size -= sizeof(start_code); 1870 1871 *dst_data = H264_NAL_SEI; 1872 dst_data++; 1873 remaining_dst_size--; 1874 1875 wrote_bytes = write_sei(sei, 1876 SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35, 1877 dst_data, 1878 remaining_dst_size); 1879 1880 if (wrote_bytes < 0) 1881 return wrote_bytes; 1882 1883 remaining_dst_size -= wrote_bytes; 1884 dst_data += wrote_bytes; 1885 1886 if (remaining_dst_size <= 0) 1887 return AVERROR_BUFFER_TOO_SMALL; 1888 1889 *dst_data = 0x80; 1890 1891 dst_data++; 1892 remaining_dst_size--; 1893 1894 wrote_sei = 1; 1895 } 1896 1897 curr_src_len = box_len + length_code_size; 1898 curr_dst_len = box_len + sizeof(start_code); 1899 1900 if (remaining_src_size < curr_src_len) { 1901 return AVERROR_BUFFER_TOO_SMALL; 1902 } 1903 1904 if (remaining_dst_size < curr_dst_len) { 1905 return AVERROR_BUFFER_TOO_SMALL; 1906 } 1907 1908 dst_box = dst_data + sizeof(start_code); 1909 1910 memcpy(dst_data, start_code, sizeof(start_code)); 1911 status = CMBlockBufferCopyDataBytes(block, 1912 src_offset + length_code_size, 1913 box_len, 1914 dst_box); 1915 1916 if (status) { 1917 av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status); 1918 return AVERROR_EXTERNAL; 1919 } 1920 1921 if (sei && !wrote_sei && nal_type == H264_NAL_SEI) { 1922 //Found SEI NAL unit - append. 1923 int wrote_bytes; 1924 int old_sei_length; 1925 int extra_bytes; 1926 uint8_t *new_sei; 1927 old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei); 1928 if (old_sei_length < 0) 1929 return status; 1930 1931 wrote_bytes = write_sei(sei, 1932 SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35, 1933 new_sei, 1934 remaining_dst_size - old_sei_length); 1935 if (wrote_bytes < 0) 1936 return wrote_bytes; 1937 1938 if (new_sei + wrote_bytes >= dst_data + remaining_dst_size) 1939 return AVERROR_BUFFER_TOO_SMALL; 1940 1941 new_sei[wrote_bytes++] = 0x80; 1942 extra_bytes = wrote_bytes - (dst_box + box_len - new_sei); 1943 1944 dst_data += extra_bytes; 1945 remaining_dst_size -= extra_bytes; 1946 1947 wrote_sei = 1; 1948 } 1949 1950 src_offset += curr_src_len; 1951 dst_data += curr_dst_len; 1952 1953 remaining_src_size -= curr_src_len; 1954 remaining_dst_size -= curr_dst_len; 1955 } 1956 1957 return 0; 1958} 1959 1960/** 1961 * Returns a sufficient number of bytes to contain the sei data. 1962 * It may be greater than the minimum required. 1963 */ 1964static int get_sei_msg_bytes(const ExtraSEI* sei, int type){ 1965 int copied_size; 1966 if (sei->size == 0) 1967 return 0; 1968 1969 copied_size = -copy_emulation_prev(sei->data, 1970 sei->size, 1971 NULL, 1972 0, 1973 0); 1974 1975 if ((sei->size % 255) == 0) //may result in an extra byte 1976 copied_size++; 1977 1978 return copied_size + sei->size / 255 + 1 + type / 255 + 1; 1979} 1980 1981static int vtenc_cm_to_avpacket( 1982 AVCodecContext *avctx, 1983 CMSampleBufferRef sample_buffer, 1984 AVPacket *pkt, 1985 ExtraSEI *sei) 1986{ 1987 VTEncContext *vtctx = avctx->priv_data; 1988 1989 int status; 1990 bool is_key_frame; 1991 bool add_header; 1992 size_t length_code_size; 1993 size_t header_size = 0; 1994 size_t in_buf_size; 1995 size_t out_buf_size; 1996 size_t sei_nalu_size = 0; 1997 int64_t dts_delta; 1998 int64_t time_base_num; 1999 int nalu_count; 2000 CMTime pts; 2001 CMTime dts; 2002 CMVideoFormatDescriptionRef vid_fmt; 2003 2004 vtenc_get_frame_info(sample_buffer, &is_key_frame); 2005 2006 if (vtctx->get_param_set_func) { 2007 status = get_length_code_size(avctx, sample_buffer, &length_code_size); 2008 if (status) return status; 2009 2010 add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER); 2011 2012 if (add_header) { 2013 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer); 2014 if (!vid_fmt) { 2015 av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n"); 2016 return AVERROR_EXTERNAL; 2017 } 2018 2019 int status = get_params_size(avctx, vid_fmt, &header_size); 2020 if (status) return status; 2021 } 2022 2023 status = count_nalus(length_code_size, sample_buffer, &nalu_count); 2024 if(status) 2025 return status; 2026 2027 if (sei) { 2028 size_t msg_size = get_sei_msg_bytes(sei, 2029 SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35); 2030 2031 sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1; 2032 } 2033 2034 in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer); 2035 out_buf_size = header_size + 2036 in_buf_size + 2037 sei_nalu_size + 2038 nalu_count * ((int)sizeof(start_code) - (int)length_code_size); 2039 2040 status = ff_get_encode_buffer(avctx, pkt, out_buf_size, 0); 2041 if (status < 0) 2042 return status; 2043 2044 if (add_header) { 2045 status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size); 2046 if(status) return status; 2047 } 2048 2049 status = copy_replace_length_codes( 2050 avctx, 2051 length_code_size, 2052 sample_buffer, 2053 sei, 2054 pkt->data + header_size, 2055 pkt->size - header_size 2056 ); 2057 2058 if (status) { 2059 av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status); 2060 return status; 2061 } 2062 } else { 2063 size_t len; 2064 CMBlockBufferRef buf = CMSampleBufferGetDataBuffer(sample_buffer); 2065 if (!buf) { 2066 av_log(avctx, AV_LOG_ERROR, "Error getting block buffer\n"); 2067 return AVERROR_EXTERNAL; 2068 } 2069 2070 len = CMBlockBufferGetDataLength(buf); 2071 2072 status = ff_get_encode_buffer(avctx, pkt, len, 0); 2073 if (status < 0) 2074 return status; 2075 2076 status = CMBlockBufferCopyDataBytes(buf, 0, len, pkt->data); 2077 if (status) { 2078 av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status); 2079 return AVERROR_EXTERNAL; 2080 } 2081 } 2082 2083 if (is_key_frame) { 2084 pkt->flags |= AV_PKT_FLAG_KEY; 2085 } 2086 2087 pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer); 2088 dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer); 2089 2090 if (CMTIME_IS_INVALID(dts)) { 2091 if (!vtctx->has_b_frames) { 2092 dts = pts; 2093 } else { 2094 av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n"); 2095 return AVERROR_EXTERNAL; 2096 } 2097 } 2098 2099 dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0; 2100 time_base_num = avctx->time_base.num; 2101 pkt->pts = pts.value / time_base_num; 2102 pkt->dts = dts.value / time_base_num - dts_delta; 2103 2104 return 0; 2105} 2106 2107/* 2108 * contiguous_buf_size is 0 if not contiguous, and the size of the buffer 2109 * containing all planes if so. 2110 */ 2111static int get_cv_pixel_info( 2112 AVCodecContext *avctx, 2113 const AVFrame *frame, 2114 int *color, 2115 int *plane_count, 2116 size_t *widths, 2117 size_t *heights, 2118 size_t *strides, 2119 size_t *contiguous_buf_size) 2120{ 2121 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt); 2122 VTEncContext *vtctx = avctx->priv_data; 2123 int av_format = frame->format; 2124 int av_color_range = frame->color_range; 2125 int i; 2126 int range_guessed; 2127 int status; 2128 2129 if (!desc) 2130 return AVERROR(EINVAL); 2131 2132 status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed); 2133 if (status) { 2134 av_log(avctx, 2135 AV_LOG_ERROR, 2136 "Could not get pixel format for color format '%s' range '%s'.\n", 2137 av_get_pix_fmt_name(av_format), 2138 av_color_range > AVCOL_RANGE_UNSPECIFIED && 2139 av_color_range < AVCOL_RANGE_NB ? 2140 av_color_range_name(av_color_range) : 2141 "Unknown"); 2142 2143 return AVERROR(EINVAL); 2144 } 2145 2146 if (range_guessed) { 2147 if (!vtctx->warned_color_range) { 2148 vtctx->warned_color_range = true; 2149 av_log(avctx, 2150 AV_LOG_WARNING, 2151 "Color range not set for %s. Using MPEG range.\n", 2152 av_get_pix_fmt_name(av_format)); 2153 } 2154 } 2155 2156 *plane_count = av_pix_fmt_count_planes(avctx->pix_fmt); 2157 2158 for (i = 0; i < desc->nb_components; i++) { 2159 int p = desc->comp[i].plane; 2160 bool hasAlpha = (desc->flags & AV_PIX_FMT_FLAG_ALPHA); 2161 bool isAlpha = hasAlpha && (p + 1 == *plane_count); 2162 bool isChroma = (p != 0) && !isAlpha; 2163 int shiftw = isChroma ? desc->log2_chroma_w : 0; 2164 int shifth = isChroma ? desc->log2_chroma_h : 0; 2165 widths[p] = (avctx->width + ((1 << shiftw) >> 1)) >> shiftw; 2166 heights[p] = (avctx->height + ((1 << shifth) >> 1)) >> shifth; 2167 strides[p] = frame->linesize[p]; 2168 } 2169 2170 *contiguous_buf_size = 0; 2171 for (i = 0; i < *plane_count; i++) { 2172 if (i < *plane_count - 1 && 2173 frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) { 2174 *contiguous_buf_size = 0; 2175 break; 2176 } 2177 2178 *contiguous_buf_size += strides[i] * heights[i]; 2179 } 2180 2181 return 0; 2182} 2183 2184//Not used on OSX - frame is never copied. 2185static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, 2186 const AVFrame *frame, 2187 CVPixelBufferRef cv_img, 2188 const size_t *plane_strides, 2189 const size_t *plane_rows) 2190{ 2191 int i, j; 2192 size_t plane_count; 2193 int status; 2194 int rows; 2195 int src_stride; 2196 int dst_stride; 2197 uint8_t *src_addr; 2198 uint8_t *dst_addr; 2199 size_t copy_bytes; 2200 2201 status = CVPixelBufferLockBaseAddress(cv_img, 0); 2202 if (status) { 2203 av_log( 2204 avctx, 2205 AV_LOG_ERROR, 2206 "Error: Could not lock base address of CVPixelBuffer: %d.\n", 2207 status 2208 ); 2209 } 2210 2211 if (CVPixelBufferIsPlanar(cv_img)) { 2212 plane_count = CVPixelBufferGetPlaneCount(cv_img); 2213 for (i = 0; frame->data[i]; i++) { 2214 if (i == plane_count) { 2215 CVPixelBufferUnlockBaseAddress(cv_img, 0); 2216 av_log(avctx, 2217 AV_LOG_ERROR, 2218 "Error: different number of planes in AVFrame and CVPixelBuffer.\n" 2219 ); 2220 2221 return AVERROR_EXTERNAL; 2222 } 2223 2224 dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i); 2225 src_addr = (uint8_t*)frame->data[i]; 2226 dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i); 2227 src_stride = plane_strides[i]; 2228 rows = plane_rows[i]; 2229 2230 if (dst_stride == src_stride) { 2231 memcpy(dst_addr, src_addr, src_stride * rows); 2232 } else { 2233 copy_bytes = dst_stride < src_stride ? dst_stride : src_stride; 2234 2235 for (j = 0; j < rows; j++) { 2236 memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes); 2237 } 2238 } 2239 } 2240 } else { 2241 if (frame->data[1]) { 2242 CVPixelBufferUnlockBaseAddress(cv_img, 0); 2243 av_log(avctx, 2244 AV_LOG_ERROR, 2245 "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n" 2246 ); 2247 2248 return AVERROR_EXTERNAL; 2249 } 2250 2251 dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img); 2252 src_addr = (uint8_t*)frame->data[0]; 2253 dst_stride = CVPixelBufferGetBytesPerRow(cv_img); 2254 src_stride = plane_strides[0]; 2255 rows = plane_rows[0]; 2256 2257 if (dst_stride == src_stride) { 2258 memcpy(dst_addr, src_addr, src_stride * rows); 2259 } else { 2260 copy_bytes = dst_stride < src_stride ? dst_stride : src_stride; 2261 2262 for (j = 0; j < rows; j++) { 2263 memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes); 2264 } 2265 } 2266 } 2267 2268 status = CVPixelBufferUnlockBaseAddress(cv_img, 0); 2269 if (status) { 2270 av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status); 2271 return AVERROR_EXTERNAL; 2272 } 2273 2274 return 0; 2275} 2276 2277static int create_cv_pixel_buffer(AVCodecContext *avctx, 2278 const AVFrame *frame, 2279 CVPixelBufferRef *cv_img) 2280{ 2281 int plane_count; 2282 int color; 2283 size_t widths [AV_NUM_DATA_POINTERS]; 2284 size_t heights[AV_NUM_DATA_POINTERS]; 2285 size_t strides[AV_NUM_DATA_POINTERS]; 2286 int status; 2287 size_t contiguous_buf_size; 2288 CVPixelBufferPoolRef pix_buf_pool; 2289 VTEncContext* vtctx = avctx->priv_data; 2290 2291 if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) { 2292 av_assert0(frame->format == AV_PIX_FMT_VIDEOTOOLBOX); 2293 2294 *cv_img = (CVPixelBufferRef)frame->data[3]; 2295 av_assert0(*cv_img); 2296 2297 CFRetain(*cv_img); 2298 return 0; 2299 } 2300 2301 memset(widths, 0, sizeof(widths)); 2302 memset(heights, 0, sizeof(heights)); 2303 memset(strides, 0, sizeof(strides)); 2304 2305 status = get_cv_pixel_info( 2306 avctx, 2307 frame, 2308 &color, 2309 &plane_count, 2310 widths, 2311 heights, 2312 strides, 2313 &contiguous_buf_size 2314 ); 2315 2316 if (status) { 2317 av_log( 2318 avctx, 2319 AV_LOG_ERROR, 2320 "Error: Cannot convert format %d color_range %d: %d\n", 2321 frame->format, 2322 frame->color_range, 2323 status 2324 ); 2325 2326 return AVERROR_EXTERNAL; 2327 } 2328 2329 pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session); 2330 if (!pix_buf_pool) { 2331 /* On iOS, the VT session is invalidated when the APP switches from 2332 * foreground to background and vice versa. Fetch the actual error code 2333 * of the VT session to detect that case and restart the VT session 2334 * accordingly. */ 2335 OSStatus vtstatus; 2336 2337 vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session); 2338 if (vtstatus == kVTInvalidSessionErr) { 2339 CFRelease(vtctx->session); 2340 vtctx->session = NULL; 2341 status = vtenc_configure_encoder(avctx); 2342 if (status == 0) 2343 pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session); 2344 } 2345 if (!pix_buf_pool) { 2346 av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n"); 2347 return AVERROR_EXTERNAL; 2348 } 2349 else 2350 av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a " 2351 "kVTInvalidSessionErr error.\n"); 2352 } 2353 2354 status = CVPixelBufferPoolCreatePixelBuffer(NULL, 2355 pix_buf_pool, 2356 cv_img); 2357 2358 2359 if (status) { 2360 av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status); 2361 return AVERROR_EXTERNAL; 2362 } 2363 2364 status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights); 2365 if (status) { 2366 CFRelease(*cv_img); 2367 *cv_img = NULL; 2368 return status; 2369 } 2370 2371 return 0; 2372} 2373 2374static int create_encoder_dict_h264(const AVFrame *frame, 2375 CFDictionaryRef* dict_out) 2376{ 2377 CFDictionaryRef dict = NULL; 2378 if (frame->pict_type == AV_PICTURE_TYPE_I) { 2379 const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame }; 2380 const void *vals[] = { kCFBooleanTrue }; 2381 2382 dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL); 2383 if(!dict) return AVERROR(ENOMEM); 2384 } 2385 2386 *dict_out = dict; 2387 return 0; 2388} 2389 2390static int vtenc_send_frame(AVCodecContext *avctx, 2391 VTEncContext *vtctx, 2392 const AVFrame *frame) 2393{ 2394 CMTime time; 2395 CFDictionaryRef frame_dict; 2396 CVPixelBufferRef cv_img = NULL; 2397 AVFrameSideData *side_data = NULL; 2398 ExtraSEI *sei = NULL; 2399 int status = create_cv_pixel_buffer(avctx, frame, &cv_img); 2400 2401 if (status) return status; 2402 2403 status = create_encoder_dict_h264(frame, &frame_dict); 2404 if (status) { 2405 CFRelease(cv_img); 2406 return status; 2407 } 2408 2409#if CONFIG_ATSC_A53 2410 side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC); 2411 if (vtctx->a53_cc && side_data && side_data->size) { 2412 sei = av_mallocz(sizeof(*sei)); 2413 if (!sei) { 2414 av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n"); 2415 } else { 2416 int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size); 2417 if (ret < 0) { 2418 av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n"); 2419 av_free(sei); 2420 sei = NULL; 2421 } 2422 } 2423 } 2424#endif 2425 2426 time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den); 2427 status = VTCompressionSessionEncodeFrame( 2428 vtctx->session, 2429 cv_img, 2430 time, 2431 kCMTimeInvalid, 2432 frame_dict, 2433 sei, 2434 NULL 2435 ); 2436 2437 if (frame_dict) CFRelease(frame_dict); 2438 CFRelease(cv_img); 2439 2440 if (status) { 2441 av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status); 2442 return AVERROR_EXTERNAL; 2443 } 2444 2445 return 0; 2446} 2447 2448static av_cold int vtenc_frame( 2449 AVCodecContext *avctx, 2450 AVPacket *pkt, 2451 const AVFrame *frame, 2452 int *got_packet) 2453{ 2454 VTEncContext *vtctx = avctx->priv_data; 2455 bool get_frame; 2456 int status; 2457 CMSampleBufferRef buf = NULL; 2458 ExtraSEI *sei = NULL; 2459 2460 if (frame) { 2461 status = vtenc_send_frame(avctx, vtctx, frame); 2462 2463 if (status) { 2464 status = AVERROR_EXTERNAL; 2465 goto end_nopkt; 2466 } 2467 2468 if (vtctx->frame_ct_in == 0) { 2469 vtctx->first_pts = frame->pts; 2470 } else if(vtctx->frame_ct_in == vtctx->has_b_frames) { 2471 vtctx->dts_delta = frame->pts - vtctx->first_pts; 2472 } 2473 2474 vtctx->frame_ct_in++; 2475 } else if(!vtctx->flushing) { 2476 vtctx->flushing = true; 2477 2478 status = VTCompressionSessionCompleteFrames(vtctx->session, 2479 kCMTimeIndefinite); 2480 2481 if (status) { 2482 av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status); 2483 status = AVERROR_EXTERNAL; 2484 goto end_nopkt; 2485 } 2486 } 2487 2488 *got_packet = 0; 2489 get_frame = vtctx->dts_delta >= 0 || !frame; 2490 if (!get_frame) { 2491 status = 0; 2492 goto end_nopkt; 2493 } 2494 2495 status = vtenc_q_pop(vtctx, !frame, &buf, &sei); 2496 if (status) goto end_nopkt; 2497 if (!buf) goto end_nopkt; 2498 2499 status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei); 2500 if (sei) { 2501 if (sei->data) av_free(sei->data); 2502 av_free(sei); 2503 } 2504 CFRelease(buf); 2505 if (status) goto end_nopkt; 2506 2507 *got_packet = 1; 2508 return 0; 2509 2510end_nopkt: 2511 av_packet_unref(pkt); 2512 return status; 2513} 2514 2515static int vtenc_populate_extradata(AVCodecContext *avctx, 2516 CMVideoCodecType codec_type, 2517 CFStringRef profile_level, 2518 CFNumberRef gamma_level, 2519 CFDictionaryRef enc_info, 2520 CFDictionaryRef pixel_buffer_info) 2521{ 2522 VTEncContext *vtctx = avctx->priv_data; 2523 int status; 2524 CVPixelBufferPoolRef pool = NULL; 2525 CVPixelBufferRef pix_buf = NULL; 2526 CMTime time; 2527 CMSampleBufferRef buf = NULL; 2528 2529 status = vtenc_create_encoder(avctx, 2530 codec_type, 2531 profile_level, 2532 gamma_level, 2533 enc_info, 2534 pixel_buffer_info, 2535 &vtctx->session); 2536 if (status) 2537 goto pe_cleanup; 2538 2539 pool = VTCompressionSessionGetPixelBufferPool(vtctx->session); 2540 if(!pool){ 2541 av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n"); 2542 goto pe_cleanup; 2543 } 2544 2545 status = CVPixelBufferPoolCreatePixelBuffer(NULL, 2546 pool, 2547 &pix_buf); 2548 2549 if(status != kCVReturnSuccess){ 2550 av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status); 2551 goto pe_cleanup; 2552 } 2553 2554 time = CMTimeMake(0, avctx->time_base.den); 2555 status = VTCompressionSessionEncodeFrame(vtctx->session, 2556 pix_buf, 2557 time, 2558 kCMTimeInvalid, 2559 NULL, 2560 NULL, 2561 NULL); 2562 2563 if (status) { 2564 av_log(avctx, 2565 AV_LOG_ERROR, 2566 "Error sending frame for extradata: %d\n", 2567 status); 2568 2569 goto pe_cleanup; 2570 } 2571 2572 //Populates extradata - output frames are flushed and param sets are available. 2573 status = VTCompressionSessionCompleteFrames(vtctx->session, 2574 kCMTimeIndefinite); 2575 2576 if (status) 2577 goto pe_cleanup; 2578 2579 status = vtenc_q_pop(vtctx, 0, &buf, NULL); 2580 if (status) { 2581 av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status); 2582 goto pe_cleanup; 2583 } 2584 2585 CFRelease(buf); 2586 2587 2588 2589pe_cleanup: 2590 CVPixelBufferRelease(pix_buf); 2591 if(vtctx->session) 2592 CFRelease(vtctx->session); 2593 2594 vtctx->session = NULL; 2595 vtctx->frame_ct_out = 0; 2596 2597 av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0)); 2598 2599 return status; 2600} 2601 2602static av_cold int vtenc_close(AVCodecContext *avctx) 2603{ 2604 VTEncContext *vtctx = avctx->priv_data; 2605 2606 if(!vtctx->session) { 2607 pthread_cond_destroy(&vtctx->cv_sample_sent); 2608 pthread_mutex_destroy(&vtctx->lock); 2609 return 0; 2610 } 2611 2612 VTCompressionSessionCompleteFrames(vtctx->session, 2613 kCMTimeIndefinite); 2614 clear_frame_queue(vtctx); 2615 pthread_cond_destroy(&vtctx->cv_sample_sent); 2616 pthread_mutex_destroy(&vtctx->lock); 2617 CFRelease(vtctx->session); 2618 vtctx->session = NULL; 2619 2620 if (vtctx->color_primaries) { 2621 CFRelease(vtctx->color_primaries); 2622 vtctx->color_primaries = NULL; 2623 } 2624 2625 if (vtctx->transfer_function) { 2626 CFRelease(vtctx->transfer_function); 2627 vtctx->transfer_function = NULL; 2628 } 2629 2630 if (vtctx->ycbcr_matrix) { 2631 CFRelease(vtctx->ycbcr_matrix); 2632 vtctx->ycbcr_matrix = NULL; 2633 } 2634 2635 return 0; 2636} 2637 2638static const enum AVPixelFormat avc_pix_fmts[] = { 2639 AV_PIX_FMT_VIDEOTOOLBOX, 2640 AV_PIX_FMT_NV12, 2641 AV_PIX_FMT_YUV420P, 2642 AV_PIX_FMT_NONE 2643}; 2644 2645static const enum AVPixelFormat hevc_pix_fmts[] = { 2646 AV_PIX_FMT_VIDEOTOOLBOX, 2647 AV_PIX_FMT_NV12, 2648 AV_PIX_FMT_YUV420P, 2649 AV_PIX_FMT_BGRA, 2650 AV_PIX_FMT_P010LE, 2651 AV_PIX_FMT_NONE 2652}; 2653 2654static const enum AVPixelFormat prores_pix_fmts[] = { 2655 AV_PIX_FMT_VIDEOTOOLBOX, 2656 AV_PIX_FMT_YUV420P, 2657#ifdef kCFCoreFoundationVersionNumber10_7 2658 AV_PIX_FMT_NV12, 2659 AV_PIX_FMT_AYUV64, 2660#endif 2661 AV_PIX_FMT_UYVY422, 2662#if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE 2663 AV_PIX_FMT_P010, 2664#endif 2665#if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE 2666 AV_PIX_FMT_NV16, 2667#endif 2668#if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE 2669 AV_PIX_FMT_P210, 2670#endif 2671#if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE 2672 AV_PIX_FMT_P216, 2673#endif 2674#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE 2675 AV_PIX_FMT_NV24, 2676#endif 2677#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE 2678 AV_PIX_FMT_P410, 2679#endif 2680#if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE 2681 AV_PIX_FMT_P416, 2682#endif 2683 AV_PIX_FMT_BGRA, 2684 AV_PIX_FMT_NONE 2685}; 2686 2687#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM 2688#define COMMON_OPTIONS \ 2689 { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \ 2690 { .i64 = 0 }, 0, 1, VE }, \ 2691 { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \ 2692 { .i64 = 0 }, 0, 1, VE }, \ 2693 { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \ 2694 OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, -1, 1, VE }, \ 2695 { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \ 2696 OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \ 2697 { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \ 2698 OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \ 2699 { "prio_speed", "prioritize encoding speed", OFFSET(prio_speed), AV_OPT_TYPE_BOOL, \ 2700 { .i64 = -1 }, -1, 1, VE }, \ 2701 2702#define OFFSET(x) offsetof(VTEncContext, x) 2703static const AVOption h264_options[] = { 2704 { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT64, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" }, 2705 { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" }, 2706 { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" }, 2707 { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" }, 2708 { "extended", "Extend Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" }, 2709 2710 { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" }, 2711 { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" }, 2712 { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" }, 2713 { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" }, 2714 { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" }, 2715 { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" }, 2716 { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" }, 2717 { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" }, 2718 { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" }, 2719 { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" }, 2720 { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" }, 2721 2722 { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" }, 2723 { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" }, 2724 { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" }, 2725 { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" }, 2726 { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" }, 2727 2728 { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE }, 2729 2730 COMMON_OPTIONS 2731 { NULL }, 2732}; 2733 2734static const AVClass h264_videotoolbox_class = { 2735 .class_name = "h264_videotoolbox", 2736 .item_name = av_default_item_name, 2737 .option = h264_options, 2738 .version = LIBAVUTIL_VERSION_INT, 2739}; 2740 2741const FFCodec ff_h264_videotoolbox_encoder = { 2742 .p.name = "h264_videotoolbox", 2743 .p.long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"), 2744 .p.type = AVMEDIA_TYPE_VIDEO, 2745 .p.id = AV_CODEC_ID_H264, 2746 .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY, 2747 .priv_data_size = sizeof(VTEncContext), 2748 .p.pix_fmts = avc_pix_fmts, 2749 .init = vtenc_init, 2750 FF_CODEC_ENCODE_CB(vtenc_frame), 2751 .close = vtenc_close, 2752 .p.priv_class = &h264_videotoolbox_class, 2753 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | 2754 FF_CODEC_CAP_INIT_CLEANUP, 2755}; 2756 2757static const AVOption hevc_options[] = { 2758 { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT64, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" }, 2759 { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" }, 2760 { "main10", "Main10 Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" }, 2761 2762 { "alpha_quality", "Compression quality for the alpha channel", OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0, VE }, 2763 2764 COMMON_OPTIONS 2765 { NULL }, 2766}; 2767 2768static const AVClass hevc_videotoolbox_class = { 2769 .class_name = "hevc_videotoolbox", 2770 .item_name = av_default_item_name, 2771 .option = hevc_options, 2772 .version = LIBAVUTIL_VERSION_INT, 2773}; 2774 2775const FFCodec ff_hevc_videotoolbox_encoder = { 2776 .p.name = "hevc_videotoolbox", 2777 .p.long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"), 2778 .p.type = AVMEDIA_TYPE_VIDEO, 2779 .p.id = AV_CODEC_ID_HEVC, 2780 .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | 2781 AV_CODEC_CAP_HARDWARE, 2782 .priv_data_size = sizeof(VTEncContext), 2783 .p.pix_fmts = hevc_pix_fmts, 2784 .init = vtenc_init, 2785 FF_CODEC_ENCODE_CB(vtenc_frame), 2786 .close = vtenc_close, 2787 .p.priv_class = &hevc_videotoolbox_class, 2788 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | 2789 FF_CODEC_CAP_INIT_CLEANUP, 2790 .p.wrapper_name = "videotoolbox", 2791}; 2792 2793static const AVOption prores_options[] = { 2794 { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT64, { .i64 = FF_PROFILE_UNKNOWN }, FF_PROFILE_UNKNOWN, FF_PROFILE_PRORES_XQ, VE, "profile" }, 2795 { "auto", "Automatically determine based on input format", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_UNKNOWN }, INT_MIN, INT_MAX, VE, "profile" }, 2796 { "proxy", "ProRes 422 Proxy", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_PROXY }, INT_MIN, INT_MAX, VE, "profile" }, 2797 { "lt", "ProRes 422 LT", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_LT }, INT_MIN, INT_MAX, VE, "profile" }, 2798 { "standard", "ProRes 422", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_STANDARD }, INT_MIN, INT_MAX, VE, "profile" }, 2799 { "hq", "ProRes 422 HQ", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_HQ }, INT_MIN, INT_MAX, VE, "profile" }, 2800 { "4444", "ProRes 4444", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_4444 }, INT_MIN, INT_MAX, VE, "profile" }, 2801 { "xq", "ProRes 4444 XQ", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_PRORES_XQ }, INT_MIN, INT_MAX, VE, "profile" }, 2802 2803 COMMON_OPTIONS 2804 { NULL }, 2805}; 2806 2807static const AVClass prores_videotoolbox_class = { 2808 .class_name = "prores_videotoolbox", 2809 .item_name = av_default_item_name, 2810 .option = prores_options, 2811 .version = LIBAVUTIL_VERSION_INT, 2812}; 2813 2814const FFCodec ff_prores_videotoolbox_encoder = { 2815 .p.name = "prores_videotoolbox", 2816 .p.long_name = NULL_IF_CONFIG_SMALL("VideoToolbox ProRes Encoder"), 2817 .p.type = AVMEDIA_TYPE_VIDEO, 2818 .p.id = AV_CODEC_ID_PRORES, 2819 .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | 2820 AV_CODEC_CAP_HARDWARE, 2821 .priv_data_size = sizeof(VTEncContext), 2822 .p.pix_fmts = prores_pix_fmts, 2823 .init = vtenc_init, 2824 FF_CODEC_ENCODE_CB(vtenc_frame), 2825 .close = vtenc_close, 2826 .p.priv_class = &prores_videotoolbox_class, 2827 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | 2828 FF_CODEC_CAP_INIT_CLEANUP, 2829 .p.wrapper_name = "videotoolbox", 2830}; 2831