/* * Copyright (c) 2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "hdecoder.h" #include #include #include #include "utils/hdf_base.h" #include "codec_omx_ext.h" #include "media_description.h" // foundation/multimedia/av_codec/interfaces/inner_api/native/ #include "sync_fence.h" // foundation/graphic/graphic_2d/utils/sync_fence/export/ #include "OMX_VideoExt.h" #include "hcodec_log.h" #include "hcodec_dfx.h" #include "type_converter.h" #include "surface_buffer.h" #include "buffer_extra_data_impl.h" // foundation/graphic/graphic_surface/surface/include/ namespace OHOS::MediaAVCodec { using namespace std; using namespace CodecHDI; HDecoder::~HDecoder() { MsgHandleLoop::Stop(); } int32_t HDecoder::OnConfigure(const Format &format) { configFormat_ = make_shared(format); SupportBufferType type; InitOMXParamExt(type); type.portIndex = OMX_DirOutput; if (GetParameter(OMX_IndexParamSupportBufferType, type) && (type.bufferTypes & CODEC_BUFFER_TYPE_DYNAMIC_HANDLE) && UseHandleOnOutputPort(true)) { HLOGI("dynamic mode"); isDynamic_ = true; } else if (UseHandleOnOutputPort(false)) { HLOGI("normal mode"); isDynamic_ = false; } else { HLOGE("invalid output buffer type"); return AVCS_ERR_UNKNOWN; } int32_t ret = SetLowLatency(format); if (ret != AVCS_ERR_OK) { return ret; } SaveTransform(format); SaveScaleMode(format); (void)SetProcessName(); (void)SetFrameRateAdaptiveMode(format); #ifdef USE_VIDEO_PROCESSING_ENGINE ret = SetVrrEnable(format); if (ret != AVCS_ERR_OK) { return ret; } #endif return SetupPort(format); } int32_t HDecoder::SetupPort(const Format &format) { int32_t width; if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width) || width <= 0) { HLOGE("format should contain width"); return AVCS_ERR_INVALID_VAL; } int32_t height; if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height) || height <= 0) { HLOGE("format should contain height"); return AVCS_ERR_INVALID_VAL; } HLOGI("user set width %d, height %d", width, height); if (!GetPixelFmtFromUser(format)) { return AVCS_ERR_INVALID_VAL; } optional frameRate = GetFrameRateFromUser(format); if (frameRate.has_value()) { codecRate_ = frameRate.value(); } else { HLOGI("user don't set valid frame rate, use default 60.0"); frameRate = 60.0; // default frame rate 60.0 } PortInfo inputPortInfo {static_cast(width), static_cast(height), codingType_, std::nullopt, frameRate.value()}; int32_t maxInputSize = 0; if (format.GetIntValue(MediaDescriptionKey::MD_KEY_MAX_INPUT_SIZE, maxInputSize)) { if (maxInputSize > 0) { inputPortInfo.inputBufSize = static_cast(maxInputSize); } else { HLOGW("user don't set valid input buffer size"); } } int32_t ret = SetVideoPortInfo(OMX_DirInput, inputPortInfo); if (ret != AVCS_ERR_OK) { return ret; } PortInfo outputPortInfo = {static_cast(width), static_cast(height), OMX_VIDEO_CodingUnused, configuredFmt_, frameRate.value()}; ret = SetVideoPortInfo(OMX_DirOutput, outputPortInfo); if (ret != AVCS_ERR_OK) { return ret; } return AVCS_ERR_OK; } int32_t HDecoder::UpdateInPortFormat() { OMX_PARAM_PORTDEFINITIONTYPE def; InitOMXParam(def); def.nPortIndex = OMX_DirInput; if (!GetParameter(OMX_IndexParamPortDefinition, def)) { HLOGE("get input port definition failed"); return AVCS_ERR_UNKNOWN; } PrintPortDefinition(def); if (inputFormat_ == nullptr) { inputFormat_ = make_shared(); } inputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, def.format.video.nFrameWidth); inputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, def.format.video.nFrameHeight); return AVCS_ERR_OK; } bool HDecoder::UpdateConfiguredFmt(OMX_COLOR_FORMATTYPE portFmt) { auto graphicFmt = static_cast(portFmt); if (graphicFmt != configuredFmt_.graphicFmt) { optional fmt = TypeConverter::GraphicFmtToFmt(graphicFmt); if (!fmt.has_value()) { return false; } HLOGI("GraphicPixelFormat need update: configured(%s) -> portdefinition(%s)", configuredFmt_.strFmt.c_str(), fmt->strFmt.c_str()); configuredFmt_ = fmt.value(); } return true; } int32_t HDecoder::UpdateOutPortFormat() { OMX_PARAM_PORTDEFINITIONTYPE def; InitOMXParam(def); def.nPortIndex = OMX_DirOutput; if (!GetParameter(OMX_IndexParamPortDefinition, def)) { HLOGE("get output port definition failed"); return AVCS_ERR_UNKNOWN; } PrintPortDefinition(def); if (def.nBufferCountActual == 0) { HLOGE("invalid bufferCount"); return AVCS_ERR_UNKNOWN; } (void)UpdateConfiguredFmt(def.format.video.eColorFormat); uint32_t w = def.format.video.nFrameWidth; uint32_t h = def.format.video.nFrameHeight; // save into member variable OHOS::Rect damage{}; GetCropFromOmx(w, h, damage); outBufferCnt_ = def.nBufferCountActual; requestCfg_.timeout = 0; // never wait when request requestCfg_.width = damage.w; requestCfg_.height = damage.h; requestCfg_.strideAlignment = STRIDE_ALIGNMENT; requestCfg_.format = configuredFmt_.graphicFmt; requestCfg_.usage = GetProducerUsage(); // save into format if (outputFormat_ == nullptr) { outputFormat_ = make_shared(); } if (!outputFormat_->ContainKey(MediaDescriptionKey::MD_KEY_WIDTH)) { outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, w); // deprecated } if (!outputFormat_->ContainKey(MediaDescriptionKey::MD_KEY_HEIGHT)) { outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, h); // deprecated } outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_DISPLAY_WIDTH, damage.w); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_DISPLAY_HEIGHT, damage.h); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_WIDTH, damage.w); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_HEIGHT, damage.h); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, static_cast(configuredFmt_.innerFmt)); HLOGI("output format: %s", outputFormat_->Stringify().c_str()); return AVCS_ERR_OK; } void HDecoder::UpdateColorAspects() { CodecVideoColorspace param; InitOMXParamExt(param); param.portIndex = OMX_DirOutput; if (!GetParameter(OMX_IndexColorAspects, param, true)) { return; } HLOGI("isFullRange %d, primary %u, transfer %u, matrix %u", param.aspects.range, param.aspects.primaries, param.aspects.transfer, param.aspects.matrixCoeffs); if (outputFormat_) { outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_RANGE_FLAG, param.aspects.range); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_COLOR_PRIMARIES, param.aspects.primaries); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_TRANSFER_CHARACTERISTICS, param.aspects.transfer); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_MATRIX_COEFFICIENTS, param.aspects.matrixCoeffs); HLOGI("output format changed: %s", outputFormat_->Stringify().c_str()); callback_->OnOutputFormatChanged(*(outputFormat_.get())); } } void HDecoder::GetCropFromOmx(uint32_t w, uint32_t h, OHOS::Rect& damage) { damage.x = 0; damage.y = 0; damage.w = static_cast(w); damage.h = static_cast(h); OMX_CONFIG_RECTTYPE rect; InitOMXParam(rect); rect.nPortIndex = OMX_DirOutput; if (!GetParameter(OMX_IndexConfigCommonOutputCrop, rect, true)) { HLOGW("get crop failed, use default"); return; } if (rect.nLeft < 0 || rect.nTop < 0 || rect.nWidth == 0 || rect.nHeight == 0 || rect.nLeft + static_cast(rect.nWidth) > static_cast(w) || rect.nTop + static_cast(rect.nHeight) > static_cast(h)) { HLOGW("wrong crop rect (%d, %d, %u, %u) vs. frame (%u," \ "%u), use default", rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight, w, h); return; } HLOGI("crop rect (%d, %d, %u, %u)", rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight); damage.x = rect.nLeft; damage.y = rect.nTop; damage.w = static_cast(rect.nWidth); damage.h = static_cast(rect.nHeight); if (outputFormat_) { outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_CROP_LEFT, rect.nLeft); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_CROP_TOP, rect.nTop); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_CROP_RIGHT, static_cast(rect.nLeft + rect.nWidth) - 1); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_CROP_BOTTOM, static_cast(rect.nTop + rect.nHeight) - 1); } } void HDecoder::OnSetOutputSurface(const MsgInfo &msg, BufferOperationMode mode) { sptr surface; (void)msg.param->GetValue("surface", surface); if (mode == KEEP_BUFFER) { ReplyErrorCode(msg.id, OnSetOutputSurfaceWhenCfg(surface)); return; } OnSetOutputSurfaceWhenRunning(surface, msg, mode); } int32_t HDecoder::OnSetOutputSurfaceWhenCfg(const sptr &surface) { SCOPED_TRACE(); HLOGI(">>"); if (surface == nullptr) { HLOGE("surface is null"); return AVCS_ERR_INVALID_VAL; } if (surface->IsConsumer()) { HLOGE("expect a producer surface but got a consumer surface"); return AVCS_ERR_INVALID_VAL; } int32_t ret = RegisterListenerToSurface(surface); if (ret != AVCS_ERR_OK) { return ret; } currSurface_ = SurfaceItem(surface); HLOGI("set surface(%" PRIu64 ")(%s) succ", surface->GetUniqueId(), surface->GetName().c_str()); return AVCS_ERR_OK; } int32_t HDecoder::OnSetParameters(const Format &format) { int32_t ret = SaveTransform(format, true); if (ret != AVCS_ERR_OK) { return ret; } ret = SaveScaleMode(format, true); if (ret != AVCS_ERR_OK) { return ret; } optional frameRate = GetFrameRateFromUser(format); if (frameRate.has_value()) { OMX_PARAM_U32TYPE framerateCfgType; InitOMXParam(framerateCfgType); framerateCfgType.nPortIndex = OMX_DirInput; framerateCfgType.nU32 = frameRate.value() * FRAME_RATE_COEFFICIENT; if (SetParameter(OMX_IndexCodecExtConfigOperatingRate, framerateCfgType, true)) { HLOGI("succ to set frameRate %.f", frameRate.value()); } else { HLOGW("succ to set frameRate %.f", frameRate.value()); } codecRate_ = frameRate.value(); } return AVCS_ERR_OK; } int32_t HDecoder::SaveTransform(const Format &format, bool set) { int32_t orientation = 0; if (format.GetIntValue(OHOS::Media::Tag::VIDEO_ORIENTATION_TYPE, orientation)) { HLOGI("GraphicTransformType = %d", orientation); transform_ = static_cast(orientation); if (set) { return SetTransform(); } return AVCS_ERR_OK; } int32_t rotate; if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_ROTATION_ANGLE, rotate)) { return AVCS_ERR_OK; } optional transform = TypeConverter::InnerRotateToDisplayRotate( static_cast(rotate)); if (!transform.has_value()) { return AVCS_ERR_INVALID_VAL; } HLOGI("VideoRotation = %d, GraphicTransformType = %d", rotate, transform.value()); transform_ = transform.value(); if (set) { return SetTransform(); } return AVCS_ERR_OK; } int32_t HDecoder::SetTransform() { if (currSurface_.surface_ == nullptr) { return AVCS_ERR_INVALID_VAL; } GSError err = currSurface_.surface_->SetTransform(transform_); if (err != GSERROR_OK) { HLOGW("set GraphicTransformType %d to surface failed", transform_); return AVCS_ERR_UNKNOWN; } HLOGI("set GraphicTransformType %d to surface succ", transform_); return AVCS_ERR_OK; } int32_t HDecoder::SaveScaleMode(const Format &format, bool set) { int scaleType; if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_SCALE_TYPE, scaleType)) { return AVCS_ERR_OK; } auto scaleMode = static_cast(scaleType); if (scaleMode != SCALING_MODE_SCALE_TO_WINDOW && scaleMode != SCALING_MODE_SCALE_CROP) { HLOGW("user set invalid scale mode %d", scaleType); return AVCS_ERR_INVALID_VAL; } HLOGI("user set ScalingType = %d", scaleType); scaleMode_ = scaleMode; if (set) { return SetScaleMode(); } return AVCS_ERR_OK; } int32_t HDecoder::SetScaleMode() { if (currSurface_.surface_ == nullptr || !scaleMode_.has_value()) { return AVCS_ERR_INVALID_VAL; } GSError err = currSurface_.surface_->SetScalingMode(scaleMode_.value()); if (err != GSERROR_OK) { HLOGW("set ScalingMode %d to surface failed", scaleMode_.value()); return AVCS_ERR_UNKNOWN; } HLOGI("set ScalingMode %d to surface succ", scaleMode_.value()); return AVCS_ERR_OK; } #ifdef USE_VIDEO_PROCESSING_ENGINE int32_t HDecoder::SetVrrEnable(const Format &format) { int32_t vrrEnable = 0; if (!format.GetIntValue(OHOS::Media::Tag::VIDEO_DECODER_ENABLE_VRR, vrrEnable) || vrrEnable == 0) { HLOGI("VRR disabled"); return AVCS_ERR_OK; } optional frameRate = GetFrameRateFromUser(format); if (frameRate.has_value() && floor(frameRate.value()) != VRR_DEFAULT_INPUT_FRAME_RATE && ceil(frameRate.value()) != VRR_DEFAULT_INPUT_FRAME_RATE) { HLOGE("VRR only support for 60fps, current frameRate = %f", frameRate.value()); return AVCS_ERR_UNSUPPORT; } OMX_CONFIG_BOOLEANTYPE param {}; InitOMXParam(param); param.bEnabled = OMX_TRUE; if (!SetParameter(OMX_IndexParamIsMvUpload, param)) { HLOGE("VRR SetIsMvUploadParam SetParameter failed"); return AVCS_ERR_UNSUPPORT; } vrrPredictor_ = OHOS::Media::VideoProcessingEngine::VideoRefreshRatePrediction::Create(); if (vrrPredictor_ == nullptr) { HLOGE("VRR VideoRefreshRatePrediction create failed"); return AVCS_ERR_UNSUPPORT; } int32_t ret = vrrPredictor_->CheckLtpoSupport(); if (ret != AVCS_ERR_OK) { HLOGE("VRR SetParameter failed"); return AVCS_ERR_UNKNOWN; } isVrrEnable_ = true; HLOGI("VRR enabled"); return AVCS_ERR_OK; } #endif int32_t HDecoder::SubmitOutputBuffersToOmxNode() { for (BufferInfo& info : outputBufferPool_) { if (info.owner != BufferOwner::OWNED_BY_US) { continue; } if (info.surfaceBuffer != nullptr) { int32_t ret = NotifyOmxToFillThisOutBuffer(info); if (ret != AVCS_ERR_OK) { return ret; } } } auto inCnt = std::count_if(inputBufferPool_.begin(), inputBufferPool_.end(), [](const BufferInfo& info) { return info.owner == BufferOwner::OWNED_BY_OMX; }); inCnt++; // at least submit one out buffer to omx while (inCnt > 0) { DynamicModeSubmitBuffer(); inCnt--; } return AVCS_ERR_OK; } bool HDecoder::UseHandleOnOutputPort(bool isDynamic) { UseBufferType useBufferTypes; InitOMXParamExt(useBufferTypes); useBufferTypes.portIndex = OMX_DirOutput; useBufferTypes.bufferType = (isDynamic ? CODEC_BUFFER_TYPE_DYNAMIC_HANDLE : CODEC_BUFFER_TYPE_HANDLE); return SetParameter(OMX_IndexParamUseBufferType, useBufferTypes); } bool HDecoder::ReadyToStart() { if (callback_ == nullptr || outputFormat_ == nullptr || inputFormat_ == nullptr) { return false; } if (currSurface_.surface_) { HLOGI("surface mode"); SetTransform(); SetScaleMode(); } else { HLOGI("buffer mode"); } return true; } int32_t HDecoder::AllocateBuffersOnPort(OMX_DIRTYPE portIndex) { if (portIndex == OMX_DirInput) { return AllocateAvLinearBuffers(portIndex); } int32_t ret; if (isDynamic_) { ret = AllocOutDynamicSurfaceBuf(); } else { ret = (currSurface_.surface_ ? AllocateOutputBuffersFromSurface() : AllocateAvSurfaceBuffers(portIndex)); } if (ret == AVCS_ERR_OK) { UpdateFormatFromSurfaceBuffer(); } return ret; } void HDecoder::SetCallerToBuffer(int fd) { if (currSurface_.surface_ == nullptr) { return; // only set on surface mode } string pid = std::to_string(calledByAvcodec_ ? avcodecCaller_.pid : playerCaller_.pid); int ret = ioctl(fd, DMA_BUF_SET_NAME_A, pid.c_str()); if (ret != 0) { HLOGW("set pid %s to fd %d failed", pid.c_str(), fd); return; } HLOGD("set pid %s to fd %d succ", pid.c_str(), fd); } void HDecoder::UpdateFormatFromSurfaceBuffer() { if (outputBufferPool_.empty()) { return; } sptr surfaceBuffer = outputBufferPool_.front().surfaceBuffer; if (surfaceBuffer == nullptr) { return; } HLOGI(">>"); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_DISPLAY_WIDTH, surfaceBuffer->GetWidth()); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_DISPLAY_HEIGHT, surfaceBuffer->GetHeight()); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_WIDTH, surfaceBuffer->GetWidth()); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_PIC_HEIGHT, surfaceBuffer->GetHeight()); int32_t stride = surfaceBuffer->GetStride(); if (stride <= 0) { HLOGW("invalid stride %d", stride); return; } outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_STRIDE, stride); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_WIDTH, stride); // deprecated OH_NativeBuffer_Planes *planes = nullptr; GSError err = surfaceBuffer->GetPlanesInfo(reinterpret_cast(&planes)); if (err != GSERROR_OK || planes == nullptr) { HLOGI("can not get plane info, ignore"); return; } for (uint32_t i = 0; i < planes->planeCount; i++) { HLOGI("plane[%u]: offset=%" PRIu64 ", rowStride=%u, columnStride=%u", i, planes->planes[i].offset, planes->planes[i].rowStride, planes->planes[i].columnStride); } int32_t sliceHeight = static_cast(static_cast(planes->planes[1].offset) / stride); HLOGI("[%dx%d][%dx%d]", surfaceBuffer->GetWidth(), surfaceBuffer->GetHeight(), stride, sliceHeight); outputFormat_->PutIntValue(OHOS::Media::Tag::VIDEO_SLICE_HEIGHT, sliceHeight); outputFormat_->PutIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, sliceHeight); } int32_t HDecoder::SubmitAllBuffersOwnedByUs() { HLOGI(">>"); if (isBufferCirculating_) { HLOGI("buffer is already circulating, no need to do again"); return AVCS_ERR_OK; } int32_t ret = SubmitOutputBuffersToOmxNode(); if (ret != AVCS_ERR_OK) { return ret; } for (BufferInfo& info : inputBufferPool_) { if (info.owner == BufferOwner::OWNED_BY_US) { NotifyUserToFillThisInBuffer(info); } } isBufferCirculating_ = true; return AVCS_ERR_OK; } void HDecoder::EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) { vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; if (i >= pool.size()) { return; } BufferInfo& info = pool[i]; if (portIndex == OMX_DirOutput && info.owner != BufferOwner::OWNED_BY_SURFACE) { CancelBufferToSurface(info); } FreeOmxBuffer(portIndex, info); ReduceOwner((portIndex == OMX_DirInput), info.owner); pool.erase(pool.begin() + i); } void HDecoder::OnClearBufferPool(OMX_DIRTYPE portIndex) { if ((portIndex == OMX_DirOutput) && currSurface_.surface_) { GSError err = currSurface_.surface_->CleanCache(); if (err != GSERROR_OK) { HLOGW("clean cache failed, GSError=%d", err); } freeList_.clear(); } } uint64_t HDecoder::GetProducerUsage() { uint64_t producerUsage = currSurface_.surface_ ? SURFACE_MODE_PRODUCER_USAGE : BUFFER_MODE_REQUEST_USAGE; GetBufferHandleUsageParams vendorUsage; InitOMXParamExt(vendorUsage); vendorUsage.portIndex = static_cast(OMX_DirOutput); if (GetParameter(OMX_IndexParamGetBufferHandleUsage, vendorUsage)) { HLOGI("vendor producer usage = 0x%" PRIx64 "", vendorUsage.usage); producerUsage |= vendorUsage.usage; } else { HLOGW("get vendor producer usage failed, add CPU_READ"); producerUsage |= BUFFER_USAGE_CPU_READ; } HLOGI("decoder producer usage = 0x%" PRIx64 "", producerUsage); return producerUsage; } void HDecoder::CombineConsumerUsage() { uint32_t consumerUsage = currSurface_.surface_->GetDefaultUsage(); uint64_t finalUsage = requestCfg_.usage | consumerUsage; HLOGI("producer usage 0x%" PRIx64 " | consumer usage 0x%x -> 0x%" PRIx64 "", requestCfg_.usage, consumerUsage, finalUsage); requestCfg_.usage = finalUsage; } int32_t HDecoder::SetQueueSize(const sptr &surface, uint32_t targetSize) { GSError err = surface->SetQueueSize(targetSize); if (err != GSERROR_OK) { HLOGE("surface(%" PRIu64 "), SetQueueSize to %u failed, GSError=%d", surface->GetUniqueId(), targetSize, err); return AVCS_ERR_UNKNOWN; } HLOGI("surface(%" PRIu64 "), SetQueueSize to %u succ", surface->GetUniqueId(), targetSize); return AVCS_ERR_OK; } int32_t HDecoder::AllocOutDynamicSurfaceBuf() { SCOPED_TRACE(); if (currSurface_.surface_) { currGeneration_++; currSurface_.surface_->CleanCache(); int32_t ret = SetQueueSize(currSurface_.surface_, outBufferCnt_); if (ret != AVCS_ERR_OK) { return ret; } } outputBufferPool_.clear(); for (uint32_t i = 0; i < outBufferCnt_; ++i) { shared_ptr omxBuffer = DynamicSurfaceBufferToOmxBuffer(); shared_ptr outBuffer = make_shared(); int32_t ret = compNode_->UseBuffer(OMX_DirOutput, *omxBuffer, *outBuffer); if (ret != HDF_SUCCESS) { HLOGE("Failed to UseBuffer on input port"); return AVCS_ERR_UNKNOWN; } BufferInfo info {}; info.isInput = false; info.owner = BufferOwner::OWNED_BY_US; info.surfaceBuffer = nullptr; info.avBuffer = (currSurface_.surface_ ? AVBuffer::CreateAVBuffer() : nullptr); info.omxBuffer = outBuffer; info.bufferId = outBuffer->bufferId; outputBufferPool_.push_back(info); } HLOGI("succ"); return AVCS_ERR_OK; } int32_t HDecoder::AllocateOutputBuffersFromSurface() { SCOPED_TRACE(); currGeneration_++; currSurface_.surface_->CleanCache(); int32_t ret = SetQueueSize(currSurface_.surface_, outBufferCnt_); if (ret != AVCS_ERR_OK) { return ret; } outputBufferPool_.clear(); CombineConsumerUsage(); for (uint32_t i = 0; i < outBufferCnt_; ++i) { sptr surfaceBuffer; sptr fence; GSError err = currSurface_.surface_->RequestBuffer(surfaceBuffer, fence, requestCfg_); if (err != GSERROR_OK || surfaceBuffer == nullptr) { HLOGE("RequestBuffer %u failed, GSError=%d", i, err); return AVCS_ERR_UNKNOWN; } shared_ptr omxBuffer = SurfaceBufferToOmxBuffer(surfaceBuffer); if (omxBuffer == nullptr) { currSurface_.surface_->CancelBuffer(surfaceBuffer); return AVCS_ERR_UNKNOWN; } shared_ptr outBuffer = make_shared(); int32_t hdfRet = compNode_->UseBuffer(OMX_DirOutput, *omxBuffer, *outBuffer); if (hdfRet != HDF_SUCCESS) { currSurface_.surface_->CancelBuffer(surfaceBuffer); HLOGE("Failed to UseBuffer with output port"); return AVCS_ERR_NO_MEMORY; } SetCallerToBuffer(surfaceBuffer->GetFileDescriptor()); outBuffer->fenceFd = -1; BufferInfo info {}; info.isInput = false; info.owner = BufferOwner::OWNED_BY_US; info.surfaceBuffer = surfaceBuffer; info.avBuffer = AVBuffer::CreateAVBuffer(); info.omxBuffer = outBuffer; info.bufferId = outBuffer->bufferId; outputBufferPool_.push_back(info); HLOGI("generation=%d, bufferId=%u, seq=%u", currGeneration_, info.bufferId, surfaceBuffer->GetSeqNum()); } return AVCS_ERR_OK; } void HDecoder::CancelBufferToSurface(BufferInfo& info) { if (currSurface_.surface_ && info.surfaceBuffer) { GSError err = currSurface_.surface_->CancelBuffer(info.surfaceBuffer); if (err != GSERROR_OK) { HLOGW("surface(%" PRIu64 "), CancelBuffer(seq=%u) failed, GSError=%d", currSurface_.surface_->GetUniqueId(), info.surfaceBuffer->GetSeqNum(), err); } else { HLOGI("surface(%" PRIu64 "), CancelBuffer(seq=%u) succ", currSurface_.surface_->GetUniqueId(), info.surfaceBuffer->GetSeqNum()); } } ChangeOwner(info, BufferOwner::OWNED_BY_SURFACE); // change owner even if cancel failed } int32_t HDecoder::RegisterListenerToSurface(const sptr &surface) { uint64_t surfaceId = surface->GetUniqueId(); std::weak_ptr weakThis = weak_from_this(); GSError err = surface->RegisterReleaseListener([weakThis, surfaceId](sptr&) { std::shared_ptr codec = weakThis.lock(); if (codec == nullptr) { LOGD("decoder is gone"); return GSERROR_OK; } return codec->OnBufferReleasedByConsumer(surfaceId); }); if (err != GSERROR_OK) { HLOGE("surface(%" PRIu64 "), RegisterReleaseListener failed, GSError=%d", surfaceId, err); return AVCS_ERR_UNKNOWN; } return AVCS_ERR_OK; } GSError HDecoder::OnBufferReleasedByConsumer(uint64_t surfaceId) { ParamSP param = make_shared(); param->SetValue("surfaceId", surfaceId); SendAsyncMsg(MsgWhat::GET_BUFFER_FROM_SURFACE, param); return GSERROR_OK; } HDecoder::SurfaceBufferItem HDecoder::RequestBuffer() { SurfaceBufferItem item{}; item.generation = currGeneration_; GSError err = currSurface_.surface_->RequestBuffer(item.buffer, item.fence, requestCfg_); if (err != GSERROR_OK || item.buffer == nullptr || item.buffer->GetBufferHandle() == nullptr) { HLOGW("RequestBuffer failed, GSError=%d", err); return { nullptr, nullptr }; } return item; } std::vector::iterator HDecoder::FindBelongTo(sptr& buffer) { BufferHandle* handle = buffer->GetBufferHandle(); return std::find_if(outputBufferPool_.begin(), outputBufferPool_.end(), [handle](const BufferInfo& info) { return (info.owner == BufferOwner::OWNED_BY_SURFACE) && info.surfaceBuffer && (info.surfaceBuffer->GetBufferHandle() == handle); }); } void HDecoder::OnGetBufferFromSurface(const ParamSP& param) { SCOPED_TRACE(); uint64_t surfaceId = 0; param->GetValue("surfaceId", surfaceId); if (!currSurface_.surface_ || currSurface_.surface_->GetUniqueId() != surfaceId) { return; } SurfaceBufferItem item = RequestBuffer(); if (item.buffer == nullptr) { return; } ScopedTrace tracePoint("requested:" + std::to_string(item.buffer->GetSeqNum())); freeList_.push_back(item); // push to list, retrive it later, to avoid wait fence too early static constexpr size_t MAX_CACHE_CNT = 2; if (item.fence == nullptr || !item.fence->IsValid() || freeList_.size() > MAX_CACHE_CNT) { SurfaceModeSubmitBuffer(); } } void HDecoder::SurfaceModeSubmitBuffer() { if (!isDynamic_) { // in surface normal mode, just get one surfacebuffer SurfaceModeSubmitBufferFromFreeList(); return; } // in dynamic mode, find slot which has null surfacebuffer auto nullSlot = std::find_if(outputBufferPool_.begin(), outputBufferPool_.end(), [](const BufferInfo& info) { return info.surfaceBuffer == nullptr; }); if (nullSlot == outputBufferPool_.end()) { // if every slot is not null, just get one surfacebuffer SurfaceModeSubmitBufferFromFreeList(); return; } SurfaceDynamicModeSubmitBuffer(nullSlot); } void HDecoder::SurfaceModeSubmitBufferFromFreeList() { while (!freeList_.empty()) { SurfaceBufferItem item = freeList_.front(); freeList_.pop_front(); if (SurfaceModeSubmitOneItem(item)) { return; } } } bool HDecoder::SurfaceModeSubmitOneItem(SurfaceBufferItem& item) { SCOPED_TRACE_WITH_ID(item.buffer->GetSeqNum()); if (item.generation != currGeneration_) { HLOGI("buffer generation %d != current generation %d, ignore", item.generation, currGeneration_); return false; } auto iter = FindBelongTo(item.buffer); if (iter == outputBufferPool_.end()) { HLOGI("seq=%u dont belong to output set, ignore", item.buffer->GetSeqNum()); return false; } WaitFence(item.fence); ChangeOwner(*iter, BufferOwner::OWNED_BY_US); NotifyOmxToFillThisOutBuffer(*iter); return true; } void HDecoder::DynamicModeSubmitBuffer() { if (!isDynamic_) { return; } auto nullSlot = std::find_if(outputBufferPool_.begin(), outputBufferPool_.end(), [](const BufferInfo& info) { return info.surfaceBuffer == nullptr; }); if (nullSlot == outputBufferPool_.end()) { return; } if (currSurface_.surface_) { SurfaceDynamicModeSubmitBuffer(nullSlot); } else { BufferDynamicModeSubmitBuffer(nullSlot); } } void HDecoder::SurfaceDynamicModeSubmitBuffer(std::vector::iterator nullSlot) { SCOPED_TRACE(); for (size_t i = 0; i < outputBufferPool_.size(); i++) { SurfaceBufferItem item{}; if (!freeList_.empty()) { item = freeList_.front(); freeList_.pop_front(); } else { item = RequestBuffer(); if (item.buffer == nullptr) { return; } } if (item.generation != currGeneration_) { HLOGI("buffer generation %d != current generation %d, ignore", item.generation, currGeneration_); continue; } auto iter = FindBelongTo(item.buffer); WaitFence(item.fence); if (iter != outputBufferPool_.end()) { // familiar buffer ChangeOwner(*iter, BufferOwner::OWNED_BY_US); NotifyOmxToFillThisOutBuffer(*iter); continue; // keep request until we got a new surfacebuffer } // unfamiliar buffer SetCallerToBuffer(item.buffer->GetFileDescriptor()); HLOGI("generation=%d, bufferId=%u, seq=%u", currGeneration_, nullSlot->bufferId, item.buffer->GetSeqNum()); WrapSurfaceBufferToSlot(*nullSlot, item.buffer, 0, 0); if (nullSlot == outputBufferPool_.begin()) { UpdateFormatFromSurfaceBuffer(); } NotifyOmxToFillThisOutBuffer(*nullSlot); nullSlot->omxBuffer->bufferhandle = nullptr; return; } } void HDecoder::BufferDynamicModeSubmitBuffer(std::vector::iterator nullSlot) { SCOPED_TRACE(); sptr buffer = SurfaceBuffer::Create(); IF_TRUE_RETURN_VOID_WITH_MSG(buffer == nullptr, "CreateSurfaceBuffer failed"); GSError err = buffer->Alloc(requestCfg_); IF_TRUE_RETURN_VOID_WITH_MSG(err != GSERROR_OK, "AllocSurfaceBuffer failed"); std::shared_ptr avBuffer = AVBuffer::CreateAVBuffer(buffer); if (avBuffer == nullptr || avBuffer->memory_ == nullptr) { HLOGE("CreateAVBuffer failed"); return; } SetCallerToBuffer(buffer->GetFileDescriptor()); HLOGI("bufferId=%u, seq=%u", nullSlot->bufferId, buffer->GetSeqNum()); WrapSurfaceBufferToSlot(*nullSlot, buffer, 0, 0); nullSlot->avBuffer = avBuffer; nullSlot->needDealWithCache = (requestCfg_.usage & BUFFER_USAGE_MEM_MMZ_CACHE); if (nullSlot == outputBufferPool_.begin()) { UpdateFormatFromSurfaceBuffer(); } NotifyOmxToFillThisOutBuffer(*nullSlot); nullSlot->omxBuffer->bufferhandle = nullptr; } int32_t HDecoder::NotifySurfaceToRenderOutputBuffer(BufferInfo &info) { SCOPED_TRACE_WITH_ID(info.omxBuffer->pts); info.lastFlushTime = GetNowUs(); if (std::abs(lastFlushRate_ - codecRate_) > std::numeric_limits::epsilon()) { sptr extraData = new BufferExtraDataImpl(); extraData->ExtraSet("VIDEO_RATE", codecRate_); info.surfaceBuffer->SetExtraData(extraData); lastFlushRate_ = codecRate_; HLOGI("flush video rate(%d)", static_cast(codecRate_)); } BufferFlushConfig cfg { .damage = {.x = 0, .y = 0, .w = info.surfaceBuffer->GetWidth(), .h = info.surfaceBuffer->GetHeight() }, .timestamp = info.omxBuffer->pts, .desiredPresentTimestamp = -1, }; if (info.avBuffer->meta_->Find(OHOS::Media::Tag::VIDEO_DECODER_DESIRED_PRESENT_TIMESTAMP) != info.avBuffer->meta_->end()) { info.avBuffer->meta_->Get( cfg.desiredPresentTimestamp); info.avBuffer->meta_->Remove(OHOS::Media::Tag::VIDEO_DECODER_DESIRED_PRESENT_TIMESTAMP); } GSError ret = currSurface_.surface_->FlushBuffer(info.surfaceBuffer, -1, cfg); if (ret != GSERROR_OK) { HLOGW("surface(%" PRIu64 "), FlushBuffer(seq=%u) failed, GSError=%d", currSurface_.surface_->GetUniqueId(), info.surfaceBuffer->GetSeqNum(), ret); return AVCS_ERR_UNKNOWN; } ChangeOwner(info, BufferOwner::OWNED_BY_SURFACE); return AVCS_ERR_OK; } void HDecoder::OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) { SCOPED_TRACE_WITH_ID(bufferId); BufferInfo *info = FindBufferInfoByID(OMX_DirInput, bufferId); if (info == nullptr) { HLOGE("unknown buffer id %u", bufferId); return; } if (info->owner != BufferOwner::OWNED_BY_OMX) { HLOGE("wrong ownership: buffer id=%d, owner=%s", bufferId, ToString(info->owner)); return; } ChangeOwner(*info, BufferOwner::OWNED_BY_US); switch (mode) { case KEEP_BUFFER: return; case RESUBMIT_BUFFER: { if (!inputPortEos_) { NotifyUserToFillThisInBuffer(*info); } return; } default: { HLOGE("SHOULD NEVER BE HERE"); return; } } } void HDecoder::OnReleaseOutputBuffer(const BufferInfo &info) { if (currSurface_.surface_) { HLOGI("outBufId = %u, discard by user, pts = %" PRId64, info.bufferId, info.omxBuffer->pts); } } void HDecoder::OnRenderOutputBuffer(const MsgInfo &msg, BufferOperationMode mode) { if (currSurface_.surface_ == nullptr) { HLOGE("can only render in surface mode"); ReplyErrorCode(msg.id, AVCS_ERR_INVALID_OPERATION); return; } uint32_t bufferId = 0; (void)msg.param->GetValue(BUFFER_ID, bufferId); SCOPED_TRACE_WITH_ID(bufferId); optional idx = FindBufferIndexByID(OMX_DirOutput, bufferId); if (!idx.has_value()) { ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL); return; } BufferInfo& info = outputBufferPool_[idx.value()]; if (info.owner != BufferOwner::OWNED_BY_USER) { HLOGE("wrong ownership: buffer id=%d, owner=%s", bufferId, ToString(info.owner)); ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL); return; } info.omxBuffer->pts = info.avBuffer->pts_; ChangeOwner(info, BufferOwner::OWNED_BY_US); ReplyErrorCode(msg.id, AVCS_ERR_OK); if (info.omxBuffer->filledLen != 0) { NotifySurfaceToRenderOutputBuffer(info); } if (mode == FREE_BUFFER) { EraseBufferFromPool(OMX_DirOutput, idx.value()); } else { SurfaceModeSubmitBuffer(); } } void HDecoder::OnEnterUninitializedState() { freeList_.clear(); currSurface_.Release(); } HDecoder::SurfaceItem::SurfaceItem(const sptr &surface) : surface_(surface), originalTransform_(surface->GetTransform()) {} void HDecoder::SurfaceItem::Release() { if (surface_) { LOGI("release surface(%" PRIu64 ")", surface_->GetUniqueId()); if (originalTransform_.has_value()) { surface_->SetTransform(originalTransform_.value()); originalTransform_ = std::nullopt; } surface_ = nullptr; } } void HDecoder::OnSetOutputSurfaceWhenRunning(const sptr &newSurface, const MsgInfo &msg, BufferOperationMode mode) { SCOPED_TRACE(); if (currSurface_.surface_ == nullptr) { HLOGE("can only switch surface on surface mode"); ReplyErrorCode(msg.id, AVCS_ERR_INVALID_OPERATION); return; } if (newSurface == nullptr) { HLOGE("surface is null"); ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL); return; } if (newSurface->IsConsumer()) { HLOGE("expect a producer surface but got a consumer surface"); ReplyErrorCode(msg.id, AVCS_ERR_INVALID_VAL); return; } uint64_t oldId = currSurface_.surface_->GetUniqueId(); uint64_t newId = newSurface->GetUniqueId(); HLOGI("surface %" PRIu64 " -> %" PRIu64, oldId, newId); if (oldId == newId) { HLOGI("same surface, no need to set again"); ReplyErrorCode(msg.id, AVCS_ERR_OK); return; } int32_t ret = RegisterListenerToSurface(newSurface); if (ret != AVCS_ERR_OK) { ReplyErrorCode(msg.id, ret); return; } ret = SetQueueSize(newSurface, outBufferCnt_); if (ret != AVCS_ERR_OK) { ReplyErrorCode(msg.id, ret); return; } SwitchBetweenSurface(newSurface, msg, mode); } void HDecoder::ConsumeFreeList(BufferOperationMode mode) { SCOPED_TRACE(); while (!freeList_.empty()) { SurfaceBufferItem item = freeList_.front(); freeList_.pop_front(); auto iter = FindBelongTo(item.buffer); if (iter == outputBufferPool_.end()) { continue; } ChangeOwner(*iter, BufferOwner::OWNED_BY_US); if (mode == RESUBMIT_BUFFER) { WaitFence(item.fence); NotifyOmxToFillThisOutBuffer(*iter); } } } void HDecoder::SwitchBetweenSurface(const sptr &newSurface, const MsgInfo &msg, BufferOperationMode mode) { SCOPED_TRACE(); newSurface->Connect(); // cleancache will work only if the surface is connected by us newSurface->CleanCache(); // make sure new surface is empty uint64_t newId = newSurface->GetUniqueId(); for (size_t i = 0; i < outputBufferPool_.size(); i++) { BufferInfo& info = outputBufferPool_[i]; if (info.surfaceBuffer == nullptr) { continue; } GSError err = newSurface->AttachBufferToQueue(info.surfaceBuffer); if (err != GSERROR_OK) { HLOGE("surface(%" PRIu64 "), AttachBufferToQueue(seq=%u) failed, GSError=%d", newId, info.surfaceBuffer->GetSeqNum(), err); ReplyErrorCode(msg.id, AVCS_ERR_UNKNOWN); return; } } ReplyErrorCode(msg.id, AVCS_ERR_OK); ConsumeFreeList(mode); map ownedBySurfaceFlushTime2BufferIndex; vector ownedByUs; for (size_t i = 0; i < outputBufferPool_.size(); i++) { BufferInfo& info = outputBufferPool_[i]; if (info.surfaceBuffer == nullptr) { continue; } if (info.owner == OWNED_BY_SURFACE) { ownedBySurfaceFlushTime2BufferIndex[info.lastFlushTime] = i; } else if (info.owner == OWNED_BY_US) { ownedByUs.push_back(i); } } SurfaceItem oldSurface = currSurface_; currSurface_ = SurfaceItem(newSurface); // if owned by old surface, we need to transfer them to new surface for (auto [flushTime, i] : ownedBySurfaceFlushTime2BufferIndex) { ChangeOwner(outputBufferPool_[i], BufferOwner::OWNED_BY_US); NotifySurfaceToRenderOutputBuffer(outputBufferPool_[i]); } // the consumer of old surface may be destroyed, so flushbuffer will fail, and they are owned by us for (size_t i : ownedByUs) { if (mode == RESUBMIT_BUFFER) { NotifyOmxToFillThisOutBuffer(outputBufferPool_[i]); } } oldSurface.surface_->CleanCache(true); // make sure old surface is empty and go black oldSurface.Release(); SetTransform(); SetScaleMode(); HLOGI("set surface(%" PRIu64 ")(%s) succ", newId, newSurface->GetName().c_str()); } #ifdef USE_VIDEO_PROCESSING_ENGINE int32_t HDecoder::VrrPrediction(BufferInfo &info) { SCOPED_TRACE(); if (vrrPredictor_ == nullptr) { HLOGE("vrrPredictor_ is nullptr"); return AVCS_ERR_INVALID_OPERATION; } int vrrMvType = Media::VideoProcessingEngine::MOTIONVECTOR_TYPE_NONE; if (static_cast(codingType_) == CODEC_OMX_VIDEO_CodingHEVC) { vrrMvType = Media::VideoProcessingEngine::MOTIONVECTOR_TYPE_HEVC; } else if (static_cast(codingType_) == OMX_VIDEO_CodingAVC) { vrrMvType = Media::VideoProcessingEngine::MOTIONVECTOR_TYPE_AVC; } else { HLOGE("VRR only support for HEVC or AVC"); return AVCS_ERR_UNSUPPORT; } vrrPredictor_->Process(info.surfaceBuffer, static_cast(codecRate_), vrrMvType); return AVCS_ERR_OK; } #endif } // namespace OHOS::MediaAVCodec