1 /*
2  * Copyright (c) 2024-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "video_encoder.h"
17 #include <cstdint>
18 #include <string>
19 #include <unordered_map>
20 #include "frame_record.h"
21 #include "surface_type.h"
22 #include "external_window.h"
23 #include "sample_callback.h"
24 #include "camera_log.h"
25 #include <chrono>
26 #include <fcntl.h>
27 #include <cinttypes>
28 #include <unistd.h>
29 #include <memory>
30 #include <sync_fence.h>
31 #include "surface_utils.h"
32 #include <cinttypes>
33 
34 namespace OHOS {
35 namespace CameraStandard {
36 
~VideoEncoder()37 VideoEncoder::~VideoEncoder()
38 {
39     MEDIA_INFO_LOG("~VideoEncoder enter");
40     Release();
41 }
42 
VideoEncoder(VideoCodecType type)43 VideoEncoder::VideoEncoder(VideoCodecType type) : videoCodecType_(type)
44 {
45     MEDIA_INFO_LOG("VideoEncoder enter");
46 }
47 
Create(const std::string &codecMime)48 int32_t VideoEncoder::Create(const std::string &codecMime)
49 {
50     std::lock_guard<std::mutex> lock(encoderMutex_);
51     encoder_ = OH_VideoEncoder_CreateByMime(codecMime.data());
52     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Create failed");
53     return 0;
54 }
55 
Config()56 int32_t VideoEncoder::Config()
57 {
58     std::lock_guard<std::mutex> lock(encoderMutex_);
59     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
60     std::unique_lock<std::mutex> contextLock(contextMutex_);
61     context_ = new CodecUserData;
62     // Configure video encoder
63     int32_t ret = Configure();
64     CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Configure failed");
65     // SetCallback for video encoder
66     ret = SetCallback(context_);
67     CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Set callback failed");
68     contextLock.unlock();
69     return 0;
70 }
71 
Start()72 int32_t VideoEncoder::Start()
73 {
74     std::lock_guard<std::mutex> lock(encoderMutex_);
75     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
76      // Prepare video encoder
77     int ret = OH_VideoEncoder_Prepare(encoder_);
78     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Prepare failed, ret: %{public}d", ret);
79     // Start video encoder
80     ret = OH_VideoEncoder_Start(encoder_);
81     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Start failed, ret: %{public}d", ret);
82     isStarted_ = true;
83     return 0;
84 }
85 
GetSurface()86 int32_t VideoEncoder::GetSurface()
87 {
88     std::lock_guard<std::mutex> lock(encoderMutex_);
89     OHNativeWindow *nativeWindow;
90     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
91     int ret = OH_VideoEncoder_GetSurface(encoder_, &nativeWindow);
92     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Get surface failed, ret: %{public}d", ret);
93     uint64_t  surfaceId;
94     ret = OH_NativeWindow_GetSurfaceId(nativeWindow, &surfaceId);
95     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Get surfaceId failed, ret: %{public}d", ret);
96     sptr<Surface> surface = SurfaceUtils::GetInstance()->GetSurface(surfaceId);
97     CHECK_AND_RETURN_RET_LOG(surface != nullptr, 1, "Surface is null");
98     surfaceMutex_.lock();
99     codecSurface_ = surface;
100     surfaceMutex_.unlock();
101     return 0;
102 }
103 
ReleaseSurfaceBuffer(sptr<FrameRecord> frameRecord)104 int32_t VideoEncoder::ReleaseSurfaceBuffer(sptr<FrameRecord> frameRecord)
105 {
106     CAMERA_SYNC_TRACE;
107     CHECK_AND_RETURN_RET_LOG(frameRecord->GetSurfaceBuffer() != nullptr, 1,
108         "SurfaceBuffer is released %{public}s", frameRecord->GetFrameId().c_str());
109     sptr<SyncFence> syncFence = SyncFence::INVALID_FENCE;
110     BufferRequestConfig requestConfig = {
111         .width = frameRecord->GetSurfaceBuffer()->GetWidth(),
112         .height = frameRecord->GetSurfaceBuffer()->GetHeight(),
113         .strideAlignment = 0x8, // default stride is 8 Bytes.
114         .format = frameRecord->GetSurfaceBuffer()->GetFormat(),
115         .usage = frameRecord->GetSurfaceBuffer()->GetUsage(),
116         .timeout = 0,
117     };
118     sptr<SurfaceBuffer> releaseBuffer;
119     {
120         std::lock_guard<std::mutex> lock(surfaceMutex_);
121         CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, 1, "codecSurface_ is null");
122         SurfaceError ret = codecSurface_->RequestBuffer(releaseBuffer, syncFence, requestConfig);
123         if (ret != SURFACE_ERROR_OK) {
124             MEDIA_ERR_LOG("RequestBuffer failed. %{public}d", ret);
125             return ret;
126         }
127         constexpr uint32_t waitForEver = -1;
128         (void)syncFence->Wait(waitForEver);
129 
130         if (!releaseBuffer) {
131             MEDIA_ERR_LOG("Failed to requestBuffer, %{public}s", frameRecord->GetFrameId().c_str());
132             return ret;
133         }
134         ret = codecSurface_->DetachBufferFromQueue(releaseBuffer);
135         if (ret != SURFACE_ERROR_OK) {
136             MEDIA_ERR_LOG("Failed to detach buffer");
137             return ret;
138         }
139     }
140     frameRecord->SetSurfaceBuffer(releaseBuffer);
141     // after request surfaceBuffer
142     frameRecord->NotifyBufferRelease();
143     MEDIA_INFO_LOG("release codec surface buffer end");
144     return 0;
145 }
146 
PushInputData(sptr<CodecAVBufferInfo> info)147 int32_t VideoEncoder::PushInputData(sptr<CodecAVBufferInfo> info)
148 {
149     std::lock_guard<std::mutex> lock(encoderMutex_);
150     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Decoder is null");
151     int32_t ret = AV_ERR_OK;
152     ret = OH_AVBuffer_SetBufferAttr(info->buffer, &info->attr);
153     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set avbuffer attr failed, ret: %{public}d", ret);
154     ret = OH_VideoEncoder_PushInputBuffer(encoder_, info->bufferIndex);
155     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Push input data failed, ret: %{public}d", ret);
156     return 0;
157 }
158 
NotifyEndOfStream()159 int32_t VideoEncoder::NotifyEndOfStream()
160 {
161     std::lock_guard<std::mutex> lock(encoderMutex_);
162     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
163     int32_t ret = OH_VideoEncoder_NotifyEndOfStream(encoder_);
164     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1,
165         "Notify end of stream failed, ret: %{public}d", ret);
166     return 0;
167 }
168 
FreeOutputData(uint32_t bufferIndex)169 int32_t VideoEncoder::FreeOutputData(uint32_t bufferIndex)
170 {
171     std::lock_guard<std::mutex> lock(encoderMutex_);
172     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
173     int32_t ret = OH_VideoEncoder_FreeOutputBuffer(encoder_, bufferIndex);
174     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1,
175         "Free output data failed, ret: %{public}d", ret);
176     return 0;
177 }
178 
Stop()179 int32_t VideoEncoder::Stop()
180 {
181     CAMERA_SYNC_TRACE;
182     std::lock_guard<std::mutex> lock(encoderMutex_);
183     CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
184     int ret = OH_VideoEncoder_Stop(encoder_);
185     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Stop failed, ret: %{public}d", ret);
186     isStarted_ = false;
187     return 0;
188 }
189 
RestartVideoCodec(shared_ptr<Size> size, int32_t rotation)190 void VideoEncoder::RestartVideoCodec(shared_ptr<Size> size, int32_t rotation)
191 {
192     Release();
193     size_ = size;
194     rotation_ = rotation;
195     MEDIA_INFO_LOG("VideoEncoder videoCodecType_ = %{public}d", videoCodecType_);
196     if (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_AVC) {
197         Create(MIME_VIDEO_AVC.data());
198     } else if (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_HEVC) {
199         Create(MIME_VIDEO_HEVC.data());
200     }
201     Config();
202     GetSurface();
203     Start();
204 }
205 
EnqueueBuffer(sptr<FrameRecord> frameRecord, int32_t keyFrameInterval)206 bool VideoEncoder::EnqueueBuffer(sptr<FrameRecord> frameRecord, int32_t keyFrameInterval)
207 {
208     if (!isStarted_ || encoder_ == nullptr || size_ == nullptr) {
209         RestartVideoCodec(frameRecord->GetFrameSize(), frameRecord->GetRotation());
210     }
211     if (keyFrameInterval == KEY_FRAME_INTERVAL) {
212         std::lock_guard<std::mutex> lock(encoderMutex_);
213         OH_AVFormat *format = OH_AVFormat_Create();
214         OH_AVFormat_SetIntValue(format, OH_MD_KEY_REQUEST_I_FRAME, true);
215         OH_VideoEncoder_SetParameter(encoder_, format);
216         OH_AVFormat_Destroy(format);
217     }
218     sptr<SurfaceBuffer> buffer = frameRecord->GetSurfaceBuffer();
219     if (buffer == nullptr) {
220         MEDIA_ERR_LOG("Enqueue video buffer is empty");
221         return false;
222     }
223     std::lock_guard<std::mutex> lock(surfaceMutex_);
224     CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, false, "codecSurface_ is null");
225     SurfaceError surfaceRet = codecSurface_->AttachBufferToQueue(buffer);
226     if (surfaceRet != SURFACE_ERROR_OK) {
227         MEDIA_ERR_LOG("Failed to attach buffer, surfaceRet: %{public}d", surfaceRet);
228         // notify release buffer when attach failed
229         frameRecord->NotifyBufferRelease();
230         return false;
231     }
232     constexpr int32_t invalidFence = -1;
233     BufferFlushConfig flushConfig = {
234         .damage = {
235             .w = buffer->GetWidth(),
236             .h = buffer->GetHeight(),
237         },
238         .timestamp = frameRecord->GetTimeStamp(),
239     };
240     surfaceRet = codecSurface_->FlushBuffer(buffer, invalidFence, flushConfig);
241     CHECK_AND_RETURN_RET_LOG(surfaceRet == 0, false, "FlushBuffer failed");
242     MEDIA_DEBUG_LOG("Success frame id is : %{public}s", frameRecord->GetFrameId().c_str());
243     return true;
244 }
245 
EncodeSurfaceBuffer(sptr<FrameRecord> frameRecord)246 bool VideoEncoder::EncodeSurfaceBuffer(sptr<FrameRecord> frameRecord)
247 {
248     keyFrameInterval_ = (keyFrameInterval_ == 0 ? KEY_FRAME_INTERVAL : keyFrameInterval_);
249     if (!EnqueueBuffer(frameRecord, keyFrameInterval_)) {
250         return false;
251     }
252     keyFrameInterval_--;
253     int32_t retryCount = 5;
254     while (retryCount > 0) {
255         retryCount--;
256         std::unique_lock<std::mutex> contextLock(contextMutex_);
257         CHECK_AND_RETURN_RET_LOG(context_ != nullptr, false, "VideoEncoder has been released");
258         std::unique_lock<std::mutex> lock(context_->outputMutex_);
259         bool condRet = context_->outputCond_.wait_for(lock, std::chrono::milliseconds(BUFFER_ENCODE_EXPIREATION_TIME),
260             [this]() { return !isStarted_ || !context_->outputBufferInfoQueue_.empty(); });
261         CHECK_AND_CONTINUE_LOG(!context_->outputBufferInfoQueue_.empty(),
262             "Buffer queue is empty, continue, cond ret: %{public}d", condRet);
263         sptr<CodecAVBufferInfo> bufferInfo = context_->outputBufferInfoQueue_.front();
264         MEDIA_INFO_LOG("Out buffer count: %{public}u, size: %{public}d, flag: %{public}u, pts:%{public}" PRId64 ", "
265             "timestamp:%{public}" PRId64, context_->outputFrameCount_, bufferInfo->attr.size, bufferInfo->attr.flags,
266             bufferInfo->attr.pts, frameRecord->GetTimeStamp());
267         context_->outputBufferInfoQueue_.pop();
268         context_->outputFrameCount_++;
269         lock.unlock();
270         contextLock.unlock();
271         if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_CODEC_DATA) {
272             // first return IDR frame
273             OH_AVBuffer *IDRBuffer = bufferInfo->GetCopyAVBuffer();
274             frameRecord->CacheBuffer(IDRBuffer);
275             frameRecord->SetIDRProperty(true);
276             successFrame_ = false;
277         } else if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_SYNC_FRAME) {
278             // then return I frame
279             OH_AVBuffer *tempBuffer = bufferInfo->AddCopyAVBuffer(frameRecord->encodedBuffer);
280             if (tempBuffer != nullptr) {
281                 frameRecord->encodedBuffer = tempBuffer;
282             }
283             successFrame_ = true;
284         } else if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_NONE) {
285             // return P frame
286             OH_AVBuffer *PBuffer = bufferInfo->GetCopyAVBuffer();
287             frameRecord->CacheBuffer(PBuffer);
288             frameRecord->SetIDRProperty(false);
289             successFrame_ = true;
290         } else {
291             MEDIA_ERR_LOG("Flag is not acceptted number: %{public}u", bufferInfo->attr.flags);
292             int32_t ret = FreeOutputData(bufferInfo->bufferIndex);
293             CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed");
294             continue;
295         }
296         int32_t ret = FreeOutputData(bufferInfo->bufferIndex);
297         CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed");
298         if (successFrame_) {
299             MEDIA_DEBUG_LOG("Success frame id is : %{public}s, refCount: %{public}d",
300                 frameRecord->GetFrameId().c_str(), frameRecord->GetSptrRefCount());
301             return true;
302         }
303     }
304     MEDIA_ERR_LOG("Failed frame id is : %{public}s", frameRecord->GetFrameId().c_str());
305     return false;
306 }
307 
Release()308 int32_t VideoEncoder::Release()
309 {
310     {
311         std::lock_guard<std::mutex> lock(encoderMutex_);
312         if (encoder_ != nullptr) {
313             OH_VideoEncoder_Destroy(encoder_);
314             encoder_ = nullptr;
315         }
316     }
317     std::unique_lock<std::mutex> contextLock(contextMutex_);
318     if (context_ != nullptr) {
319         delete context_;
320         context_ = nullptr;
321     }
322     isStarted_ = false;
323     return 0;
324 }
325 
SetCallback(CodecUserData *codecUserData)326 int32_t VideoEncoder::SetCallback(CodecUserData *codecUserData)
327 {
328     int32_t ret = AV_ERR_OK;
329     ret = OH_VideoEncoder_RegisterCallback(encoder_,
330         {SampleCallback::OnCodecError, SampleCallback::OnCodecFormatChange,
331          SampleCallback::OnNeedInputBuffer, SampleCallback::OnNewOutputBuffer}, codecUserData);
332     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set callback failed, ret: %{public}d", ret);
333     return 0;
334 }
335 
Configure()336 int32_t VideoEncoder::Configure()
337 {
338     OH_AVFormat *format = OH_AVFormat_Create();
339     CHECK_AND_RETURN_RET_LOG(format != nullptr, 1, "AVFormat create failed");
340     int32_t bitrate = static_cast<int32_t>(pow(float(size_->width) * float(size_->height) / DEFAULT_SIZE,
341         VIDEO_BITRATE_CONSTANT) * BITRATE_22M);
342     bitrate_ = (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_AVC
343         ? static_cast<int32_t>(bitrate * HEVC_TO_AVC_FACTOR) : bitrate);
344     MEDIA_INFO_LOG("Current resolution is : %{public}d*%{public}d, encode type : %{public}d, set bitrate : %{public}d",
345         size_->width, size_->height, videoCodecType_, bitrate_);
346 
347     OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, size_->width);
348     OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, size_->height);
349     OH_AVFormat_SetIntValue(format, OH_MD_KEY_ROTATION, rotation_);
350     OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, VIDOE_FRAME_RATE);
351     OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, VBR);
352     OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, bitrate_);
353     OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, VIDOE_PIXEL_FORMAT);
354     OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, INT_MAX);
355 
356     int ret = OH_VideoEncoder_Configure(encoder_, format);
357     OH_AVFormat_Destroy(format);
358     format = nullptr;
359     CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Config failed, ret: %{public}d", ret);
360 
361     return 0;
362 }
363 } // CameraStandard
364 } // OHOS