1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "subtitle_sink.h"
17 
18 #include "common/log.h"
19 #include "syspara/parameters.h"
20 #include "meta/format.h"
21 
22 namespace {
23 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "SubtitleSink" };
24 }
25 
26 namespace OHOS {
27 namespace Media {
28 namespace {
29 constexpr bool SUBTITME_LOOP_RUNNING = true;
30 }
31 
SubtitleSink()32 SubtitleSink::SubtitleSink()
33 {
34     MEDIA_LOG_I("SubtitleSink ctor");
35     syncerPriority_ = IMediaSynchronizer::SUBTITLE_SINK;
36 }
37 
~SubtitleSink()38 SubtitleSink::~SubtitleSink()
39 {
40     MEDIA_LOG_I("SubtitleSink dtor");
41     {
42         std::unique_lock<std::mutex> lock(mutex_);
43         isThreadExit_ = true;
44     }
45     updateCond_.notify_all();
46     if (readThread_ != nullptr && readThread_->joinable()) {
47         readThread_->join();
48         readThread_ = nullptr;
49     }
50 
51     if (inputBufferQueueProducer_ != nullptr) {
52         for (auto &buffer : inputBufferVector_) {
53             inputBufferQueueProducer_->DetachBuffer(buffer);
54         }
55         inputBufferVector_.clear();
56         inputBufferQueueProducer_->SetQueueSize(0);
57     }
58 }
59 
NotifySeek()60 void SubtitleSink::NotifySeek()
61 {
62     Flush();
63 }
64 
GetTargetSubtitleIndex(int64_t currentTime)65 void SubtitleSink::GetTargetSubtitleIndex(int64_t currentTime)
66 {
67     int32_t left = 0;
68     int32_t right = subtitleInfoVec_.size();
69     while (left < right) {
70         int32_t mid = (left + right) / 2;
71         int64_t startTime = subtitleInfoVec_.at(mid).pts_;
72         int64_t endTime = subtitleInfoVec_.at(mid).duration_ + startTime;
73         if (startTime > currentTime) {
74             right = mid;
75             continue;
76         } else if (endTime < currentTime) {
77             left = mid + 1;
78             continue;
79         } else {
80             left = mid;
81             break;
82         }
83     }
84     currentInfoIndex_ = left;
85 }
86 
Init(std::shared_ptr<Meta> &meta, const std::shared_ptr<Pipeline::EventReceiver> &receiver)87 Status SubtitleSink::Init(std::shared_ptr<Meta> &meta, const std::shared_ptr<Pipeline::EventReceiver> &receiver)
88 {
89     state_ = Pipeline::FilterState::INITIALIZED;
90     if (meta != nullptr) {
91         meta->SetData(Tag::APP_PID, appPid_);
92         meta->SetData(Tag::APP_UID, appUid_);
93     }
94     return Status::OK;
95 }
96 
GetBufferQueueProducer()97 sptr<AVBufferQueueProducer> SubtitleSink::GetBufferQueueProducer()
98 {
99     if (state_ != Pipeline::FilterState::READY) {
100         return nullptr;
101     }
102     return inputBufferQueueProducer_;
103 }
104 
GetBufferQueueConsumer()105 sptr<AVBufferQueueConsumer> SubtitleSink::GetBufferQueueConsumer()
106 {
107     if (state_ != Pipeline::FilterState::READY) {
108         return nullptr;
109     }
110     return inputBufferQueueConsumer_;
111 }
112 
SetParameter(const std::shared_ptr<Meta> &meta)113 Status SubtitleSink::SetParameter(const std::shared_ptr<Meta> &meta)
114 {
115     return Status::OK;
116 }
117 
GetParameter(std::shared_ptr<Meta> &meta)118 Status SubtitleSink::GetParameter(std::shared_ptr<Meta> &meta)
119 {
120     return Status::OK;
121 }
122 
Prepare()123 Status SubtitleSink::Prepare()
124 {
125     state_ = Pipeline::FilterState::PREPARING;
126     Status ret = PrepareInputBufferQueue();
127     if (ret != Status::OK) {
128         state_ = Pipeline::FilterState::INITIALIZED;
129         return ret;
130     }
131     state_ = Pipeline::FilterState::READY;
132     return ret;
133 }
134 
Start()135 Status SubtitleSink::Start()
136 {
137     isEos_ = false;
138     state_ = Pipeline::FilterState::RUNNING;
139     readThread_ = std::make_unique<std::thread>(&SubtitleSink::RenderLoop, this);
140     pthread_setname_np(readThread_->native_handle(), "SubtitleRenderLoop");
141     return Status::OK;
142 }
143 
Stop()144 Status SubtitleSink::Stop()
145 {
146     updateCond_.notify_all();
147     state_ = Pipeline::FilterState::INITIALIZED;
148     return Status::OK;
149 }
150 
Pause()151 Status SubtitleSink::Pause()
152 {
153     state_ = Pipeline::FilterState::PAUSED;
154     return Status::OK;
155 }
156 
Resume()157 Status SubtitleSink::Resume()
158 {
159     {
160         std::unique_lock<std::mutex> lock(mutex_);
161         isEos_ = false;
162         state_ = Pipeline::FilterState::RUNNING;
163     }
164     updateCond_.notify_all();
165     return Status::OK;
166 }
167 
Flush()168 Status SubtitleSink::Flush()
169 {
170     {
171         std::unique_lock<std::mutex> lock(mutex_);
172         shouldUpdate_ = true;
173         if (subtitleInfoVec_.size() > 0) {
174             inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
175             subtitleInfoVec_.clear();
176         }
177     }
178     isFlush_.store(true);
179     updateCond_.notify_all();
180     return Status::OK;
181 }
182 
Release()183 Status SubtitleSink::Release()
184 {
185     return Status::OK;
186 }
187 
SetIsTransitent(bool isTransitent)188 Status SubtitleSink::SetIsTransitent(bool isTransitent)
189 {
190     isTransitent_ = isTransitent;
191     return Status::OK;
192 }
193 
PrepareInputBufferQueue()194 Status SubtitleSink::PrepareInputBufferQueue()
195 {
196     if (inputBufferQueue_ != nullptr && inputBufferQueue_->GetQueueSize() > 0) {
197         MEDIA_LOG_I("InputBufferQueue already create");
198         return Status::ERROR_INVALID_OPERATION;
199     }
200     int32_t inputBufferNum = 1;
201     int32_t capacity = 1024;
202     MemoryType memoryType = MemoryType::SHARED_MEMORY;
203 #ifndef MEDIA_OHOS
204     memoryType = MemoryType::VIRTUAL_MEMORY;
205 #endif
206     MEDIA_LOG_I("PrepareInputBufferQueue");
207     if (inputBufferQueue_ == nullptr) {
208         inputBufferQueue_ = AVBufferQueue::Create(inputBufferNum, memoryType, INPUT_BUFFER_QUEUE_NAME);
209     }
210     FALSE_RETURN_V_MSG_E(inputBufferQueue_ != nullptr, Status::ERROR_UNKNOWN, "inputBufferQueue_ is nullptr");
211 
212     inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
213     inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
214 
215     for (int i = 0; i < inputBufferNum; i++) {
216         std::shared_ptr<AVAllocator> avAllocator;
217 #ifndef MEDIA_OHOS
218         MEDIA_LOG_D("CreateVirtualAllocator,i=%{public}d capacity=%{public}d", i, capacity);
219         avAllocator = AVAllocatorFactory::CreateVirtualAllocator();
220 #else
221         MEDIA_LOG_D("CreateSharedAllocator,i=%{public}d capacity=%{public}d", i, capacity);
222         avAllocator = AVAllocatorFactory::CreateSharedAllocator(MemoryFlag::MEMORY_READ_WRITE);
223 #endif
224         std::shared_ptr<AVBuffer> inputBuffer = AVBuffer::CreateAVBuffer(avAllocator, capacity);
225         FALSE_RETURN_V_MSG_E(inputBuffer != nullptr, Status::ERROR_UNKNOWN,
226                              "inputBuffer is nullptr");
227         FALSE_RETURN_V_MSG_E(inputBufferQueueProducer_ != nullptr, Status::ERROR_UNKNOWN,
228                              "inputBufferQueueProducer_ is nullptr");
229         inputBufferQueueProducer_->AttachBuffer(inputBuffer, false);
230         MEDIA_LOG_I("Attach intput buffer. index: %{public}d, bufferId: %{public}" PRIu64,
231             i, inputBuffer->GetUniqueId());
232         inputBufferVector_.push_back(inputBuffer);
233     }
234     return Status::OK;
235 }
236 
DrainOutputBuffer(bool flushed)237 void SubtitleSink::DrainOutputBuffer(bool flushed)
238 {
239     Status ret;
240     FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
241     FALSE_RETURN(!isEos_.load());
242     ret = inputBufferQueueConsumer_->AcquireBuffer(filledOutputBuffer_);
243     if (filledOutputBuffer_->flag_ & BUFFER_FLAG_EOS) {
244         isEos_ = true;
245     }
246     if (ret != Status::OK || filledOutputBuffer_ == nullptr || filledOutputBuffer_->memory_ == nullptr) {
247         return;
248     }
249     std::string subtitleText(reinterpret_cast<const char *>(filledOutputBuffer_->memory_->GetAddr()),
250                              filledOutputBuffer_->memory_->GetSize());
251     SubtitleInfo subtitleInfo{ subtitleText, filledOutputBuffer_->pts_, filledOutputBuffer_->duration_ };
252     {
253         std::unique_lock<std::mutex> lock(mutex_);
254         subtitleInfoVec_.push_back(subtitleInfo);
255     }
256     updateCond_.notify_all();
257 }
258 
RenderLoop()259 void SubtitleSink::RenderLoop()
260 {
261     while (SUBTITME_LOOP_RUNNING) {
262         std::unique_lock<std::mutex> lock(mutex_);
263         updateCond_.wait(lock, [this] {
264             return isThreadExit_.load() ||
265                    (subtitleInfoVec_.size() > 0 && state_ == Pipeline::FilterState::RUNNING);
266         });
267         if (isFlush_) {
268             MEDIA_LOG_I("SubtitleSink RenderLoop flush");
269             isFlush_.store(false);
270             continue;
271         }
272         FALSE_RETURN(!isThreadExit_.load());
273         // wait timeout, seek or stop
274         SubtitleInfo tempSubtitleInfo = subtitleInfoVec_.back();
275         SubtitleInfo subtitleInfo{ tempSubtitleInfo.text_, tempSubtitleInfo.pts_, tempSubtitleInfo.duration_ };
276         int64_t waitTime = CalcWaitTime(subtitleInfo);
277         updateCond_.wait_for(lock, std::chrono::microseconds(waitTime),
278                              [this] { return isThreadExit_.load() || shouldUpdate_; });
279         MEDIA_LOG_I("SubtitleSink NotifyRender buffer. pts = " PUBLIC_LOG_D64 " waitTime: " PUBLIC_LOG_D64,
280             subtitleInfo.pts_, waitTime);
281         if (isFlush_) {
282             MEDIA_LOG_I("SubtitleSink RenderLoop flush");
283             isFlush_.store(false);
284             continue;
285         }
286         FALSE_RETURN(!isThreadExit_.load());
287         auto actionToDo = ActionToDo(subtitleInfo);
288         if (actionToDo == SubtitleBufferState::DROP) {
289             inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
290             subtitleInfoVec_.clear();
291             continue;
292         } else if (actionToDo == SubtitleBufferState::WAIT) {
293             continue;
294         } else {}
295         NotifyRender(subtitleInfo);
296         inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer_);
297         subtitleInfoVec_.clear();
298     }
299 }
300 
ResetSyncInfo()301 void SubtitleSink::ResetSyncInfo()
302 {
303     auto syncCenter = syncCenter_.lock();
304     if (syncCenter) {
305         syncCenter->Reset();
306     }
307     lastReportedClockTime_ = HST_TIME_NONE;
308 }
309 
CalcWaitTime(SubtitleInfo &subtitleInfo)310 uint64_t SubtitleSink::CalcWaitTime(SubtitleInfo &subtitleInfo)
311 {
312     int64_t curTime;
313     if (shouldUpdate_.load()) {
314         shouldUpdate_ = false;
315     }
316     curTime = GetMediaTime();
317     if (subtitleInfo.pts_ < curTime) {
318         return 0;
319     }
320     return (subtitleInfo.pts_ - curTime) / speed_;
321 }
322 
ActionToDo(SubtitleInfo &subtitleInfo)323 uint32_t SubtitleSink::ActionToDo(SubtitleInfo &subtitleInfo)
324 {
325     auto curTime = GetMediaTime();
326     if (shouldUpdate_ || subtitleInfo.pts_ + subtitleInfo.duration_ < curTime) {
327         return SubtitleBufferState::DROP;
328     }
329     if (subtitleInfo.pts_ > curTime || state_ != Pipeline::FilterState::RUNNING) {
330         return SubtitleBufferState::WAIT;
331     }
332     subtitleInfo.duration_ -= curTime - subtitleInfo.pts_;
333     return SubtitleBufferState::SHOW;
334 }
335 
DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer> &buffer)336 int64_t SubtitleSink::DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer> &buffer)
337 {
338     (void)buffer;
339     return 0;
340 }
341 
NotifyRender(SubtitleInfo &subtitleInfo)342 void SubtitleSink::NotifyRender(SubtitleInfo &subtitleInfo)
343 {
344     Format format;
345     (void)format.PutStringValue(Tag::SUBTITLE_TEXT, subtitleInfo.text_);
346     (void)format.PutIntValue(Tag::SUBTITLE_PTS, Plugins::Us2Ms(subtitleInfo.pts_));
347     (void)format.PutIntValue(Tag::SUBTITLE_DURATION, Plugins::Us2Ms(subtitleInfo.duration_));
348     Event event{ .srcFilter = "SubtitleSink", .type = EventType::EVENT_SUBTITLE_TEXT_UPDATE, .param = format };
349     FALSE_RETURN(playerEventReceiver_ != nullptr);
350     playerEventReceiver_->OnEvent(event);
351 }
352 
SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver)353 void SubtitleSink::SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver)
354 {
355     FALSE_RETURN(receiver != nullptr);
356     playerEventReceiver_ = receiver;
357 }
358 
SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)359 void SubtitleSink::SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)
360 {
361     syncCenter_ = syncCenter;
362     MediaSynchronousSink::Init();
363 }
364 
SetSpeed(float speed)365 Status SubtitleSink::SetSpeed(float speed)
366 {
367     FALSE_RETURN_V_MSG_W(speed > 0, Status::OK, "Invalid speed %{public}f", speed);
368     {
369         std::unique_lock<std::mutex> lock(mutex_);
370         speed_ = speed;
371         shouldUpdate_ = true;
372     }
373     updateCond_.notify_all();
374     return Status::OK;
375 }
376 
GetMediaTime()377 int64_t SubtitleSink::GetMediaTime()
378 {
379     auto syncCenter = syncCenter_.lock();
380     if (!syncCenter) {
381         return 0;
382     }
383     return syncCenter->GetMediaTimeNow();
384 }
385 }  // namespace MEDIA
386 }  // namespace OHOS