1 /*
2  * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <atomic>
17 #include <chrono>
18 #include <fstream>
19 #include <memory>
20 #include <iostream>
21 #include <thread>
22 #include <utility>
23 #include <condition_variable>
24 #include <sys/mman.h>
25 #ifdef HIDUMPER_ENABLED
26 #include <iservice_registry.h>
27 #include <system_ability_load_callback_stub.h>
28 #include "idump_broker.h"
29 #include "dump_broker_proxy.h"
30 #include "system_ability_definition.h"
31 #endif
32 #include "accessibility_event_info.h"
33 #include "accessibility_ui_test_ability.h"
34 #include "ability_manager_client.h"
35 #include "display_manager.h"
36 #include "screen_manager.h"
37 #include "input_manager.h"
38 #include "png.h"
39 #include "wm_common.h"
40 #include "element_node_iterator_impl.h"
41 #include "system_ui_controller.h"
42 #include "test_server_client.h"
43 
44 using namespace std;
45 using namespace chrono;
46 
47 namespace OHOS::uitest {
48     using namespace std;
49     using namespace nlohmann;
50     using namespace OHOS::MMI;
51     using namespace OHOS::Accessibility;
52     using namespace OHOS::Rosen;
53     using namespace OHOS::Media;
54     using namespace OHOS::HiviewDFX;
55     using namespace OHOS;
56 
57     class UiEventMonitor final : public AccessibleAbilityListener {
58     public:
59         virtual ~UiEventMonitor() override = default;
60 
61         void OnAbilityConnected() override;
62 
63         void OnAbilityDisconnected() override;
64 
65         void OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) override;
66 
67         void SetOnAbilityConnectCallback(function<void()> onConnectCb);
68 
69         void SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb);
70 
71         bool OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> &keyEvent) override
72         {
73             return false;
74         }
75 
76         uint64_t GetLastEventMillis();
77 
78         bool WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs);
79 
80         void WaitScrollCompelete();
81 
82         void RegisterUiEventListener(shared_ptr<UiEventListener> listerner);
83 
84     private:
85         function<void()> onConnectCallback_ = nullptr;
86         function<void()> onDisConnectCallback_ = nullptr;
87         atomic<uint64_t> lastEventMillis_ = 0;
88         atomic<uint64_t> lastScrollBeginEventMillis_ = 0;
89         atomic<bool> scrollCompelete_ = true;
90         vector<shared_ptr<UiEventListener>> listeners_;
91     };
92 
93     struct EventSpec {
94         std::string_view componentTyep;
95         int32_t eventType;
96         std::string_view event;
97     };
98 
99     const std::map<char, int32_t> SingleKeySymbalMap = {
100         {' ', OHOS::MMI::KeyEvent::KEYCODE_SPACE},
101         {'`', OHOS::MMI::KeyEvent::KEYCODE_GRAVE},
102         {'[', OHOS::MMI::KeyEvent::KEYCODE_LEFT_BRACKET},
103         {']', OHOS::MMI::KeyEvent::KEYCODE_RIGHT_BRACKET},
104         {'\\', OHOS::MMI::KeyEvent::KEYCODE_BACKSLASH},
105         {',', OHOS::MMI::KeyEvent::KEYCODE_COMMA},
106         {';', OHOS::MMI::KeyEvent::KEYCODE_SEMICOLON},
107         {'\'', OHOS::MMI::KeyEvent::KEYCODE_APOSTROPHE},
108         {'/', OHOS::MMI::KeyEvent::KEYCODE_SLASH},
109         {'*', OHOS::MMI::KeyEvent::KEYCODE_NUMPAD_MULTIPLY},
110         {'-', OHOS::MMI::KeyEvent::KEYCODE_MINUS},
111         {'.', OHOS::MMI::KeyEvent::KEYCODE_PERIOD},
112         {'=', OHOS::MMI::KeyEvent::KEYCODE_EQUALS}
113     };
114 
115     const std::map<char, int32_t> MultiKeySymbalMap = {
116         {'~', OHOS::MMI::KeyEvent::KEYCODE_GRAVE},
117         {'!', OHOS::MMI::KeyEvent::KEYCODE_1},
118         {'@', OHOS::MMI::KeyEvent::KEYCODE_2},
119         {'#', OHOS::MMI::KeyEvent::KEYCODE_3},
120         {'$', OHOS::MMI::KeyEvent::KEYCODE_4},
121         {'%', OHOS::MMI::KeyEvent::KEYCODE_5},
122         {'^', OHOS::MMI::KeyEvent::KEYCODE_6},
123         {'&', OHOS::MMI::KeyEvent::KEYCODE_7},
124         {'(', OHOS::MMI::KeyEvent::KEYCODE_9},
125         {')', OHOS::MMI::KeyEvent::KEYCODE_0},
126         {'+', OHOS::MMI::KeyEvent::KEYCODE_EQUALS},
127         {'_', OHOS::MMI::KeyEvent::KEYCODE_MINUS},
128         {':', OHOS::MMI::KeyEvent::KEYCODE_SEMICOLON},
129         {'"', OHOS::MMI::KeyEvent::KEYCODE_APOSTROPHE},
130         {'<', OHOS::MMI::KeyEvent::KEYCODE_COMMA},
131         {'>', OHOS::MMI::KeyEvent::KEYCODE_PERIOD},
132         {'?', OHOS::MMI::KeyEvent::KEYCODE_SLASH},
133         {'{', OHOS::MMI::KeyEvent::KEYCODE_LEFT_BRACKET},
134         {'}', OHOS::MMI::KeyEvent::KEYCODE_RIGHT_BRACKET},
135         {'|', OHOS::MMI::KeyEvent::KEYCODE_BACKSLASH}
136     };
137 
138     static constexpr EventSpec WATCHED_EVENTS[] = {
139         {"Toast", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "toastShow"},
140         {"AlertDialog", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "dialogShow"}
141     };
142 
GetWatchedEvent(const AccessibilityEventInfo &eventInfo)143     static std::string GetWatchedEvent(const AccessibilityEventInfo &eventInfo)
144     {
145         auto eventCounts = sizeof(WATCHED_EVENTS) / sizeof(EventSpec);
146         for (unsigned long index = 0; index < eventCounts; index++) {
147             if (WATCHED_EVENTS[index].componentTyep == eventInfo.GetComponentType() &&
148                 WATCHED_EVENTS[index].eventType == eventInfo.GetWindowContentChangeTypes()) {
149                 LOG_W("Capture event: %{public}s", WATCHED_EVENTS[index].event.data());
150                 return string(WATCHED_EVENTS[index].event);
151             }
152         }
153         return "undefine";
154     }
155 
156     // UiEventMonitor instance.
157     static shared_ptr<UiEventMonitor> g_monitorInstance_ = make_shared<UiEventMonitor>();
158 
SetOnAbilityConnectCallback(function<void()> onConnectCb)159     void UiEventMonitor::SetOnAbilityConnectCallback(function<void()> onConnectCb)
160     {
161         onConnectCallback_ = std::move(onConnectCb);
162     }
163 
SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb)164     void UiEventMonitor::SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb)
165     {
166         onDisConnectCallback_ = std::move(onDisConnectCb);
167     }
168 
OnAbilityConnected()169     void UiEventMonitor::OnAbilityConnected()
170     {
171         if (onConnectCallback_ != nullptr) {
172             onConnectCallback_();
173         }
174     }
175 
OnAbilityDisconnected()176     void UiEventMonitor::OnAbilityDisconnected()
177     {
178         if (onDisConnectCallback_ != nullptr) {
179             onDisConnectCallback_();
180         }
181     }
182 
183     // the monitored events
184     static constexpr uint32_t EVENT_MASK = EventType::TYPE_VIEW_TEXT_UPDATE_EVENT |
185                                            EventType::TYPE_PAGE_STATE_UPDATE |
186                                            EventType::TYPE_PAGE_CONTENT_UPDATE |
187                                            EventType::TYPE_VIEW_SCROLLED_EVENT |
188                                            EventType::TYPE_WINDOW_UPDATE;
189 
RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner)190     void UiEventMonitor::RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner)
191     {
192         listeners_.emplace_back(listerner);
193     }
194 
OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo)195     void UiEventMonitor::OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo)
196     {
197         auto eventType = eventInfo.GetEventType();
198         LOG_D("OnEvent:0x%{public}x", eventType);
199         auto capturedEvent = GetWatchedEvent(eventInfo);
200         if (eventType == Accessibility::EventType::TYPE_VIEW_SCROLLED_START) {
201             LOG_I("Capture scroll begin");
202             scrollCompelete_.store(false);
203             lastScrollBeginEventMillis_.store(GetCurrentMillisecond());
204         }
205         if (eventType == Accessibility::EventType::TYPE_VIEW_SCROLLED_EVENT) {
206             LOG_I("Capture scroll end");
207             scrollCompelete_.store(true);
208         }
209         if (capturedEvent != "undefine") {
210             auto bundleName = eventInfo.GetBundleName();
211             auto contentList = eventInfo.GetContentList();
212             auto text = !contentList.empty() ? contentList[0] : "";
213             auto type = eventInfo.GetComponentType();
214             UiEventSourceInfo uiEventSourceInfo = {bundleName, text, type};
215             for (auto &listener : listeners_) {
216                 listener->OnEvent(capturedEvent, uiEventSourceInfo);
217             }
218         }
219         if ((eventInfo.GetEventType() & EVENT_MASK) > 0) {
220             lastEventMillis_.store(GetCurrentMillisecond());
221         }
222     }
223 
GetLastEventMillis()224     uint64_t UiEventMonitor::GetLastEventMillis()
225     {
226         if (lastEventMillis_.load() <= 0) {
227             lastEventMillis_.store(GetCurrentMillisecond());
228         }
229         return lastEventMillis_.load();
230     }
231 
WaitScrollCompelete()232     void UiEventMonitor::WaitScrollCompelete()
233     {
234         if (scrollCompelete_.load()) {
235             return;
236         }
237         auto currentMs = GetCurrentMillisecond();
238         if (lastScrollBeginEventMillis_.load() <= 0) {
239             lastScrollBeginEventMillis_.store(currentMs);
240         }
241         const auto idleThresholdMs = 10000;
242         static constexpr auto sliceMs = 10;
243         while (currentMs - lastScrollBeginEventMillis_.load() < idleThresholdMs) {
244             if (scrollCompelete_.load()) {
245                 return;
246             }
247             this_thread::sleep_for(chrono::milliseconds(sliceMs));
248             currentMs = GetCurrentMillisecond();
249         }
250         LOG_E("wait for scrollEnd event timeout.");
251         scrollCompelete_.store(true);
252         return;
253     }
254 
WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs)255     bool UiEventMonitor::WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs)
256     {
257         const auto currentMs = GetCurrentMillisecond();
258         if (lastEventMillis_.load() <= 0) {
259             lastEventMillis_.store(currentMs);
260         }
261         if (currentMs - lastEventMillis_.load() >= idleThresholdMs) {
262             return true;
263         }
264         static constexpr auto sliceMs = 10;
265         this_thread::sleep_for(chrono::milliseconds(sliceMs));
266         if (timeoutMs <= sliceMs) {
267             return false;
268         }
269         return WaitEventIdle(idleThresholdMs, timeoutMs - sliceMs);
270     }
271 
SysUiController()272     SysUiController::SysUiController() : UiController() {}
273 
~SysUiController()274     SysUiController::~SysUiController()
275     {
276         DisConnectFromSysAbility();
277     }
278 
Initialize(ApiCallErr &error)279     bool SysUiController::Initialize(ApiCallErr &error)
280     {
281         return this->ConnectToSysAbility(error);
282     }
283 
GetVisibleRect(Rect windowBounds, Accessibility::Rect nodeBounds)284     static Rect GetVisibleRect(Rect windowBounds, Accessibility::Rect nodeBounds)
285     {
286         auto leftX = nodeBounds.GetLeftTopXScreenPostion();
287         auto topY = nodeBounds.GetLeftTopYScreenPostion();
288         auto rightX = nodeBounds.GetRightBottomXScreenPostion();
289         auto bottomY = nodeBounds.GetRightBottomYScreenPostion();
290         Rect newBounds((leftX < windowBounds.left_) ? windowBounds.left_ : leftX,
291                        (rightX > windowBounds.right_) ? windowBounds.right_ : rightX,
292                        (topY < windowBounds.top_) ? windowBounds.top_ : topY,
293                        (bottomY > windowBounds.bottom_) ? windowBounds.bottom_ : bottomY);
294         return newBounds;
295     }
296 
InflateWindowInfo(AccessibilityWindowInfo& node, Window& info)297     static void InflateWindowInfo(AccessibilityWindowInfo& node, Window& info)
298     {
299         info.focused_ = node.IsFocused();
300         info.actived_ = node.IsActive();
301         info.decoratorEnabled_ = node.IsDecorEnable();
302         // get bundle name by root node
303         AccessibilityElementInfo element;
304         LOG_D("Start Get Bundle Name by WindowId %{public}d", node.GetWindowId());
305         if (AccessibilityUITestAbility::GetInstance()->GetRootByWindow(node, element) != RET_OK) {
306             LOG_E("Failed Get Bundle Name by WindowId %{public}d", node.GetWindowId());
307         } else {
308             std::string app = element.GetBundleName();
309             LOG_I("End Get Bundle Name by WindowId %{public}d, app is %{public}s", node.GetWindowId(), app.data());
310             info.bundleName_ = app;
311             const auto foreAbility = AAFwk::AbilityManagerClient::GetInstance()->GetTopAbility();
312             info.abilityName_ = (app == foreAbility.GetBundleName()) ? foreAbility.GetAbilityName() : "";
313             info.pagePath_ = (app == foreAbility.GetBundleName()) ? element.GetPagePath() : "";
314         }
315         info.mode_ = WindowMode::UNKNOWN;
316         const auto origMode = static_cast<OHOS::Rosen::WindowMode>(node.GetWindowMode());
317         switch (origMode) {
318             case OHOS::Rosen::WindowMode::WINDOW_MODE_FULLSCREEN:
319                 info.mode_ = WindowMode::FULLSCREEN;
320                 break;
321             case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_PRIMARY:
322                 info.mode_ = WindowMode::SPLIT_PRIMARY;
323                 break;
324             case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_SECONDARY:
325                 info.mode_ = WindowMode::SPLIT_SECONDARY;
326                 break;
327             case OHOS::Rosen::WindowMode::WINDOW_MODE_FLOATING:
328                 info.mode_ = WindowMode::FLOATING;
329                 break;
330             case OHOS::Rosen::WindowMode::WINDOW_MODE_PIP:
331                 info.mode_ = WindowMode::PIP;
332                 break;
333             default:
334                 info.mode_ = WindowMode::UNKNOWN;
335                 break;
336         }
337     }
338 
GetAamsWindowInfos(vector<AccessibilityWindowInfo> &windows)339     static bool GetAamsWindowInfos(vector<AccessibilityWindowInfo> &windows)
340     {
341         auto ability = AccessibilityUITestAbility::GetInstance();
342         g_monitorInstance_->WaitScrollCompelete();
343         if (ability->GetWindows(windows) != RET_OK) {
344             LOG_W("GetWindows from AccessibilityUITestAbility failed");
345             return false;
346         }
347         sort(windows.begin(), windows.end(), [](auto &w1, auto &w2) -> bool {
348             return w1.GetWindowLayer() > w2.GetWindowLayer();
349         });
350         return true;
351     }
352 
GetUiWindows(std::vector<Window> &out)353     void SysUiController::GetUiWindows(std::vector<Window> &out)
354     {
355         std::lock_guard<std::mutex> dumpLocker(dumpMtx); // disallow concurrent dumpUi
356         ApiCallErr error = ApiCallErr(NO_ERROR);
357         if (!connected_ && !ConnectToSysAbility(error)) {
358             LOG_E("%{public}s", error.message_.c_str());
359             return;
360         }
361         vector<AccessibilityWindowInfo> windows;
362         LOG_D("Get Window root info");
363         if (!GetAamsWindowInfos(windows)) {
364             return;
365         }
366         LOG_D("End Get Window root info");
367         auto screenSize = GetDisplaySize();
368         auto screenRect = Rect(0, screenSize.px_, 0, screenSize.py_);
369         std::vector<Rect> overplays;
370         // window wrapper
371         for (auto &win : windows) {
372             Rect winRectInScreen = GetVisibleRect(screenRect, win.GetRectInScreen());
373             Rect visibleArea = winRectInScreen;
374             if (!RectAlgorithm::ComputeMaxVisibleRegion(winRectInScreen, overplays, visibleArea)) {
375                 LOG_I("window is covered, windowId : %{public}d, layer is %{public}d", win.GetWindowId(),
376                       win.GetWindowLayer());
377                 continue;
378             }
379             LOG_I("window is visible, windowId: %{public}d, active: %{public}d, focus: %{public}d, layer: %{public}d",
380                 win.GetWindowId(), win.IsActive(), win.IsFocused(), win.GetWindowLayer());
381             Window winWrapper{win.GetWindowId()};
382             InflateWindowInfo(win, winWrapper);
383             winWrapper.bounds_ = winRectInScreen;
384             for (const auto &overWin : overplays) {
385                 Rect intersectionRect{0, 0, 0, 0};
386                 if (RectAlgorithm::ComputeIntersection(winRectInScreen, overWin, intersectionRect)) {
387                     winWrapper.invisibleBoundsVec_.emplace_back(overWin);
388                 }
389             }
390             RectAlgorithm::ComputeMaxVisibleRegion(winWrapper.bounds_, overplays, winWrapper.visibleBounds_);
391             overplays.emplace_back(winRectInScreen);
392             out.emplace_back(std::move(winWrapper));
393         }
394     }
395 
GetWidgetsInWindow(const Window &winInfo, unique_ptr<ElementNodeIterator> &elementIterator)396     bool SysUiController::GetWidgetsInWindow(const Window &winInfo, unique_ptr<ElementNodeIterator> &elementIterator)
397     {
398         std::lock_guard<std::mutex> dumpLocker(dumpMtx); // disallow concurrent dumpUi
399         if (!connected_) {
400             LOG_W("Connect to AccessibilityUITestAbility failed");
401             return false;
402         }
403         std::vector<AccessibilityElementInfo> elementInfos;
404         AccessibilityWindowInfo window;
405         LOG_D("Get Window by WindowId %{public}d", winInfo.id_);
406         if (AccessibilityUITestAbility::GetInstance()->GetWindow(winInfo.id_, window) != RET_OK) {
407             LOG_E("GetWindowInfo failed, windowId: %{public}d", winInfo.id_);
408             return false;
409         }
410         LOG_D("Start Get nodes from window by WindowId %{public}d", winInfo.id_);
411         if (AccessibilityUITestAbility::GetInstance()->GetRootByWindowBatch(window, elementInfos) != RET_OK) {
412             LOG_E("GetRootByWindowBatch failed, windowId: %{public}d", winInfo.id_);
413             return false;
414         } else {
415             LOG_I("End Get nodes from window by WindowId %{public}d, node size is %{public}zu, appId: %{public}s",
416                   winInfo.id_, elementInfos.size(), winInfo.bundleName_.data());
417             elementIterator = std::make_unique<ElementNodeIteratorImpl>(elementInfos);
418         }
419         return true;
420     }
421 
AddPinterItems(PointerEvent &event, const vector<pair<bool, Point>> &fingerStatus, uint32_t currentFinger)422     static void AddPinterItems(PointerEvent &event, const vector<pair<bool, Point>> &fingerStatus,
423         uint32_t currentFinger)
424     {
425         PointerEvent::PointerItem pinterItem1;
426         pinterItem1.SetPointerId(currentFinger);
427         pinterItem1.SetOriginPointerId(currentFinger);
428         pinterItem1.SetDisplayX(fingerStatus[currentFinger].second.px_);
429         pinterItem1.SetDisplayY(fingerStatus[currentFinger].second.py_);
430         pinterItem1.SetPressed(fingerStatus[currentFinger].first);
431         event.UpdatePointerItem(currentFinger, pinterItem1);
432         LOG_D("Add touchItem, finger:%{public}d, pressed:%{public}d, location:%{public}d, %{public}d",
433             currentFinger, fingerStatus[currentFinger].first, fingerStatus[currentFinger].second.px_,
434             fingerStatus[currentFinger].second.py_);
435         // update pinterItem of other fingers which in pressed state.
436         for (uint32_t index = 0; index < fingerStatus.size(); index++) {
437             if (index == currentFinger) {
438                 continue;
439             }
440             if (fingerStatus[index].first) {
441                 PointerEvent::PointerItem pinterItem;
442                 pinterItem.SetPointerId(index);
443                 pinterItem.SetOriginPointerId(index);
444                 pinterItem.SetDisplayX(fingerStatus[index].second.px_);
445                 pinterItem.SetDisplayY(fingerStatus[index].second.py_);
446                 pinterItem.SetPressed(true);
447                 event.UpdatePointerItem(index, pinterItem);
448                 LOG_D("Add touchItem, finger:%{public}d, pressed:%{public}d, location:%{public}d, %{public}d",
449                     index, fingerStatus[index].first, fingerStatus[index].second.px_,
450                     fingerStatus[index].second.py_);
451             }
452         }
453     }
454 
InjectTouchEventSequence(const PointerMatrix &events) const455     void SysUiController::InjectTouchEventSequence(const PointerMatrix &events) const
456     {
457         // fingerStatus stores the press status and coordinates of each finger.
458         vector<pair<bool, Point>> fingerStatus(events.GetFingers(), make_pair(false, Point(0,0)));
459         for (uint32_t step = 0; step < events.GetSteps(); step++) {
460             for (uint32_t finger = 0; finger < events.GetFingers(); finger++) {
461                 auto pointerEvent = PointerEvent::Create();
462                 if (pointerEvent == nullptr) {
463                     LOG_E("Creat PointerEvent failed.");
464                     return;
465                 }
466                 bool isPressed = events.At(finger, step).stage_ != ActionStage::UP;
467                 fingerStatus[finger] = make_pair(isPressed, events.At(finger, step).point_);
468                 pointerEvent->SetPointerId(finger);
469                 switch (events.At(finger, step).stage_) {
470                     case ActionStage::DOWN:
471                         pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_DOWN);
472                         break;
473                     case ActionStage::MOVE:
474                         pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_MOVE);
475                         break;
476                     case ActionStage::UP:
477                         pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_UP);
478                         break;
479                     default:
480                         break;
481                 }
482                 AddPinterItems(*pointerEvent, fingerStatus, finger);
483                 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_TOUCHSCREEN);
484                 DisplayManager &displayMgr = DisplayManager::GetInstance();
485                 pointerEvent->SetTargetDisplayId(displayMgr.GetDefaultDisplayId());
486                 InputManager::GetInstance()->SimulateInputEvent(pointerEvent);
487                 LOG_D("Inject touchEvent");
488                 if (events.At(finger, step).holdMs_ > 0) {
489                     this_thread::sleep_for(chrono::milliseconds(events.At(finger, step).holdMs_));
490                 }
491             }
492         }
493     }
494 
SetMousePointerItemAttr(const MouseEvent &event, PointerEvent::PointerItem &item)495     static void SetMousePointerItemAttr(const MouseEvent &event, PointerEvent::PointerItem &item)
496     {
497         item.SetPointerId(0);
498         item.SetOriginPointerId(0);
499         item.SetToolType(PointerEvent::TOOL_TYPE_MOUSE);
500         item.SetDisplayX(event.point_.px_);
501         item.SetDisplayY(event.point_.py_);
502         item.SetPressed(false);
503         item.SetDownTime(0);
504         LOG_D("Inject mouseEvent, pressed:%{public}d, location:%{public}d, %{public}d",
505             event.stage_ == ActionStage::DOWN, event.point_.px_, event.point_.py_);
506     }
507 
InjectMouseEvent(const MouseEvent &event) const508     void SysUiController::InjectMouseEvent(const MouseEvent &event) const
509     {
510         auto pointerEvent = PointerEvent::Create();
511         if (pointerEvent == nullptr) {
512             return;
513         }
514         PointerEvent::PointerItem item;
515         pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_MOUSE);
516         pointerEvent->SetPointerId(0);
517         pointerEvent->SetButtonId(event.button_);
518         SetMousePointerItemAttr(event, item);
519         constexpr double axialValue = 15;
520         static bool flag = true;
521         auto injectAxialValue = axialValue;
522         switch (event.stage_) {
523             case ActionStage::DOWN:
524                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_DOWN);
525                 pointerEvent->SetButtonId(event.button_);
526                 pointerEvent->SetButtonPressed(event.button_);
527                 item.SetPressed(true);
528                 break;
529             case ActionStage::MOVE:
530                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_MOVE);
531                 break;
532             case ActionStage::UP:
533                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_UP);
534                 pointerEvent->SetButtonId(event.button_);
535                 pointerEvent->SetButtonPressed(event.button_);
536                 break;
537             case ActionStage::AXIS_UP:
538                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_BEGIN);
539                 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, -axialValue);
540                 flag = false;
541                 break;
542             case ActionStage::AXIS_DOWN:
543                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_BEGIN);
544                 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, axialValue);
545                 flag = true;
546                 break;
547             case ActionStage::AXIS_STOP:
548                 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_END);
549                 injectAxialValue = flag ? axialValue : -axialValue;
550                 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, injectAxialValue);
551                 break;
552             default:
553                 break;
554         }
555         pointerEvent->AddPointerItem(item);
556         InputManager::GetInstance()->SimulateInputEvent(pointerEvent);
557         this_thread::sleep_for(chrono::milliseconds(event.holdMs_));
558     }
559 
InjectMouseEventSequence(const vector<MouseEvent> &events) const560     void SysUiController::InjectMouseEventSequence(const vector<MouseEvent> &events) const
561     {
562         for (auto &event : events) {
563             auto keyEvents = event.keyEvents_;
564             if (!keyEvents.empty() && keyEvents.front().stage_ == ActionStage::DOWN) {
565                 InjectKeyEventSequence(keyEvents);
566                 InjectMouseEvent(event);
567             } else {
568                 InjectMouseEvent(event);
569                 InjectKeyEventSequence(keyEvents);
570             }
571         }
572     }
573 
InjectKeyEventSequence(const vector<KeyEvent> &events) const574     void SysUiController::InjectKeyEventSequence(const vector<KeyEvent> &events) const
575     {
576         static vector<int32_t> downKeys;
577         for (auto &event : events) {
578             if (event.code_ == KEYCODE_NONE) {
579                 continue;
580             }
581             auto keyEvent = OHOS::MMI::KeyEvent::Create();
582             if (keyEvent == nullptr) {
583                 LOG_E("Creat KeyEvent failed.");
584                 return;
585             }
586             if (event.stage_ == ActionStage::UP) {
587                 auto iter = std::find(downKeys.begin(), downKeys.end(), event.code_);
588                 if (iter == downKeys.end()) {
589                     LOG_W("Cannot release a not-pressed key: %{public}d", event.code_);
590                     continue;
591                 }
592                 downKeys.erase(iter);
593                 keyEvent->SetKeyCode(event.code_);
594                 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_UP);
595                 OHOS::MMI::KeyEvent::KeyItem keyItem;
596                 keyItem.SetKeyCode(event.code_);
597                 keyItem.SetPressed(true);
598                 keyEvent->AddKeyItem(keyItem);
599                 InputManager::GetInstance()->SimulateInputEvent(keyEvent);
600                 LOG_D("Inject keyEvent up, keycode:%{public}d", event.code_);
601             } else {
602                 downKeys.push_back(event.code_);
603                 for (auto downKey : downKeys) {
604                     keyEvent->SetKeyCode(downKey);
605                     keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_DOWN);
606                     OHOS::MMI::KeyEvent::KeyItem keyItem;
607                     keyItem.SetKeyCode(downKey);
608                     keyItem.SetPressed(true);
609                     keyEvent->AddKeyItem(keyItem);
610                 }
611                 InputManager::GetInstance()->SimulateInputEvent(keyEvent);
612                 LOG_D("Inject keyEvent down, keycode:%{public}d", event.code_);
613                 if (event.holdMs_ > 0) {
614                     this_thread::sleep_for(chrono::milliseconds(event.holdMs_));
615                 }
616             }
617         }
618         // check not released keys
619         for (auto downKey : downKeys) {
620             LOG_W("Key event sequence injections done with not-released key: %{public}d", downKey);
621         }
622     }
623 
PutTextToClipboard(string_view text) const624     void SysUiController::PutTextToClipboard(string_view text) const
625     {
626         OHOS::testserver::TestServerClient::GetInstance().SetPasteData(string(text));
627     }
628 
IsWorkable() const629     bool SysUiController::IsWorkable() const
630     {
631         return connected_;
632     }
633 
GetCharKeyCode(char ch, int32_t &code, int32_t &ctrlCode) const634     bool SysUiController::GetCharKeyCode(char ch, int32_t &code, int32_t &ctrlCode) const
635     {
636         ctrlCode = KEYCODE_NONE;
637         if (ch >= 'a' && ch <= 'z') {
638             code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'a');
639         } else if (ch >= 'A' && ch <= 'Z') {
640             ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT;
641             code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'A');
642         } else if (ch >= '0' && ch <= '9') {
643             code = OHOS::MMI::KeyEvent::KEYCODE_0 + static_cast<int32_t>(ch - '0');
644         } else if (SingleKeySymbalMap.find(ch) != SingleKeySymbalMap.end()) {
645             code = SingleKeySymbalMap.find(ch)->second;
646         } else if (MultiKeySymbalMap.find(ch) != MultiKeySymbalMap.end()) {
647             ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT;
648             code = MultiKeySymbalMap.find(ch)->second;
649         } else {
650             return false;
651         }
652         return true;
653     }
654 
TakeScreenCap(int32_t fd, std::stringstream &errReceiver, Rect rect) const655     bool SysUiController::TakeScreenCap(int32_t fd, std::stringstream &errReceiver, Rect rect) const
656     {
657         DisplayManager &displayMgr = DisplayManager::GetInstance();
658         // get PixelMap from DisplayManager API
659         shared_ptr<PixelMap> pixelMap;
660         if (rect.GetWidth() == 0) {
661             pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId());
662         } else {
663             Media::Rect region = {.left = rect.left_, .top = rect.top_,
664                 .width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_};
665             Media::Size size = {.width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_};
666             pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId(), region, size, 0);
667         }
668         if (pixelMap == nullptr) {
669             errReceiver << "Failed to get display pixelMap";
670             return false;
671         }
672         FILE *fp = fdopen(fd, "wb");
673         if (fp == nullptr) {
674             perror("File opening failed");
675             errReceiver << "File opening failed";
676             return false;
677         }
678         png_structp pngStruct = png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
679         if (pngStruct == nullptr) {
680             fclose(fp);
681             return false;
682         }
683         png_infop pngInfo = png_create_info_struct(pngStruct);
684         if (pngInfo == nullptr) {
685             fclose(fp);
686             png_destroy_write_struct(&pngStruct, nullptr);
687             return false;
688         }
689         png_init_io(pngStruct, fp);
690         auto width = static_cast<uint32_t>(pixelMap->GetWidth());
691         auto height = static_cast<uint32_t>(pixelMap->GetHeight());
692         auto data = pixelMap->GetPixels();
693         auto stride = static_cast<uint32_t>(pixelMap->GetRowBytes());
694         // set png header
695         static constexpr int bitmapDepth = 8;
696         png_set_IHDR(pngStruct, pngInfo, width, height, bitmapDepth, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE,
697                      PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE);
698         png_set_packing(pngStruct); // set packing info
699         png_write_info(pngStruct, pngInfo); // write to header
700         for (uint32_t column = 0; column < height; column++) {
701             png_write_row(pngStruct, data + (column * stride));
702         }
703         // free/close
704         png_write_end(pngStruct, pngInfo);
705         png_destroy_write_struct(&pngStruct, &pngInfo);
706         (void)fclose(fp);
707         return true;
708     }
709 
ConnectToSysAbility(ApiCallErr &error)710     bool SysUiController::ConnectToSysAbility(ApiCallErr &error)
711     {
712         if (connected_) {
713             return true;
714         }
715         mutex mtx;
716         unique_lock<mutex> uLock(mtx);
717         std::shared_ptr<condition_variable> condition = make_shared<condition_variable>();
718         auto onConnectCallback = [condition]() {
719             LOG_I("Success connect to AccessibilityUITestAbility");
720             condition->notify_all();
721         };
722         auto onDisConnectCallback = [this]() { this->connected_ = false; };
723         if (g_monitorInstance_ == nullptr) {
724             g_monitorInstance_ = make_shared<UiEventMonitor>();
725         }
726         g_monitorInstance_->SetOnAbilityConnectCallback(onConnectCallback);
727         g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback);
728         auto ability = AccessibilityUITestAbility::GetInstance();
729         if (ability->RegisterAbilityListener(g_monitorInstance_) != RET_OK) {
730             error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS, REGISTER_LISTENER_FAILED");
731             return false;
732         }
733         auto ret = ability->Connect();
734         LOG_I("Connect to AAMS, result: %{public}d", ret);
735         if (ret != RET_OK) {
736             error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS");
737             if (ret == RET_ERR_CONNECTION_EXIST) {
738                 error.message_ += ", RET_ERR_CONNECTION_EXIST";
739             } else {
740                 error.message_ += ", RET_ERR_AAMS";
741             }
742             return false;
743         }
744         const auto timeout = chrono::milliseconds(7000);
745         if (condition->wait_for(uLock, timeout) == cv_status::timeout) {
746             LOG_E("Wait connection to AccessibilityUITestAbility timed out");
747             error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS, RET_TIMEOUT");
748             return false;
749         }
750         connected_ = true;
751         return true;
752     }
753 
RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const754     void SysUiController::RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const
755     {
756         g_monitorInstance_->RegisterUiEventListener(listener);
757     }
758 
WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutMs) const759     bool SysUiController::WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutMs) const
760     {
761         return g_monitorInstance_->WaitEventIdle(idleThresholdMs, timeoutMs);
762     }
763 
DisConnectFromSysAbility()764     void SysUiController::DisConnectFromSysAbility()
765     {
766         if (!connected_ || g_monitorInstance_ == nullptr) {
767             return;
768         }
769         connected_ = false;
770         mutex mtx;
771         unique_lock<mutex> uLock(mtx);
772         condition_variable condition;
773         auto onDisConnectCallback = [&condition]() {
774             LOG_I("Success disconnect from AccessibilityUITestAbility");
775             condition.notify_all();
776         };
777         g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback);
778         auto ability = AccessibilityUITestAbility::GetInstance();
779         LOG_I("Start disconnect from AccessibilityUITestAbility");
780         if (ability->Disconnect() != RET_OK) {
781             LOG_E("Failed to disconnect from AccessibilityUITestAbility");
782             return;
783         }
784         const auto timeout = chrono::milliseconds(200);
785         if (condition.wait_for(uLock, timeout) == cv_status::timeout) {
786             LOG_E("Wait disconnection from AccessibilityUITestAbility timed out");
787             return;
788         }
789     }
790 
SetDisplayRotation(DisplayRotation rotation) const791     void SysUiController::SetDisplayRotation(DisplayRotation rotation) const
792     {
793         auto display = DisplayManager::GetInstance().GetDefaultDisplay();
794         if (display == nullptr) {
795             LOG_E("DisplayManager init fail");
796             return;
797         }
798         auto screenId = display->GetScreenId();
799         ScreenManager &screenMgr = ScreenManager::GetInstance();
800         bool isLocked = false;
801         screenMgr.IsScreenRotationLocked(isLocked);
802         if (isLocked) {
803             screenMgr.SetScreenRotationLocked(false);
804         }
805         auto screen = screenMgr.GetScreenById(screenId);
806         if (screen == nullptr) {
807             LOG_E("ScreenManager init fail");
808             return;
809         }
810         switch (rotation) {
811             case ROTATION_0 :
812                 screen->SetOrientation(Orientation::VERTICAL);
813                 break;
814             case ROTATION_90 :
815                 screen->SetOrientation(Orientation::HORIZONTAL);
816                 break;
817             case ROTATION_180 :
818                 screen->SetOrientation(Orientation::REVERSE_VERTICAL);
819                 break;
820             case ROTATION_270 :
821                 screen->SetOrientation(Orientation::REVERSE_HORIZONTAL);
822                 break;
823             default :
824                 break;
825         }
826     }
827 
GetDisplayRotation() const828     DisplayRotation SysUiController::GetDisplayRotation() const
829     {
830         auto display = DisplayManager::GetInstance().GetDefaultDisplay();
831         if (display == nullptr) {
832             LOG_E("DisplayManager init fail");
833             return DisplayRotation::ROTATION_0;
834         }
835         auto rotation = (DisplayRotation)display->GetRotation();
836         return rotation;
837     }
838 
SetDisplayRotationEnabled(bool enabled) const839     void SysUiController::SetDisplayRotationEnabled(bool enabled) const
840     {
841         ScreenManager &screenMgr = ScreenManager::GetInstance();
842         screenMgr.SetScreenRotationLocked(!enabled);
843     }
844 
GetDisplaySize() const845     Point SysUiController::GetDisplaySize() const
846     {
847         auto display = DisplayManager::GetInstance().GetDefaultDisplay();
848         if (display == nullptr) {
849             LOG_E("DisplayManager init fail");
850             return {0, 0};
851         }
852         auto width = display->GetWidth();
853         auto height = display->GetHeight();
854         LOG_D("GetDisplaysize, width: %{public}d, height: %{public}d", width, height);
855         Point result(width, height);
856         return result;
857     }
858 
GetDisplayDensity() const859     Point SysUiController::GetDisplayDensity() const
860     {
861         auto display = DisplayManager::GetInstance().GetDefaultDisplay();
862         if (display == nullptr) {
863             LOG_E("DisplayManager init fail");
864             return {0, 0};
865         }
866         auto rate = display->GetVirtualPixelRatio();
867         Point displaySize = GetDisplaySize();
868         Point result(displaySize.px_ * rate, displaySize.py_ * rate);
869         return result;
870     }
871 
IsScreenOn() const872     bool SysUiController::IsScreenOn() const
873     {
874         DisplayManager &displayMgr = DisplayManager::GetInstance();
875         auto displayId = displayMgr.GetDefaultDisplayId();
876         auto state = displayMgr.GetDisplayState(displayId);
877         return (state != DisplayState::OFF);
878     }
879 
880     class OnSaLoadCallback : public SystemAbilityLoadCallbackStub {
881     public:
OnSaLoadCallback(mutex &mutex)882         explicit OnSaLoadCallback(mutex &mutex): mutex_(mutex) {};
~OnSaLoadCallback()883         ~OnSaLoadCallback() {};
884         void OnLoadSystemAbilitySuccess(int32_t systemAbilityId, const sptr<IRemoteObject>& remoteObject) override
885         {
886             if (systemAbilityId == OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID) {
887                 remoteObject_ = remoteObject;
888                 mutex_.unlock();
889             }
890         }
891         void OnLoadSystemAbilityFail(int32_t systemAbilityId) override
892         {
893             if (systemAbilityId == OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID) {
894                 mutex_.unlock();
895             }
896         }
897 
GetSaObject()898         sptr<IRemoteObject> GetSaObject()
899         {
900             return remoteObject_;
901         }
902 
903     private:
904         mutex &mutex_;
905         sptr<IRemoteObject> remoteObject_ = nullptr;
906     };
907 
CreateHidumperCmd(const std::string &windowId, vector<u16string> &result)908     static void CreateHidumperCmd(const std::string &windowId, vector<u16string> &result)
909     {
910         result.emplace_back(u"hidumper");
911         result.emplace_back(u"-s");
912         result.emplace_back(u"WindowManagerService");
913         result.emplace_back(u"-a");
914         auto winIdInUtf16 = std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>{}.from_bytes(windowId);
915         auto arg = u16string(u"-w ").append(winIdInUtf16).append(u" -default -lastpage");
916         result.emplace_back(move(arg));
917     }
918 
GetHidumperInfo(std::string windowId, char **buf, size_t &len)919     void SysUiController::GetHidumperInfo(std::string windowId, char **buf, size_t &len)
920     {
921 #ifdef HIDUMPER_ENABLED
922         auto sam = SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager();
923         // wati SA start
924         constexpr auto delayMs = 2000;
925         this_thread::sleep_for(chrono::milliseconds(delayMs));
926         if (sam == nullptr) {
927             LOG_E("Get samgr failed");
928             return;
929         }
930         auto remoteObject = sam->CheckSystemAbility(OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID);
931         if (remoteObject == nullptr) {
932             mutex lock;
933             lock.lock();
934             sptr<OnSaLoadCallback> loadCallback = new OnSaLoadCallback(lock);
935             if (sam->LoadSystemAbility(OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID, loadCallback) != ERR_OK) {
936                 LOG_E("Schedule LoadSystemAbility failed");
937                 lock.unlock();
938                 return;
939             }
940             LOG_E("Schedule LoadSystemAbility succeed");
941             lock.unlock();
942             remoteObject = loadCallback->GetSaObject();
943             LOG_E("LoadSystemAbility callbacked, result = %{public}s", remoteObject == nullptr ? "FAIL" : "SUCCESS");
944         }
945         if (remoteObject == nullptr) {
946             LOG_E("remoteObject is null");
947             return;
948         }
949         // run dump command
950         sptr<IDumpBroker> client = iface_cast<IDumpBroker>(remoteObject);
951         if (client == nullptr) {
952             LOG_E("IDumpBroker converts failed");
953             return;
954         }
955         auto fd = memfd_create("dummy_file", 2);
956         ftruncate(fd, 0);
957         vector<u16string> args;
958         CreateHidumperCmd(windowId, args);
959         client->Request(args, fd);
960         auto size = lseek(fd, 0, SEEK_END);
961         char *tempBuf = new char[size + 1];
962         lseek(fd, 0, SEEK_SET);
963         read(fd, tempBuf, size);
964         *buf = tempBuf;
965         len = size;
966         close(fd);
967 #else
968         *buf = nullptr;
969         len = 0;
970 #endif
971     }
972 } // namespace OHOS::uitest