1/*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <unistd.h>
17#include "cpp/mutex.h"
18#ifndef _GNU_SOURCE
19#define _GNU_SOURCE
20#endif
21
22#include <map>
23#include <functional>
24#include "sync/sync.h"
25#include "eu/co_routine.h"
26#include "internal_inc/osal.h"
27#include "internal_inc/types.h"
28#include "sync/mutex_private.h"
29#include "dfx/log/ffrt_log_api.h"
30#include "ffrt_trace.h"
31#include "tm/cpu_task.h"
32
33namespace ffrt {
34bool mutexPrivate::try_lock()
35{
36    int v = sync_detail::UNLOCK;
37    bool ret = l.compare_exchange_strong(v, sync_detail::LOCK, std::memory_order_acquire, std::memory_order_relaxed);
38#ifdef FFRT_MUTEX_DEADLOCK_CHECK
39    if (ret) {
40        uint64_t task = ExecuteCtx::Cur()->task ? reinterpret_cast<uint64_t>(ExecuteCtx::Cur()->task) : GetTid();
41        MutexGraph::Instance().AddNode(task, 0, false);
42        owner.store(task, std::memory_order_relaxed);
43    }
44#endif
45    return ret;
46}
47
48void mutexPrivate::lock()
49{
50#ifdef FFRT_MUTEX_DEADLOCK_CHECK
51    uint64_t task;
52    uint64_t ownerTask;
53    task = ExecuteCtx::Cur()->task ? reinterpret_cast<uint64_t>(ExecuteCtx::Cur()->task) : GetTid();
54    ownerTask = owner.load(std::memory_order_relaxed);
55    if (ownerTask) {
56        MutexGraph::Instance().AddNode(task, ownerTask, true);
57    } else {
58        MutexGraph::Instance().AddNode(task, 0, false);
59    }
60#endif
61    int v = sync_detail::UNLOCK;
62    if (l.compare_exchange_strong(v, sync_detail::LOCK, std::memory_order_acquire, std::memory_order_relaxed)) {
63        goto lock_out;
64    }
65    if (l.load(std::memory_order_relaxed) == sync_detail::WAIT) {
66        wait();
67    }
68    while (l.exchange(sync_detail::WAIT, std::memory_order_acquire) != sync_detail::UNLOCK) {
69        wait();
70    }
71
72lock_out:
73#ifdef FFRT_MUTEX_DEADLOCK_CHECK
74    owner.store(task, std::memory_order_relaxed);
75#endif
76    return;
77}
78
79bool RecursiveMutexPrivate::try_lock()
80{
81    auto ctx = ExecuteCtx::Cur();
82    auto task = ctx->task;
83    if ((!USE_COROUTINE) || (task == nullptr)) {
84        fMutex.lock();
85        if (taskLockNums.first == UINT64_MAX) {
86            fMutex.unlock();
87            if (mt.try_lock()) {
88                fMutex.lock();
89                taskLockNums = std::make_pair(GetTid(), 1);
90                fMutex.unlock();
91                return true;
92            } else {
93                return false;
94            }
95        }
96
97        if (taskLockNums.first == GetTid()) {
98            taskLockNums.second += 1;
99            fMutex.unlock();
100            return true;
101        }
102
103        fMutex.unlock();
104        return false;
105    }
106
107    fMutex.lock();
108    if (taskLockNums.first == UINT64_MAX) {
109        fMutex.unlock();
110        if (mt.try_lock()) {
111            fMutex.lock();
112            taskLockNums = std::make_pair(task->gid | 0x8000000000000000, 1);
113            fMutex.unlock();
114            return true;
115        } else {
116            return false;
117        }
118    }
119
120    if (taskLockNums.first == (task->gid | 0x8000000000000000)) {
121        taskLockNums.second += 1;
122        fMutex.unlock();
123        return true;
124    }
125
126    fMutex.unlock();
127    return false;
128}
129
130void RecursiveMutexPrivate::lock()
131{
132    auto ctx = ExecuteCtx::Cur();
133    auto task = ctx->task;
134    if ((!USE_COROUTINE) || (task == nullptr)) {
135        fMutex.lock();
136        if (taskLockNums.first != GetTid()) {
137            fMutex.unlock();
138            mt.lock();
139            fMutex.lock();
140            taskLockNums = std::make_pair(GetTid(), 1);
141            fMutex.unlock();
142            return;
143        }
144
145        taskLockNums.second += 1;
146        fMutex.unlock();
147        return;
148    }
149
150    fMutex.lock();
151    if (taskLockNums.first != (task->gid | 0x8000000000000000)) {
152        fMutex.unlock();
153        mt.lock();
154        fMutex.lock();
155        taskLockNums = std::make_pair(task->gid | 0x8000000000000000, 1);
156        fMutex.unlock();
157        return;
158    }
159
160    taskLockNums.second += 1;
161    fMutex.unlock();
162}
163
164void RecursiveMutexPrivate::unlock()
165{
166    auto ctx = ExecuteCtx::Cur();
167    auto task = ctx->task;
168    if ((!USE_COROUTINE) || (task == nullptr)) {
169        fMutex.lock();
170        if (taskLockNums.first != GetTid()) {
171            fMutex.unlock();
172            return;
173        }
174
175        if (taskLockNums.second == 1) {
176            taskLockNums = std::make_pair(UINT64_MAX, 0);
177            fMutex.unlock();
178            mt.unlock();
179            return;
180        }
181
182        taskLockNums.second -= 1;
183        fMutex.unlock();
184        return;
185    }
186
187    fMutex.lock();
188    if (taskLockNums.first != (task->gid | 0x8000000000000000)) {
189        fMutex.unlock();
190        return;
191    }
192
193    if (taskLockNums.second == 1) {
194        taskLockNums = std::make_pair(UINT64_MAX, 0);
195        fMutex.unlock();
196        mt.unlock();
197        return;
198    }
199
200    taskLockNums.second -= 1;
201    fMutex.unlock();
202}
203
204void mutexPrivate::unlock()
205{
206#ifdef FFRT_MUTEX_DEADLOCK_CHECK
207    uint64_t ownerTask = owner.load(std::memory_order_relaxed);
208    owner.store(0, std::memory_order_relaxed);
209    MutexGraph::Instance().RemoveNode(ownerTask);
210#endif
211    if (l.exchange(sync_detail::UNLOCK, std::memory_order_release) == sync_detail::WAIT) {
212        wake();
213    }
214}
215
216void mutexPrivate::wait()
217{
218    auto ctx = ExecuteCtx::Cur();
219    auto task = ctx->task;
220    if (ThreadWaitMode(task)) {
221        wlock.lock();
222        if (l.load(std::memory_order_relaxed) != sync_detail::WAIT) {
223            wlock.unlock();
224            return;
225        }
226        list.PushBack(ctx->wn.node);
227        std::unique_lock<std::mutex> lk(ctx->wn.wl);
228        if (FFRT_UNLIKELY(LegacyMode(task))) {
229            task->blockType = BlockType::BLOCK_THREAD;
230            ctx->wn.task = task;
231        }
232        wlock.unlock();
233        ctx->wn.cv.wait(lk);
234        return;
235    } else {
236        FFRT_BLOCK_TRACER(task->gid, mtx);
237        CoWait([this](CPUEUTask* task) -> bool {
238            wlock.lock();
239            if (l.load(std::memory_order_relaxed) != sync_detail::WAIT) {
240                wlock.unlock();
241                return false;
242            }
243            list.PushBack(task->fq_we.node);
244            wlock.unlock();
245            // The ownership of the task belongs to ReadyTaskQueue, and the task cannot be accessed any more.
246            return true;
247        });
248    }
249}
250
251void mutexPrivate::wake()
252{
253    wlock.lock();
254    if (list.Empty()) {
255        wlock.unlock();
256        return;
257    }
258    WaitEntry* we = list.PopFront(&WaitEntry::node);
259    if (we == nullptr) {
260        wlock.unlock();
261        return;
262    }
263    CPUEUTask* task = we->task;
264    if (ThreadNotifyMode(task) || we->weType == 2) {
265        WaitUntilEntry* wue = static_cast<WaitUntilEntry*>(we);
266        std::unique_lock lk(wue->wl);
267        if (BlockThread(task)) {
268            task->blockType = BlockType::BLOCK_COROUTINE;
269            we->task = nullptr;
270        }
271        wlock.unlock();
272        wue->cv.notify_one();
273    } else {
274        wlock.unlock();
275        CoRoutineFactory::CoWakeFunc(task, false);
276    }
277}
278} // namespace ffrt
279
280#ifdef __cplusplus
281extern "C" {
282#endif
283API_ATTRIBUTE((visibility("default")))
284int ffrt_mutexattr_init(ffrt_mutexattr_t* attr)
285{
286    if (attr == nullptr) {
287        FFRT_LOGE("attr should not be empty");
288        return ffrt_error_inval;
289    }
290    attr->storage = static_cast<long>(ffrt_mutex_default);
291    return ffrt_success;
292}
293
294API_ATTRIBUTE((visibility("default")))
295int ffrt_mutexattr_settype(ffrt_mutexattr_t* attr, int type)
296{
297    if (attr == nullptr) {
298        FFRT_LOGE("attr should not be empty");
299        return ffrt_error_inval;
300    }
301    if (type != ffrt_mutex_normal && type != ffrt_mutex_recursive && type != ffrt_mutex_default) {
302        FFRT_LOGE("mutex type is invaild");
303        return ffrt_error_inval;
304    }
305    attr->storage = static_cast<long>(type);
306    return ffrt_success;
307}
308
309API_ATTRIBUTE((visibility("default")))
310int ffrt_mutexattr_gettype(ffrt_mutexattr_t* attr, int* type)
311{
312    if (attr == nullptr || type == nullptr) {
313        FFRT_LOGE("attr or type should not be empty");
314        return ffrt_error_inval;
315    }
316    *type = static_cast<int>(attr->storage);
317    return ffrt_success;
318}
319
320API_ATTRIBUTE((visibility("default")))
321int ffrt_mutexattr_destroy(ffrt_mutexattr_t* attr)
322{
323    if (attr == nullptr) {
324        FFRT_LOGE("attr should not be empty");
325        return ffrt_error_inval;
326    }
327    return ffrt_success;
328}
329
330API_ATTRIBUTE((visibility("default")))
331int ffrt_mutex_init(ffrt_mutex_t* mutex, const ffrt_mutexattr_t* attr)
332{
333    if (!mutex) {
334        FFRT_LOGE("mutex should not be empty");
335        return ffrt_error_inval;
336    }
337    if (attr == nullptr || attr->storage == static_cast<long>(ffrt_mutex_normal)) {
338        static_assert(sizeof(ffrt::mutexPrivate) <= ffrt_mutex_storage_size,
339        "size must be less than ffrt_mutex_storage_size");
340        new (mutex)ffrt::mutexPrivate();
341        return ffrt_success;
342    } else if (attr->storage == static_cast<long>(ffrt_mutex_recursive)) {
343        static_assert(sizeof(ffrt::RecursiveMutexPrivate) <= ffrt_mutex_storage_size,
344        "size must be less than ffrt_mutex_storage_size");
345        new (mutex)ffrt::RecursiveMutexPrivate();
346        return ffrt_success;
347    }
348    return ffrt_error_inval;
349}
350
351API_ATTRIBUTE((visibility("default")))
352int ffrt_mutex_lock(ffrt_mutex_t* mutex)
353{
354    if (!mutex) {
355        FFRT_LOGE("mutex should not be empty");
356        return ffrt_error_inval;
357    }
358    auto p = reinterpret_cast<ffrt::mutexBase*>(mutex);
359    p->lock();
360    return ffrt_success;
361}
362
363API_ATTRIBUTE((visibility("default")))
364int ffrt_mutex_unlock(ffrt_mutex_t* mutex)
365{
366    if (!mutex) {
367        FFRT_LOGE("mutex should not be empty");
368        return ffrt_error_inval;
369    }
370    auto p = reinterpret_cast<ffrt::mutexBase*>(mutex);
371    p->unlock();
372    return ffrt_success;
373}
374
375API_ATTRIBUTE((visibility("default")))
376int ffrt_mutex_trylock(ffrt_mutex_t* mutex)
377{
378    if (!mutex) {
379        FFRT_LOGE("mutex should not be empty");
380        return ffrt_error_inval;
381    }
382    auto p = reinterpret_cast<ffrt::mutexBase*>(mutex);
383    return p->try_lock() ? ffrt_success : ffrt_error_busy;
384}
385
386API_ATTRIBUTE((visibility("default")))
387int ffrt_mutex_destroy(ffrt_mutex_t* mutex)
388{
389    if (!mutex) {
390        FFRT_LOGE("mutex should not be empty");
391        return ffrt_error_inval;
392    }
393    auto p = reinterpret_cast<ffrt::mutexBase*>(mutex);
394    p->~mutexBase();
395    return ffrt_success;
396}
397
398#ifdef __cplusplus
399}
400#endif
401