1/* 2 * Copyright © 2015 Intel 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#ifndef _SIMPLE_MTX_H 25#define _SIMPLE_MTX_H 26 27#include "util/futex.h" 28#include "util/macros.h" 29#include "u_atomic.h" 30 31#include "c11/threads.h" 32 33#if UTIL_FUTEX_SUPPORTED 34 35#if defined(HAVE_VALGRIND) && !defined(NDEBUG) 36# include <valgrind.h> 37# include <helgrind.h> 38# define HG(x) x 39#else 40# define HG(x) 41#endif 42 43/* mtx_t - Fast, simple mutex 44 * 45 * While modern pthread mutexes are very fast (implemented using futex), they 46 * still incur a call to an external DSO and overhead of the generality and 47 * features of pthread mutexes. Most mutexes in mesa only needs lock/unlock, 48 * and the idea here is that we can inline the atomic operation and make the 49 * fast case just two intructions. Mutexes are subtle and finicky to 50 * implement, so we carefully copy the implementation from Ulrich Dreppers 51 * well-written and well-reviewed paper: 52 * 53 * "Futexes Are Tricky" 54 * http://www.akkadia.org/drepper/futex.pdf 55 * 56 * We implement "mutex3", which gives us a mutex that has no syscalls on 57 * uncontended lock or unlock. Further, the uncontended case boils down to a 58 * locked cmpxchg and an untaken branch, the uncontended unlock is just a 59 * locked decr and an untaken branch. We use __builtin_expect() to indicate 60 * that contention is unlikely so that gcc will put the contention code out of 61 * the main code flow. 62 * 63 * A fast mutex only supports lock/unlock, can't be recursive or used with 64 * condition variables. 65 */ 66 67typedef struct { 68 uint32_t val; 69} simple_mtx_t; 70 71#define _SIMPLE_MTX_INITIALIZER_NP { 0 } 72 73#define _SIMPLE_MTX_INVALID_VALUE 0xd0d0d0d0 74 75static inline void 76simple_mtx_init(simple_mtx_t *mtx, ASSERTED int type) 77{ 78 assert(type == mtx_plain); 79 80 mtx->val = 0; 81 82 HG(ANNOTATE_RWLOCK_CREATE(mtx)); 83} 84 85static inline void 86simple_mtx_destroy(ASSERTED simple_mtx_t *mtx) 87{ 88 HG(ANNOTATE_RWLOCK_DESTROY(mtx)); 89#ifndef NDEBUG 90 mtx->val = _SIMPLE_MTX_INVALID_VALUE; 91#endif 92} 93 94static inline void 95simple_mtx_lock(simple_mtx_t *mtx) 96{ 97 uint32_t c; 98 99 c = p_atomic_cmpxchg(&mtx->val, 0, 1); 100 101 assert(c != _SIMPLE_MTX_INVALID_VALUE); 102 103 if (__builtin_expect(c != 0, 0)) { 104 if (c != 2) 105 c = p_atomic_xchg(&mtx->val, 2); 106 while (c != 0) { 107 futex_wait(&mtx->val, 2, NULL); 108 c = p_atomic_xchg(&mtx->val, 2); 109 } 110 } 111 112 HG(ANNOTATE_RWLOCK_ACQUIRED(mtx, 1)); 113} 114 115static inline void 116simple_mtx_unlock(simple_mtx_t *mtx) 117{ 118 uint32_t c; 119 120 HG(ANNOTATE_RWLOCK_RELEASED(mtx, 1)); 121 122 c = p_atomic_fetch_add(&mtx->val, -1); 123 124 assert(c != _SIMPLE_MTX_INVALID_VALUE); 125 126 if (__builtin_expect(c != 1, 0)) { 127 mtx->val = 0; 128 futex_wake(&mtx->val, 1); 129 } 130} 131 132static inline void 133simple_mtx_assert_locked(simple_mtx_t *mtx) 134{ 135 assert(mtx->val); 136} 137 138#else 139 140typedef mtx_t simple_mtx_t; 141 142#define _SIMPLE_MTX_INITIALIZER_NP _MTX_INITIALIZER_NP 143 144static inline void 145simple_mtx_init(simple_mtx_t *mtx, int type) 146{ 147 mtx_init(mtx, type); 148} 149 150static inline void 151simple_mtx_destroy(simple_mtx_t *mtx) 152{ 153 mtx_destroy(mtx); 154} 155 156static inline void 157simple_mtx_lock(simple_mtx_t *mtx) 158{ 159 mtx_lock(mtx); 160} 161 162static inline void 163simple_mtx_unlock(simple_mtx_t *mtx) 164{ 165 mtx_unlock(mtx); 166} 167 168static inline void 169simple_mtx_assert_locked(simple_mtx_t *mtx) 170{ 171#ifndef NDEBUG 172 /* NOTE: this would not work for recursive mutexes, but 173 * mtx_t doesn't support those 174 */ 175 int ret = mtx_trylock(mtx); 176 assert(ret == thrd_busy); 177 if (ret == thrd_success) 178 mtx_unlock(mtx); 179#else 180 (void)mtx; 181#endif 182} 183 184#endif 185 186#endif 187