18c2ecf20Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-or-later */ 28c2ecf20Sopenharmony_ci/* 38c2ecf20Sopenharmony_ci * atomic64_t for 586+ 48c2ecf20Sopenharmony_ci * 58c2ecf20Sopenharmony_ci * Copyright © 2010 Luca Barbieri 68c2ecf20Sopenharmony_ci */ 78c2ecf20Sopenharmony_ci 88c2ecf20Sopenharmony_ci#include <linux/linkage.h> 98c2ecf20Sopenharmony_ci#include <asm/alternative.h> 108c2ecf20Sopenharmony_ci 118c2ecf20Sopenharmony_ci.macro read64 reg 128c2ecf20Sopenharmony_ci movl %ebx, %eax 138c2ecf20Sopenharmony_ci movl %ecx, %edx 148c2ecf20Sopenharmony_ci/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 158c2ecf20Sopenharmony_ci LOCK_PREFIX 168c2ecf20Sopenharmony_ci cmpxchg8b (\reg) 178c2ecf20Sopenharmony_ci.endm 188c2ecf20Sopenharmony_ci 198c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_read_cx8) 208c2ecf20Sopenharmony_ci read64 %ecx 218c2ecf20Sopenharmony_ci RET 228c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_read_cx8) 238c2ecf20Sopenharmony_ci 248c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_set_cx8) 258c2ecf20Sopenharmony_ci1: 268c2ecf20Sopenharmony_ci/* we don't need LOCK_PREFIX since aligned 64-bit writes 278c2ecf20Sopenharmony_ci * are atomic on 586 and newer */ 288c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 298c2ecf20Sopenharmony_ci jne 1b 308c2ecf20Sopenharmony_ci 318c2ecf20Sopenharmony_ci RET 328c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_set_cx8) 338c2ecf20Sopenharmony_ci 348c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_xchg_cx8) 358c2ecf20Sopenharmony_ci1: 368c2ecf20Sopenharmony_ci LOCK_PREFIX 378c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 388c2ecf20Sopenharmony_ci jne 1b 398c2ecf20Sopenharmony_ci 408c2ecf20Sopenharmony_ci RET 418c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_xchg_cx8) 428c2ecf20Sopenharmony_ci 438c2ecf20Sopenharmony_ci.macro addsub_return func ins insc 448c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_\func\()_return_cx8) 458c2ecf20Sopenharmony_ci pushl %ebp 468c2ecf20Sopenharmony_ci pushl %ebx 478c2ecf20Sopenharmony_ci pushl %esi 488c2ecf20Sopenharmony_ci pushl %edi 498c2ecf20Sopenharmony_ci 508c2ecf20Sopenharmony_ci movl %eax, %esi 518c2ecf20Sopenharmony_ci movl %edx, %edi 528c2ecf20Sopenharmony_ci movl %ecx, %ebp 538c2ecf20Sopenharmony_ci 548c2ecf20Sopenharmony_ci read64 %ecx 558c2ecf20Sopenharmony_ci1: 568c2ecf20Sopenharmony_ci movl %eax, %ebx 578c2ecf20Sopenharmony_ci movl %edx, %ecx 588c2ecf20Sopenharmony_ci \ins\()l %esi, %ebx 598c2ecf20Sopenharmony_ci \insc\()l %edi, %ecx 608c2ecf20Sopenharmony_ci LOCK_PREFIX 618c2ecf20Sopenharmony_ci cmpxchg8b (%ebp) 628c2ecf20Sopenharmony_ci jne 1b 638c2ecf20Sopenharmony_ci 648c2ecf20Sopenharmony_ci10: 658c2ecf20Sopenharmony_ci movl %ebx, %eax 668c2ecf20Sopenharmony_ci movl %ecx, %edx 678c2ecf20Sopenharmony_ci popl %edi 688c2ecf20Sopenharmony_ci popl %esi 698c2ecf20Sopenharmony_ci popl %ebx 708c2ecf20Sopenharmony_ci popl %ebp 718c2ecf20Sopenharmony_ci RET 728c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_\func\()_return_cx8) 738c2ecf20Sopenharmony_ci.endm 748c2ecf20Sopenharmony_ci 758c2ecf20Sopenharmony_ciaddsub_return add add adc 768c2ecf20Sopenharmony_ciaddsub_return sub sub sbb 778c2ecf20Sopenharmony_ci 788c2ecf20Sopenharmony_ci.macro incdec_return func ins insc 798c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_\func\()_return_cx8) 808c2ecf20Sopenharmony_ci pushl %ebx 818c2ecf20Sopenharmony_ci 828c2ecf20Sopenharmony_ci read64 %esi 838c2ecf20Sopenharmony_ci1: 848c2ecf20Sopenharmony_ci movl %eax, %ebx 858c2ecf20Sopenharmony_ci movl %edx, %ecx 868c2ecf20Sopenharmony_ci \ins\()l $1, %ebx 878c2ecf20Sopenharmony_ci \insc\()l $0, %ecx 888c2ecf20Sopenharmony_ci LOCK_PREFIX 898c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 908c2ecf20Sopenharmony_ci jne 1b 918c2ecf20Sopenharmony_ci 928c2ecf20Sopenharmony_ci10: 938c2ecf20Sopenharmony_ci movl %ebx, %eax 948c2ecf20Sopenharmony_ci movl %ecx, %edx 958c2ecf20Sopenharmony_ci popl %ebx 968c2ecf20Sopenharmony_ci RET 978c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_\func\()_return_cx8) 988c2ecf20Sopenharmony_ci.endm 998c2ecf20Sopenharmony_ci 1008c2ecf20Sopenharmony_ciincdec_return inc add adc 1018c2ecf20Sopenharmony_ciincdec_return dec sub sbb 1028c2ecf20Sopenharmony_ci 1038c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_dec_if_positive_cx8) 1048c2ecf20Sopenharmony_ci pushl %ebx 1058c2ecf20Sopenharmony_ci 1068c2ecf20Sopenharmony_ci read64 %esi 1078c2ecf20Sopenharmony_ci1: 1088c2ecf20Sopenharmony_ci movl %eax, %ebx 1098c2ecf20Sopenharmony_ci movl %edx, %ecx 1108c2ecf20Sopenharmony_ci subl $1, %ebx 1118c2ecf20Sopenharmony_ci sbb $0, %ecx 1128c2ecf20Sopenharmony_ci js 2f 1138c2ecf20Sopenharmony_ci LOCK_PREFIX 1148c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 1158c2ecf20Sopenharmony_ci jne 1b 1168c2ecf20Sopenharmony_ci 1178c2ecf20Sopenharmony_ci2: 1188c2ecf20Sopenharmony_ci movl %ebx, %eax 1198c2ecf20Sopenharmony_ci movl %ecx, %edx 1208c2ecf20Sopenharmony_ci popl %ebx 1218c2ecf20Sopenharmony_ci RET 1228c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_dec_if_positive_cx8) 1238c2ecf20Sopenharmony_ci 1248c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_add_unless_cx8) 1258c2ecf20Sopenharmony_ci pushl %ebp 1268c2ecf20Sopenharmony_ci pushl %ebx 1278c2ecf20Sopenharmony_ci/* these just push these two parameters on the stack */ 1288c2ecf20Sopenharmony_ci pushl %edi 1298c2ecf20Sopenharmony_ci pushl %ecx 1308c2ecf20Sopenharmony_ci 1318c2ecf20Sopenharmony_ci movl %eax, %ebp 1328c2ecf20Sopenharmony_ci movl %edx, %edi 1338c2ecf20Sopenharmony_ci 1348c2ecf20Sopenharmony_ci read64 %esi 1358c2ecf20Sopenharmony_ci1: 1368c2ecf20Sopenharmony_ci cmpl %eax, 0(%esp) 1378c2ecf20Sopenharmony_ci je 4f 1388c2ecf20Sopenharmony_ci2: 1398c2ecf20Sopenharmony_ci movl %eax, %ebx 1408c2ecf20Sopenharmony_ci movl %edx, %ecx 1418c2ecf20Sopenharmony_ci addl %ebp, %ebx 1428c2ecf20Sopenharmony_ci adcl %edi, %ecx 1438c2ecf20Sopenharmony_ci LOCK_PREFIX 1448c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 1458c2ecf20Sopenharmony_ci jne 1b 1468c2ecf20Sopenharmony_ci 1478c2ecf20Sopenharmony_ci movl $1, %eax 1488c2ecf20Sopenharmony_ci3: 1498c2ecf20Sopenharmony_ci addl $8, %esp 1508c2ecf20Sopenharmony_ci popl %ebx 1518c2ecf20Sopenharmony_ci popl %ebp 1528c2ecf20Sopenharmony_ci RET 1538c2ecf20Sopenharmony_ci4: 1548c2ecf20Sopenharmony_ci cmpl %edx, 4(%esp) 1558c2ecf20Sopenharmony_ci jne 2b 1568c2ecf20Sopenharmony_ci xorl %eax, %eax 1578c2ecf20Sopenharmony_ci jmp 3b 1588c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_add_unless_cx8) 1598c2ecf20Sopenharmony_ci 1608c2ecf20Sopenharmony_ciSYM_FUNC_START(atomic64_inc_not_zero_cx8) 1618c2ecf20Sopenharmony_ci pushl %ebx 1628c2ecf20Sopenharmony_ci 1638c2ecf20Sopenharmony_ci read64 %esi 1648c2ecf20Sopenharmony_ci1: 1658c2ecf20Sopenharmony_ci movl %eax, %ecx 1668c2ecf20Sopenharmony_ci orl %edx, %ecx 1678c2ecf20Sopenharmony_ci jz 3f 1688c2ecf20Sopenharmony_ci movl %eax, %ebx 1698c2ecf20Sopenharmony_ci xorl %ecx, %ecx 1708c2ecf20Sopenharmony_ci addl $1, %ebx 1718c2ecf20Sopenharmony_ci adcl %edx, %ecx 1728c2ecf20Sopenharmony_ci LOCK_PREFIX 1738c2ecf20Sopenharmony_ci cmpxchg8b (%esi) 1748c2ecf20Sopenharmony_ci jne 1b 1758c2ecf20Sopenharmony_ci 1768c2ecf20Sopenharmony_ci movl $1, %eax 1778c2ecf20Sopenharmony_ci3: 1788c2ecf20Sopenharmony_ci popl %ebx 1798c2ecf20Sopenharmony_ci RET 1808c2ecf20Sopenharmony_ciSYM_FUNC_END(atomic64_inc_not_zero_cx8) 181