162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-or-later */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * atomic64_t for 586+ 462306a36Sopenharmony_ci * 562306a36Sopenharmony_ci * Copyright © 2010 Luca Barbieri 662306a36Sopenharmony_ci */ 762306a36Sopenharmony_ci 862306a36Sopenharmony_ci#include <linux/linkage.h> 962306a36Sopenharmony_ci#include <asm/alternative.h> 1062306a36Sopenharmony_ci 1162306a36Sopenharmony_ci.macro read64 reg 1262306a36Sopenharmony_ci movl %ebx, %eax 1362306a36Sopenharmony_ci movl %ecx, %edx 1462306a36Sopenharmony_ci/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 1562306a36Sopenharmony_ci LOCK_PREFIX 1662306a36Sopenharmony_ci cmpxchg8b (\reg) 1762306a36Sopenharmony_ci.endm 1862306a36Sopenharmony_ci 1962306a36Sopenharmony_ciSYM_FUNC_START(atomic64_read_cx8) 2062306a36Sopenharmony_ci read64 %ecx 2162306a36Sopenharmony_ci RET 2262306a36Sopenharmony_ciSYM_FUNC_END(atomic64_read_cx8) 2362306a36Sopenharmony_ci 2462306a36Sopenharmony_ciSYM_FUNC_START(atomic64_set_cx8) 2562306a36Sopenharmony_ci1: 2662306a36Sopenharmony_ci/* we don't need LOCK_PREFIX since aligned 64-bit writes 2762306a36Sopenharmony_ci * are atomic on 586 and newer */ 2862306a36Sopenharmony_ci cmpxchg8b (%esi) 2962306a36Sopenharmony_ci jne 1b 3062306a36Sopenharmony_ci 3162306a36Sopenharmony_ci RET 3262306a36Sopenharmony_ciSYM_FUNC_END(atomic64_set_cx8) 3362306a36Sopenharmony_ci 3462306a36Sopenharmony_ciSYM_FUNC_START(atomic64_xchg_cx8) 3562306a36Sopenharmony_ci1: 3662306a36Sopenharmony_ci LOCK_PREFIX 3762306a36Sopenharmony_ci cmpxchg8b (%esi) 3862306a36Sopenharmony_ci jne 1b 3962306a36Sopenharmony_ci 4062306a36Sopenharmony_ci RET 4162306a36Sopenharmony_ciSYM_FUNC_END(atomic64_xchg_cx8) 4262306a36Sopenharmony_ci 4362306a36Sopenharmony_ci.macro addsub_return func ins insc 4462306a36Sopenharmony_ciSYM_FUNC_START(atomic64_\func\()_return_cx8) 4562306a36Sopenharmony_ci pushl %ebp 4662306a36Sopenharmony_ci pushl %ebx 4762306a36Sopenharmony_ci pushl %esi 4862306a36Sopenharmony_ci pushl %edi 4962306a36Sopenharmony_ci 5062306a36Sopenharmony_ci movl %eax, %esi 5162306a36Sopenharmony_ci movl %edx, %edi 5262306a36Sopenharmony_ci movl %ecx, %ebp 5362306a36Sopenharmony_ci 5462306a36Sopenharmony_ci read64 %ecx 5562306a36Sopenharmony_ci1: 5662306a36Sopenharmony_ci movl %eax, %ebx 5762306a36Sopenharmony_ci movl %edx, %ecx 5862306a36Sopenharmony_ci \ins\()l %esi, %ebx 5962306a36Sopenharmony_ci \insc\()l %edi, %ecx 6062306a36Sopenharmony_ci LOCK_PREFIX 6162306a36Sopenharmony_ci cmpxchg8b (%ebp) 6262306a36Sopenharmony_ci jne 1b 6362306a36Sopenharmony_ci 6462306a36Sopenharmony_ci10: 6562306a36Sopenharmony_ci movl %ebx, %eax 6662306a36Sopenharmony_ci movl %ecx, %edx 6762306a36Sopenharmony_ci popl %edi 6862306a36Sopenharmony_ci popl %esi 6962306a36Sopenharmony_ci popl %ebx 7062306a36Sopenharmony_ci popl %ebp 7162306a36Sopenharmony_ci RET 7262306a36Sopenharmony_ciSYM_FUNC_END(atomic64_\func\()_return_cx8) 7362306a36Sopenharmony_ci.endm 7462306a36Sopenharmony_ci 7562306a36Sopenharmony_ciaddsub_return add add adc 7662306a36Sopenharmony_ciaddsub_return sub sub sbb 7762306a36Sopenharmony_ci 7862306a36Sopenharmony_ci.macro incdec_return func ins insc 7962306a36Sopenharmony_ciSYM_FUNC_START(atomic64_\func\()_return_cx8) 8062306a36Sopenharmony_ci pushl %ebx 8162306a36Sopenharmony_ci 8262306a36Sopenharmony_ci read64 %esi 8362306a36Sopenharmony_ci1: 8462306a36Sopenharmony_ci movl %eax, %ebx 8562306a36Sopenharmony_ci movl %edx, %ecx 8662306a36Sopenharmony_ci \ins\()l $1, %ebx 8762306a36Sopenharmony_ci \insc\()l $0, %ecx 8862306a36Sopenharmony_ci LOCK_PREFIX 8962306a36Sopenharmony_ci cmpxchg8b (%esi) 9062306a36Sopenharmony_ci jne 1b 9162306a36Sopenharmony_ci 9262306a36Sopenharmony_ci10: 9362306a36Sopenharmony_ci movl %ebx, %eax 9462306a36Sopenharmony_ci movl %ecx, %edx 9562306a36Sopenharmony_ci popl %ebx 9662306a36Sopenharmony_ci RET 9762306a36Sopenharmony_ciSYM_FUNC_END(atomic64_\func\()_return_cx8) 9862306a36Sopenharmony_ci.endm 9962306a36Sopenharmony_ci 10062306a36Sopenharmony_ciincdec_return inc add adc 10162306a36Sopenharmony_ciincdec_return dec sub sbb 10262306a36Sopenharmony_ci 10362306a36Sopenharmony_ciSYM_FUNC_START(atomic64_dec_if_positive_cx8) 10462306a36Sopenharmony_ci pushl %ebx 10562306a36Sopenharmony_ci 10662306a36Sopenharmony_ci read64 %esi 10762306a36Sopenharmony_ci1: 10862306a36Sopenharmony_ci movl %eax, %ebx 10962306a36Sopenharmony_ci movl %edx, %ecx 11062306a36Sopenharmony_ci subl $1, %ebx 11162306a36Sopenharmony_ci sbb $0, %ecx 11262306a36Sopenharmony_ci js 2f 11362306a36Sopenharmony_ci LOCK_PREFIX 11462306a36Sopenharmony_ci cmpxchg8b (%esi) 11562306a36Sopenharmony_ci jne 1b 11662306a36Sopenharmony_ci 11762306a36Sopenharmony_ci2: 11862306a36Sopenharmony_ci movl %ebx, %eax 11962306a36Sopenharmony_ci movl %ecx, %edx 12062306a36Sopenharmony_ci popl %ebx 12162306a36Sopenharmony_ci RET 12262306a36Sopenharmony_ciSYM_FUNC_END(atomic64_dec_if_positive_cx8) 12362306a36Sopenharmony_ci 12462306a36Sopenharmony_ciSYM_FUNC_START(atomic64_add_unless_cx8) 12562306a36Sopenharmony_ci pushl %ebp 12662306a36Sopenharmony_ci pushl %ebx 12762306a36Sopenharmony_ci/* these just push these two parameters on the stack */ 12862306a36Sopenharmony_ci pushl %edi 12962306a36Sopenharmony_ci pushl %ecx 13062306a36Sopenharmony_ci 13162306a36Sopenharmony_ci movl %eax, %ebp 13262306a36Sopenharmony_ci movl %edx, %edi 13362306a36Sopenharmony_ci 13462306a36Sopenharmony_ci read64 %esi 13562306a36Sopenharmony_ci1: 13662306a36Sopenharmony_ci cmpl %eax, 0(%esp) 13762306a36Sopenharmony_ci je 4f 13862306a36Sopenharmony_ci2: 13962306a36Sopenharmony_ci movl %eax, %ebx 14062306a36Sopenharmony_ci movl %edx, %ecx 14162306a36Sopenharmony_ci addl %ebp, %ebx 14262306a36Sopenharmony_ci adcl %edi, %ecx 14362306a36Sopenharmony_ci LOCK_PREFIX 14462306a36Sopenharmony_ci cmpxchg8b (%esi) 14562306a36Sopenharmony_ci jne 1b 14662306a36Sopenharmony_ci 14762306a36Sopenharmony_ci movl $1, %eax 14862306a36Sopenharmony_ci3: 14962306a36Sopenharmony_ci addl $8, %esp 15062306a36Sopenharmony_ci popl %ebx 15162306a36Sopenharmony_ci popl %ebp 15262306a36Sopenharmony_ci RET 15362306a36Sopenharmony_ci4: 15462306a36Sopenharmony_ci cmpl %edx, 4(%esp) 15562306a36Sopenharmony_ci jne 2b 15662306a36Sopenharmony_ci xorl %eax, %eax 15762306a36Sopenharmony_ci jmp 3b 15862306a36Sopenharmony_ciSYM_FUNC_END(atomic64_add_unless_cx8) 15962306a36Sopenharmony_ci 16062306a36Sopenharmony_ciSYM_FUNC_START(atomic64_inc_not_zero_cx8) 16162306a36Sopenharmony_ci pushl %ebx 16262306a36Sopenharmony_ci 16362306a36Sopenharmony_ci read64 %esi 16462306a36Sopenharmony_ci1: 16562306a36Sopenharmony_ci movl %eax, %ecx 16662306a36Sopenharmony_ci orl %edx, %ecx 16762306a36Sopenharmony_ci jz 3f 16862306a36Sopenharmony_ci movl %eax, %ebx 16962306a36Sopenharmony_ci xorl %ecx, %ecx 17062306a36Sopenharmony_ci addl $1, %ebx 17162306a36Sopenharmony_ci adcl %edx, %ecx 17262306a36Sopenharmony_ci LOCK_PREFIX 17362306a36Sopenharmony_ci cmpxchg8b (%esi) 17462306a36Sopenharmony_ci jne 1b 17562306a36Sopenharmony_ci 17662306a36Sopenharmony_ci movl $1, %eax 17762306a36Sopenharmony_ci3: 17862306a36Sopenharmony_ci popl %ebx 17962306a36Sopenharmony_ci RET 18062306a36Sopenharmony_ciSYM_FUNC_END(atomic64_inc_not_zero_cx8) 181