1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright(c) 2017 IBM Corporation. All rights reserved. 4 */ 5 6#include <linux/string.h> 7#include <linux/export.h> 8#include <linux/uaccess.h> 9 10#include <asm/cacheflush.h> 11 12static inline void __clean_pmem_range(unsigned long start, unsigned long stop) 13{ 14 unsigned long shift = l1_dcache_shift(); 15 unsigned long bytes = l1_dcache_bytes(); 16 void *addr = (void *)(start & ~(bytes - 1)); 17 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 18 unsigned long i; 19 20 for (i = 0; i < size >> shift; i++, addr += bytes) 21 asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory"); 22} 23 24static inline void __flush_pmem_range(unsigned long start, unsigned long stop) 25{ 26 unsigned long shift = l1_dcache_shift(); 27 unsigned long bytes = l1_dcache_bytes(); 28 void *addr = (void *)(start & ~(bytes - 1)); 29 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 30 unsigned long i; 31 32 for (i = 0; i < size >> shift; i++, addr += bytes) 33 asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory"); 34} 35 36static inline void clean_pmem_range(unsigned long start, unsigned long stop) 37{ 38 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 39 return __clean_pmem_range(start, stop); 40} 41 42static inline void flush_pmem_range(unsigned long start, unsigned long stop) 43{ 44 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 45 return __flush_pmem_range(start, stop); 46} 47 48/* 49 * CONFIG_ARCH_HAS_PMEM_API symbols 50 */ 51void arch_wb_cache_pmem(void *addr, size_t size) 52{ 53 unsigned long start = (unsigned long) addr; 54 clean_pmem_range(start, start + size); 55} 56EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); 57 58void arch_invalidate_pmem(void *addr, size_t size) 59{ 60 unsigned long start = (unsigned long) addr; 61 flush_pmem_range(start, start + size); 62} 63EXPORT_SYMBOL_GPL(arch_invalidate_pmem); 64 65/* 66 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols 67 */ 68long __copy_from_user_flushcache(void *dest, const void __user *src, 69 unsigned size) 70{ 71 unsigned long copied, start = (unsigned long) dest; 72 73 copied = __copy_from_user(dest, src, size); 74 clean_pmem_range(start, start + size); 75 76 return copied; 77} 78 79void memcpy_flushcache(void *dest, const void *src, size_t size) 80{ 81 unsigned long start = (unsigned long) dest; 82 83 memcpy(dest, src, size); 84 clean_pmem_range(start, start + size); 85} 86EXPORT_SYMBOL(memcpy_flushcache); 87 88void memcpy_page_flushcache(char *to, struct page *page, size_t offset, 89 size_t len) 90{ 91 memcpy_flushcache(to, page_to_virt(page) + offset, len); 92} 93EXPORT_SYMBOL(memcpy_page_flushcache); 94