1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights 4 * reserved. 5 */ 6 7/* 8 * SN Platform Special Memory (mspec) Support 9 * 10 * This driver exports the SN special memory (mspec) facility to user 11 * processes. 12 * There are two types of memory made available thru this driver: 13 * uncached and cached. 14 * 15 * Uncached are used for memory write combining feature of the ia64 16 * cpu. 17 * 18 * Cached are used for areas of memory that are used as cached addresses 19 * on our partition and used as uncached addresses from other partitions. 20 * Due to a design constraint of the SN2 Shub, you can not have processors 21 * on the same FSB perform both a cached and uncached reference to the 22 * same cache line. These special memory cached regions prevent the 23 * kernel from ever dropping in a TLB entry and therefore prevent the 24 * processor from ever speculating a cache line from this page. 25 */ 26 27#include <linux/types.h> 28#include <linux/kernel.h> 29#include <linux/module.h> 30#include <linux/init.h> 31#include <linux/errno.h> 32#include <linux/miscdevice.h> 33#include <linux/spinlock.h> 34#include <linux/mm.h> 35#include <linux/fs.h> 36#include <linux/vmalloc.h> 37#include <linux/string.h> 38#include <linux/slab.h> 39#include <linux/numa.h> 40#include <linux/refcount.h> 41#include <asm/page.h> 42#include <linux/atomic.h> 43#include <asm/tlbflush.h> 44#include <asm/uncached.h> 45 46 47#define CACHED_ID "Cached," 48#define UNCACHED_ID "Uncached" 49#define REVISION "4.0" 50#define MSPEC_BASENAME "mspec" 51 52/* 53 * Page types allocated by the device. 54 */ 55enum mspec_page_type { 56 MSPEC_CACHED = 2, 57 MSPEC_UNCACHED 58}; 59 60/* 61 * One of these structures is allocated when an mspec region is mmaped. The 62 * structure is pointed to by the vma->vm_private_data field in the vma struct. 63 * This structure is used to record the addresses of the mspec pages. 64 * This structure is shared by all vma's that are split off from the 65 * original vma when split_vma()'s are done. 66 * 67 * The refcnt is incremented atomically because mm->mmap_lock does not 68 * protect in fork case where multiple tasks share the vma_data. 69 */ 70struct vma_data { 71 refcount_t refcnt; /* Number of vmas sharing the data. */ 72 spinlock_t lock; /* Serialize access to this structure. */ 73 int count; /* Number of pages allocated. */ 74 enum mspec_page_type type; /* Type of pages allocated. */ 75 unsigned long vm_start; /* Original (unsplit) base. */ 76 unsigned long vm_end; /* Original (unsplit) end. */ 77 unsigned long maddr[]; /* Array of MSPEC addresses. */ 78}; 79 80/* 81 * mspec_open 82 * 83 * Called when a device mapping is created by a means other than mmap 84 * (via fork, munmap, etc.). Increments the reference count on the 85 * underlying mspec data so it is not freed prematurely. 86 */ 87static void 88mspec_open(struct vm_area_struct *vma) 89{ 90 struct vma_data *vdata; 91 92 vdata = vma->vm_private_data; 93 refcount_inc(&vdata->refcnt); 94} 95 96/* 97 * mspec_close 98 * 99 * Called when unmapping a device mapping. Frees all mspec pages 100 * belonging to all the vma's sharing this vma_data structure. 101 */ 102static void 103mspec_close(struct vm_area_struct *vma) 104{ 105 struct vma_data *vdata; 106 int index, last_index; 107 unsigned long my_page; 108 109 vdata = vma->vm_private_data; 110 111 if (!refcount_dec_and_test(&vdata->refcnt)) 112 return; 113 114 last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT; 115 for (index = 0; index < last_index; index++) { 116 if (vdata->maddr[index] == 0) 117 continue; 118 /* 119 * Clear the page before sticking it back 120 * into the pool. 121 */ 122 my_page = vdata->maddr[index]; 123 vdata->maddr[index] = 0; 124 memset((char *)my_page, 0, PAGE_SIZE); 125 uncached_free_page(my_page, 1); 126 } 127 128 kvfree(vdata); 129} 130 131/* 132 * mspec_fault 133 * 134 * Creates a mspec page and maps it to user space. 135 */ 136static vm_fault_t 137mspec_fault(struct vm_fault *vmf) 138{ 139 unsigned long paddr, maddr; 140 unsigned long pfn; 141 pgoff_t index = vmf->pgoff; 142 struct vma_data *vdata = vmf->vma->vm_private_data; 143 144 maddr = (volatile unsigned long) vdata->maddr[index]; 145 if (maddr == 0) { 146 maddr = uncached_alloc_page(numa_node_id(), 1); 147 if (maddr == 0) 148 return VM_FAULT_OOM; 149 150 spin_lock(&vdata->lock); 151 if (vdata->maddr[index] == 0) { 152 vdata->count++; 153 vdata->maddr[index] = maddr; 154 } else { 155 uncached_free_page(maddr, 1); 156 maddr = vdata->maddr[index]; 157 } 158 spin_unlock(&vdata->lock); 159 } 160 161 paddr = maddr & ~__IA64_UNCACHED_OFFSET; 162 pfn = paddr >> PAGE_SHIFT; 163 164 return vmf_insert_pfn(vmf->vma, vmf->address, pfn); 165} 166 167static const struct vm_operations_struct mspec_vm_ops = { 168 .open = mspec_open, 169 .close = mspec_close, 170 .fault = mspec_fault, 171}; 172 173/* 174 * mspec_mmap 175 * 176 * Called when mmapping the device. Initializes the vma with a fault handler 177 * and private data structure necessary to allocate, track, and free the 178 * underlying pages. 179 */ 180static int 181mspec_mmap(struct file *file, struct vm_area_struct *vma, 182 enum mspec_page_type type) 183{ 184 struct vma_data *vdata; 185 int pages, vdata_size; 186 187 if (vma->vm_pgoff != 0) 188 return -EINVAL; 189 190 if ((vma->vm_flags & VM_SHARED) == 0) 191 return -EINVAL; 192 193 if ((vma->vm_flags & VM_WRITE) == 0) 194 return -EPERM; 195 196 pages = vma_pages(vma); 197 vdata_size = sizeof(struct vma_data) + pages * sizeof(long); 198 vdata = kvzalloc(vdata_size, GFP_KERNEL); 199 if (!vdata) 200 return -ENOMEM; 201 202 vdata->vm_start = vma->vm_start; 203 vdata->vm_end = vma->vm_end; 204 vdata->type = type; 205 spin_lock_init(&vdata->lock); 206 refcount_set(&vdata->refcnt, 1); 207 vma->vm_private_data = vdata; 208 209 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 210 if (vdata->type == MSPEC_UNCACHED) 211 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 212 vma->vm_ops = &mspec_vm_ops; 213 214 return 0; 215} 216 217static int 218cached_mmap(struct file *file, struct vm_area_struct *vma) 219{ 220 return mspec_mmap(file, vma, MSPEC_CACHED); 221} 222 223static int 224uncached_mmap(struct file *file, struct vm_area_struct *vma) 225{ 226 return mspec_mmap(file, vma, MSPEC_UNCACHED); 227} 228 229static const struct file_operations cached_fops = { 230 .owner = THIS_MODULE, 231 .mmap = cached_mmap, 232 .llseek = noop_llseek, 233}; 234 235static struct miscdevice cached_miscdev = { 236 .minor = MISC_DYNAMIC_MINOR, 237 .name = "mspec_cached", 238 .fops = &cached_fops 239}; 240 241static const struct file_operations uncached_fops = { 242 .owner = THIS_MODULE, 243 .mmap = uncached_mmap, 244 .llseek = noop_llseek, 245}; 246 247static struct miscdevice uncached_miscdev = { 248 .minor = MISC_DYNAMIC_MINOR, 249 .name = "mspec_uncached", 250 .fops = &uncached_fops 251}; 252 253/* 254 * mspec_init 255 * 256 * Called at boot time to initialize the mspec facility. 257 */ 258static int __init 259mspec_init(void) 260{ 261 int ret; 262 263 ret = misc_register(&cached_miscdev); 264 if (ret) { 265 printk(KERN_ERR "%s: failed to register device %i\n", 266 CACHED_ID, ret); 267 return ret; 268 } 269 ret = misc_register(&uncached_miscdev); 270 if (ret) { 271 printk(KERN_ERR "%s: failed to register device %i\n", 272 UNCACHED_ID, ret); 273 misc_deregister(&cached_miscdev); 274 return ret; 275 } 276 277 printk(KERN_INFO "%s %s initialized devices: %s %s\n", 278 MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID); 279 280 return 0; 281} 282 283static void __exit 284mspec_exit(void) 285{ 286 misc_deregister(&uncached_miscdev); 287 misc_deregister(&cached_miscdev); 288} 289 290module_init(mspec_init); 291module_exit(mspec_exit); 292 293MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>"); 294MODULE_DESCRIPTION("Driver for SGI SN special memory operations"); 295MODULE_LICENSE("GPL"); 296