1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
4 *
5 * Copyright (C) 2003 David Gibson, IBM Corporation.
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
11#include <linux/mm.h>
12#include <linux/hugetlb.h>
13#include <asm/cacheflush.h>
14#include <asm/machdep.h>
15
16unsigned int hpage_shift;
17EXPORT_SYMBOL(hpage_shift);
18
19extern long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
20				  unsigned long pa, unsigned long rlags,
21				  unsigned long vflags, int psize, int ssize);
22
23int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
24		     pte_t *ptep, unsigned long trap, unsigned long flags,
25		     int ssize, unsigned int shift, unsigned int mmu_psize)
26{
27	real_pte_t rpte;
28	unsigned long vpn;
29	unsigned long old_pte, new_pte;
30	unsigned long rflags, pa;
31	long slot, offset;
32
33	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
34
35	/* Search the Linux page table for a match with va */
36	vpn = hpt_vpn(ea, vsid, ssize);
37
38	/*
39	 * At this point, we have a pte (old_pte) which can be used to build
40	 * or update an HPTE. There are 2 cases:
41	 *
42	 * 1. There is a valid (present) pte with no associated HPTE (this is
43	 *	the most common case)
44	 * 2. There is a valid (present) pte with an associated HPTE. The
45	 *	current values of the pp bits in the HPTE prevent access
46	 *	because we are doing software DIRTY bit management and the
47	 *	page is currently not DIRTY.
48	 */
49
50
51	do {
52		old_pte = pte_val(*ptep);
53		/* If PTE busy, retry the access */
54		if (unlikely(old_pte & H_PAGE_BUSY))
55			return 0;
56		/* If PTE permissions don't match, take page fault */
57		if (unlikely(!check_pte_access(access, old_pte)))
58			return 1;
59
60		/*
61		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
62		 * a write access
63		 */
64		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
65		if (access & _PAGE_WRITE)
66			new_pte |= _PAGE_DIRTY;
67	} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
68
69	/* Make sure this is a hugetlb entry */
70	if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))
71		return 0;
72
73	rflags = htab_convert_pte_flags(new_pte);
74	if (unlikely(mmu_psize == MMU_PAGE_16G))
75		offset = PTRS_PER_PUD;
76	else
77		offset = PTRS_PER_PMD;
78	rpte = __real_pte(__pte(old_pte), ptep, offset);
79
80	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
81		/*
82		 * No CPU has hugepages but lacks no execute, so we
83		 * don't need to worry about that case
84		 */
85		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
86
87	/* Check if pte already has an hpte (case 2) */
88	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
89		/* There MIGHT be an HPTE for this pte */
90		unsigned long gslot;
91
92		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
93		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
94					       mmu_psize, ssize, flags) == -1)
95			old_pte &= ~_PAGE_HPTEFLAGS;
96	}
97
98	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
99		unsigned long hash = hpt_hash(vpn, shift, ssize);
100
101		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
102
103		/* clear HPTE slot informations in new PTE */
104		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
105
106		slot = hpte_insert_repeating(hash, vpn, pa, rflags, 0,
107					     mmu_psize, ssize);
108
109		/*
110		 * Hypervisor failure. Restore old pte and return -1
111		 * similar to __hash_page_*
112		 */
113		if (unlikely(slot == -2)) {
114			*ptep = __pte(old_pte);
115			hash_failure_debug(ea, access, vsid, trap, ssize,
116					   mmu_psize, mmu_psize, old_pte);
117			return -1;
118		}
119
120		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
121	}
122
123	/*
124	 * No need to use ldarx/stdcx here
125	 */
126	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
127	return 0;
128}
129
130pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
131				  unsigned long addr, pte_t *ptep)
132{
133	unsigned long pte_val;
134	/*
135	 * Clear the _PAGE_PRESENT so that no hardware parallel update is
136	 * possible. Also keep the pte_present true so that we don't take
137	 * wrong fault.
138	 */
139	pte_val = pte_update(vma->vm_mm, addr, ptep,
140			     _PAGE_PRESENT, _PAGE_INVALID, 1);
141
142	return __pte(pte_val);
143}
144
145void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
146				  pte_t *ptep, pte_t old_pte, pte_t pte)
147{
148
149	if (radix_enabled())
150		return radix__huge_ptep_modify_prot_commit(vma, addr, ptep,
151							   old_pte, pte);
152	set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
153}
154
155void hugetlbpage_init_default(void)
156{
157	/* Set default large page size. Currently, we pick 16M or 1M
158	 * depending on what is available
159	 */
160	if (mmu_psize_defs[MMU_PAGE_16M].shift)
161		hpage_shift = mmu_psize_defs[MMU_PAGE_16M].shift;
162	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
163		hpage_shift = mmu_psize_defs[MMU_PAGE_1M].shift;
164	else if (mmu_psize_defs[MMU_PAGE_2M].shift)
165		hpage_shift = mmu_psize_defs[MMU_PAGE_2M].shift;
166}
167