1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020 Loongson Technology Corporation Limited
4 */
5#include <linux/fs.h>
6#include <linux/kernel.h>
7#include <linux/linkage.h>
8#include <linux/export.h>
9#include <linux/sched.h>
10#include <linux/syscalls.h>
11#include <linux/mm.h>
12#include <linux/cacheinfo.h>
13#include <linux/cpu_pm.h>
14#include <linux/highmem.h>
15
16#include <asm/cacheflush.h>
17#include <asm/processor.h>
18#include <asm/cpu.h>
19#include <asm/cpu-features.h>
20#include <asm/numa.h>
21#include <asm/setup.h>
22#include <asm/dma.h>
23#include <loongson.h>
24
25void cache_error_setup(void)
26{
27	extern char __weak except_vec_cex;
28	set_merr_handler(0x0, &except_vec_cex, 0x80);
29}
30
31/*
32 * LoongArch maintains ICache/DCache coherency by hardware,
33 * we just need "ibar" to avoid instruction hazard here.
34 */
35void local_flush_icache_range(unsigned long start, unsigned long end)
36{
37	asm volatile ("\tibar 0\n"::);
38}
39EXPORT_SYMBOL(local_flush_icache_range);
40
41static void flush_cache_leaf(unsigned int leaf)
42{
43	int i, j, nr_nodes;
44	uint64_t addr = CSR_DMW0_BASE;
45	struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
46
47	nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
48
49	do {
50		for (i = 0; i < cdesc->sets; i++) {
51			for (j = 0; j < cdesc->ways; j++) {
52				flush_cache_line(leaf, addr);
53				addr++;
54			}
55
56			addr -= cdesc->ways;
57			addr += cdesc->linesz;
58		}
59		addr += (1ULL << NODE_ADDRSPACE_SHIFT);
60	} while (--nr_nodes > 0);
61}
62
63asmlinkage __visible void __flush_cache_all(void)
64{
65	int leaf;
66	struct cache_desc *cdesc = current_cpu_data.cache_leaves;
67	unsigned int cache_present = current_cpu_data.cache_leaves_present;
68
69	leaf = cache_present - 1;
70	if (cache_inclusive(cdesc + leaf)) {
71		flush_cache_leaf(leaf);
72		return;
73	}
74
75	for (leaf = 0; leaf < cache_present; leaf++)
76		flush_cache_leaf(leaf);
77}
78
79#define L1IUPRE		(1 << 0)
80#define L1IUUNIFY	(1 << 1)
81#define L1DPRE		(1 << 2)
82
83#define LXIUPRE		(1 << 0)
84#define LXIUUNIFY	(1 << 1)
85#define LXIUPRIV	(1 << 2)
86#define LXIUINCL	(1 << 3)
87#define LXDPRE		(1 << 4)
88#define LXDPRIV		(1 << 5)
89#define LXDINCL		(1 << 6)
90
91#define populate_cache_properties(cfg0, cdesc, level, leaf)				\
92do {											\
93	unsigned int cfg1;								\
94											\
95	cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf);					\
96	if (level == 1)	{								\
97		cdesc->flags |= CACHE_PRIVATE;						\
98	} else {									\
99		if (cfg0 & LXIUPRIV)							\
100			cdesc->flags |= CACHE_PRIVATE;					\
101		if (cfg0 & LXIUINCL)							\
102			cdesc->flags |= CACHE_INCLUSIVE;				\
103	}										\
104	cdesc->level = level;								\
105	cdesc->flags |= CACHE_PRESENT;							\
106	cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1;		\
107	cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS);		\
108	cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE);	\
109	cdesc++; leaf++;								\
110} while (0)
111
112void cpu_cache_init(void)
113{
114	unsigned int leaf = 0, level = 1;
115	unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
116	struct cache_desc *cdesc = current_cpu_data.cache_leaves;
117
118	if (config & L1IUPRE) {
119		if (config & L1IUUNIFY)
120			cdesc->type = CACHE_TYPE_UNIFIED;
121		else
122			cdesc->type = CACHE_TYPE_INST;
123		populate_cache_properties(config, cdesc, level, leaf);
124	}
125
126	if (config & L1DPRE) {
127		cdesc->type = CACHE_TYPE_DATA;
128		populate_cache_properties(config, cdesc, level, leaf);
129	}
130
131	config = config >> 3;
132	for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
133		if (!config)
134			break;
135
136		if (config & LXIUPRE) {
137			if (config & LXIUUNIFY)
138				cdesc->type = CACHE_TYPE_UNIFIED;
139			else
140				cdesc->type = CACHE_TYPE_INST;
141			populate_cache_properties(config, cdesc, level, leaf);
142		}
143
144		if (config & LXDPRE) {
145			cdesc->type = CACHE_TYPE_DATA;
146			populate_cache_properties(config, cdesc, level, leaf);
147		}
148
149		config = config >> 7;
150	}
151
152	BUG_ON(leaf > CACHE_LEAVES_MAX);
153
154	current_cpu_data.cache_leaves_present = leaf;
155	current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
156}
157