1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <linux/module.h>
27#include <linux/pci.h>
28
29#include "amdgpu.h"
30#include "amdgpu_atombios.h"
31#include "amdgpu_ih.h"
32#include "amdgpu_uvd.h"
33#include "amdgpu_vce.h"
34#include "cikd.h"
35#include "atom.h"
36#include "amd_pcie.h"
37
38#include "cik.h"
39#include "gmc_v7_0.h"
40#include "cik_ih.h"
41#include "dce_v8_0.h"
42#include "gfx_v7_0.h"
43#include "cik_sdma.h"
44#include "uvd_v4_2.h"
45#include "vce_v2_0.h"
46#include "cik_dpm.h"
47
48#include "uvd/uvd_4_2_d.h"
49
50#include "smu/smu_7_0_1_d.h"
51#include "smu/smu_7_0_1_sh_mask.h"
52
53#include "dce/dce_8_0_d.h"
54#include "dce/dce_8_0_sh_mask.h"
55
56#include "bif/bif_4_1_d.h"
57#include "bif/bif_4_1_sh_mask.h"
58
59#include "gca/gfx_7_2_d.h"
60#include "gca/gfx_7_2_enum.h"
61#include "gca/gfx_7_2_sh_mask.h"
62
63#include "gmc/gmc_7_1_d.h"
64#include "gmc/gmc_7_1_sh_mask.h"
65
66#include "oss/oss_2_0_d.h"
67#include "oss/oss_2_0_sh_mask.h"
68
69#include "amdgpu_dm.h"
70#include "amdgpu_amdkfd.h"
71#include "dce_virtual.h"
72
73/*
74 * Indirect registers accessor
75 */
76static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
77{
78	unsigned long flags;
79	u32 r;
80
81	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
82	WREG32(mmPCIE_INDEX, reg);
83	(void)RREG32(mmPCIE_INDEX);
84	r = RREG32(mmPCIE_DATA);
85	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
86	return r;
87}
88
89static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
90{
91	unsigned long flags;
92
93	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
94	WREG32(mmPCIE_INDEX, reg);
95	(void)RREG32(mmPCIE_INDEX);
96	WREG32(mmPCIE_DATA, v);
97	(void)RREG32(mmPCIE_DATA);
98	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
99}
100
101static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
102{
103	unsigned long flags;
104	u32 r;
105
106	spin_lock_irqsave(&adev->smc_idx_lock, flags);
107	WREG32(mmSMC_IND_INDEX_0, (reg));
108	r = RREG32(mmSMC_IND_DATA_0);
109	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
110	return r;
111}
112
113static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
114{
115	unsigned long flags;
116
117	spin_lock_irqsave(&adev->smc_idx_lock, flags);
118	WREG32(mmSMC_IND_INDEX_0, (reg));
119	WREG32(mmSMC_IND_DATA_0, (v));
120	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
121}
122
123static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
124{
125	unsigned long flags;
126	u32 r;
127
128	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
129	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
130	r = RREG32(mmUVD_CTX_DATA);
131	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
132	return r;
133}
134
135static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
136{
137	unsigned long flags;
138
139	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
140	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
141	WREG32(mmUVD_CTX_DATA, (v));
142	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
143}
144
145static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
146{
147	unsigned long flags;
148	u32 r;
149
150	spin_lock_irqsave(&adev->didt_idx_lock, flags);
151	WREG32(mmDIDT_IND_INDEX, (reg));
152	r = RREG32(mmDIDT_IND_DATA);
153	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
154	return r;
155}
156
157static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
158{
159	unsigned long flags;
160
161	spin_lock_irqsave(&adev->didt_idx_lock, flags);
162	WREG32(mmDIDT_IND_INDEX, (reg));
163	WREG32(mmDIDT_IND_DATA, (v));
164	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
165}
166
167static const u32 bonaire_golden_spm_registers[] =
168{
169	0xc200, 0xe0ffffff, 0xe0000000
170};
171
172static const u32 bonaire_golden_common_registers[] =
173{
174	0x31dc, 0xffffffff, 0x00000800,
175	0x31dd, 0xffffffff, 0x00000800,
176	0x31e6, 0xffffffff, 0x00007fbf,
177	0x31e7, 0xffffffff, 0x00007faf
178};
179
180static const u32 bonaire_golden_registers[] =
181{
182	0xcd5, 0x00000333, 0x00000333,
183	0xcd4, 0x000c0fc0, 0x00040200,
184	0x2684, 0x00010000, 0x00058208,
185	0xf000, 0xffff1fff, 0x00140000,
186	0xf080, 0xfdfc0fff, 0x00000100,
187	0xf08d, 0x40000000, 0x40000200,
188	0x260c, 0xffffffff, 0x00000000,
189	0x260d, 0xf00fffff, 0x00000400,
190	0x260e, 0x0002021c, 0x00020200,
191	0x31e, 0x00000080, 0x00000000,
192	0x16ec, 0x000000f0, 0x00000070,
193	0x16f0, 0xf0311fff, 0x80300000,
194	0x263e, 0x73773777, 0x12010001,
195	0xd43, 0x00810000, 0x408af000,
196	0x1c0c, 0x31000111, 0x00000011,
197	0xbd2, 0x73773777, 0x12010001,
198	0x883, 0x00007fb6, 0x0021a1b1,
199	0x884, 0x00007fb6, 0x002021b1,
200	0x860, 0x00007fb6, 0x00002191,
201	0x886, 0x00007fb6, 0x002121b1,
202	0x887, 0x00007fb6, 0x002021b1,
203	0x877, 0x00007fb6, 0x00002191,
204	0x878, 0x00007fb6, 0x00002191,
205	0xd8a, 0x0000003f, 0x0000000a,
206	0xd8b, 0x0000003f, 0x0000000a,
207	0xab9, 0x00073ffe, 0x000022a2,
208	0x903, 0x000007ff, 0x00000000,
209	0x2285, 0xf000003f, 0x00000007,
210	0x22fc, 0x00002001, 0x00000001,
211	0x22c9, 0xffffffff, 0x00ffffff,
212	0xc281, 0x0000ff0f, 0x00000000,
213	0xa293, 0x07ffffff, 0x06000000,
214	0x136, 0x00000fff, 0x00000100,
215	0xf9e, 0x00000001, 0x00000002,
216	0x2440, 0x03000000, 0x0362c688,
217	0x2300, 0x000000ff, 0x00000001,
218	0x390, 0x00001fff, 0x00001fff,
219	0x2418, 0x0000007f, 0x00000020,
220	0x2542, 0x00010000, 0x00010000,
221	0x2b05, 0x000003ff, 0x000000f3,
222	0x2b03, 0xffffffff, 0x00001032
223};
224
225static const u32 bonaire_mgcg_cgcg_init[] =
226{
227	0x3108, 0xffffffff, 0xfffffffc,
228	0xc200, 0xffffffff, 0xe0000000,
229	0xf0a8, 0xffffffff, 0x00000100,
230	0xf082, 0xffffffff, 0x00000100,
231	0xf0b0, 0xffffffff, 0xc0000100,
232	0xf0b2, 0xffffffff, 0xc0000100,
233	0xf0b1, 0xffffffff, 0xc0000100,
234	0x1579, 0xffffffff, 0x00600100,
235	0xf0a0, 0xffffffff, 0x00000100,
236	0xf085, 0xffffffff, 0x06000100,
237	0xf088, 0xffffffff, 0x00000100,
238	0xf086, 0xffffffff, 0x06000100,
239	0xf081, 0xffffffff, 0x00000100,
240	0xf0b8, 0xffffffff, 0x00000100,
241	0xf089, 0xffffffff, 0x00000100,
242	0xf080, 0xffffffff, 0x00000100,
243	0xf08c, 0xffffffff, 0x00000100,
244	0xf08d, 0xffffffff, 0x00000100,
245	0xf094, 0xffffffff, 0x00000100,
246	0xf095, 0xffffffff, 0x00000100,
247	0xf096, 0xffffffff, 0x00000100,
248	0xf097, 0xffffffff, 0x00000100,
249	0xf098, 0xffffffff, 0x00000100,
250	0xf09f, 0xffffffff, 0x00000100,
251	0xf09e, 0xffffffff, 0x00000100,
252	0xf084, 0xffffffff, 0x06000100,
253	0xf0a4, 0xffffffff, 0x00000100,
254	0xf09d, 0xffffffff, 0x00000100,
255	0xf0ad, 0xffffffff, 0x00000100,
256	0xf0ac, 0xffffffff, 0x00000100,
257	0xf09c, 0xffffffff, 0x00000100,
258	0xc200, 0xffffffff, 0xe0000000,
259	0xf008, 0xffffffff, 0x00010000,
260	0xf009, 0xffffffff, 0x00030002,
261	0xf00a, 0xffffffff, 0x00040007,
262	0xf00b, 0xffffffff, 0x00060005,
263	0xf00c, 0xffffffff, 0x00090008,
264	0xf00d, 0xffffffff, 0x00010000,
265	0xf00e, 0xffffffff, 0x00030002,
266	0xf00f, 0xffffffff, 0x00040007,
267	0xf010, 0xffffffff, 0x00060005,
268	0xf011, 0xffffffff, 0x00090008,
269	0xf012, 0xffffffff, 0x00010000,
270	0xf013, 0xffffffff, 0x00030002,
271	0xf014, 0xffffffff, 0x00040007,
272	0xf015, 0xffffffff, 0x00060005,
273	0xf016, 0xffffffff, 0x00090008,
274	0xf017, 0xffffffff, 0x00010000,
275	0xf018, 0xffffffff, 0x00030002,
276	0xf019, 0xffffffff, 0x00040007,
277	0xf01a, 0xffffffff, 0x00060005,
278	0xf01b, 0xffffffff, 0x00090008,
279	0xf01c, 0xffffffff, 0x00010000,
280	0xf01d, 0xffffffff, 0x00030002,
281	0xf01e, 0xffffffff, 0x00040007,
282	0xf01f, 0xffffffff, 0x00060005,
283	0xf020, 0xffffffff, 0x00090008,
284	0xf021, 0xffffffff, 0x00010000,
285	0xf022, 0xffffffff, 0x00030002,
286	0xf023, 0xffffffff, 0x00040007,
287	0xf024, 0xffffffff, 0x00060005,
288	0xf025, 0xffffffff, 0x00090008,
289	0xf026, 0xffffffff, 0x00010000,
290	0xf027, 0xffffffff, 0x00030002,
291	0xf028, 0xffffffff, 0x00040007,
292	0xf029, 0xffffffff, 0x00060005,
293	0xf02a, 0xffffffff, 0x00090008,
294	0xf000, 0xffffffff, 0x96e00200,
295	0x21c2, 0xffffffff, 0x00900100,
296	0x3109, 0xffffffff, 0x0020003f,
297	0xe, 0xffffffff, 0x0140001c,
298	0xf, 0x000f0000, 0x000f0000,
299	0x88, 0xffffffff, 0xc060000c,
300	0x89, 0xc0000fff, 0x00000100,
301	0x3e4, 0xffffffff, 0x00000100,
302	0x3e6, 0x00000101, 0x00000000,
303	0x82a, 0xffffffff, 0x00000104,
304	0x1579, 0xff000fff, 0x00000100,
305	0xc33, 0xc0000fff, 0x00000104,
306	0x3079, 0x00000001, 0x00000001,
307	0x3403, 0xff000ff0, 0x00000100,
308	0x3603, 0xff000ff0, 0x00000100
309};
310
311static const u32 spectre_golden_spm_registers[] =
312{
313	0xc200, 0xe0ffffff, 0xe0000000
314};
315
316static const u32 spectre_golden_common_registers[] =
317{
318	0x31dc, 0xffffffff, 0x00000800,
319	0x31dd, 0xffffffff, 0x00000800,
320	0x31e6, 0xffffffff, 0x00007fbf,
321	0x31e7, 0xffffffff, 0x00007faf
322};
323
324static const u32 spectre_golden_registers[] =
325{
326	0xf000, 0xffff1fff, 0x96940200,
327	0xf003, 0xffff0001, 0xff000000,
328	0xf080, 0xfffc0fff, 0x00000100,
329	0x1bb6, 0x00010101, 0x00010000,
330	0x260d, 0xf00fffff, 0x00000400,
331	0x260e, 0xfffffffc, 0x00020200,
332	0x16ec, 0x000000f0, 0x00000070,
333	0x16f0, 0xf0311fff, 0x80300000,
334	0x263e, 0x73773777, 0x12010001,
335	0x26df, 0x00ff0000, 0x00fc0000,
336	0xbd2, 0x73773777, 0x12010001,
337	0x2285, 0xf000003f, 0x00000007,
338	0x22c9, 0xffffffff, 0x00ffffff,
339	0xa0d4, 0x3f3f3fff, 0x00000082,
340	0xa0d5, 0x0000003f, 0x00000000,
341	0xf9e, 0x00000001, 0x00000002,
342	0x244f, 0xffff03df, 0x00000004,
343	0x31da, 0x00000008, 0x00000008,
344	0x2300, 0x000008ff, 0x00000800,
345	0x2542, 0x00010000, 0x00010000,
346	0x2b03, 0xffffffff, 0x54763210,
347	0x853e, 0x01ff01ff, 0x00000002,
348	0x8526, 0x007ff800, 0x00200000,
349	0x8057, 0xffffffff, 0x00000f40,
350	0xc24d, 0xffffffff, 0x00000001
351};
352
353static const u32 spectre_mgcg_cgcg_init[] =
354{
355	0x3108, 0xffffffff, 0xfffffffc,
356	0xc200, 0xffffffff, 0xe0000000,
357	0xf0a8, 0xffffffff, 0x00000100,
358	0xf082, 0xffffffff, 0x00000100,
359	0xf0b0, 0xffffffff, 0x00000100,
360	0xf0b2, 0xffffffff, 0x00000100,
361	0xf0b1, 0xffffffff, 0x00000100,
362	0x1579, 0xffffffff, 0x00600100,
363	0xf0a0, 0xffffffff, 0x00000100,
364	0xf085, 0xffffffff, 0x06000100,
365	0xf088, 0xffffffff, 0x00000100,
366	0xf086, 0xffffffff, 0x06000100,
367	0xf081, 0xffffffff, 0x00000100,
368	0xf0b8, 0xffffffff, 0x00000100,
369	0xf089, 0xffffffff, 0x00000100,
370	0xf080, 0xffffffff, 0x00000100,
371	0xf08c, 0xffffffff, 0x00000100,
372	0xf08d, 0xffffffff, 0x00000100,
373	0xf094, 0xffffffff, 0x00000100,
374	0xf095, 0xffffffff, 0x00000100,
375	0xf096, 0xffffffff, 0x00000100,
376	0xf097, 0xffffffff, 0x00000100,
377	0xf098, 0xffffffff, 0x00000100,
378	0xf09f, 0xffffffff, 0x00000100,
379	0xf09e, 0xffffffff, 0x00000100,
380	0xf084, 0xffffffff, 0x06000100,
381	0xf0a4, 0xffffffff, 0x00000100,
382	0xf09d, 0xffffffff, 0x00000100,
383	0xf0ad, 0xffffffff, 0x00000100,
384	0xf0ac, 0xffffffff, 0x00000100,
385	0xf09c, 0xffffffff, 0x00000100,
386	0xc200, 0xffffffff, 0xe0000000,
387	0xf008, 0xffffffff, 0x00010000,
388	0xf009, 0xffffffff, 0x00030002,
389	0xf00a, 0xffffffff, 0x00040007,
390	0xf00b, 0xffffffff, 0x00060005,
391	0xf00c, 0xffffffff, 0x00090008,
392	0xf00d, 0xffffffff, 0x00010000,
393	0xf00e, 0xffffffff, 0x00030002,
394	0xf00f, 0xffffffff, 0x00040007,
395	0xf010, 0xffffffff, 0x00060005,
396	0xf011, 0xffffffff, 0x00090008,
397	0xf012, 0xffffffff, 0x00010000,
398	0xf013, 0xffffffff, 0x00030002,
399	0xf014, 0xffffffff, 0x00040007,
400	0xf015, 0xffffffff, 0x00060005,
401	0xf016, 0xffffffff, 0x00090008,
402	0xf017, 0xffffffff, 0x00010000,
403	0xf018, 0xffffffff, 0x00030002,
404	0xf019, 0xffffffff, 0x00040007,
405	0xf01a, 0xffffffff, 0x00060005,
406	0xf01b, 0xffffffff, 0x00090008,
407	0xf01c, 0xffffffff, 0x00010000,
408	0xf01d, 0xffffffff, 0x00030002,
409	0xf01e, 0xffffffff, 0x00040007,
410	0xf01f, 0xffffffff, 0x00060005,
411	0xf020, 0xffffffff, 0x00090008,
412	0xf021, 0xffffffff, 0x00010000,
413	0xf022, 0xffffffff, 0x00030002,
414	0xf023, 0xffffffff, 0x00040007,
415	0xf024, 0xffffffff, 0x00060005,
416	0xf025, 0xffffffff, 0x00090008,
417	0xf026, 0xffffffff, 0x00010000,
418	0xf027, 0xffffffff, 0x00030002,
419	0xf028, 0xffffffff, 0x00040007,
420	0xf029, 0xffffffff, 0x00060005,
421	0xf02a, 0xffffffff, 0x00090008,
422	0xf02b, 0xffffffff, 0x00010000,
423	0xf02c, 0xffffffff, 0x00030002,
424	0xf02d, 0xffffffff, 0x00040007,
425	0xf02e, 0xffffffff, 0x00060005,
426	0xf02f, 0xffffffff, 0x00090008,
427	0xf000, 0xffffffff, 0x96e00200,
428	0x21c2, 0xffffffff, 0x00900100,
429	0x3109, 0xffffffff, 0x0020003f,
430	0xe, 0xffffffff, 0x0140001c,
431	0xf, 0x000f0000, 0x000f0000,
432	0x88, 0xffffffff, 0xc060000c,
433	0x89, 0xc0000fff, 0x00000100,
434	0x3e4, 0xffffffff, 0x00000100,
435	0x3e6, 0x00000101, 0x00000000,
436	0x82a, 0xffffffff, 0x00000104,
437	0x1579, 0xff000fff, 0x00000100,
438	0xc33, 0xc0000fff, 0x00000104,
439	0x3079, 0x00000001, 0x00000001,
440	0x3403, 0xff000ff0, 0x00000100,
441	0x3603, 0xff000ff0, 0x00000100
442};
443
444static const u32 kalindi_golden_spm_registers[] =
445{
446	0xc200, 0xe0ffffff, 0xe0000000
447};
448
449static const u32 kalindi_golden_common_registers[] =
450{
451	0x31dc, 0xffffffff, 0x00000800,
452	0x31dd, 0xffffffff, 0x00000800,
453	0x31e6, 0xffffffff, 0x00007fbf,
454	0x31e7, 0xffffffff, 0x00007faf
455};
456
457static const u32 kalindi_golden_registers[] =
458{
459	0xf000, 0xffffdfff, 0x6e944040,
460	0x1579, 0xff607fff, 0xfc000100,
461	0xf088, 0xff000fff, 0x00000100,
462	0xf089, 0xff000fff, 0x00000100,
463	0xf080, 0xfffc0fff, 0x00000100,
464	0x1bb6, 0x00010101, 0x00010000,
465	0x260c, 0xffffffff, 0x00000000,
466	0x260d, 0xf00fffff, 0x00000400,
467	0x16ec, 0x000000f0, 0x00000070,
468	0x16f0, 0xf0311fff, 0x80300000,
469	0x263e, 0x73773777, 0x12010001,
470	0x263f, 0xffffffff, 0x00000010,
471	0x26df, 0x00ff0000, 0x00fc0000,
472	0x200c, 0x00001f0f, 0x0000100a,
473	0xbd2, 0x73773777, 0x12010001,
474	0x902, 0x000fffff, 0x000c007f,
475	0x2285, 0xf000003f, 0x00000007,
476	0x22c9, 0x3fff3fff, 0x00ffcfff,
477	0xc281, 0x0000ff0f, 0x00000000,
478	0xa293, 0x07ffffff, 0x06000000,
479	0x136, 0x00000fff, 0x00000100,
480	0xf9e, 0x00000001, 0x00000002,
481	0x31da, 0x00000008, 0x00000008,
482	0x2300, 0x000000ff, 0x00000003,
483	0x853e, 0x01ff01ff, 0x00000002,
484	0x8526, 0x007ff800, 0x00200000,
485	0x8057, 0xffffffff, 0x00000f40,
486	0x2231, 0x001f3ae3, 0x00000082,
487	0x2235, 0x0000001f, 0x00000010,
488	0xc24d, 0xffffffff, 0x00000000
489};
490
491static const u32 kalindi_mgcg_cgcg_init[] =
492{
493	0x3108, 0xffffffff, 0xfffffffc,
494	0xc200, 0xffffffff, 0xe0000000,
495	0xf0a8, 0xffffffff, 0x00000100,
496	0xf082, 0xffffffff, 0x00000100,
497	0xf0b0, 0xffffffff, 0x00000100,
498	0xf0b2, 0xffffffff, 0x00000100,
499	0xf0b1, 0xffffffff, 0x00000100,
500	0x1579, 0xffffffff, 0x00600100,
501	0xf0a0, 0xffffffff, 0x00000100,
502	0xf085, 0xffffffff, 0x06000100,
503	0xf088, 0xffffffff, 0x00000100,
504	0xf086, 0xffffffff, 0x06000100,
505	0xf081, 0xffffffff, 0x00000100,
506	0xf0b8, 0xffffffff, 0x00000100,
507	0xf089, 0xffffffff, 0x00000100,
508	0xf080, 0xffffffff, 0x00000100,
509	0xf08c, 0xffffffff, 0x00000100,
510	0xf08d, 0xffffffff, 0x00000100,
511	0xf094, 0xffffffff, 0x00000100,
512	0xf095, 0xffffffff, 0x00000100,
513	0xf096, 0xffffffff, 0x00000100,
514	0xf097, 0xffffffff, 0x00000100,
515	0xf098, 0xffffffff, 0x00000100,
516	0xf09f, 0xffffffff, 0x00000100,
517	0xf09e, 0xffffffff, 0x00000100,
518	0xf084, 0xffffffff, 0x06000100,
519	0xf0a4, 0xffffffff, 0x00000100,
520	0xf09d, 0xffffffff, 0x00000100,
521	0xf0ad, 0xffffffff, 0x00000100,
522	0xf0ac, 0xffffffff, 0x00000100,
523	0xf09c, 0xffffffff, 0x00000100,
524	0xc200, 0xffffffff, 0xe0000000,
525	0xf008, 0xffffffff, 0x00010000,
526	0xf009, 0xffffffff, 0x00030002,
527	0xf00a, 0xffffffff, 0x00040007,
528	0xf00b, 0xffffffff, 0x00060005,
529	0xf00c, 0xffffffff, 0x00090008,
530	0xf00d, 0xffffffff, 0x00010000,
531	0xf00e, 0xffffffff, 0x00030002,
532	0xf00f, 0xffffffff, 0x00040007,
533	0xf010, 0xffffffff, 0x00060005,
534	0xf011, 0xffffffff, 0x00090008,
535	0xf000, 0xffffffff, 0x96e00200,
536	0x21c2, 0xffffffff, 0x00900100,
537	0x3109, 0xffffffff, 0x0020003f,
538	0xe, 0xffffffff, 0x0140001c,
539	0xf, 0x000f0000, 0x000f0000,
540	0x88, 0xffffffff, 0xc060000c,
541	0x89, 0xc0000fff, 0x00000100,
542	0x82a, 0xffffffff, 0x00000104,
543	0x1579, 0xff000fff, 0x00000100,
544	0xc33, 0xc0000fff, 0x00000104,
545	0x3079, 0x00000001, 0x00000001,
546	0x3403, 0xff000ff0, 0x00000100,
547	0x3603, 0xff000ff0, 0x00000100
548};
549
550static const u32 hawaii_golden_spm_registers[] =
551{
552	0xc200, 0xe0ffffff, 0xe0000000
553};
554
555static const u32 hawaii_golden_common_registers[] =
556{
557	0xc200, 0xffffffff, 0xe0000000,
558	0xa0d4, 0xffffffff, 0x3a00161a,
559	0xa0d5, 0xffffffff, 0x0000002e,
560	0x2684, 0xffffffff, 0x00018208,
561	0x263e, 0xffffffff, 0x12011003
562};
563
564static const u32 hawaii_golden_registers[] =
565{
566	0xcd5, 0x00000333, 0x00000333,
567	0x2684, 0x00010000, 0x00058208,
568	0x260c, 0xffffffff, 0x00000000,
569	0x260d, 0xf00fffff, 0x00000400,
570	0x260e, 0x0002021c, 0x00020200,
571	0x31e, 0x00000080, 0x00000000,
572	0x16ec, 0x000000f0, 0x00000070,
573	0x16f0, 0xf0311fff, 0x80300000,
574	0xd43, 0x00810000, 0x408af000,
575	0x1c0c, 0x31000111, 0x00000011,
576	0xbd2, 0x73773777, 0x12010001,
577	0x848, 0x0000007f, 0x0000001b,
578	0x877, 0x00007fb6, 0x00002191,
579	0xd8a, 0x0000003f, 0x0000000a,
580	0xd8b, 0x0000003f, 0x0000000a,
581	0xab9, 0x00073ffe, 0x000022a2,
582	0x903, 0x000007ff, 0x00000000,
583	0x22fc, 0x00002001, 0x00000001,
584	0x22c9, 0xffffffff, 0x00ffffff,
585	0xc281, 0x0000ff0f, 0x00000000,
586	0xa293, 0x07ffffff, 0x06000000,
587	0xf9e, 0x00000001, 0x00000002,
588	0x31da, 0x00000008, 0x00000008,
589	0x31dc, 0x00000f00, 0x00000800,
590	0x31dd, 0x00000f00, 0x00000800,
591	0x31e6, 0x00ffffff, 0x00ff7fbf,
592	0x31e7, 0x00ffffff, 0x00ff7faf,
593	0x2300, 0x000000ff, 0x00000800,
594	0x390, 0x00001fff, 0x00001fff,
595	0x2418, 0x0000007f, 0x00000020,
596	0x2542, 0x00010000, 0x00010000,
597	0x2b80, 0x00100000, 0x000ff07c,
598	0x2b05, 0x000003ff, 0x0000000f,
599	0x2b04, 0xffffffff, 0x7564fdec,
600	0x2b03, 0xffffffff, 0x3120b9a8,
601	0x2b02, 0x20000000, 0x0f9c0000
602};
603
604static const u32 hawaii_mgcg_cgcg_init[] =
605{
606	0x3108, 0xffffffff, 0xfffffffd,
607	0xc200, 0xffffffff, 0xe0000000,
608	0xf0a8, 0xffffffff, 0x00000100,
609	0xf082, 0xffffffff, 0x00000100,
610	0xf0b0, 0xffffffff, 0x00000100,
611	0xf0b2, 0xffffffff, 0x00000100,
612	0xf0b1, 0xffffffff, 0x00000100,
613	0x1579, 0xffffffff, 0x00200100,
614	0xf0a0, 0xffffffff, 0x00000100,
615	0xf085, 0xffffffff, 0x06000100,
616	0xf088, 0xffffffff, 0x00000100,
617	0xf086, 0xffffffff, 0x06000100,
618	0xf081, 0xffffffff, 0x00000100,
619	0xf0b8, 0xffffffff, 0x00000100,
620	0xf089, 0xffffffff, 0x00000100,
621	0xf080, 0xffffffff, 0x00000100,
622	0xf08c, 0xffffffff, 0x00000100,
623	0xf08d, 0xffffffff, 0x00000100,
624	0xf094, 0xffffffff, 0x00000100,
625	0xf095, 0xffffffff, 0x00000100,
626	0xf096, 0xffffffff, 0x00000100,
627	0xf097, 0xffffffff, 0x00000100,
628	0xf098, 0xffffffff, 0x00000100,
629	0xf09f, 0xffffffff, 0x00000100,
630	0xf09e, 0xffffffff, 0x00000100,
631	0xf084, 0xffffffff, 0x06000100,
632	0xf0a4, 0xffffffff, 0x00000100,
633	0xf09d, 0xffffffff, 0x00000100,
634	0xf0ad, 0xffffffff, 0x00000100,
635	0xf0ac, 0xffffffff, 0x00000100,
636	0xf09c, 0xffffffff, 0x00000100,
637	0xc200, 0xffffffff, 0xe0000000,
638	0xf008, 0xffffffff, 0x00010000,
639	0xf009, 0xffffffff, 0x00030002,
640	0xf00a, 0xffffffff, 0x00040007,
641	0xf00b, 0xffffffff, 0x00060005,
642	0xf00c, 0xffffffff, 0x00090008,
643	0xf00d, 0xffffffff, 0x00010000,
644	0xf00e, 0xffffffff, 0x00030002,
645	0xf00f, 0xffffffff, 0x00040007,
646	0xf010, 0xffffffff, 0x00060005,
647	0xf011, 0xffffffff, 0x00090008,
648	0xf012, 0xffffffff, 0x00010000,
649	0xf013, 0xffffffff, 0x00030002,
650	0xf014, 0xffffffff, 0x00040007,
651	0xf015, 0xffffffff, 0x00060005,
652	0xf016, 0xffffffff, 0x00090008,
653	0xf017, 0xffffffff, 0x00010000,
654	0xf018, 0xffffffff, 0x00030002,
655	0xf019, 0xffffffff, 0x00040007,
656	0xf01a, 0xffffffff, 0x00060005,
657	0xf01b, 0xffffffff, 0x00090008,
658	0xf01c, 0xffffffff, 0x00010000,
659	0xf01d, 0xffffffff, 0x00030002,
660	0xf01e, 0xffffffff, 0x00040007,
661	0xf01f, 0xffffffff, 0x00060005,
662	0xf020, 0xffffffff, 0x00090008,
663	0xf021, 0xffffffff, 0x00010000,
664	0xf022, 0xffffffff, 0x00030002,
665	0xf023, 0xffffffff, 0x00040007,
666	0xf024, 0xffffffff, 0x00060005,
667	0xf025, 0xffffffff, 0x00090008,
668	0xf026, 0xffffffff, 0x00010000,
669	0xf027, 0xffffffff, 0x00030002,
670	0xf028, 0xffffffff, 0x00040007,
671	0xf029, 0xffffffff, 0x00060005,
672	0xf02a, 0xffffffff, 0x00090008,
673	0xf02b, 0xffffffff, 0x00010000,
674	0xf02c, 0xffffffff, 0x00030002,
675	0xf02d, 0xffffffff, 0x00040007,
676	0xf02e, 0xffffffff, 0x00060005,
677	0xf02f, 0xffffffff, 0x00090008,
678	0xf030, 0xffffffff, 0x00010000,
679	0xf031, 0xffffffff, 0x00030002,
680	0xf032, 0xffffffff, 0x00040007,
681	0xf033, 0xffffffff, 0x00060005,
682	0xf034, 0xffffffff, 0x00090008,
683	0xf035, 0xffffffff, 0x00010000,
684	0xf036, 0xffffffff, 0x00030002,
685	0xf037, 0xffffffff, 0x00040007,
686	0xf038, 0xffffffff, 0x00060005,
687	0xf039, 0xffffffff, 0x00090008,
688	0xf03a, 0xffffffff, 0x00010000,
689	0xf03b, 0xffffffff, 0x00030002,
690	0xf03c, 0xffffffff, 0x00040007,
691	0xf03d, 0xffffffff, 0x00060005,
692	0xf03e, 0xffffffff, 0x00090008,
693	0x30c6, 0xffffffff, 0x00020200,
694	0xcd4, 0xffffffff, 0x00000200,
695	0x570, 0xffffffff, 0x00000400,
696	0x157a, 0xffffffff, 0x00000000,
697	0xbd4, 0xffffffff, 0x00000902,
698	0xf000, 0xffffffff, 0x96940200,
699	0x21c2, 0xffffffff, 0x00900100,
700	0x3109, 0xffffffff, 0x0020003f,
701	0xe, 0xffffffff, 0x0140001c,
702	0xf, 0x000f0000, 0x000f0000,
703	0x88, 0xffffffff, 0xc060000c,
704	0x89, 0xc0000fff, 0x00000100,
705	0x3e4, 0xffffffff, 0x00000100,
706	0x3e6, 0x00000101, 0x00000000,
707	0x82a, 0xffffffff, 0x00000104,
708	0x1579, 0xff000fff, 0x00000100,
709	0xc33, 0xc0000fff, 0x00000104,
710	0x3079, 0x00000001, 0x00000001,
711	0x3403, 0xff000ff0, 0x00000100,
712	0x3603, 0xff000ff0, 0x00000100
713};
714
715static const u32 godavari_golden_registers[] =
716{
717	0x1579, 0xff607fff, 0xfc000100,
718	0x1bb6, 0x00010101, 0x00010000,
719	0x260c, 0xffffffff, 0x00000000,
720	0x260c0, 0xf00fffff, 0x00000400,
721	0x184c, 0xffffffff, 0x00010000,
722	0x16ec, 0x000000f0, 0x00000070,
723	0x16f0, 0xf0311fff, 0x80300000,
724	0x263e, 0x73773777, 0x12010001,
725	0x263f, 0xffffffff, 0x00000010,
726	0x200c, 0x00001f0f, 0x0000100a,
727	0xbd2, 0x73773777, 0x12010001,
728	0x902, 0x000fffff, 0x000c007f,
729	0x2285, 0xf000003f, 0x00000007,
730	0x22c9, 0xffffffff, 0x00ff0fff,
731	0xc281, 0x0000ff0f, 0x00000000,
732	0xa293, 0x07ffffff, 0x06000000,
733	0x136, 0x00000fff, 0x00000100,
734	0x3405, 0x00010000, 0x00810001,
735	0x3605, 0x00010000, 0x00810001,
736	0xf9e, 0x00000001, 0x00000002,
737	0x31da, 0x00000008, 0x00000008,
738	0x31dc, 0x00000f00, 0x00000800,
739	0x31dd, 0x00000f00, 0x00000800,
740	0x31e6, 0x00ffffff, 0x00ff7fbf,
741	0x31e7, 0x00ffffff, 0x00ff7faf,
742	0x2300, 0x000000ff, 0x00000001,
743	0x853e, 0x01ff01ff, 0x00000002,
744	0x8526, 0x007ff800, 0x00200000,
745	0x8057, 0xffffffff, 0x00000f40,
746	0x2231, 0x001f3ae3, 0x00000082,
747	0x2235, 0x0000001f, 0x00000010,
748	0xc24d, 0xffffffff, 0x00000000
749};
750
751static void cik_init_golden_registers(struct amdgpu_device *adev)
752{
753	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
754	mutex_lock(&adev->grbm_idx_mutex);
755
756	switch (adev->asic_type) {
757	case CHIP_BONAIRE:
758		amdgpu_device_program_register_sequence(adev,
759							bonaire_mgcg_cgcg_init,
760							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
761		amdgpu_device_program_register_sequence(adev,
762							bonaire_golden_registers,
763							ARRAY_SIZE(bonaire_golden_registers));
764		amdgpu_device_program_register_sequence(adev,
765							bonaire_golden_common_registers,
766							ARRAY_SIZE(bonaire_golden_common_registers));
767		amdgpu_device_program_register_sequence(adev,
768							bonaire_golden_spm_registers,
769							ARRAY_SIZE(bonaire_golden_spm_registers));
770		break;
771	case CHIP_KABINI:
772		amdgpu_device_program_register_sequence(adev,
773							kalindi_mgcg_cgcg_init,
774							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
775		amdgpu_device_program_register_sequence(adev,
776							kalindi_golden_registers,
777							ARRAY_SIZE(kalindi_golden_registers));
778		amdgpu_device_program_register_sequence(adev,
779							kalindi_golden_common_registers,
780							ARRAY_SIZE(kalindi_golden_common_registers));
781		amdgpu_device_program_register_sequence(adev,
782							kalindi_golden_spm_registers,
783							ARRAY_SIZE(kalindi_golden_spm_registers));
784		break;
785	case CHIP_MULLINS:
786		amdgpu_device_program_register_sequence(adev,
787							kalindi_mgcg_cgcg_init,
788							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
789		amdgpu_device_program_register_sequence(adev,
790							godavari_golden_registers,
791							ARRAY_SIZE(godavari_golden_registers));
792		amdgpu_device_program_register_sequence(adev,
793							kalindi_golden_common_registers,
794							ARRAY_SIZE(kalindi_golden_common_registers));
795		amdgpu_device_program_register_sequence(adev,
796							kalindi_golden_spm_registers,
797							ARRAY_SIZE(kalindi_golden_spm_registers));
798		break;
799	case CHIP_KAVERI:
800		amdgpu_device_program_register_sequence(adev,
801							spectre_mgcg_cgcg_init,
802							ARRAY_SIZE(spectre_mgcg_cgcg_init));
803		amdgpu_device_program_register_sequence(adev,
804							spectre_golden_registers,
805							ARRAY_SIZE(spectre_golden_registers));
806		amdgpu_device_program_register_sequence(adev,
807							spectre_golden_common_registers,
808							ARRAY_SIZE(spectre_golden_common_registers));
809		amdgpu_device_program_register_sequence(adev,
810							spectre_golden_spm_registers,
811							ARRAY_SIZE(spectre_golden_spm_registers));
812		break;
813	case CHIP_HAWAII:
814		amdgpu_device_program_register_sequence(adev,
815							hawaii_mgcg_cgcg_init,
816							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
817		amdgpu_device_program_register_sequence(adev,
818							hawaii_golden_registers,
819							ARRAY_SIZE(hawaii_golden_registers));
820		amdgpu_device_program_register_sequence(adev,
821							hawaii_golden_common_registers,
822							ARRAY_SIZE(hawaii_golden_common_registers));
823		amdgpu_device_program_register_sequence(adev,
824							hawaii_golden_spm_registers,
825							ARRAY_SIZE(hawaii_golden_spm_registers));
826		break;
827	default:
828		break;
829	}
830	mutex_unlock(&adev->grbm_idx_mutex);
831}
832
833/**
834 * cik_get_xclk - get the xclk
835 *
836 * @adev: amdgpu_device pointer
837 *
838 * Returns the reference clock used by the gfx engine
839 * (CIK).
840 */
841static u32 cik_get_xclk(struct amdgpu_device *adev)
842{
843	u32 reference_clock = adev->clock.spll.reference_freq;
844
845	if (adev->flags & AMD_IS_APU) {
846		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
847			return reference_clock / 2;
848	} else {
849		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
850			return reference_clock / 4;
851	}
852	return reference_clock;
853}
854
855/**
856 * cik_srbm_select - select specific register instances
857 *
858 * @adev: amdgpu_device pointer
859 * @me: selected ME (micro engine)
860 * @pipe: pipe
861 * @queue: queue
862 * @vmid: VMID
863 *
864 * Switches the currently active registers instances.  Some
865 * registers are instanced per VMID, others are instanced per
866 * me/pipe/queue combination.
867 */
868void cik_srbm_select(struct amdgpu_device *adev,
869		     u32 me, u32 pipe, u32 queue, u32 vmid)
870{
871	u32 srbm_gfx_cntl =
872		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
873		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
874		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
875		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
876	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
877}
878
879static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
880{
881	uint32_t tmp;
882
883	tmp = RREG32(mmCONFIG_CNTL);
884	if (!state)
885		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
886	else
887		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
888	WREG32(mmCONFIG_CNTL, tmp);
889}
890
891static bool cik_read_disabled_bios(struct amdgpu_device *adev)
892{
893	u32 bus_cntl;
894	u32 d1vga_control = 0;
895	u32 d2vga_control = 0;
896	u32 vga_render_control = 0;
897	u32 rom_cntl;
898	bool r;
899
900	bus_cntl = RREG32(mmBUS_CNTL);
901	if (adev->mode_info.num_crtc) {
902		d1vga_control = RREG32(mmD1VGA_CONTROL);
903		d2vga_control = RREG32(mmD2VGA_CONTROL);
904		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
905	}
906	rom_cntl = RREG32_SMC(ixROM_CNTL);
907
908	/* enable the rom */
909	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
910	if (adev->mode_info.num_crtc) {
911		/* Disable VGA mode */
912		WREG32(mmD1VGA_CONTROL,
913		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
914					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
915		WREG32(mmD2VGA_CONTROL,
916		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
917					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
918		WREG32(mmVGA_RENDER_CONTROL,
919		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
920	}
921	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
922
923	r = amdgpu_read_bios(adev);
924
925	/* restore regs */
926	WREG32(mmBUS_CNTL, bus_cntl);
927	if (adev->mode_info.num_crtc) {
928		WREG32(mmD1VGA_CONTROL, d1vga_control);
929		WREG32(mmD2VGA_CONTROL, d2vga_control);
930		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
931	}
932	WREG32_SMC(ixROM_CNTL, rom_cntl);
933	return r;
934}
935
936static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
937				   u8 *bios, u32 length_bytes)
938{
939	u32 *dw_ptr;
940	unsigned long flags;
941	u32 i, length_dw;
942
943	if (bios == NULL)
944		return false;
945	if (length_bytes == 0)
946		return false;
947	/* APU vbios image is part of sbios image */
948	if (adev->flags & AMD_IS_APU)
949		return false;
950
951	dw_ptr = (u32 *)bios;
952	length_dw = ALIGN(length_bytes, 4) / 4;
953	/* take the smc lock since we are using the smc index */
954	spin_lock_irqsave(&adev->smc_idx_lock, flags);
955	/* set rom index to 0 */
956	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
957	WREG32(mmSMC_IND_DATA_0, 0);
958	/* set index to data for continous read */
959	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
960	for (i = 0; i < length_dw; i++)
961		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
962	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
963
964	return true;
965}
966
967static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
968	{mmGRBM_STATUS},
969	{mmGRBM_STATUS2},
970	{mmGRBM_STATUS_SE0},
971	{mmGRBM_STATUS_SE1},
972	{mmGRBM_STATUS_SE2},
973	{mmGRBM_STATUS_SE3},
974	{mmSRBM_STATUS},
975	{mmSRBM_STATUS2},
976	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
977	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
978	{mmCP_STAT},
979	{mmCP_STALLED_STAT1},
980	{mmCP_STALLED_STAT2},
981	{mmCP_STALLED_STAT3},
982	{mmCP_CPF_BUSY_STAT},
983	{mmCP_CPF_STALLED_STAT1},
984	{mmCP_CPF_STATUS},
985	{mmCP_CPC_BUSY_STAT},
986	{mmCP_CPC_STALLED_STAT1},
987	{mmCP_CPC_STATUS},
988	{mmGB_ADDR_CONFIG},
989	{mmMC_ARB_RAMCFG},
990	{mmGB_TILE_MODE0},
991	{mmGB_TILE_MODE1},
992	{mmGB_TILE_MODE2},
993	{mmGB_TILE_MODE3},
994	{mmGB_TILE_MODE4},
995	{mmGB_TILE_MODE5},
996	{mmGB_TILE_MODE6},
997	{mmGB_TILE_MODE7},
998	{mmGB_TILE_MODE8},
999	{mmGB_TILE_MODE9},
1000	{mmGB_TILE_MODE10},
1001	{mmGB_TILE_MODE11},
1002	{mmGB_TILE_MODE12},
1003	{mmGB_TILE_MODE13},
1004	{mmGB_TILE_MODE14},
1005	{mmGB_TILE_MODE15},
1006	{mmGB_TILE_MODE16},
1007	{mmGB_TILE_MODE17},
1008	{mmGB_TILE_MODE18},
1009	{mmGB_TILE_MODE19},
1010	{mmGB_TILE_MODE20},
1011	{mmGB_TILE_MODE21},
1012	{mmGB_TILE_MODE22},
1013	{mmGB_TILE_MODE23},
1014	{mmGB_TILE_MODE24},
1015	{mmGB_TILE_MODE25},
1016	{mmGB_TILE_MODE26},
1017	{mmGB_TILE_MODE27},
1018	{mmGB_TILE_MODE28},
1019	{mmGB_TILE_MODE29},
1020	{mmGB_TILE_MODE30},
1021	{mmGB_TILE_MODE31},
1022	{mmGB_MACROTILE_MODE0},
1023	{mmGB_MACROTILE_MODE1},
1024	{mmGB_MACROTILE_MODE2},
1025	{mmGB_MACROTILE_MODE3},
1026	{mmGB_MACROTILE_MODE4},
1027	{mmGB_MACROTILE_MODE5},
1028	{mmGB_MACROTILE_MODE6},
1029	{mmGB_MACROTILE_MODE7},
1030	{mmGB_MACROTILE_MODE8},
1031	{mmGB_MACROTILE_MODE9},
1032	{mmGB_MACROTILE_MODE10},
1033	{mmGB_MACROTILE_MODE11},
1034	{mmGB_MACROTILE_MODE12},
1035	{mmGB_MACROTILE_MODE13},
1036	{mmGB_MACROTILE_MODE14},
1037	{mmGB_MACROTILE_MODE15},
1038	{mmCC_RB_BACKEND_DISABLE, true},
1039	{mmGC_USER_RB_BACKEND_DISABLE, true},
1040	{mmGB_BACKEND_MAP, false},
1041	{mmPA_SC_RASTER_CONFIG, true},
1042	{mmPA_SC_RASTER_CONFIG_1, true},
1043};
1044
1045
1046static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1047				       bool indexed, u32 se_num,
1048				       u32 sh_num, u32 reg_offset)
1049{
1050	if (indexed) {
1051		uint32_t val;
1052		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1053		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1054
1055		switch (reg_offset) {
1056		case mmCC_RB_BACKEND_DISABLE:
1057			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1058		case mmGC_USER_RB_BACKEND_DISABLE:
1059			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1060		case mmPA_SC_RASTER_CONFIG:
1061			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1062		case mmPA_SC_RASTER_CONFIG_1:
1063			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1064		}
1065
1066		mutex_lock(&adev->grbm_idx_mutex);
1067		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1068			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1069
1070		val = RREG32(reg_offset);
1071
1072		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1073			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1074		mutex_unlock(&adev->grbm_idx_mutex);
1075		return val;
1076	} else {
1077		unsigned idx;
1078
1079		switch (reg_offset) {
1080		case mmGB_ADDR_CONFIG:
1081			return adev->gfx.config.gb_addr_config;
1082		case mmMC_ARB_RAMCFG:
1083			return adev->gfx.config.mc_arb_ramcfg;
1084		case mmGB_TILE_MODE0:
1085		case mmGB_TILE_MODE1:
1086		case mmGB_TILE_MODE2:
1087		case mmGB_TILE_MODE3:
1088		case mmGB_TILE_MODE4:
1089		case mmGB_TILE_MODE5:
1090		case mmGB_TILE_MODE6:
1091		case mmGB_TILE_MODE7:
1092		case mmGB_TILE_MODE8:
1093		case mmGB_TILE_MODE9:
1094		case mmGB_TILE_MODE10:
1095		case mmGB_TILE_MODE11:
1096		case mmGB_TILE_MODE12:
1097		case mmGB_TILE_MODE13:
1098		case mmGB_TILE_MODE14:
1099		case mmGB_TILE_MODE15:
1100		case mmGB_TILE_MODE16:
1101		case mmGB_TILE_MODE17:
1102		case mmGB_TILE_MODE18:
1103		case mmGB_TILE_MODE19:
1104		case mmGB_TILE_MODE20:
1105		case mmGB_TILE_MODE21:
1106		case mmGB_TILE_MODE22:
1107		case mmGB_TILE_MODE23:
1108		case mmGB_TILE_MODE24:
1109		case mmGB_TILE_MODE25:
1110		case mmGB_TILE_MODE26:
1111		case mmGB_TILE_MODE27:
1112		case mmGB_TILE_MODE28:
1113		case mmGB_TILE_MODE29:
1114		case mmGB_TILE_MODE30:
1115		case mmGB_TILE_MODE31:
1116			idx = (reg_offset - mmGB_TILE_MODE0);
1117			return adev->gfx.config.tile_mode_array[idx];
1118		case mmGB_MACROTILE_MODE0:
1119		case mmGB_MACROTILE_MODE1:
1120		case mmGB_MACROTILE_MODE2:
1121		case mmGB_MACROTILE_MODE3:
1122		case mmGB_MACROTILE_MODE4:
1123		case mmGB_MACROTILE_MODE5:
1124		case mmGB_MACROTILE_MODE6:
1125		case mmGB_MACROTILE_MODE7:
1126		case mmGB_MACROTILE_MODE8:
1127		case mmGB_MACROTILE_MODE9:
1128		case mmGB_MACROTILE_MODE10:
1129		case mmGB_MACROTILE_MODE11:
1130		case mmGB_MACROTILE_MODE12:
1131		case mmGB_MACROTILE_MODE13:
1132		case mmGB_MACROTILE_MODE14:
1133		case mmGB_MACROTILE_MODE15:
1134			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1135			return adev->gfx.config.macrotile_mode_array[idx];
1136		default:
1137			return RREG32(reg_offset);
1138		}
1139	}
1140}
1141
1142static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1143			     u32 sh_num, u32 reg_offset, u32 *value)
1144{
1145	uint32_t i;
1146
1147	*value = 0;
1148	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1149		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1150
1151		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1152			continue;
1153
1154		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1155						reg_offset);
1156		return 0;
1157	}
1158	return -EINVAL;
1159}
1160
1161struct kv_reset_save_regs {
1162	u32 gmcon_reng_execute;
1163	u32 gmcon_misc;
1164	u32 gmcon_misc3;
1165};
1166
1167static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1168				   struct kv_reset_save_regs *save)
1169{
1170	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1171	save->gmcon_misc = RREG32(mmGMCON_MISC);
1172	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1173
1174	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1175		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1176	WREG32(mmGMCON_MISC, save->gmcon_misc &
1177		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1178			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1179}
1180
1181static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1182				      struct kv_reset_save_regs *save)
1183{
1184	int i;
1185
1186	WREG32(mmGMCON_PGFSM_WRITE, 0);
1187	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1188
1189	for (i = 0; i < 5; i++)
1190		WREG32(mmGMCON_PGFSM_WRITE, 0);
1191
1192	WREG32(mmGMCON_PGFSM_WRITE, 0);
1193	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1194
1195	for (i = 0; i < 5; i++)
1196		WREG32(mmGMCON_PGFSM_WRITE, 0);
1197
1198	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1199	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1200
1201	for (i = 0; i < 5; i++)
1202		WREG32(mmGMCON_PGFSM_WRITE, 0);
1203
1204	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1205	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1206
1207	for (i = 0; i < 5; i++)
1208		WREG32(mmGMCON_PGFSM_WRITE, 0);
1209
1210	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1211	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1212
1213	for (i = 0; i < 5; i++)
1214		WREG32(mmGMCON_PGFSM_WRITE, 0);
1215
1216	WREG32(mmGMCON_PGFSM_WRITE, 0);
1217	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1218
1219	for (i = 0; i < 5; i++)
1220		WREG32(mmGMCON_PGFSM_WRITE, 0);
1221
1222	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1223	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1224
1225	for (i = 0; i < 5; i++)
1226		WREG32(mmGMCON_PGFSM_WRITE, 0);
1227
1228	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1229	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1230
1231	for (i = 0; i < 5; i++)
1232		WREG32(mmGMCON_PGFSM_WRITE, 0);
1233
1234	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1235	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1236
1237	for (i = 0; i < 5; i++)
1238		WREG32(mmGMCON_PGFSM_WRITE, 0);
1239
1240	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1241	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1242
1243	for (i = 0; i < 5; i++)
1244		WREG32(mmGMCON_PGFSM_WRITE, 0);
1245
1246	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1247	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1248
1249	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1250	WREG32(mmGMCON_MISC, save->gmcon_misc);
1251	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1252}
1253
1254static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1255{
1256	struct kv_reset_save_regs kv_save = { 0 };
1257	u32 i;
1258	int r = -EINVAL;
1259
1260	dev_info(adev->dev, "GPU pci config reset\n");
1261
1262	if (adev->flags & AMD_IS_APU)
1263		kv_save_regs_for_reset(adev, &kv_save);
1264
1265	/* disable BM */
1266	pci_clear_master(adev->pdev);
1267	/* reset */
1268	amdgpu_device_pci_config_reset(adev);
1269
1270	udelay(100);
1271
1272	/* wait for asic to come out of reset */
1273	for (i = 0; i < adev->usec_timeout; i++) {
1274		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1275			/* enable BM */
1276			pci_set_master(adev->pdev);
1277			adev->has_hw_reset = true;
1278			r = 0;
1279			break;
1280		}
1281		udelay(1);
1282	}
1283
1284	/* does asic init need to be run first??? */
1285	if (adev->flags & AMD_IS_APU)
1286		kv_restore_regs_for_reset(adev, &kv_save);
1287
1288	return r;
1289}
1290
1291/**
1292 * cik_asic_pci_config_reset - soft reset GPU
1293 *
1294 * @adev: amdgpu_device pointer
1295 *
1296 * Use PCI Config method to reset the GPU.
1297 *
1298 * Returns 0 for success.
1299 */
1300static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1301{
1302	int r;
1303
1304	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1305
1306	r = cik_gpu_pci_config_reset(adev);
1307
1308	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1309
1310	return r;
1311}
1312
1313static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1314{
1315	switch (adev->asic_type) {
1316	case CHIP_BONAIRE:
1317	case CHIP_HAWAII:
1318		return amdgpu_dpm_is_baco_supported(adev);
1319	default:
1320		return false;
1321	}
1322}
1323
1324static enum amd_reset_method
1325cik_asic_reset_method(struct amdgpu_device *adev)
1326{
1327	bool baco_reset;
1328
1329	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1330	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1331		return amdgpu_reset_method;
1332
1333	if (amdgpu_reset_method != -1)
1334		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1335				  amdgpu_reset_method);
1336
1337	switch (adev->asic_type) {
1338	case CHIP_BONAIRE:
1339		/* disable baco reset until it works */
1340		/* smu7_asic_get_baco_capability(adev, &baco_reset); */
1341		baco_reset = false;
1342		break;
1343	case CHIP_HAWAII:
1344		baco_reset = cik_asic_supports_baco(adev);
1345		break;
1346	default:
1347		baco_reset = false;
1348		break;
1349	}
1350
1351	if (baco_reset)
1352		return AMD_RESET_METHOD_BACO;
1353	else
1354		return AMD_RESET_METHOD_LEGACY;
1355}
1356
1357/**
1358 * cik_asic_reset - soft reset GPU
1359 *
1360 * @adev: amdgpu_device pointer
1361 *
1362 * Look up which blocks are hung and attempt
1363 * to reset them.
1364 * Returns 0 for success.
1365 */
1366static int cik_asic_reset(struct amdgpu_device *adev)
1367{
1368	int r;
1369
1370	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1371		dev_info(adev->dev, "BACO reset\n");
1372		r = amdgpu_dpm_baco_reset(adev);
1373	} else {
1374		dev_info(adev->dev, "PCI CONFIG reset\n");
1375		r = cik_asic_pci_config_reset(adev);
1376	}
1377
1378	return r;
1379}
1380
1381static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1382{
1383	return RREG32(mmCONFIG_MEMSIZE);
1384}
1385
1386static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1387			      u32 cntl_reg, u32 status_reg)
1388{
1389	int r, i;
1390	struct atom_clock_dividers dividers;
1391	uint32_t tmp;
1392
1393	r = amdgpu_atombios_get_clock_dividers(adev,
1394					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1395					       clock, false, &dividers);
1396	if (r)
1397		return r;
1398
1399	tmp = RREG32_SMC(cntl_reg);
1400	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1401		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1402	tmp |= dividers.post_divider;
1403	WREG32_SMC(cntl_reg, tmp);
1404
1405	for (i = 0; i < 100; i++) {
1406		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1407			break;
1408		mdelay(10);
1409	}
1410	if (i == 100)
1411		return -ETIMEDOUT;
1412
1413	return 0;
1414}
1415
1416static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1417{
1418	int r = 0;
1419
1420	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1421	if (r)
1422		return r;
1423
1424	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1425	return r;
1426}
1427
1428static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1429{
1430	int r, i;
1431	struct atom_clock_dividers dividers;
1432	u32 tmp;
1433
1434	r = amdgpu_atombios_get_clock_dividers(adev,
1435					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1436					       ecclk, false, &dividers);
1437	if (r)
1438		return r;
1439
1440	for (i = 0; i < 100; i++) {
1441		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1442			break;
1443		mdelay(10);
1444	}
1445	if (i == 100)
1446		return -ETIMEDOUT;
1447
1448	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1449	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1450		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1451	tmp |= dividers.post_divider;
1452	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1453
1454	for (i = 0; i < 100; i++) {
1455		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1456			break;
1457		mdelay(10);
1458	}
1459	if (i == 100)
1460		return -ETIMEDOUT;
1461
1462	return 0;
1463}
1464
1465static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1466{
1467	struct pci_dev *root = adev->pdev->bus->self;
1468	u32 speed_cntl, current_data_rate;
1469	int i;
1470	u16 tmp16;
1471
1472	if (pci_is_root_bus(adev->pdev->bus))
1473		return;
1474
1475	if (amdgpu_pcie_gen2 == 0)
1476		return;
1477
1478	if (adev->flags & AMD_IS_APU)
1479		return;
1480
1481	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1482					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1483		return;
1484
1485	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1486	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1487		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1488	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1489		if (current_data_rate == 2) {
1490			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1491			return;
1492		}
1493		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1494	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1495		if (current_data_rate == 1) {
1496			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1497			return;
1498		}
1499		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1500	}
1501
1502	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1503		return;
1504
1505	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1506		/* re-try equalization if gen3 is not already enabled */
1507		if (current_data_rate != 2) {
1508			u16 bridge_cfg, gpu_cfg;
1509			u16 bridge_cfg2, gpu_cfg2;
1510			u32 max_lw, current_lw, tmp;
1511
1512			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1513			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1514
1515			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1516			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1517				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1518			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1519				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1520
1521			if (current_lw < max_lw) {
1522				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1523				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1524					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1525						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1526					tmp |= (max_lw <<
1527						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1528					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1529					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1530					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1531					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1532				}
1533			}
1534
1535			for (i = 0; i < 10; i++) {
1536				/* check status */
1537				pcie_capability_read_word(adev->pdev,
1538							  PCI_EXP_DEVSTA,
1539							  &tmp16);
1540				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1541					break;
1542
1543				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1544							  &bridge_cfg);
1545				pcie_capability_read_word(adev->pdev,
1546							  PCI_EXP_LNKCTL,
1547							  &gpu_cfg);
1548
1549				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1550							  &bridge_cfg2);
1551				pcie_capability_read_word(adev->pdev,
1552							  PCI_EXP_LNKCTL2,
1553							  &gpu_cfg2);
1554
1555				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1556				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1557				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1558
1559				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1560				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1561				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1562
1563				msleep(100);
1564
1565				/* linkctl */
1566				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1567								   PCI_EXP_LNKCTL_HAWD,
1568								   bridge_cfg &
1569								   PCI_EXP_LNKCTL_HAWD);
1570				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1571								   PCI_EXP_LNKCTL_HAWD,
1572								   gpu_cfg &
1573								   PCI_EXP_LNKCTL_HAWD);
1574
1575				/* linkctl2 */
1576				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1577							  &tmp16);
1578				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1579					   PCI_EXP_LNKCTL2_TX_MARGIN);
1580				tmp16 |= (bridge_cfg2 &
1581					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1582					   PCI_EXP_LNKCTL2_TX_MARGIN));
1583				pcie_capability_write_word(root,
1584							   PCI_EXP_LNKCTL2,
1585							   tmp16);
1586
1587				pcie_capability_read_word(adev->pdev,
1588							  PCI_EXP_LNKCTL2,
1589							  &tmp16);
1590				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1591					   PCI_EXP_LNKCTL2_TX_MARGIN);
1592				tmp16 |= (gpu_cfg2 &
1593					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1594					   PCI_EXP_LNKCTL2_TX_MARGIN));
1595				pcie_capability_write_word(adev->pdev,
1596							   PCI_EXP_LNKCTL2,
1597							   tmp16);
1598
1599				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1600				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1601				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1602			}
1603		}
1604	}
1605
1606	/* set the link speed */
1607	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1608		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1609	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1610	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1611
1612	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1613	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1614
1615	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1616		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1617	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1618		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1619	else
1620		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1621	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1622
1623	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1624	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1625	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1626
1627	for (i = 0; i < adev->usec_timeout; i++) {
1628		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1629		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1630			break;
1631		udelay(1);
1632	}
1633}
1634
1635static void cik_program_aspm(struct amdgpu_device *adev)
1636{
1637	u32 data, orig;
1638	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1639	bool disable_clkreq = false;
1640
1641	if (amdgpu_aspm == 0)
1642		return;
1643
1644	if (pci_is_root_bus(adev->pdev->bus))
1645		return;
1646
1647	/* XXX double check APUs */
1648	if (adev->flags & AMD_IS_APU)
1649		return;
1650
1651	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1652	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1653	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1654		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1655	if (orig != data)
1656		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1657
1658	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1659	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1660	if (orig != data)
1661		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1662
1663	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1664	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1665	if (orig != data)
1666		WREG32_PCIE(ixPCIE_P_CNTL, data);
1667
1668	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1669	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1670		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1671	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1672	if (!disable_l0s)
1673		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1674
1675	if (!disable_l1) {
1676		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1677		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1678		if (orig != data)
1679			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1680
1681		if (!disable_plloff_in_l1) {
1682			bool clk_req_support;
1683
1684			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1685			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1686				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1687			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1688				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1689			if (orig != data)
1690				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1691
1692			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1693			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1694				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1695			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1696				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1697			if (orig != data)
1698				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1699
1700			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1701			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1702				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1703			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1704				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1705			if (orig != data)
1706				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1707
1708			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1709			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1710				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1711			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1712				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1713			if (orig != data)
1714				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1715
1716			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1717			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1718			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1719			if (orig != data)
1720				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1721
1722			if (!disable_clkreq) {
1723				struct pci_dev *root = adev->pdev->bus->self;
1724				u32 lnkcap;
1725
1726				clk_req_support = false;
1727				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1728				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1729					clk_req_support = true;
1730			} else {
1731				clk_req_support = false;
1732			}
1733
1734			if (clk_req_support) {
1735				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1736				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1737					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1738				if (orig != data)
1739					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1740
1741				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1742				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1743					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1744				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1745					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1746				if (orig != data)
1747					WREG32_SMC(ixTHM_CLK_CNTL, data);
1748
1749				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1750				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1751					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1752				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1753					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1754				if (orig != data)
1755					WREG32_SMC(ixMISC_CLK_CTRL, data);
1756
1757				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1758				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1759				if (orig != data)
1760					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1761
1762				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1763				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1764				if (orig != data)
1765					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1766
1767				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1768				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1769				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1770				if (orig != data)
1771					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1772			}
1773		}
1774	} else {
1775		if (orig != data)
1776			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1777	}
1778
1779	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1780	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1781		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1782		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1783	if (orig != data)
1784		WREG32_PCIE(ixPCIE_CNTL2, data);
1785
1786	if (!disable_l0s) {
1787		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1788		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1789				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1790			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1791			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1792			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1793				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1794				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1795				if (orig != data)
1796					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1797			}
1798		}
1799	}
1800}
1801
1802static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1803{
1804	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1805		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1806}
1807
1808static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1809{
1810	if (!ring || !ring->funcs->emit_wreg) {
1811		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1812		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1813	} else {
1814		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1815	}
1816}
1817
1818static void cik_invalidate_hdp(struct amdgpu_device *adev,
1819			       struct amdgpu_ring *ring)
1820{
1821	if (!ring || !ring->funcs->emit_wreg) {
1822		WREG32(mmHDP_DEBUG0, 1);
1823		RREG32(mmHDP_DEBUG0);
1824	} else {
1825		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1826	}
1827}
1828
1829static bool cik_need_full_reset(struct amdgpu_device *adev)
1830{
1831	/* change this when we support soft reset */
1832	return true;
1833}
1834
1835static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1836			       uint64_t *count1)
1837{
1838	uint32_t perfctr = 0;
1839	uint64_t cnt0_of, cnt1_of;
1840	int tmp;
1841
1842	/* This reports 0 on APUs, so return to avoid writing/reading registers
1843	 * that may or may not be different from their GPU counterparts
1844	 */
1845	if (adev->flags & AMD_IS_APU)
1846		return;
1847
1848	/* Set the 2 events that we wish to watch, defined above */
1849	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1850	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1851	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1852
1853	/* Write to enable desired perf counters */
1854	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1855	/* Zero out and enable the perf counters
1856	 * Write 0x5:
1857	 * Bit 0 = Start all counters(1)
1858	 * Bit 2 = Global counter reset enable(1)
1859	 */
1860	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1861
1862	msleep(1000);
1863
1864	/* Load the shadow and disable the perf counters
1865	 * Write 0x2:
1866	 * Bit 0 = Stop counters(0)
1867	 * Bit 1 = Load the shadow counters(1)
1868	 */
1869	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1870
1871	/* Read register values to get any >32bit overflow */
1872	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1873	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1874	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1875
1876	/* Get the values and add the overflow */
1877	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1878	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1879}
1880
1881static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1882{
1883	u32 clock_cntl, pc;
1884
1885	if (adev->flags & AMD_IS_APU)
1886		return false;
1887
1888	/* check if the SMC is already running */
1889	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1890	pc = RREG32_SMC(ixSMC_PC_C);
1891	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1892	    (0x20100 <= pc))
1893		return true;
1894
1895	return false;
1896}
1897
1898static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1899{
1900	uint64_t nak_r, nak_g;
1901
1902	/* Get the number of NAKs received and generated */
1903	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1904	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1905
1906	/* Add the total number of NAKs, i.e the number of replays */
1907	return (nak_r + nak_g);
1908}
1909
1910static void cik_pre_asic_init(struct amdgpu_device *adev)
1911{
1912}
1913
1914static const struct amdgpu_asic_funcs cik_asic_funcs =
1915{
1916	.read_disabled_bios = &cik_read_disabled_bios,
1917	.read_bios_from_rom = &cik_read_bios_from_rom,
1918	.read_register = &cik_read_register,
1919	.reset = &cik_asic_reset,
1920	.reset_method = &cik_asic_reset_method,
1921	.set_vga_state = &cik_vga_set_state,
1922	.get_xclk = &cik_get_xclk,
1923	.set_uvd_clocks = &cik_set_uvd_clocks,
1924	.set_vce_clocks = &cik_set_vce_clocks,
1925	.get_config_memsize = &cik_get_config_memsize,
1926	.flush_hdp = &cik_flush_hdp,
1927	.invalidate_hdp = &cik_invalidate_hdp,
1928	.need_full_reset = &cik_need_full_reset,
1929	.init_doorbell_index = &legacy_doorbell_index_init,
1930	.get_pcie_usage = &cik_get_pcie_usage,
1931	.need_reset_on_init = &cik_need_reset_on_init,
1932	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1933	.supports_baco = &cik_asic_supports_baco,
1934	.pre_asic_init = &cik_pre_asic_init,
1935};
1936
1937static int cik_common_early_init(void *handle)
1938{
1939	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1940
1941	adev->smc_rreg = &cik_smc_rreg;
1942	adev->smc_wreg = &cik_smc_wreg;
1943	adev->pcie_rreg = &cik_pcie_rreg;
1944	adev->pcie_wreg = &cik_pcie_wreg;
1945	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1946	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1947	adev->didt_rreg = &cik_didt_rreg;
1948	adev->didt_wreg = &cik_didt_wreg;
1949
1950	adev->asic_funcs = &cik_asic_funcs;
1951
1952	adev->rev_id = cik_get_rev_id(adev);
1953	adev->external_rev_id = 0xFF;
1954	switch (adev->asic_type) {
1955	case CHIP_BONAIRE:
1956		adev->cg_flags =
1957			AMD_CG_SUPPORT_GFX_MGCG |
1958			AMD_CG_SUPPORT_GFX_MGLS |
1959			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1960			AMD_CG_SUPPORT_GFX_CGLS |
1961			AMD_CG_SUPPORT_GFX_CGTS |
1962			AMD_CG_SUPPORT_GFX_CGTS_LS |
1963			AMD_CG_SUPPORT_GFX_CP_LS |
1964			AMD_CG_SUPPORT_MC_LS |
1965			AMD_CG_SUPPORT_MC_MGCG |
1966			AMD_CG_SUPPORT_SDMA_MGCG |
1967			AMD_CG_SUPPORT_SDMA_LS |
1968			AMD_CG_SUPPORT_BIF_LS |
1969			AMD_CG_SUPPORT_VCE_MGCG |
1970			AMD_CG_SUPPORT_UVD_MGCG |
1971			AMD_CG_SUPPORT_HDP_LS |
1972			AMD_CG_SUPPORT_HDP_MGCG;
1973		adev->pg_flags = 0;
1974		adev->external_rev_id = adev->rev_id + 0x14;
1975		break;
1976	case CHIP_HAWAII:
1977		adev->cg_flags =
1978			AMD_CG_SUPPORT_GFX_MGCG |
1979			AMD_CG_SUPPORT_GFX_MGLS |
1980			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1981			AMD_CG_SUPPORT_GFX_CGLS |
1982			AMD_CG_SUPPORT_GFX_CGTS |
1983			AMD_CG_SUPPORT_GFX_CP_LS |
1984			AMD_CG_SUPPORT_MC_LS |
1985			AMD_CG_SUPPORT_MC_MGCG |
1986			AMD_CG_SUPPORT_SDMA_MGCG |
1987			AMD_CG_SUPPORT_SDMA_LS |
1988			AMD_CG_SUPPORT_BIF_LS |
1989			AMD_CG_SUPPORT_VCE_MGCG |
1990			AMD_CG_SUPPORT_UVD_MGCG |
1991			AMD_CG_SUPPORT_HDP_LS |
1992			AMD_CG_SUPPORT_HDP_MGCG;
1993		adev->pg_flags = 0;
1994		adev->external_rev_id = 0x28;
1995		break;
1996	case CHIP_KAVERI:
1997		adev->cg_flags =
1998			AMD_CG_SUPPORT_GFX_MGCG |
1999			AMD_CG_SUPPORT_GFX_MGLS |
2000			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2001			AMD_CG_SUPPORT_GFX_CGLS |
2002			AMD_CG_SUPPORT_GFX_CGTS |
2003			AMD_CG_SUPPORT_GFX_CGTS_LS |
2004			AMD_CG_SUPPORT_GFX_CP_LS |
2005			AMD_CG_SUPPORT_SDMA_MGCG |
2006			AMD_CG_SUPPORT_SDMA_LS |
2007			AMD_CG_SUPPORT_BIF_LS |
2008			AMD_CG_SUPPORT_VCE_MGCG |
2009			AMD_CG_SUPPORT_UVD_MGCG |
2010			AMD_CG_SUPPORT_HDP_LS |
2011			AMD_CG_SUPPORT_HDP_MGCG;
2012		adev->pg_flags =
2013			/*AMD_PG_SUPPORT_GFX_PG |
2014			  AMD_PG_SUPPORT_GFX_SMG |
2015			  AMD_PG_SUPPORT_GFX_DMG |*/
2016			AMD_PG_SUPPORT_UVD |
2017			AMD_PG_SUPPORT_VCE |
2018			/*  AMD_PG_SUPPORT_CP |
2019			  AMD_PG_SUPPORT_GDS |
2020			  AMD_PG_SUPPORT_RLC_SMU_HS |
2021			  AMD_PG_SUPPORT_ACP |
2022			  AMD_PG_SUPPORT_SAMU |*/
2023			0;
2024		if (adev->pdev->device == 0x1312 ||
2025			adev->pdev->device == 0x1316 ||
2026			adev->pdev->device == 0x1317)
2027			adev->external_rev_id = 0x41;
2028		else
2029			adev->external_rev_id = 0x1;
2030		break;
2031	case CHIP_KABINI:
2032	case CHIP_MULLINS:
2033		adev->cg_flags =
2034			AMD_CG_SUPPORT_GFX_MGCG |
2035			AMD_CG_SUPPORT_GFX_MGLS |
2036			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2037			AMD_CG_SUPPORT_GFX_CGLS |
2038			AMD_CG_SUPPORT_GFX_CGTS |
2039			AMD_CG_SUPPORT_GFX_CGTS_LS |
2040			AMD_CG_SUPPORT_GFX_CP_LS |
2041			AMD_CG_SUPPORT_SDMA_MGCG |
2042			AMD_CG_SUPPORT_SDMA_LS |
2043			AMD_CG_SUPPORT_BIF_LS |
2044			AMD_CG_SUPPORT_VCE_MGCG |
2045			AMD_CG_SUPPORT_UVD_MGCG |
2046			AMD_CG_SUPPORT_HDP_LS |
2047			AMD_CG_SUPPORT_HDP_MGCG;
2048		adev->pg_flags =
2049			/*AMD_PG_SUPPORT_GFX_PG |
2050			  AMD_PG_SUPPORT_GFX_SMG | */
2051			AMD_PG_SUPPORT_UVD |
2052			/*AMD_PG_SUPPORT_VCE |
2053			  AMD_PG_SUPPORT_CP |
2054			  AMD_PG_SUPPORT_GDS |
2055			  AMD_PG_SUPPORT_RLC_SMU_HS |
2056			  AMD_PG_SUPPORT_SAMU |*/
2057			0;
2058		if (adev->asic_type == CHIP_KABINI) {
2059			if (adev->rev_id == 0)
2060				adev->external_rev_id = 0x81;
2061			else if (adev->rev_id == 1)
2062				adev->external_rev_id = 0x82;
2063			else if (adev->rev_id == 2)
2064				adev->external_rev_id = 0x85;
2065		} else
2066			adev->external_rev_id = adev->rev_id + 0xa1;
2067		break;
2068	default:
2069		/* FIXME: not supported yet */
2070		return -EINVAL;
2071	}
2072
2073	return 0;
2074}
2075
2076static int cik_common_sw_init(void *handle)
2077{
2078	return 0;
2079}
2080
2081static int cik_common_sw_fini(void *handle)
2082{
2083	return 0;
2084}
2085
2086static int cik_common_hw_init(void *handle)
2087{
2088	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2089
2090	/* move the golden regs per IP block */
2091	cik_init_golden_registers(adev);
2092	/* enable pcie gen2/3 link */
2093	cik_pcie_gen3_enable(adev);
2094	/* enable aspm */
2095	cik_program_aspm(adev);
2096
2097	return 0;
2098}
2099
2100static int cik_common_hw_fini(void *handle)
2101{
2102	return 0;
2103}
2104
2105static int cik_common_suspend(void *handle)
2106{
2107	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2108
2109	return cik_common_hw_fini(adev);
2110}
2111
2112static int cik_common_resume(void *handle)
2113{
2114	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2115
2116	return cik_common_hw_init(adev);
2117}
2118
2119static bool cik_common_is_idle(void *handle)
2120{
2121	return true;
2122}
2123
2124static int cik_common_wait_for_idle(void *handle)
2125{
2126	return 0;
2127}
2128
2129static int cik_common_soft_reset(void *handle)
2130{
2131	/* XXX hard reset?? */
2132	return 0;
2133}
2134
2135static int cik_common_set_clockgating_state(void *handle,
2136					    enum amd_clockgating_state state)
2137{
2138	return 0;
2139}
2140
2141static int cik_common_set_powergating_state(void *handle,
2142					    enum amd_powergating_state state)
2143{
2144	return 0;
2145}
2146
2147static const struct amd_ip_funcs cik_common_ip_funcs = {
2148	.name = "cik_common",
2149	.early_init = cik_common_early_init,
2150	.late_init = NULL,
2151	.sw_init = cik_common_sw_init,
2152	.sw_fini = cik_common_sw_fini,
2153	.hw_init = cik_common_hw_init,
2154	.hw_fini = cik_common_hw_fini,
2155	.suspend = cik_common_suspend,
2156	.resume = cik_common_resume,
2157	.is_idle = cik_common_is_idle,
2158	.wait_for_idle = cik_common_wait_for_idle,
2159	.soft_reset = cik_common_soft_reset,
2160	.set_clockgating_state = cik_common_set_clockgating_state,
2161	.set_powergating_state = cik_common_set_powergating_state,
2162};
2163
2164static const struct amdgpu_ip_block_version cik_common_ip_block =
2165{
2166	.type = AMD_IP_BLOCK_TYPE_COMMON,
2167	.major = 1,
2168	.minor = 0,
2169	.rev = 0,
2170	.funcs = &cik_common_ip_funcs,
2171};
2172
2173int cik_set_ip_blocks(struct amdgpu_device *adev)
2174{
2175	switch (adev->asic_type) {
2176	case CHIP_BONAIRE:
2177		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2178		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2179		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2180		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2181		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2182		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2183		if (adev->enable_virtual_display)
2184			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2185#if defined(CONFIG_DRM_AMD_DC)
2186		else if (amdgpu_device_has_dc_support(adev))
2187			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2188#endif
2189		else
2190			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2191		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2192		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2193		break;
2194	case CHIP_HAWAII:
2195		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2196		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2197		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2198		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2199		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2200		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2201		if (adev->enable_virtual_display)
2202			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2203#if defined(CONFIG_DRM_AMD_DC)
2204		else if (amdgpu_device_has_dc_support(adev))
2205			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2206#endif
2207		else
2208			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2209		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2210		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2211		break;
2212	case CHIP_KAVERI:
2213		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2214		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2215		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2216		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2217		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2218		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2219		if (adev->enable_virtual_display)
2220			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2221#if defined(CONFIG_DRM_AMD_DC)
2222		else if (amdgpu_device_has_dc_support(adev))
2223			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2224#endif
2225		else
2226			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2227
2228		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2229		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2230		break;
2231	case CHIP_KABINI:
2232	case CHIP_MULLINS:
2233		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2234		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2235		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2236		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2237		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2238		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2239		if (adev->enable_virtual_display)
2240			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2241#if defined(CONFIG_DRM_AMD_DC)
2242		else if (amdgpu_device_has_dc_support(adev))
2243			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2244#endif
2245		else
2246			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2247		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2248		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2249		break;
2250	default:
2251		/* FIXME: not supported yet */
2252		return -EINVAL;
2253	}
2254	return 0;
2255}
2256