1/* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 25#include <linux/firmware.h> 26#include <linux/pci.h> 27#include <linux/slab.h> 28 29#include <drm/drm_vblank.h> 30#include <drm/radeon_drm.h> 31 32#include "atom.h" 33#include "avivod.h" 34#include "evergreen_blit_shaders.h" 35#include "evergreen_reg.h" 36#include "evergreend.h" 37#include "radeon.h" 38#include "radeon_asic.h" 39#include "radeon_audio.h" 40#include "radeon_ucode.h" 41 42#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc)) 43#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc)) 44#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc)) 45 46/* 47 * Indirect registers accessor 48 */ 49u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg) 50{ 51 unsigned long flags; 52 u32 r; 53 54 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 55 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 56 r = RREG32(EVERGREEN_CG_IND_DATA); 57 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 58 return r; 59} 60 61void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v) 62{ 63 unsigned long flags; 64 65 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 66 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 67 WREG32(EVERGREEN_CG_IND_DATA, (v)); 68 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 69} 70 71u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg) 72{ 73 unsigned long flags; 74 u32 r; 75 76 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 77 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 78 r = RREG32(EVERGREEN_PIF_PHY0_DATA); 79 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 80 return r; 81} 82 83void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v) 84{ 85 unsigned long flags; 86 87 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 88 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 89 WREG32(EVERGREEN_PIF_PHY0_DATA, (v)); 90 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 91} 92 93u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg) 94{ 95 unsigned long flags; 96 u32 r; 97 98 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 99 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 100 r = RREG32(EVERGREEN_PIF_PHY1_DATA); 101 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 102 return r; 103} 104 105void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v) 106{ 107 unsigned long flags; 108 109 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 110 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 111 WREG32(EVERGREEN_PIF_PHY1_DATA, (v)); 112 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 113} 114 115static const u32 crtc_offsets[6] = 116{ 117 EVERGREEN_CRTC0_REGISTER_OFFSET, 118 EVERGREEN_CRTC1_REGISTER_OFFSET, 119 EVERGREEN_CRTC2_REGISTER_OFFSET, 120 EVERGREEN_CRTC3_REGISTER_OFFSET, 121 EVERGREEN_CRTC4_REGISTER_OFFSET, 122 EVERGREEN_CRTC5_REGISTER_OFFSET 123}; 124 125#include "clearstate_evergreen.h" 126 127static const u32 sumo_rlc_save_restore_register_list[] = 128{ 129 0x98fc, 130 0x9830, 131 0x9834, 132 0x9838, 133 0x9870, 134 0x9874, 135 0x8a14, 136 0x8b24, 137 0x8bcc, 138 0x8b10, 139 0x8d00, 140 0x8d04, 141 0x8c00, 142 0x8c04, 143 0x8c08, 144 0x8c0c, 145 0x8d8c, 146 0x8c20, 147 0x8c24, 148 0x8c28, 149 0x8c18, 150 0x8c1c, 151 0x8cf0, 152 0x8e2c, 153 0x8e38, 154 0x8c30, 155 0x9508, 156 0x9688, 157 0x9608, 158 0x960c, 159 0x9610, 160 0x9614, 161 0x88c4, 162 0x88d4, 163 0xa008, 164 0x900c, 165 0x9100, 166 0x913c, 167 0x98f8, 168 0x98f4, 169 0x9b7c, 170 0x3f8c, 171 0x8950, 172 0x8954, 173 0x8a18, 174 0x8b28, 175 0x9144, 176 0x9148, 177 0x914c, 178 0x3f90, 179 0x3f94, 180 0x915c, 181 0x9160, 182 0x9178, 183 0x917c, 184 0x9180, 185 0x918c, 186 0x9190, 187 0x9194, 188 0x9198, 189 0x919c, 190 0x91a8, 191 0x91ac, 192 0x91b0, 193 0x91b4, 194 0x91b8, 195 0x91c4, 196 0x91c8, 197 0x91cc, 198 0x91d0, 199 0x91d4, 200 0x91e0, 201 0x91e4, 202 0x91ec, 203 0x91f0, 204 0x91f4, 205 0x9200, 206 0x9204, 207 0x929c, 208 0x9150, 209 0x802c, 210}; 211 212static void evergreen_gpu_init(struct radeon_device *rdev); 213void evergreen_fini(struct radeon_device *rdev); 214void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 215void evergreen_program_aspm(struct radeon_device *rdev); 216extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, 217 int ring, u32 cp_int_cntl); 218extern void cayman_vm_decode_fault(struct radeon_device *rdev, 219 u32 status, u32 addr); 220void cik_init_cp_pg_table(struct radeon_device *rdev); 221 222extern u32 si_get_csb_size(struct radeon_device *rdev); 223extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 224extern u32 cik_get_csb_size(struct radeon_device *rdev); 225extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 226extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); 227 228static const u32 evergreen_golden_registers[] = 229{ 230 0x3f90, 0xffff0000, 0xff000000, 231 0x9148, 0xffff0000, 0xff000000, 232 0x3f94, 0xffff0000, 0xff000000, 233 0x914c, 0xffff0000, 0xff000000, 234 0x9b7c, 0xffffffff, 0x00000000, 235 0x8a14, 0xffffffff, 0x00000007, 236 0x8b10, 0xffffffff, 0x00000000, 237 0x960c, 0xffffffff, 0x54763210, 238 0x88c4, 0xffffffff, 0x000000c2, 239 0x88d4, 0xffffffff, 0x00000010, 240 0x8974, 0xffffffff, 0x00000000, 241 0xc78, 0x00000080, 0x00000080, 242 0x5eb4, 0xffffffff, 0x00000002, 243 0x5e78, 0xffffffff, 0x001000f0, 244 0x6104, 0x01000300, 0x00000000, 245 0x5bc0, 0x00300000, 0x00000000, 246 0x7030, 0xffffffff, 0x00000011, 247 0x7c30, 0xffffffff, 0x00000011, 248 0x10830, 0xffffffff, 0x00000011, 249 0x11430, 0xffffffff, 0x00000011, 250 0x12030, 0xffffffff, 0x00000011, 251 0x12c30, 0xffffffff, 0x00000011, 252 0xd02c, 0xffffffff, 0x08421000, 253 0x240c, 0xffffffff, 0x00000380, 254 0x8b24, 0xffffffff, 0x00ff0fff, 255 0x28a4c, 0x06000000, 0x06000000, 256 0x10c, 0x00000001, 0x00000001, 257 0x8d00, 0xffffffff, 0x100e4848, 258 0x8d04, 0xffffffff, 0x00164745, 259 0x8c00, 0xffffffff, 0xe4000003, 260 0x8c04, 0xffffffff, 0x40600060, 261 0x8c08, 0xffffffff, 0x001c001c, 262 0x8cf0, 0xffffffff, 0x08e00620, 263 0x8c20, 0xffffffff, 0x00800080, 264 0x8c24, 0xffffffff, 0x00800080, 265 0x8c18, 0xffffffff, 0x20202078, 266 0x8c1c, 0xffffffff, 0x00001010, 267 0x28350, 0xffffffff, 0x00000000, 268 0xa008, 0xffffffff, 0x00010000, 269 0x5c4, 0xffffffff, 0x00000001, 270 0x9508, 0xffffffff, 0x00000002, 271 0x913c, 0x0000000f, 0x0000000a 272}; 273 274static const u32 evergreen_golden_registers2[] = 275{ 276 0x2f4c, 0xffffffff, 0x00000000, 277 0x54f4, 0xffffffff, 0x00000000, 278 0x54f0, 0xffffffff, 0x00000000, 279 0x5498, 0xffffffff, 0x00000000, 280 0x549c, 0xffffffff, 0x00000000, 281 0x5494, 0xffffffff, 0x00000000, 282 0x53cc, 0xffffffff, 0x00000000, 283 0x53c8, 0xffffffff, 0x00000000, 284 0x53c4, 0xffffffff, 0x00000000, 285 0x53c0, 0xffffffff, 0x00000000, 286 0x53bc, 0xffffffff, 0x00000000, 287 0x53b8, 0xffffffff, 0x00000000, 288 0x53b4, 0xffffffff, 0x00000000, 289 0x53b0, 0xffffffff, 0x00000000 290}; 291 292static const u32 cypress_mgcg_init[] = 293{ 294 0x802c, 0xffffffff, 0xc0000000, 295 0x5448, 0xffffffff, 0x00000100, 296 0x55e4, 0xffffffff, 0x00000100, 297 0x160c, 0xffffffff, 0x00000100, 298 0x5644, 0xffffffff, 0x00000100, 299 0xc164, 0xffffffff, 0x00000100, 300 0x8a18, 0xffffffff, 0x00000100, 301 0x897c, 0xffffffff, 0x06000100, 302 0x8b28, 0xffffffff, 0x00000100, 303 0x9144, 0xffffffff, 0x00000100, 304 0x9a60, 0xffffffff, 0x00000100, 305 0x9868, 0xffffffff, 0x00000100, 306 0x8d58, 0xffffffff, 0x00000100, 307 0x9510, 0xffffffff, 0x00000100, 308 0x949c, 0xffffffff, 0x00000100, 309 0x9654, 0xffffffff, 0x00000100, 310 0x9030, 0xffffffff, 0x00000100, 311 0x9034, 0xffffffff, 0x00000100, 312 0x9038, 0xffffffff, 0x00000100, 313 0x903c, 0xffffffff, 0x00000100, 314 0x9040, 0xffffffff, 0x00000100, 315 0xa200, 0xffffffff, 0x00000100, 316 0xa204, 0xffffffff, 0x00000100, 317 0xa208, 0xffffffff, 0x00000100, 318 0xa20c, 0xffffffff, 0x00000100, 319 0x971c, 0xffffffff, 0x00000100, 320 0x977c, 0xffffffff, 0x00000100, 321 0x3f80, 0xffffffff, 0x00000100, 322 0xa210, 0xffffffff, 0x00000100, 323 0xa214, 0xffffffff, 0x00000100, 324 0x4d8, 0xffffffff, 0x00000100, 325 0x9784, 0xffffffff, 0x00000100, 326 0x9698, 0xffffffff, 0x00000100, 327 0x4d4, 0xffffffff, 0x00000200, 328 0x30cc, 0xffffffff, 0x00000100, 329 0xd0c0, 0xffffffff, 0xff000100, 330 0x802c, 0xffffffff, 0x40000000, 331 0x915c, 0xffffffff, 0x00010000, 332 0x9160, 0xffffffff, 0x00030002, 333 0x9178, 0xffffffff, 0x00070000, 334 0x917c, 0xffffffff, 0x00030002, 335 0x9180, 0xffffffff, 0x00050004, 336 0x918c, 0xffffffff, 0x00010006, 337 0x9190, 0xffffffff, 0x00090008, 338 0x9194, 0xffffffff, 0x00070000, 339 0x9198, 0xffffffff, 0x00030002, 340 0x919c, 0xffffffff, 0x00050004, 341 0x91a8, 0xffffffff, 0x00010006, 342 0x91ac, 0xffffffff, 0x00090008, 343 0x91b0, 0xffffffff, 0x00070000, 344 0x91b4, 0xffffffff, 0x00030002, 345 0x91b8, 0xffffffff, 0x00050004, 346 0x91c4, 0xffffffff, 0x00010006, 347 0x91c8, 0xffffffff, 0x00090008, 348 0x91cc, 0xffffffff, 0x00070000, 349 0x91d0, 0xffffffff, 0x00030002, 350 0x91d4, 0xffffffff, 0x00050004, 351 0x91e0, 0xffffffff, 0x00010006, 352 0x91e4, 0xffffffff, 0x00090008, 353 0x91e8, 0xffffffff, 0x00000000, 354 0x91ec, 0xffffffff, 0x00070000, 355 0x91f0, 0xffffffff, 0x00030002, 356 0x91f4, 0xffffffff, 0x00050004, 357 0x9200, 0xffffffff, 0x00010006, 358 0x9204, 0xffffffff, 0x00090008, 359 0x9208, 0xffffffff, 0x00070000, 360 0x920c, 0xffffffff, 0x00030002, 361 0x9210, 0xffffffff, 0x00050004, 362 0x921c, 0xffffffff, 0x00010006, 363 0x9220, 0xffffffff, 0x00090008, 364 0x9224, 0xffffffff, 0x00070000, 365 0x9228, 0xffffffff, 0x00030002, 366 0x922c, 0xffffffff, 0x00050004, 367 0x9238, 0xffffffff, 0x00010006, 368 0x923c, 0xffffffff, 0x00090008, 369 0x9240, 0xffffffff, 0x00070000, 370 0x9244, 0xffffffff, 0x00030002, 371 0x9248, 0xffffffff, 0x00050004, 372 0x9254, 0xffffffff, 0x00010006, 373 0x9258, 0xffffffff, 0x00090008, 374 0x925c, 0xffffffff, 0x00070000, 375 0x9260, 0xffffffff, 0x00030002, 376 0x9264, 0xffffffff, 0x00050004, 377 0x9270, 0xffffffff, 0x00010006, 378 0x9274, 0xffffffff, 0x00090008, 379 0x9278, 0xffffffff, 0x00070000, 380 0x927c, 0xffffffff, 0x00030002, 381 0x9280, 0xffffffff, 0x00050004, 382 0x928c, 0xffffffff, 0x00010006, 383 0x9290, 0xffffffff, 0x00090008, 384 0x9294, 0xffffffff, 0x00000000, 385 0x929c, 0xffffffff, 0x00000001, 386 0x802c, 0xffffffff, 0x40010000, 387 0x915c, 0xffffffff, 0x00010000, 388 0x9160, 0xffffffff, 0x00030002, 389 0x9178, 0xffffffff, 0x00070000, 390 0x917c, 0xffffffff, 0x00030002, 391 0x9180, 0xffffffff, 0x00050004, 392 0x918c, 0xffffffff, 0x00010006, 393 0x9190, 0xffffffff, 0x00090008, 394 0x9194, 0xffffffff, 0x00070000, 395 0x9198, 0xffffffff, 0x00030002, 396 0x919c, 0xffffffff, 0x00050004, 397 0x91a8, 0xffffffff, 0x00010006, 398 0x91ac, 0xffffffff, 0x00090008, 399 0x91b0, 0xffffffff, 0x00070000, 400 0x91b4, 0xffffffff, 0x00030002, 401 0x91b8, 0xffffffff, 0x00050004, 402 0x91c4, 0xffffffff, 0x00010006, 403 0x91c8, 0xffffffff, 0x00090008, 404 0x91cc, 0xffffffff, 0x00070000, 405 0x91d0, 0xffffffff, 0x00030002, 406 0x91d4, 0xffffffff, 0x00050004, 407 0x91e0, 0xffffffff, 0x00010006, 408 0x91e4, 0xffffffff, 0x00090008, 409 0x91e8, 0xffffffff, 0x00000000, 410 0x91ec, 0xffffffff, 0x00070000, 411 0x91f0, 0xffffffff, 0x00030002, 412 0x91f4, 0xffffffff, 0x00050004, 413 0x9200, 0xffffffff, 0x00010006, 414 0x9204, 0xffffffff, 0x00090008, 415 0x9208, 0xffffffff, 0x00070000, 416 0x920c, 0xffffffff, 0x00030002, 417 0x9210, 0xffffffff, 0x00050004, 418 0x921c, 0xffffffff, 0x00010006, 419 0x9220, 0xffffffff, 0x00090008, 420 0x9224, 0xffffffff, 0x00070000, 421 0x9228, 0xffffffff, 0x00030002, 422 0x922c, 0xffffffff, 0x00050004, 423 0x9238, 0xffffffff, 0x00010006, 424 0x923c, 0xffffffff, 0x00090008, 425 0x9240, 0xffffffff, 0x00070000, 426 0x9244, 0xffffffff, 0x00030002, 427 0x9248, 0xffffffff, 0x00050004, 428 0x9254, 0xffffffff, 0x00010006, 429 0x9258, 0xffffffff, 0x00090008, 430 0x925c, 0xffffffff, 0x00070000, 431 0x9260, 0xffffffff, 0x00030002, 432 0x9264, 0xffffffff, 0x00050004, 433 0x9270, 0xffffffff, 0x00010006, 434 0x9274, 0xffffffff, 0x00090008, 435 0x9278, 0xffffffff, 0x00070000, 436 0x927c, 0xffffffff, 0x00030002, 437 0x9280, 0xffffffff, 0x00050004, 438 0x928c, 0xffffffff, 0x00010006, 439 0x9290, 0xffffffff, 0x00090008, 440 0x9294, 0xffffffff, 0x00000000, 441 0x929c, 0xffffffff, 0x00000001, 442 0x802c, 0xffffffff, 0xc0000000 443}; 444 445static const u32 redwood_mgcg_init[] = 446{ 447 0x802c, 0xffffffff, 0xc0000000, 448 0x5448, 0xffffffff, 0x00000100, 449 0x55e4, 0xffffffff, 0x00000100, 450 0x160c, 0xffffffff, 0x00000100, 451 0x5644, 0xffffffff, 0x00000100, 452 0xc164, 0xffffffff, 0x00000100, 453 0x8a18, 0xffffffff, 0x00000100, 454 0x897c, 0xffffffff, 0x06000100, 455 0x8b28, 0xffffffff, 0x00000100, 456 0x9144, 0xffffffff, 0x00000100, 457 0x9a60, 0xffffffff, 0x00000100, 458 0x9868, 0xffffffff, 0x00000100, 459 0x8d58, 0xffffffff, 0x00000100, 460 0x9510, 0xffffffff, 0x00000100, 461 0x949c, 0xffffffff, 0x00000100, 462 0x9654, 0xffffffff, 0x00000100, 463 0x9030, 0xffffffff, 0x00000100, 464 0x9034, 0xffffffff, 0x00000100, 465 0x9038, 0xffffffff, 0x00000100, 466 0x903c, 0xffffffff, 0x00000100, 467 0x9040, 0xffffffff, 0x00000100, 468 0xa200, 0xffffffff, 0x00000100, 469 0xa204, 0xffffffff, 0x00000100, 470 0xa208, 0xffffffff, 0x00000100, 471 0xa20c, 0xffffffff, 0x00000100, 472 0x971c, 0xffffffff, 0x00000100, 473 0x977c, 0xffffffff, 0x00000100, 474 0x3f80, 0xffffffff, 0x00000100, 475 0xa210, 0xffffffff, 0x00000100, 476 0xa214, 0xffffffff, 0x00000100, 477 0x4d8, 0xffffffff, 0x00000100, 478 0x9784, 0xffffffff, 0x00000100, 479 0x9698, 0xffffffff, 0x00000100, 480 0x4d4, 0xffffffff, 0x00000200, 481 0x30cc, 0xffffffff, 0x00000100, 482 0xd0c0, 0xffffffff, 0xff000100, 483 0x802c, 0xffffffff, 0x40000000, 484 0x915c, 0xffffffff, 0x00010000, 485 0x9160, 0xffffffff, 0x00030002, 486 0x9178, 0xffffffff, 0x00070000, 487 0x917c, 0xffffffff, 0x00030002, 488 0x9180, 0xffffffff, 0x00050004, 489 0x918c, 0xffffffff, 0x00010006, 490 0x9190, 0xffffffff, 0x00090008, 491 0x9194, 0xffffffff, 0x00070000, 492 0x9198, 0xffffffff, 0x00030002, 493 0x919c, 0xffffffff, 0x00050004, 494 0x91a8, 0xffffffff, 0x00010006, 495 0x91ac, 0xffffffff, 0x00090008, 496 0x91b0, 0xffffffff, 0x00070000, 497 0x91b4, 0xffffffff, 0x00030002, 498 0x91b8, 0xffffffff, 0x00050004, 499 0x91c4, 0xffffffff, 0x00010006, 500 0x91c8, 0xffffffff, 0x00090008, 501 0x91cc, 0xffffffff, 0x00070000, 502 0x91d0, 0xffffffff, 0x00030002, 503 0x91d4, 0xffffffff, 0x00050004, 504 0x91e0, 0xffffffff, 0x00010006, 505 0x91e4, 0xffffffff, 0x00090008, 506 0x91e8, 0xffffffff, 0x00000000, 507 0x91ec, 0xffffffff, 0x00070000, 508 0x91f0, 0xffffffff, 0x00030002, 509 0x91f4, 0xffffffff, 0x00050004, 510 0x9200, 0xffffffff, 0x00010006, 511 0x9204, 0xffffffff, 0x00090008, 512 0x9294, 0xffffffff, 0x00000000, 513 0x929c, 0xffffffff, 0x00000001, 514 0x802c, 0xffffffff, 0xc0000000 515}; 516 517static const u32 cedar_golden_registers[] = 518{ 519 0x3f90, 0xffff0000, 0xff000000, 520 0x9148, 0xffff0000, 0xff000000, 521 0x3f94, 0xffff0000, 0xff000000, 522 0x914c, 0xffff0000, 0xff000000, 523 0x9b7c, 0xffffffff, 0x00000000, 524 0x8a14, 0xffffffff, 0x00000007, 525 0x8b10, 0xffffffff, 0x00000000, 526 0x960c, 0xffffffff, 0x54763210, 527 0x88c4, 0xffffffff, 0x000000c2, 528 0x88d4, 0xffffffff, 0x00000000, 529 0x8974, 0xffffffff, 0x00000000, 530 0xc78, 0x00000080, 0x00000080, 531 0x5eb4, 0xffffffff, 0x00000002, 532 0x5e78, 0xffffffff, 0x001000f0, 533 0x6104, 0x01000300, 0x00000000, 534 0x5bc0, 0x00300000, 0x00000000, 535 0x7030, 0xffffffff, 0x00000011, 536 0x7c30, 0xffffffff, 0x00000011, 537 0x10830, 0xffffffff, 0x00000011, 538 0x11430, 0xffffffff, 0x00000011, 539 0xd02c, 0xffffffff, 0x08421000, 540 0x240c, 0xffffffff, 0x00000380, 541 0x8b24, 0xffffffff, 0x00ff0fff, 542 0x28a4c, 0x06000000, 0x06000000, 543 0x10c, 0x00000001, 0x00000001, 544 0x8d00, 0xffffffff, 0x100e4848, 545 0x8d04, 0xffffffff, 0x00164745, 546 0x8c00, 0xffffffff, 0xe4000003, 547 0x8c04, 0xffffffff, 0x40600060, 548 0x8c08, 0xffffffff, 0x001c001c, 549 0x8cf0, 0xffffffff, 0x08e00410, 550 0x8c20, 0xffffffff, 0x00800080, 551 0x8c24, 0xffffffff, 0x00800080, 552 0x8c18, 0xffffffff, 0x20202078, 553 0x8c1c, 0xffffffff, 0x00001010, 554 0x28350, 0xffffffff, 0x00000000, 555 0xa008, 0xffffffff, 0x00010000, 556 0x5c4, 0xffffffff, 0x00000001, 557 0x9508, 0xffffffff, 0x00000002 558}; 559 560static const u32 cedar_mgcg_init[] = 561{ 562 0x802c, 0xffffffff, 0xc0000000, 563 0x5448, 0xffffffff, 0x00000100, 564 0x55e4, 0xffffffff, 0x00000100, 565 0x160c, 0xffffffff, 0x00000100, 566 0x5644, 0xffffffff, 0x00000100, 567 0xc164, 0xffffffff, 0x00000100, 568 0x8a18, 0xffffffff, 0x00000100, 569 0x897c, 0xffffffff, 0x06000100, 570 0x8b28, 0xffffffff, 0x00000100, 571 0x9144, 0xffffffff, 0x00000100, 572 0x9a60, 0xffffffff, 0x00000100, 573 0x9868, 0xffffffff, 0x00000100, 574 0x8d58, 0xffffffff, 0x00000100, 575 0x9510, 0xffffffff, 0x00000100, 576 0x949c, 0xffffffff, 0x00000100, 577 0x9654, 0xffffffff, 0x00000100, 578 0x9030, 0xffffffff, 0x00000100, 579 0x9034, 0xffffffff, 0x00000100, 580 0x9038, 0xffffffff, 0x00000100, 581 0x903c, 0xffffffff, 0x00000100, 582 0x9040, 0xffffffff, 0x00000100, 583 0xa200, 0xffffffff, 0x00000100, 584 0xa204, 0xffffffff, 0x00000100, 585 0xa208, 0xffffffff, 0x00000100, 586 0xa20c, 0xffffffff, 0x00000100, 587 0x971c, 0xffffffff, 0x00000100, 588 0x977c, 0xffffffff, 0x00000100, 589 0x3f80, 0xffffffff, 0x00000100, 590 0xa210, 0xffffffff, 0x00000100, 591 0xa214, 0xffffffff, 0x00000100, 592 0x4d8, 0xffffffff, 0x00000100, 593 0x9784, 0xffffffff, 0x00000100, 594 0x9698, 0xffffffff, 0x00000100, 595 0x4d4, 0xffffffff, 0x00000200, 596 0x30cc, 0xffffffff, 0x00000100, 597 0xd0c0, 0xffffffff, 0xff000100, 598 0x802c, 0xffffffff, 0x40000000, 599 0x915c, 0xffffffff, 0x00010000, 600 0x9178, 0xffffffff, 0x00050000, 601 0x917c, 0xffffffff, 0x00030002, 602 0x918c, 0xffffffff, 0x00010004, 603 0x9190, 0xffffffff, 0x00070006, 604 0x9194, 0xffffffff, 0x00050000, 605 0x9198, 0xffffffff, 0x00030002, 606 0x91a8, 0xffffffff, 0x00010004, 607 0x91ac, 0xffffffff, 0x00070006, 608 0x91e8, 0xffffffff, 0x00000000, 609 0x9294, 0xffffffff, 0x00000000, 610 0x929c, 0xffffffff, 0x00000001, 611 0x802c, 0xffffffff, 0xc0000000 612}; 613 614static const u32 juniper_mgcg_init[] = 615{ 616 0x802c, 0xffffffff, 0xc0000000, 617 0x5448, 0xffffffff, 0x00000100, 618 0x55e4, 0xffffffff, 0x00000100, 619 0x160c, 0xffffffff, 0x00000100, 620 0x5644, 0xffffffff, 0x00000100, 621 0xc164, 0xffffffff, 0x00000100, 622 0x8a18, 0xffffffff, 0x00000100, 623 0x897c, 0xffffffff, 0x06000100, 624 0x8b28, 0xffffffff, 0x00000100, 625 0x9144, 0xffffffff, 0x00000100, 626 0x9a60, 0xffffffff, 0x00000100, 627 0x9868, 0xffffffff, 0x00000100, 628 0x8d58, 0xffffffff, 0x00000100, 629 0x9510, 0xffffffff, 0x00000100, 630 0x949c, 0xffffffff, 0x00000100, 631 0x9654, 0xffffffff, 0x00000100, 632 0x9030, 0xffffffff, 0x00000100, 633 0x9034, 0xffffffff, 0x00000100, 634 0x9038, 0xffffffff, 0x00000100, 635 0x903c, 0xffffffff, 0x00000100, 636 0x9040, 0xffffffff, 0x00000100, 637 0xa200, 0xffffffff, 0x00000100, 638 0xa204, 0xffffffff, 0x00000100, 639 0xa208, 0xffffffff, 0x00000100, 640 0xa20c, 0xffffffff, 0x00000100, 641 0x971c, 0xffffffff, 0x00000100, 642 0xd0c0, 0xffffffff, 0xff000100, 643 0x802c, 0xffffffff, 0x40000000, 644 0x915c, 0xffffffff, 0x00010000, 645 0x9160, 0xffffffff, 0x00030002, 646 0x9178, 0xffffffff, 0x00070000, 647 0x917c, 0xffffffff, 0x00030002, 648 0x9180, 0xffffffff, 0x00050004, 649 0x918c, 0xffffffff, 0x00010006, 650 0x9190, 0xffffffff, 0x00090008, 651 0x9194, 0xffffffff, 0x00070000, 652 0x9198, 0xffffffff, 0x00030002, 653 0x919c, 0xffffffff, 0x00050004, 654 0x91a8, 0xffffffff, 0x00010006, 655 0x91ac, 0xffffffff, 0x00090008, 656 0x91b0, 0xffffffff, 0x00070000, 657 0x91b4, 0xffffffff, 0x00030002, 658 0x91b8, 0xffffffff, 0x00050004, 659 0x91c4, 0xffffffff, 0x00010006, 660 0x91c8, 0xffffffff, 0x00090008, 661 0x91cc, 0xffffffff, 0x00070000, 662 0x91d0, 0xffffffff, 0x00030002, 663 0x91d4, 0xffffffff, 0x00050004, 664 0x91e0, 0xffffffff, 0x00010006, 665 0x91e4, 0xffffffff, 0x00090008, 666 0x91e8, 0xffffffff, 0x00000000, 667 0x91ec, 0xffffffff, 0x00070000, 668 0x91f0, 0xffffffff, 0x00030002, 669 0x91f4, 0xffffffff, 0x00050004, 670 0x9200, 0xffffffff, 0x00010006, 671 0x9204, 0xffffffff, 0x00090008, 672 0x9208, 0xffffffff, 0x00070000, 673 0x920c, 0xffffffff, 0x00030002, 674 0x9210, 0xffffffff, 0x00050004, 675 0x921c, 0xffffffff, 0x00010006, 676 0x9220, 0xffffffff, 0x00090008, 677 0x9224, 0xffffffff, 0x00070000, 678 0x9228, 0xffffffff, 0x00030002, 679 0x922c, 0xffffffff, 0x00050004, 680 0x9238, 0xffffffff, 0x00010006, 681 0x923c, 0xffffffff, 0x00090008, 682 0x9240, 0xffffffff, 0x00070000, 683 0x9244, 0xffffffff, 0x00030002, 684 0x9248, 0xffffffff, 0x00050004, 685 0x9254, 0xffffffff, 0x00010006, 686 0x9258, 0xffffffff, 0x00090008, 687 0x925c, 0xffffffff, 0x00070000, 688 0x9260, 0xffffffff, 0x00030002, 689 0x9264, 0xffffffff, 0x00050004, 690 0x9270, 0xffffffff, 0x00010006, 691 0x9274, 0xffffffff, 0x00090008, 692 0x9278, 0xffffffff, 0x00070000, 693 0x927c, 0xffffffff, 0x00030002, 694 0x9280, 0xffffffff, 0x00050004, 695 0x928c, 0xffffffff, 0x00010006, 696 0x9290, 0xffffffff, 0x00090008, 697 0x9294, 0xffffffff, 0x00000000, 698 0x929c, 0xffffffff, 0x00000001, 699 0x802c, 0xffffffff, 0xc0000000, 700 0x977c, 0xffffffff, 0x00000100, 701 0x3f80, 0xffffffff, 0x00000100, 702 0xa210, 0xffffffff, 0x00000100, 703 0xa214, 0xffffffff, 0x00000100, 704 0x4d8, 0xffffffff, 0x00000100, 705 0x9784, 0xffffffff, 0x00000100, 706 0x9698, 0xffffffff, 0x00000100, 707 0x4d4, 0xffffffff, 0x00000200, 708 0x30cc, 0xffffffff, 0x00000100, 709 0x802c, 0xffffffff, 0xc0000000 710}; 711 712static const u32 supersumo_golden_registers[] = 713{ 714 0x5eb4, 0xffffffff, 0x00000002, 715 0x5c4, 0xffffffff, 0x00000001, 716 0x7030, 0xffffffff, 0x00000011, 717 0x7c30, 0xffffffff, 0x00000011, 718 0x6104, 0x01000300, 0x00000000, 719 0x5bc0, 0x00300000, 0x00000000, 720 0x8c04, 0xffffffff, 0x40600060, 721 0x8c08, 0xffffffff, 0x001c001c, 722 0x8c20, 0xffffffff, 0x00800080, 723 0x8c24, 0xffffffff, 0x00800080, 724 0x8c18, 0xffffffff, 0x20202078, 725 0x8c1c, 0xffffffff, 0x00001010, 726 0x918c, 0xffffffff, 0x00010006, 727 0x91a8, 0xffffffff, 0x00010006, 728 0x91c4, 0xffffffff, 0x00010006, 729 0x91e0, 0xffffffff, 0x00010006, 730 0x9200, 0xffffffff, 0x00010006, 731 0x9150, 0xffffffff, 0x6e944040, 732 0x917c, 0xffffffff, 0x00030002, 733 0x9180, 0xffffffff, 0x00050004, 734 0x9198, 0xffffffff, 0x00030002, 735 0x919c, 0xffffffff, 0x00050004, 736 0x91b4, 0xffffffff, 0x00030002, 737 0x91b8, 0xffffffff, 0x00050004, 738 0x91d0, 0xffffffff, 0x00030002, 739 0x91d4, 0xffffffff, 0x00050004, 740 0x91f0, 0xffffffff, 0x00030002, 741 0x91f4, 0xffffffff, 0x00050004, 742 0x915c, 0xffffffff, 0x00010000, 743 0x9160, 0xffffffff, 0x00030002, 744 0x3f90, 0xffff0000, 0xff000000, 745 0x9178, 0xffffffff, 0x00070000, 746 0x9194, 0xffffffff, 0x00070000, 747 0x91b0, 0xffffffff, 0x00070000, 748 0x91cc, 0xffffffff, 0x00070000, 749 0x91ec, 0xffffffff, 0x00070000, 750 0x9148, 0xffff0000, 0xff000000, 751 0x9190, 0xffffffff, 0x00090008, 752 0x91ac, 0xffffffff, 0x00090008, 753 0x91c8, 0xffffffff, 0x00090008, 754 0x91e4, 0xffffffff, 0x00090008, 755 0x9204, 0xffffffff, 0x00090008, 756 0x3f94, 0xffff0000, 0xff000000, 757 0x914c, 0xffff0000, 0xff000000, 758 0x929c, 0xffffffff, 0x00000001, 759 0x8a18, 0xffffffff, 0x00000100, 760 0x8b28, 0xffffffff, 0x00000100, 761 0x9144, 0xffffffff, 0x00000100, 762 0x5644, 0xffffffff, 0x00000100, 763 0x9b7c, 0xffffffff, 0x00000000, 764 0x8030, 0xffffffff, 0x0000100a, 765 0x8a14, 0xffffffff, 0x00000007, 766 0x8b24, 0xffffffff, 0x00ff0fff, 767 0x8b10, 0xffffffff, 0x00000000, 768 0x28a4c, 0x06000000, 0x06000000, 769 0x4d8, 0xffffffff, 0x00000100, 770 0x913c, 0xffff000f, 0x0100000a, 771 0x960c, 0xffffffff, 0x54763210, 772 0x88c4, 0xffffffff, 0x000000c2, 773 0x88d4, 0xffffffff, 0x00000010, 774 0x8974, 0xffffffff, 0x00000000, 775 0xc78, 0x00000080, 0x00000080, 776 0x5e78, 0xffffffff, 0x001000f0, 777 0xd02c, 0xffffffff, 0x08421000, 778 0xa008, 0xffffffff, 0x00010000, 779 0x8d00, 0xffffffff, 0x100e4848, 780 0x8d04, 0xffffffff, 0x00164745, 781 0x8c00, 0xffffffff, 0xe4000003, 782 0x8cf0, 0x1fffffff, 0x08e00620, 783 0x28350, 0xffffffff, 0x00000000, 784 0x9508, 0xffffffff, 0x00000002 785}; 786 787static const u32 sumo_golden_registers[] = 788{ 789 0x900c, 0x00ffffff, 0x0017071f, 790 0x8c18, 0xffffffff, 0x10101060, 791 0x8c1c, 0xffffffff, 0x00001010, 792 0x8c30, 0x0000000f, 0x00000005, 793 0x9688, 0x0000000f, 0x00000007 794}; 795 796static const u32 wrestler_golden_registers[] = 797{ 798 0x5eb4, 0xffffffff, 0x00000002, 799 0x5c4, 0xffffffff, 0x00000001, 800 0x7030, 0xffffffff, 0x00000011, 801 0x7c30, 0xffffffff, 0x00000011, 802 0x6104, 0x01000300, 0x00000000, 803 0x5bc0, 0x00300000, 0x00000000, 804 0x918c, 0xffffffff, 0x00010006, 805 0x91a8, 0xffffffff, 0x00010006, 806 0x9150, 0xffffffff, 0x6e944040, 807 0x917c, 0xffffffff, 0x00030002, 808 0x9198, 0xffffffff, 0x00030002, 809 0x915c, 0xffffffff, 0x00010000, 810 0x3f90, 0xffff0000, 0xff000000, 811 0x9178, 0xffffffff, 0x00070000, 812 0x9194, 0xffffffff, 0x00070000, 813 0x9148, 0xffff0000, 0xff000000, 814 0x9190, 0xffffffff, 0x00090008, 815 0x91ac, 0xffffffff, 0x00090008, 816 0x3f94, 0xffff0000, 0xff000000, 817 0x914c, 0xffff0000, 0xff000000, 818 0x929c, 0xffffffff, 0x00000001, 819 0x8a18, 0xffffffff, 0x00000100, 820 0x8b28, 0xffffffff, 0x00000100, 821 0x9144, 0xffffffff, 0x00000100, 822 0x9b7c, 0xffffffff, 0x00000000, 823 0x8030, 0xffffffff, 0x0000100a, 824 0x8a14, 0xffffffff, 0x00000001, 825 0x8b24, 0xffffffff, 0x00ff0fff, 826 0x8b10, 0xffffffff, 0x00000000, 827 0x28a4c, 0x06000000, 0x06000000, 828 0x4d8, 0xffffffff, 0x00000100, 829 0x913c, 0xffff000f, 0x0100000a, 830 0x960c, 0xffffffff, 0x54763210, 831 0x88c4, 0xffffffff, 0x000000c2, 832 0x88d4, 0xffffffff, 0x00000010, 833 0x8974, 0xffffffff, 0x00000000, 834 0xc78, 0x00000080, 0x00000080, 835 0x5e78, 0xffffffff, 0x001000f0, 836 0xd02c, 0xffffffff, 0x08421000, 837 0xa008, 0xffffffff, 0x00010000, 838 0x8d00, 0xffffffff, 0x100e4848, 839 0x8d04, 0xffffffff, 0x00164745, 840 0x8c00, 0xffffffff, 0xe4000003, 841 0x8cf0, 0x1fffffff, 0x08e00410, 842 0x28350, 0xffffffff, 0x00000000, 843 0x9508, 0xffffffff, 0x00000002, 844 0x900c, 0xffffffff, 0x0017071f, 845 0x8c18, 0xffffffff, 0x10101060, 846 0x8c1c, 0xffffffff, 0x00001010 847}; 848 849static const u32 barts_golden_registers[] = 850{ 851 0x5eb4, 0xffffffff, 0x00000002, 852 0x5e78, 0x8f311ff1, 0x001000f0, 853 0x3f90, 0xffff0000, 0xff000000, 854 0x9148, 0xffff0000, 0xff000000, 855 0x3f94, 0xffff0000, 0xff000000, 856 0x914c, 0xffff0000, 0xff000000, 857 0xc78, 0x00000080, 0x00000080, 858 0xbd4, 0x70073777, 0x00010001, 859 0xd02c, 0xbfffff1f, 0x08421000, 860 0xd0b8, 0x03773777, 0x02011003, 861 0x5bc0, 0x00200000, 0x50100000, 862 0x98f8, 0x33773777, 0x02011003, 863 0x98fc, 0xffffffff, 0x76543210, 864 0x7030, 0x31000311, 0x00000011, 865 0x2f48, 0x00000007, 0x02011003, 866 0x6b28, 0x00000010, 0x00000012, 867 0x7728, 0x00000010, 0x00000012, 868 0x10328, 0x00000010, 0x00000012, 869 0x10f28, 0x00000010, 0x00000012, 870 0x11b28, 0x00000010, 0x00000012, 871 0x12728, 0x00000010, 0x00000012, 872 0x240c, 0x000007ff, 0x00000380, 873 0x8a14, 0xf000001f, 0x00000007, 874 0x8b24, 0x3fff3fff, 0x00ff0fff, 875 0x8b10, 0x0000ff0f, 0x00000000, 876 0x28a4c, 0x07ffffff, 0x06000000, 877 0x10c, 0x00000001, 0x00010003, 878 0xa02c, 0xffffffff, 0x0000009b, 879 0x913c, 0x0000000f, 0x0100000a, 880 0x8d00, 0xffff7f7f, 0x100e4848, 881 0x8d04, 0x00ffffff, 0x00164745, 882 0x8c00, 0xfffc0003, 0xe4000003, 883 0x8c04, 0xf8ff00ff, 0x40600060, 884 0x8c08, 0x00ff00ff, 0x001c001c, 885 0x8cf0, 0x1fff1fff, 0x08e00620, 886 0x8c20, 0x0fff0fff, 0x00800080, 887 0x8c24, 0x0fff0fff, 0x00800080, 888 0x8c18, 0xffffffff, 0x20202078, 889 0x8c1c, 0x0000ffff, 0x00001010, 890 0x28350, 0x00000f01, 0x00000000, 891 0x9508, 0x3700001f, 0x00000002, 892 0x960c, 0xffffffff, 0x54763210, 893 0x88c4, 0x001f3ae3, 0x000000c2, 894 0x88d4, 0x0000001f, 0x00000010, 895 0x8974, 0xffffffff, 0x00000000 896}; 897 898static const u32 turks_golden_registers[] = 899{ 900 0x5eb4, 0xffffffff, 0x00000002, 901 0x5e78, 0x8f311ff1, 0x001000f0, 902 0x8c8, 0x00003000, 0x00001070, 903 0x8cc, 0x000fffff, 0x00040035, 904 0x3f90, 0xffff0000, 0xfff00000, 905 0x9148, 0xffff0000, 0xfff00000, 906 0x3f94, 0xffff0000, 0xfff00000, 907 0x914c, 0xffff0000, 0xfff00000, 908 0xc78, 0x00000080, 0x00000080, 909 0xbd4, 0x00073007, 0x00010002, 910 0xd02c, 0xbfffff1f, 0x08421000, 911 0xd0b8, 0x03773777, 0x02010002, 912 0x5bc0, 0x00200000, 0x50100000, 913 0x98f8, 0x33773777, 0x00010002, 914 0x98fc, 0xffffffff, 0x33221100, 915 0x7030, 0x31000311, 0x00000011, 916 0x2f48, 0x33773777, 0x00010002, 917 0x6b28, 0x00000010, 0x00000012, 918 0x7728, 0x00000010, 0x00000012, 919 0x10328, 0x00000010, 0x00000012, 920 0x10f28, 0x00000010, 0x00000012, 921 0x11b28, 0x00000010, 0x00000012, 922 0x12728, 0x00000010, 0x00000012, 923 0x240c, 0x000007ff, 0x00000380, 924 0x8a14, 0xf000001f, 0x00000007, 925 0x8b24, 0x3fff3fff, 0x00ff0fff, 926 0x8b10, 0x0000ff0f, 0x00000000, 927 0x28a4c, 0x07ffffff, 0x06000000, 928 0x10c, 0x00000001, 0x00010003, 929 0xa02c, 0xffffffff, 0x0000009b, 930 0x913c, 0x0000000f, 0x0100000a, 931 0x8d00, 0xffff7f7f, 0x100e4848, 932 0x8d04, 0x00ffffff, 0x00164745, 933 0x8c00, 0xfffc0003, 0xe4000003, 934 0x8c04, 0xf8ff00ff, 0x40600060, 935 0x8c08, 0x00ff00ff, 0x001c001c, 936 0x8cf0, 0x1fff1fff, 0x08e00410, 937 0x8c20, 0x0fff0fff, 0x00800080, 938 0x8c24, 0x0fff0fff, 0x00800080, 939 0x8c18, 0xffffffff, 0x20202078, 940 0x8c1c, 0x0000ffff, 0x00001010, 941 0x28350, 0x00000f01, 0x00000000, 942 0x9508, 0x3700001f, 0x00000002, 943 0x960c, 0xffffffff, 0x54763210, 944 0x88c4, 0x001f3ae3, 0x000000c2, 945 0x88d4, 0x0000001f, 0x00000010, 946 0x8974, 0xffffffff, 0x00000000 947}; 948 949static const u32 caicos_golden_registers[] = 950{ 951 0x5eb4, 0xffffffff, 0x00000002, 952 0x5e78, 0x8f311ff1, 0x001000f0, 953 0x8c8, 0x00003420, 0x00001450, 954 0x8cc, 0x000fffff, 0x00040035, 955 0x3f90, 0xffff0000, 0xfffc0000, 956 0x9148, 0xffff0000, 0xfffc0000, 957 0x3f94, 0xffff0000, 0xfffc0000, 958 0x914c, 0xffff0000, 0xfffc0000, 959 0xc78, 0x00000080, 0x00000080, 960 0xbd4, 0x00073007, 0x00010001, 961 0xd02c, 0xbfffff1f, 0x08421000, 962 0xd0b8, 0x03773777, 0x02010001, 963 0x5bc0, 0x00200000, 0x50100000, 964 0x98f8, 0x33773777, 0x02010001, 965 0x98fc, 0xffffffff, 0x33221100, 966 0x7030, 0x31000311, 0x00000011, 967 0x2f48, 0x33773777, 0x02010001, 968 0x6b28, 0x00000010, 0x00000012, 969 0x7728, 0x00000010, 0x00000012, 970 0x10328, 0x00000010, 0x00000012, 971 0x10f28, 0x00000010, 0x00000012, 972 0x11b28, 0x00000010, 0x00000012, 973 0x12728, 0x00000010, 0x00000012, 974 0x240c, 0x000007ff, 0x00000380, 975 0x8a14, 0xf000001f, 0x00000001, 976 0x8b24, 0x3fff3fff, 0x00ff0fff, 977 0x8b10, 0x0000ff0f, 0x00000000, 978 0x28a4c, 0x07ffffff, 0x06000000, 979 0x10c, 0x00000001, 0x00010003, 980 0xa02c, 0xffffffff, 0x0000009b, 981 0x913c, 0x0000000f, 0x0100000a, 982 0x8d00, 0xffff7f7f, 0x100e4848, 983 0x8d04, 0x00ffffff, 0x00164745, 984 0x8c00, 0xfffc0003, 0xe4000003, 985 0x8c04, 0xf8ff00ff, 0x40600060, 986 0x8c08, 0x00ff00ff, 0x001c001c, 987 0x8cf0, 0x1fff1fff, 0x08e00410, 988 0x8c20, 0x0fff0fff, 0x00800080, 989 0x8c24, 0x0fff0fff, 0x00800080, 990 0x8c18, 0xffffffff, 0x20202078, 991 0x8c1c, 0x0000ffff, 0x00001010, 992 0x28350, 0x00000f01, 0x00000000, 993 0x9508, 0x3700001f, 0x00000002, 994 0x960c, 0xffffffff, 0x54763210, 995 0x88c4, 0x001f3ae3, 0x000000c2, 996 0x88d4, 0x0000001f, 0x00000010, 997 0x8974, 0xffffffff, 0x00000000 998}; 999 1000static void evergreen_init_golden_registers(struct radeon_device *rdev) 1001{ 1002 switch (rdev->family) { 1003 case CHIP_CYPRESS: 1004 case CHIP_HEMLOCK: 1005 radeon_program_register_sequence(rdev, 1006 evergreen_golden_registers, 1007 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1008 radeon_program_register_sequence(rdev, 1009 evergreen_golden_registers2, 1010 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1011 radeon_program_register_sequence(rdev, 1012 cypress_mgcg_init, 1013 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 1014 break; 1015 case CHIP_JUNIPER: 1016 radeon_program_register_sequence(rdev, 1017 evergreen_golden_registers, 1018 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1019 radeon_program_register_sequence(rdev, 1020 evergreen_golden_registers2, 1021 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1022 radeon_program_register_sequence(rdev, 1023 juniper_mgcg_init, 1024 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 1025 break; 1026 case CHIP_REDWOOD: 1027 radeon_program_register_sequence(rdev, 1028 evergreen_golden_registers, 1029 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1030 radeon_program_register_sequence(rdev, 1031 evergreen_golden_registers2, 1032 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1033 radeon_program_register_sequence(rdev, 1034 redwood_mgcg_init, 1035 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 1036 break; 1037 case CHIP_CEDAR: 1038 radeon_program_register_sequence(rdev, 1039 cedar_golden_registers, 1040 (const u32)ARRAY_SIZE(cedar_golden_registers)); 1041 radeon_program_register_sequence(rdev, 1042 evergreen_golden_registers2, 1043 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1044 radeon_program_register_sequence(rdev, 1045 cedar_mgcg_init, 1046 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 1047 break; 1048 case CHIP_PALM: 1049 radeon_program_register_sequence(rdev, 1050 wrestler_golden_registers, 1051 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 1052 break; 1053 case CHIP_SUMO: 1054 radeon_program_register_sequence(rdev, 1055 supersumo_golden_registers, 1056 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1057 break; 1058 case CHIP_SUMO2: 1059 radeon_program_register_sequence(rdev, 1060 supersumo_golden_registers, 1061 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1062 radeon_program_register_sequence(rdev, 1063 sumo_golden_registers, 1064 (const u32)ARRAY_SIZE(sumo_golden_registers)); 1065 break; 1066 case CHIP_BARTS: 1067 radeon_program_register_sequence(rdev, 1068 barts_golden_registers, 1069 (const u32)ARRAY_SIZE(barts_golden_registers)); 1070 break; 1071 case CHIP_TURKS: 1072 radeon_program_register_sequence(rdev, 1073 turks_golden_registers, 1074 (const u32)ARRAY_SIZE(turks_golden_registers)); 1075 break; 1076 case CHIP_CAICOS: 1077 radeon_program_register_sequence(rdev, 1078 caicos_golden_registers, 1079 (const u32)ARRAY_SIZE(caicos_golden_registers)); 1080 break; 1081 default: 1082 break; 1083 } 1084} 1085 1086/** 1087 * evergreen_get_allowed_info_register - fetch the register for the info ioctl 1088 * 1089 * @rdev: radeon_device pointer 1090 * @reg: register offset in bytes 1091 * @val: register value 1092 * 1093 * Returns 0 for success or -EINVAL for an invalid register 1094 * 1095 */ 1096int evergreen_get_allowed_info_register(struct radeon_device *rdev, 1097 u32 reg, u32 *val) 1098{ 1099 switch (reg) { 1100 case GRBM_STATUS: 1101 case GRBM_STATUS_SE0: 1102 case GRBM_STATUS_SE1: 1103 case SRBM_STATUS: 1104 case SRBM_STATUS2: 1105 case DMA_STATUS_REG: 1106 case UVD_STATUS: 1107 *val = RREG32(reg); 1108 return 0; 1109 default: 1110 return -EINVAL; 1111 } 1112} 1113 1114void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 1115 unsigned *bankh, unsigned *mtaspect, 1116 unsigned *tile_split) 1117{ 1118 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 1119 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 1120 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1121 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1122 switch (*bankw) { 1123 default: 1124 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1125 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1126 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1127 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1128 } 1129 switch (*bankh) { 1130 default: 1131 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1132 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1133 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1134 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1135 } 1136 switch (*mtaspect) { 1137 default: 1138 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1139 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1140 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1141 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1142 } 1143} 1144 1145static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1146 u32 cntl_reg, u32 status_reg) 1147{ 1148 int r, i; 1149 struct atom_clock_dividers dividers; 1150 1151 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1152 clock, false, ÷rs); 1153 if (r) 1154 return r; 1155 1156 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1157 1158 for (i = 0; i < 100; i++) { 1159 if (RREG32(status_reg) & DCLK_STATUS) 1160 break; 1161 mdelay(10); 1162 } 1163 if (i == 100) 1164 return -ETIMEDOUT; 1165 1166 return 0; 1167} 1168 1169int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1170{ 1171 int r = 0; 1172 u32 cg_scratch = RREG32(CG_SCRATCH1); 1173 1174 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1175 if (r) 1176 goto done; 1177 cg_scratch &= 0xffff0000; 1178 cg_scratch |= vclk / 100; /* Mhz */ 1179 1180 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1181 if (r) 1182 goto done; 1183 cg_scratch &= 0x0000ffff; 1184 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1185 1186done: 1187 WREG32(CG_SCRATCH1, cg_scratch); 1188 1189 return r; 1190} 1191 1192int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1193{ 1194 /* start off with something large */ 1195 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1196 int r; 1197 1198 /* bypass vclk and dclk with bclk */ 1199 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1200 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1201 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1202 1203 /* put PLL in bypass mode */ 1204 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1205 1206 if (!vclk || !dclk) { 1207 /* keep the Bypass mode, put PLL to sleep */ 1208 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1209 return 0; 1210 } 1211 1212 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1213 16384, 0x03FFFFFF, 0, 128, 5, 1214 &fb_div, &vclk_div, &dclk_div); 1215 if (r) 1216 return r; 1217 1218 /* set VCO_MODE to 1 */ 1219 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1220 1221 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1222 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1223 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1224 1225 /* deassert UPLL_RESET */ 1226 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1227 1228 mdelay(1); 1229 1230 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1231 if (r) 1232 return r; 1233 1234 /* assert UPLL_RESET again */ 1235 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1236 1237 /* disable spread spectrum. */ 1238 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1239 1240 /* set feedback divider */ 1241 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1242 1243 /* set ref divider to 0 */ 1244 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1245 1246 if (fb_div < 307200) 1247 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1248 else 1249 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1250 1251 /* set PDIV_A and PDIV_B */ 1252 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1253 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1254 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1255 1256 /* give the PLL some time to settle */ 1257 mdelay(15); 1258 1259 /* deassert PLL_RESET */ 1260 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1261 1262 mdelay(15); 1263 1264 /* switch from bypass mode to normal mode */ 1265 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1266 1267 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1268 if (r) 1269 return r; 1270 1271 /* switch VCLK and DCLK selection */ 1272 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1273 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1274 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1275 1276 mdelay(100); 1277 1278 return 0; 1279} 1280 1281void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1282{ 1283 int readrq; 1284 u16 v; 1285 1286 readrq = pcie_get_readrq(rdev->pdev); 1287 v = ffs(readrq) - 8; 1288 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1289 * to avoid hangs or perfomance issues 1290 */ 1291 if ((v == 0) || (v == 6) || (v == 7)) 1292 pcie_set_readrq(rdev->pdev, 512); 1293} 1294 1295void dce4_program_fmt(struct drm_encoder *encoder) 1296{ 1297 struct drm_device *dev = encoder->dev; 1298 struct radeon_device *rdev = dev->dev_private; 1299 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1300 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1301 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1302 int bpc = 0; 1303 u32 tmp = 0; 1304 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1305 1306 if (connector) { 1307 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1308 bpc = radeon_get_monitor_bpc(connector); 1309 dither = radeon_connector->dither; 1310 } 1311 1312 /* LVDS/eDP FMT is set up by atom */ 1313 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1314 return; 1315 1316 /* not needed for analog */ 1317 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1318 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1319 return; 1320 1321 if (bpc == 0) 1322 return; 1323 1324 switch (bpc) { 1325 case 6: 1326 if (dither == RADEON_FMT_DITHER_ENABLE) 1327 /* XXX sort out optimal dither settings */ 1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1329 FMT_SPATIAL_DITHER_EN); 1330 else 1331 tmp |= FMT_TRUNCATE_EN; 1332 break; 1333 case 8: 1334 if (dither == RADEON_FMT_DITHER_ENABLE) 1335 /* XXX sort out optimal dither settings */ 1336 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1337 FMT_RGB_RANDOM_ENABLE | 1338 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1339 else 1340 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1341 break; 1342 case 10: 1343 default: 1344 /* not needed */ 1345 break; 1346 } 1347 1348 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1349} 1350 1351static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1352{ 1353 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1354 return true; 1355 else 1356 return false; 1357} 1358 1359static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1360{ 1361 u32 pos1, pos2; 1362 1363 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1364 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1365 1366 if (pos1 != pos2) 1367 return true; 1368 else 1369 return false; 1370} 1371 1372/** 1373 * dce4_wait_for_vblank - vblank wait asic callback. 1374 * 1375 * @rdev: radeon_device pointer 1376 * @crtc: crtc to wait for vblank on 1377 * 1378 * Wait for vblank on the requested crtc (evergreen+). 1379 */ 1380void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1381{ 1382 unsigned i = 0; 1383 1384 if (crtc >= rdev->num_crtc) 1385 return; 1386 1387 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1388 return; 1389 1390 /* depending on when we hit vblank, we may be close to active; if so, 1391 * wait for another frame. 1392 */ 1393 while (dce4_is_in_vblank(rdev, crtc)) { 1394 if (i++ % 100 == 0) { 1395 if (!dce4_is_counter_moving(rdev, crtc)) 1396 break; 1397 } 1398 } 1399 1400 while (!dce4_is_in_vblank(rdev, crtc)) { 1401 if (i++ % 100 == 0) { 1402 if (!dce4_is_counter_moving(rdev, crtc)) 1403 break; 1404 } 1405 } 1406} 1407 1408/** 1409 * evergreen_page_flip - pageflip callback. 1410 * 1411 * @rdev: radeon_device pointer 1412 * @crtc_id: crtc to cleanup pageflip on 1413 * @crtc_base: new address of the crtc (GPU MC address) 1414 * 1415 * Triggers the actual pageflip by updating the primary 1416 * surface base address (evergreen+). 1417 */ 1418void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, 1419 bool async) 1420{ 1421 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1422 1423 /* update the scanout addresses */ 1424 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, 1425 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); 1426 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1427 upper_32_bits(crtc_base)); 1428 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1429 (u32)crtc_base); 1430 /* post the write */ 1431 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset); 1432} 1433 1434/** 1435 * evergreen_page_flip_pending - check if page flip is still pending 1436 * 1437 * @rdev: radeon_device pointer 1438 * @crtc_id: crtc to check 1439 * 1440 * Returns the current update pending status. 1441 */ 1442bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1443{ 1444 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1445 1446 /* Return current update_pending status: */ 1447 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1448 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1449} 1450 1451/* get temperature in millidegrees */ 1452int evergreen_get_temp(struct radeon_device *rdev) 1453{ 1454 u32 temp, toffset; 1455 int actual_temp = 0; 1456 1457 if (rdev->family == CHIP_JUNIPER) { 1458 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1459 TOFFSET_SHIFT; 1460 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1461 TS0_ADC_DOUT_SHIFT; 1462 1463 if (toffset & 0x100) 1464 actual_temp = temp / 2 - (0x200 - toffset); 1465 else 1466 actual_temp = temp / 2 + toffset; 1467 1468 actual_temp = actual_temp * 1000; 1469 1470 } else { 1471 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1472 ASIC_T_SHIFT; 1473 1474 if (temp & 0x400) 1475 actual_temp = -256; 1476 else if (temp & 0x200) 1477 actual_temp = 255; 1478 else if (temp & 0x100) { 1479 actual_temp = temp & 0x1ff; 1480 actual_temp |= ~0x1ff; 1481 } else 1482 actual_temp = temp & 0xff; 1483 1484 actual_temp = (actual_temp * 1000) / 2; 1485 } 1486 1487 return actual_temp; 1488} 1489 1490int sumo_get_temp(struct radeon_device *rdev) 1491{ 1492 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1493 int actual_temp = temp - 49; 1494 1495 return actual_temp * 1000; 1496} 1497 1498/** 1499 * sumo_pm_init_profile - Initialize power profiles callback. 1500 * 1501 * @rdev: radeon_device pointer 1502 * 1503 * Initialize the power states used in profile mode 1504 * (sumo, trinity, SI). 1505 * Used for profile mode only. 1506 */ 1507void sumo_pm_init_profile(struct radeon_device *rdev) 1508{ 1509 int idx; 1510 1511 /* default */ 1512 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1513 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1514 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1516 1517 /* low,mid sh/mh */ 1518 if (rdev->flags & RADEON_IS_MOBILITY) 1519 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1520 else 1521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1522 1523 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1524 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1525 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1527 1528 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1529 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1530 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1532 1533 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1534 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1535 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1537 1538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1542 1543 /* high sh/mh */ 1544 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1545 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1546 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1547 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1549 rdev->pm.power_state[idx].num_clock_modes - 1; 1550 1551 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1552 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1553 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1555 rdev->pm.power_state[idx].num_clock_modes - 1; 1556} 1557 1558/** 1559 * btc_pm_init_profile - Initialize power profiles callback. 1560 * 1561 * @rdev: radeon_device pointer 1562 * 1563 * Initialize the power states used in profile mode 1564 * (BTC, cayman). 1565 * Used for profile mode only. 1566 */ 1567void btc_pm_init_profile(struct radeon_device *rdev) 1568{ 1569 int idx; 1570 1571 /* default */ 1572 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1573 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1574 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1576 /* starting with BTC, there is one state that is used for both 1577 * MH and SH. Difference is that we always use the high clock index for 1578 * mclk. 1579 */ 1580 if (rdev->flags & RADEON_IS_MOBILITY) 1581 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1582 else 1583 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1584 /* low sh */ 1585 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1586 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1587 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1589 /* mid sh */ 1590 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1591 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1592 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1594 /* high sh */ 1595 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1596 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1597 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1599 /* low mh */ 1600 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1601 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1602 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1604 /* mid mh */ 1605 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1606 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1607 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1609 /* high mh */ 1610 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1611 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1612 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1614} 1615 1616/** 1617 * evergreen_pm_misc - set additional pm hw parameters callback. 1618 * 1619 * @rdev: radeon_device pointer 1620 * 1621 * Set non-clock parameters associated with a power state 1622 * (voltage, etc.) (evergreen+). 1623 */ 1624void evergreen_pm_misc(struct radeon_device *rdev) 1625{ 1626 int req_ps_idx = rdev->pm.requested_power_state_index; 1627 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1628 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1629 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1630 1631 if (voltage->type == VOLTAGE_SW) { 1632 /* 0xff0x are flags rather then an actual voltage */ 1633 if ((voltage->voltage & 0xff00) == 0xff00) 1634 return; 1635 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1636 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1637 rdev->pm.current_vddc = voltage->voltage; 1638 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1639 } 1640 1641 /* starting with BTC, there is one state that is used for both 1642 * MH and SH. Difference is that we always use the high clock index for 1643 * mclk and vddci. 1644 */ 1645 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1646 (rdev->family >= CHIP_BARTS) && 1647 rdev->pm.active_crtc_count && 1648 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1649 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1650 voltage = &rdev->pm.power_state[req_ps_idx]. 1651 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1652 1653 /* 0xff0x are flags rather then an actual voltage */ 1654 if ((voltage->vddci & 0xff00) == 0xff00) 1655 return; 1656 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1657 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1658 rdev->pm.current_vddci = voltage->vddci; 1659 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1660 } 1661 } 1662} 1663 1664/** 1665 * evergreen_pm_prepare - pre-power state change callback. 1666 * 1667 * @rdev: radeon_device pointer 1668 * 1669 * Prepare for a power state change (evergreen+). 1670 */ 1671void evergreen_pm_prepare(struct radeon_device *rdev) 1672{ 1673 struct drm_device *ddev = rdev->ddev; 1674 struct drm_crtc *crtc; 1675 struct radeon_crtc *radeon_crtc; 1676 u32 tmp; 1677 1678 /* disable any active CRTCs */ 1679 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1680 radeon_crtc = to_radeon_crtc(crtc); 1681 if (radeon_crtc->enabled) { 1682 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1683 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1684 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1685 } 1686 } 1687} 1688 1689/** 1690 * evergreen_pm_finish - post-power state change callback. 1691 * 1692 * @rdev: radeon_device pointer 1693 * 1694 * Clean up after a power state change (evergreen+). 1695 */ 1696void evergreen_pm_finish(struct radeon_device *rdev) 1697{ 1698 struct drm_device *ddev = rdev->ddev; 1699 struct drm_crtc *crtc; 1700 struct radeon_crtc *radeon_crtc; 1701 u32 tmp; 1702 1703 /* enable any active CRTCs */ 1704 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1705 radeon_crtc = to_radeon_crtc(crtc); 1706 if (radeon_crtc->enabled) { 1707 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1708 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1709 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1710 } 1711 } 1712} 1713 1714/** 1715 * evergreen_hpd_sense - hpd sense callback. 1716 * 1717 * @rdev: radeon_device pointer 1718 * @hpd: hpd (hotplug detect) pin 1719 * 1720 * Checks if a digital monitor is connected (evergreen+). 1721 * Returns true if connected, false if not connected. 1722 */ 1723bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1724{ 1725 if (hpd == RADEON_HPD_NONE) 1726 return false; 1727 1728 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE); 1729} 1730 1731/** 1732 * evergreen_hpd_set_polarity - hpd set polarity callback. 1733 * 1734 * @rdev: radeon_device pointer 1735 * @hpd: hpd (hotplug detect) pin 1736 * 1737 * Set the polarity of the hpd pin (evergreen+). 1738 */ 1739void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1740 enum radeon_hpd_id hpd) 1741{ 1742 bool connected = evergreen_hpd_sense(rdev, hpd); 1743 1744 if (hpd == RADEON_HPD_NONE) 1745 return; 1746 1747 if (connected) 1748 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY); 1749 else 1750 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY); 1751} 1752 1753/** 1754 * evergreen_hpd_init - hpd setup callback. 1755 * 1756 * @rdev: radeon_device pointer 1757 * 1758 * Setup the hpd pins used by the card (evergreen+). 1759 * Enable the pin, set the polarity, and enable the hpd interrupts. 1760 */ 1761void evergreen_hpd_init(struct radeon_device *rdev) 1762{ 1763 struct drm_device *dev = rdev->ddev; 1764 struct drm_connector *connector; 1765 unsigned enabled = 0; 1766 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1767 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1768 1769 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1770 enum radeon_hpd_id hpd = 1771 to_radeon_connector(connector)->hpd.hpd; 1772 1773 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1774 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1775 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1776 * aux dp channel on imac and help (but not completely fix) 1777 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1778 * also avoid interrupt storms during dpms. 1779 */ 1780 continue; 1781 } 1782 1783 if (hpd == RADEON_HPD_NONE) 1784 continue; 1785 1786 WREG32(DC_HPDx_CONTROL(hpd), tmp); 1787 enabled |= 1 << hpd; 1788 1789 radeon_hpd_set_polarity(rdev, hpd); 1790 } 1791 radeon_irq_kms_enable_hpd(rdev, enabled); 1792} 1793 1794/** 1795 * evergreen_hpd_fini - hpd tear down callback. 1796 * 1797 * @rdev: radeon_device pointer 1798 * 1799 * Tear down the hpd pins used by the card (evergreen+). 1800 * Disable the hpd interrupts. 1801 */ 1802void evergreen_hpd_fini(struct radeon_device *rdev) 1803{ 1804 struct drm_device *dev = rdev->ddev; 1805 struct drm_connector *connector; 1806 unsigned disabled = 0; 1807 1808 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1809 enum radeon_hpd_id hpd = 1810 to_radeon_connector(connector)->hpd.hpd; 1811 1812 if (hpd == RADEON_HPD_NONE) 1813 continue; 1814 1815 WREG32(DC_HPDx_CONTROL(hpd), 0); 1816 disabled |= 1 << hpd; 1817 } 1818 radeon_irq_kms_disable_hpd(rdev, disabled); 1819} 1820 1821/* watermark setup */ 1822 1823static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1824 struct radeon_crtc *radeon_crtc, 1825 struct drm_display_mode *mode, 1826 struct drm_display_mode *other_mode) 1827{ 1828 u32 tmp, buffer_alloc, i; 1829 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1830 /* 1831 * Line Buffer Setup 1832 * There are 3 line buffers, each one shared by 2 display controllers. 1833 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1834 * the display controllers. The paritioning is done via one of four 1835 * preset allocations specified in bits 2:0: 1836 * first display controller 1837 * 0 - first half of lb (3840 * 2) 1838 * 1 - first 3/4 of lb (5760 * 2) 1839 * 2 - whole lb (7680 * 2), other crtc must be disabled 1840 * 3 - first 1/4 of lb (1920 * 2) 1841 * second display controller 1842 * 4 - second half of lb (3840 * 2) 1843 * 5 - second 3/4 of lb (5760 * 2) 1844 * 6 - whole lb (7680 * 2), other crtc must be disabled 1845 * 7 - last 1/4 of lb (1920 * 2) 1846 */ 1847 /* this can get tricky if we have two large displays on a paired group 1848 * of crtcs. Ideally for multiple large displays we'd assign them to 1849 * non-linked crtcs for maximum line buffer allocation. 1850 */ 1851 if (radeon_crtc->base.enabled && mode) { 1852 if (other_mode) { 1853 tmp = 0; /* 1/2 */ 1854 buffer_alloc = 1; 1855 } else { 1856 tmp = 2; /* whole */ 1857 buffer_alloc = 2; 1858 } 1859 } else { 1860 tmp = 0; 1861 buffer_alloc = 0; 1862 } 1863 1864 /* second controller of the pair uses second half of the lb */ 1865 if (radeon_crtc->crtc_id % 2) 1866 tmp += 4; 1867 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1868 1869 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1870 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1871 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1872 for (i = 0; i < rdev->usec_timeout; i++) { 1873 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1874 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1875 break; 1876 udelay(1); 1877 } 1878 } 1879 1880 if (radeon_crtc->base.enabled && mode) { 1881 switch (tmp) { 1882 case 0: 1883 case 4: 1884 default: 1885 if (ASIC_IS_DCE5(rdev)) 1886 return 4096 * 2; 1887 else 1888 return 3840 * 2; 1889 case 1: 1890 case 5: 1891 if (ASIC_IS_DCE5(rdev)) 1892 return 6144 * 2; 1893 else 1894 return 5760 * 2; 1895 case 2: 1896 case 6: 1897 if (ASIC_IS_DCE5(rdev)) 1898 return 8192 * 2; 1899 else 1900 return 7680 * 2; 1901 case 3: 1902 case 7: 1903 if (ASIC_IS_DCE5(rdev)) 1904 return 2048 * 2; 1905 else 1906 return 1920 * 2; 1907 } 1908 } 1909 1910 /* controller not enabled, so no lb used */ 1911 return 0; 1912} 1913 1914u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 1915{ 1916 u32 tmp = RREG32(MC_SHARED_CHMAP); 1917 1918 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1919 case 0: 1920 default: 1921 return 1; 1922 case 1: 1923 return 2; 1924 case 2: 1925 return 4; 1926 case 3: 1927 return 8; 1928 } 1929} 1930 1931struct evergreen_wm_params { 1932 u32 dram_channels; /* number of dram channels */ 1933 u32 yclk; /* bandwidth per dram data pin in kHz */ 1934 u32 sclk; /* engine clock in kHz */ 1935 u32 disp_clk; /* display clock in kHz */ 1936 u32 src_width; /* viewport width */ 1937 u32 active_time; /* active display time in ns */ 1938 u32 blank_time; /* blank time in ns */ 1939 bool interlaced; /* mode is interlaced */ 1940 fixed20_12 vsc; /* vertical scale ratio */ 1941 u32 num_heads; /* number of active crtcs */ 1942 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1943 u32 lb_size; /* line buffer allocated to pipe */ 1944 u32 vtaps; /* vertical scaler taps */ 1945}; 1946 1947static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 1948{ 1949 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1950 fixed20_12 dram_efficiency; /* 0.7 */ 1951 fixed20_12 yclk, dram_channels, bandwidth; 1952 fixed20_12 a; 1953 1954 a.full = dfixed_const(1000); 1955 yclk.full = dfixed_const(wm->yclk); 1956 yclk.full = dfixed_div(yclk, a); 1957 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1958 a.full = dfixed_const(10); 1959 dram_efficiency.full = dfixed_const(7); 1960 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1961 bandwidth.full = dfixed_mul(dram_channels, yclk); 1962 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1963 1964 return dfixed_trunc(bandwidth); 1965} 1966 1967static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 1968{ 1969 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1970 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1971 fixed20_12 yclk, dram_channels, bandwidth; 1972 fixed20_12 a; 1973 1974 a.full = dfixed_const(1000); 1975 yclk.full = dfixed_const(wm->yclk); 1976 yclk.full = dfixed_div(yclk, a); 1977 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1978 a.full = dfixed_const(10); 1979 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1980 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1981 bandwidth.full = dfixed_mul(dram_channels, yclk); 1982 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1983 1984 return dfixed_trunc(bandwidth); 1985} 1986 1987static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 1988{ 1989 /* Calculate the display Data return Bandwidth */ 1990 fixed20_12 return_efficiency; /* 0.8 */ 1991 fixed20_12 sclk, bandwidth; 1992 fixed20_12 a; 1993 1994 a.full = dfixed_const(1000); 1995 sclk.full = dfixed_const(wm->sclk); 1996 sclk.full = dfixed_div(sclk, a); 1997 a.full = dfixed_const(10); 1998 return_efficiency.full = dfixed_const(8); 1999 return_efficiency.full = dfixed_div(return_efficiency, a); 2000 a.full = dfixed_const(32); 2001 bandwidth.full = dfixed_mul(a, sclk); 2002 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2003 2004 return dfixed_trunc(bandwidth); 2005} 2006 2007static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2008{ 2009 /* Calculate the DMIF Request Bandwidth */ 2010 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2011 fixed20_12 disp_clk, bandwidth; 2012 fixed20_12 a; 2013 2014 a.full = dfixed_const(1000); 2015 disp_clk.full = dfixed_const(wm->disp_clk); 2016 disp_clk.full = dfixed_div(disp_clk, a); 2017 a.full = dfixed_const(10); 2018 disp_clk_request_efficiency.full = dfixed_const(8); 2019 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2020 a.full = dfixed_const(32); 2021 bandwidth.full = dfixed_mul(a, disp_clk); 2022 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2023 2024 return dfixed_trunc(bandwidth); 2025} 2026 2027static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2028{ 2029 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2030 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2031 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2032 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2033 2034 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2035} 2036 2037static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2038{ 2039 /* Calculate the display mode Average Bandwidth 2040 * DisplayMode should contain the source and destination dimensions, 2041 * timing, etc. 2042 */ 2043 fixed20_12 bpp; 2044 fixed20_12 line_time; 2045 fixed20_12 src_width; 2046 fixed20_12 bandwidth; 2047 fixed20_12 a; 2048 2049 a.full = dfixed_const(1000); 2050 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2051 line_time.full = dfixed_div(line_time, a); 2052 bpp.full = dfixed_const(wm->bytes_per_pixel); 2053 src_width.full = dfixed_const(wm->src_width); 2054 bandwidth.full = dfixed_mul(src_width, bpp); 2055 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2056 bandwidth.full = dfixed_div(bandwidth, line_time); 2057 2058 return dfixed_trunc(bandwidth); 2059} 2060 2061static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2062{ 2063 /* First calcualte the latency in ns */ 2064 u32 mc_latency = 2000; /* 2000 ns. */ 2065 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2066 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2067 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2068 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2069 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2070 (wm->num_heads * cursor_line_pair_return_time); 2071 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2072 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2073 fixed20_12 a, b, c; 2074 2075 if (wm->num_heads == 0) 2076 return 0; 2077 2078 a.full = dfixed_const(2); 2079 b.full = dfixed_const(1); 2080 if ((wm->vsc.full > a.full) || 2081 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2082 (wm->vtaps >= 5) || 2083 ((wm->vsc.full >= a.full) && wm->interlaced)) 2084 max_src_lines_per_dst_line = 4; 2085 else 2086 max_src_lines_per_dst_line = 2; 2087 2088 a.full = dfixed_const(available_bandwidth); 2089 b.full = dfixed_const(wm->num_heads); 2090 a.full = dfixed_div(a, b); 2091 2092 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000); 2093 2094 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2095 b.full = dfixed_const(1000); 2096 c.full = dfixed_const(lb_fill_bw); 2097 b.full = dfixed_div(c, b); 2098 a.full = dfixed_div(a, b); 2099 line_fill_time = dfixed_trunc(a); 2100 2101 if (line_fill_time < wm->active_time) 2102 return latency; 2103 else 2104 return latency + (line_fill_time - wm->active_time); 2105 2106} 2107 2108static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2109{ 2110 if (evergreen_average_bandwidth(wm) <= 2111 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2112 return true; 2113 else 2114 return false; 2115}; 2116 2117static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2118{ 2119 if (evergreen_average_bandwidth(wm) <= 2120 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2121 return true; 2122 else 2123 return false; 2124}; 2125 2126static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2127{ 2128 u32 lb_partitions = wm->lb_size / wm->src_width; 2129 u32 line_time = wm->active_time + wm->blank_time; 2130 u32 latency_tolerant_lines; 2131 u32 latency_hiding; 2132 fixed20_12 a; 2133 2134 a.full = dfixed_const(1); 2135 if (wm->vsc.full > a.full) 2136 latency_tolerant_lines = 1; 2137 else { 2138 if (lb_partitions <= (wm->vtaps + 1)) 2139 latency_tolerant_lines = 1; 2140 else 2141 latency_tolerant_lines = 2; 2142 } 2143 2144 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2145 2146 if (evergreen_latency_watermark(wm) <= latency_hiding) 2147 return true; 2148 else 2149 return false; 2150} 2151 2152static void evergreen_program_watermarks(struct radeon_device *rdev, 2153 struct radeon_crtc *radeon_crtc, 2154 u32 lb_size, u32 num_heads) 2155{ 2156 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2157 struct evergreen_wm_params wm_low, wm_high; 2158 u32 dram_channels; 2159 u32 active_time; 2160 u32 line_time = 0; 2161 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2162 u32 priority_a_mark = 0, priority_b_mark = 0; 2163 u32 priority_a_cnt = PRIORITY_OFF; 2164 u32 priority_b_cnt = PRIORITY_OFF; 2165 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2166 u32 tmp, arb_control3; 2167 fixed20_12 a, b, c; 2168 2169 if (radeon_crtc->base.enabled && num_heads && mode) { 2170 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000, 2171 (u32)mode->clock); 2172 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000, 2173 (u32)mode->clock); 2174 line_time = min(line_time, (u32)65535); 2175 priority_a_cnt = 0; 2176 priority_b_cnt = 0; 2177 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2178 2179 /* watermark for high clocks */ 2180 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2181 wm_high.yclk = 2182 radeon_dpm_get_mclk(rdev, false) * 10; 2183 wm_high.sclk = 2184 radeon_dpm_get_sclk(rdev, false) * 10; 2185 } else { 2186 wm_high.yclk = rdev->pm.current_mclk * 10; 2187 wm_high.sclk = rdev->pm.current_sclk * 10; 2188 } 2189 2190 wm_high.disp_clk = mode->clock; 2191 wm_high.src_width = mode->crtc_hdisplay; 2192 wm_high.active_time = active_time; 2193 wm_high.blank_time = line_time - wm_high.active_time; 2194 wm_high.interlaced = false; 2195 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2196 wm_high.interlaced = true; 2197 wm_high.vsc = radeon_crtc->vsc; 2198 wm_high.vtaps = 1; 2199 if (radeon_crtc->rmx_type != RMX_OFF) 2200 wm_high.vtaps = 2; 2201 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2202 wm_high.lb_size = lb_size; 2203 wm_high.dram_channels = dram_channels; 2204 wm_high.num_heads = num_heads; 2205 2206 /* watermark for low clocks */ 2207 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2208 wm_low.yclk = 2209 radeon_dpm_get_mclk(rdev, true) * 10; 2210 wm_low.sclk = 2211 radeon_dpm_get_sclk(rdev, true) * 10; 2212 } else { 2213 wm_low.yclk = rdev->pm.current_mclk * 10; 2214 wm_low.sclk = rdev->pm.current_sclk * 10; 2215 } 2216 2217 wm_low.disp_clk = mode->clock; 2218 wm_low.src_width = mode->crtc_hdisplay; 2219 wm_low.active_time = active_time; 2220 wm_low.blank_time = line_time - wm_low.active_time; 2221 wm_low.interlaced = false; 2222 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2223 wm_low.interlaced = true; 2224 wm_low.vsc = radeon_crtc->vsc; 2225 wm_low.vtaps = 1; 2226 if (radeon_crtc->rmx_type != RMX_OFF) 2227 wm_low.vtaps = 2; 2228 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2229 wm_low.lb_size = lb_size; 2230 wm_low.dram_channels = dram_channels; 2231 wm_low.num_heads = num_heads; 2232 2233 /* set for high clocks */ 2234 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2235 /* set for low clocks */ 2236 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2237 2238 /* possibly force display priority to high */ 2239 /* should really do this at mode validation time... */ 2240 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2241 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2242 !evergreen_check_latency_hiding(&wm_high) || 2243 (rdev->disp_priority == 2)) { 2244 DRM_DEBUG_KMS("force priority a to high\n"); 2245 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2246 } 2247 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2248 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2249 !evergreen_check_latency_hiding(&wm_low) || 2250 (rdev->disp_priority == 2)) { 2251 DRM_DEBUG_KMS("force priority b to high\n"); 2252 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2253 } 2254 2255 a.full = dfixed_const(1000); 2256 b.full = dfixed_const(mode->clock); 2257 b.full = dfixed_div(b, a); 2258 c.full = dfixed_const(latency_watermark_a); 2259 c.full = dfixed_mul(c, b); 2260 c.full = dfixed_mul(c, radeon_crtc->hsc); 2261 c.full = dfixed_div(c, a); 2262 a.full = dfixed_const(16); 2263 c.full = dfixed_div(c, a); 2264 priority_a_mark = dfixed_trunc(c); 2265 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2266 2267 a.full = dfixed_const(1000); 2268 b.full = dfixed_const(mode->clock); 2269 b.full = dfixed_div(b, a); 2270 c.full = dfixed_const(latency_watermark_b); 2271 c.full = dfixed_mul(c, b); 2272 c.full = dfixed_mul(c, radeon_crtc->hsc); 2273 c.full = dfixed_div(c, a); 2274 a.full = dfixed_const(16); 2275 c.full = dfixed_div(c, a); 2276 priority_b_mark = dfixed_trunc(c); 2277 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2278 2279 /* Save number of lines the linebuffer leads before the scanout */ 2280 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 2281 } 2282 2283 /* select wm A */ 2284 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2285 tmp = arb_control3; 2286 tmp &= ~LATENCY_WATERMARK_MASK(3); 2287 tmp |= LATENCY_WATERMARK_MASK(1); 2288 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2289 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2290 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2291 LATENCY_HIGH_WATERMARK(line_time))); 2292 /* select wm B */ 2293 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2294 tmp &= ~LATENCY_WATERMARK_MASK(3); 2295 tmp |= LATENCY_WATERMARK_MASK(2); 2296 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2297 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2298 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2299 LATENCY_HIGH_WATERMARK(line_time))); 2300 /* restore original selection */ 2301 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2302 2303 /* write the priority marks */ 2304 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2305 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2306 2307 /* save values for DPM */ 2308 radeon_crtc->line_time = line_time; 2309 radeon_crtc->wm_high = latency_watermark_a; 2310 radeon_crtc->wm_low = latency_watermark_b; 2311} 2312 2313/** 2314 * evergreen_bandwidth_update - update display watermarks callback. 2315 * 2316 * @rdev: radeon_device pointer 2317 * 2318 * Update the display watermarks based on the requested mode(s) 2319 * (evergreen+). 2320 */ 2321void evergreen_bandwidth_update(struct radeon_device *rdev) 2322{ 2323 struct drm_display_mode *mode0 = NULL; 2324 struct drm_display_mode *mode1 = NULL; 2325 u32 num_heads = 0, lb_size; 2326 int i; 2327 2328 if (!rdev->mode_info.mode_config_initialized) 2329 return; 2330 2331 radeon_update_display_priority(rdev); 2332 2333 for (i = 0; i < rdev->num_crtc; i++) { 2334 if (rdev->mode_info.crtcs[i]->base.enabled) 2335 num_heads++; 2336 } 2337 for (i = 0; i < rdev->num_crtc; i += 2) { 2338 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2339 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2340 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2341 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2342 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2343 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2344 } 2345} 2346 2347/** 2348 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2349 * 2350 * @rdev: radeon_device pointer 2351 * 2352 * Wait for the MC (memory controller) to be idle. 2353 * (evergreen+). 2354 * Returns 0 if the MC is idle, -1 if not. 2355 */ 2356int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2357{ 2358 unsigned i; 2359 u32 tmp; 2360 2361 for (i = 0; i < rdev->usec_timeout; i++) { 2362 /* read MC_STATUS */ 2363 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2364 if (!tmp) 2365 return 0; 2366 udelay(1); 2367 } 2368 return -1; 2369} 2370 2371/* 2372 * GART 2373 */ 2374void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2375{ 2376 unsigned i; 2377 u32 tmp; 2378 2379 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2380 2381 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2382 for (i = 0; i < rdev->usec_timeout; i++) { 2383 /* read MC_STATUS */ 2384 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2385 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2386 if (tmp == 2) { 2387 pr_warn("[drm] r600 flush TLB failed\n"); 2388 return; 2389 } 2390 if (tmp) { 2391 return; 2392 } 2393 udelay(1); 2394 } 2395} 2396 2397static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2398{ 2399 u32 tmp; 2400 int r; 2401 2402 if (rdev->gart.robj == NULL) { 2403 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2404 return -EINVAL; 2405 } 2406 r = radeon_gart_table_vram_pin(rdev); 2407 if (r) 2408 return r; 2409 /* Setup L2 cache */ 2410 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2411 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2412 EFFECTIVE_L2_QUEUE_SIZE(7)); 2413 WREG32(VM_L2_CNTL2, 0); 2414 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2415 /* Setup TLB control */ 2416 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2417 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2418 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2419 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2420 if (rdev->flags & RADEON_IS_IGP) { 2421 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2422 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2423 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2424 } else { 2425 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2426 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2427 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2428 if ((rdev->family == CHIP_JUNIPER) || 2429 (rdev->family == CHIP_CYPRESS) || 2430 (rdev->family == CHIP_HEMLOCK) || 2431 (rdev->family == CHIP_BARTS)) 2432 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2433 } 2434 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2435 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2436 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2437 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2438 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2439 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2440 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2441 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2442 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2443 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2444 (u32)(rdev->dummy_page.addr >> 12)); 2445 WREG32(VM_CONTEXT1_CNTL, 0); 2446 2447 evergreen_pcie_gart_tlb_flush(rdev); 2448 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2449 (unsigned)(rdev->mc.gtt_size >> 20), 2450 (unsigned long long)rdev->gart.table_addr); 2451 rdev->gart.ready = true; 2452 return 0; 2453} 2454 2455static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2456{ 2457 u32 tmp; 2458 2459 /* Disable all tables */ 2460 WREG32(VM_CONTEXT0_CNTL, 0); 2461 WREG32(VM_CONTEXT1_CNTL, 0); 2462 2463 /* Setup L2 cache */ 2464 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2465 EFFECTIVE_L2_QUEUE_SIZE(7)); 2466 WREG32(VM_L2_CNTL2, 0); 2467 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2468 /* Setup TLB control */ 2469 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2470 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2471 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2472 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2473 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2474 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2475 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2476 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2477 radeon_gart_table_vram_unpin(rdev); 2478} 2479 2480static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2481{ 2482 evergreen_pcie_gart_disable(rdev); 2483 radeon_gart_table_vram_free(rdev); 2484 radeon_gart_fini(rdev); 2485} 2486 2487 2488static void evergreen_agp_enable(struct radeon_device *rdev) 2489{ 2490 u32 tmp; 2491 2492 /* Setup L2 cache */ 2493 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2494 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2495 EFFECTIVE_L2_QUEUE_SIZE(7)); 2496 WREG32(VM_L2_CNTL2, 0); 2497 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2498 /* Setup TLB control */ 2499 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2500 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2501 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2502 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2503 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2504 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2505 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2506 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2507 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2508 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2509 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2510 WREG32(VM_CONTEXT0_CNTL, 0); 2511 WREG32(VM_CONTEXT1_CNTL, 0); 2512} 2513 2514static const unsigned ni_dig_offsets[] = 2515{ 2516 NI_DIG0_REGISTER_OFFSET, 2517 NI_DIG1_REGISTER_OFFSET, 2518 NI_DIG2_REGISTER_OFFSET, 2519 NI_DIG3_REGISTER_OFFSET, 2520 NI_DIG4_REGISTER_OFFSET, 2521 NI_DIG5_REGISTER_OFFSET 2522}; 2523 2524static const unsigned ni_tx_offsets[] = 2525{ 2526 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1, 2527 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1, 2528 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1, 2529 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1, 2530 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1, 2531 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 2532}; 2533 2534static const unsigned evergreen_dp_offsets[] = 2535{ 2536 EVERGREEN_DP0_REGISTER_OFFSET, 2537 EVERGREEN_DP1_REGISTER_OFFSET, 2538 EVERGREEN_DP2_REGISTER_OFFSET, 2539 EVERGREEN_DP3_REGISTER_OFFSET, 2540 EVERGREEN_DP4_REGISTER_OFFSET, 2541 EVERGREEN_DP5_REGISTER_OFFSET 2542}; 2543 2544static const unsigned evergreen_disp_int_status[] = 2545{ 2546 DISP_INTERRUPT_STATUS, 2547 DISP_INTERRUPT_STATUS_CONTINUE, 2548 DISP_INTERRUPT_STATUS_CONTINUE2, 2549 DISP_INTERRUPT_STATUS_CONTINUE3, 2550 DISP_INTERRUPT_STATUS_CONTINUE4, 2551 DISP_INTERRUPT_STATUS_CONTINUE5 2552}; 2553 2554/* 2555 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc 2556 * We go from crtc to connector and it is not relible since it 2557 * should be an opposite direction .If crtc is enable then 2558 * find the dig_fe which selects this crtc and insure that it enable. 2559 * if such dig_fe is found then find dig_be which selects found dig_be and 2560 * insure that it enable and in DP_SST mode. 2561 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing 2562 * from dp symbols clocks . 2563 */ 2564static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev, 2565 unsigned crtc_id, unsigned *ret_dig_fe) 2566{ 2567 unsigned i; 2568 unsigned dig_fe; 2569 unsigned dig_be; 2570 unsigned dig_en_be; 2571 unsigned uniphy_pll; 2572 unsigned digs_fe_selected; 2573 unsigned dig_be_mode; 2574 unsigned dig_fe_mask; 2575 bool is_enabled = false; 2576 bool found_crtc = false; 2577 2578 /* loop through all running dig_fe to find selected crtc */ 2579 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2580 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]); 2581 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON && 2582 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) { 2583 /* found running pipe */ 2584 found_crtc = true; 2585 dig_fe_mask = 1 << i; 2586 dig_fe = i; 2587 break; 2588 } 2589 } 2590 2591 if (found_crtc) { 2592 /* loop through all running dig_be to find selected dig_fe */ 2593 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2594 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]); 2595 /* if dig_fe_selected by dig_be? */ 2596 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be); 2597 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be); 2598 if (dig_fe_mask & digs_fe_selected && 2599 /* if dig_be in sst mode? */ 2600 dig_be_mode == NI_DIG_BE_DPSST) { 2601 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL + 2602 ni_dig_offsets[i]); 2603 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 + 2604 ni_tx_offsets[i]); 2605 /* dig_be enable and tx is running */ 2606 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE && 2607 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON && 2608 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) { 2609 is_enabled = true; 2610 *ret_dig_fe = dig_fe; 2611 break; 2612 } 2613 } 2614 } 2615 } 2616 2617 return is_enabled; 2618} 2619 2620/* 2621 * Blank dig when in dp sst mode 2622 * Dig ignores crtc timing 2623 */ 2624static void evergreen_blank_dp_output(struct radeon_device *rdev, 2625 unsigned dig_fe) 2626{ 2627 unsigned stream_ctrl; 2628 unsigned fifo_ctrl; 2629 unsigned counter = 0; 2630 2631 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) { 2632 DRM_ERROR("invalid dig_fe %d\n", dig_fe); 2633 return; 2634 } 2635 2636 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2637 evergreen_dp_offsets[dig_fe]); 2638 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) { 2639 DRM_ERROR("dig %d , should be enable\n", dig_fe); 2640 return; 2641 } 2642 2643 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE; 2644 WREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2645 evergreen_dp_offsets[dig_fe], stream_ctrl); 2646 2647 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2648 evergreen_dp_offsets[dig_fe]); 2649 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) { 2650 msleep(1); 2651 counter++; 2652 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2653 evergreen_dp_offsets[dig_fe]); 2654 } 2655 if (counter >= 32 ) 2656 DRM_ERROR("counter exceeds %d\n", counter); 2657 2658 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]); 2659 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET; 2660 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl); 2661 2662} 2663 2664void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2665{ 2666 u32 crtc_enabled, tmp, frame_count, blackout; 2667 int i, j; 2668 unsigned dig_fe; 2669 2670 if (!ASIC_IS_NODCE(rdev)) { 2671 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2672 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2673 2674 /* disable VGA render */ 2675 WREG32(VGA_RENDER_CONTROL, 0); 2676 } 2677 /* blank the display controllers */ 2678 for (i = 0; i < rdev->num_crtc; i++) { 2679 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2680 if (crtc_enabled) { 2681 save->crtc_enabled[i] = true; 2682 if (ASIC_IS_DCE6(rdev)) { 2683 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2684 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2685 radeon_wait_for_vblank(rdev, i); 2686 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2687 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2688 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2690 } 2691 } else { 2692 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2693 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2694 radeon_wait_for_vblank(rdev, i); 2695 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2696 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2697 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2698 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2699 } 2700 } 2701 /* wait for the next frame */ 2702 frame_count = radeon_get_vblank_counter(rdev, i); 2703 for (j = 0; j < rdev->usec_timeout; j++) { 2704 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2705 break; 2706 udelay(1); 2707 } 2708 /*we should disable dig if it drives dp sst*/ 2709 /*but we are in radeon_device_init and the topology is unknown*/ 2710 /*and it is available after radeon_modeset_init*/ 2711 /*the following method radeon_atom_encoder_dpms_dig*/ 2712 /*does the job if we initialize it properly*/ 2713 /*for now we do it this manually*/ 2714 /**/ 2715 if (ASIC_IS_DCE5(rdev) && 2716 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe)) 2717 evergreen_blank_dp_output(rdev, dig_fe); 2718 /*we could remove 6 lines below*/ 2719 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2720 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2721 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2722 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2723 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2724 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2725 save->crtc_enabled[i] = false; 2726 /* ***** */ 2727 } else { 2728 save->crtc_enabled[i] = false; 2729 } 2730 } 2731 2732 radeon_mc_wait_for_idle(rdev); 2733 2734 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2735 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2736 /* Block CPU access */ 2737 WREG32(BIF_FB_EN, 0); 2738 /* blackout the MC */ 2739 blackout &= ~BLACKOUT_MODE_MASK; 2740 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2741 } 2742 /* wait for the MC to settle */ 2743 udelay(100); 2744 2745 /* lock double buffered regs */ 2746 for (i = 0; i < rdev->num_crtc; i++) { 2747 if (save->crtc_enabled[i]) { 2748 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2749 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2750 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2751 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2752 } 2753 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2754 if (!(tmp & 1)) { 2755 tmp |= 1; 2756 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2757 } 2758 } 2759 } 2760} 2761 2762void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2763{ 2764 u32 tmp, frame_count; 2765 int i, j; 2766 2767 /* update crtc base addresses */ 2768 for (i = 0; i < rdev->num_crtc; i++) { 2769 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2770 upper_32_bits(rdev->mc.vram_start)); 2771 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2772 upper_32_bits(rdev->mc.vram_start)); 2773 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2774 (u32)rdev->mc.vram_start); 2775 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2776 (u32)rdev->mc.vram_start); 2777 } 2778 2779 if (!ASIC_IS_NODCE(rdev)) { 2780 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2781 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2782 } 2783 2784 /* unlock regs and wait for update */ 2785 for (i = 0; i < rdev->num_crtc; i++) { 2786 if (save->crtc_enabled[i]) { 2787 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2788 if ((tmp & 0x7) != 0) { 2789 tmp &= ~0x7; 2790 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2791 } 2792 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2793 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2794 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2795 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2796 } 2797 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2798 if (tmp & 1) { 2799 tmp &= ~1; 2800 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2801 } 2802 for (j = 0; j < rdev->usec_timeout; j++) { 2803 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2804 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2805 break; 2806 udelay(1); 2807 } 2808 } 2809 } 2810 2811 /* unblackout the MC */ 2812 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2813 tmp &= ~BLACKOUT_MODE_MASK; 2814 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2815 /* allow CPU access */ 2816 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2817 2818 for (i = 0; i < rdev->num_crtc; i++) { 2819 if (save->crtc_enabled[i]) { 2820 if (ASIC_IS_DCE6(rdev)) { 2821 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2822 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2823 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2824 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2825 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2826 } else { 2827 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2828 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2829 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2830 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2831 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2832 } 2833 /* wait for the next frame */ 2834 frame_count = radeon_get_vblank_counter(rdev, i); 2835 for (j = 0; j < rdev->usec_timeout; j++) { 2836 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2837 break; 2838 udelay(1); 2839 } 2840 } 2841 } 2842 if (!ASIC_IS_NODCE(rdev)) { 2843 /* Unlock vga access */ 2844 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2845 mdelay(1); 2846 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2847 } 2848} 2849 2850void evergreen_mc_program(struct radeon_device *rdev) 2851{ 2852 struct evergreen_mc_save save; 2853 u32 tmp; 2854 int i, j; 2855 2856 /* Initialize HDP */ 2857 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2858 WREG32((0x2c14 + j), 0x00000000); 2859 WREG32((0x2c18 + j), 0x00000000); 2860 WREG32((0x2c1c + j), 0x00000000); 2861 WREG32((0x2c20 + j), 0x00000000); 2862 WREG32((0x2c24 + j), 0x00000000); 2863 } 2864 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2865 2866 evergreen_mc_stop(rdev, &save); 2867 if (evergreen_mc_wait_for_idle(rdev)) { 2868 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2869 } 2870 /* Lockout access through VGA aperture*/ 2871 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2872 /* Update configuration */ 2873 if (rdev->flags & RADEON_IS_AGP) { 2874 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2875 /* VRAM before AGP */ 2876 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2877 rdev->mc.vram_start >> 12); 2878 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2879 rdev->mc.gtt_end >> 12); 2880 } else { 2881 /* VRAM after AGP */ 2882 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2883 rdev->mc.gtt_start >> 12); 2884 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2885 rdev->mc.vram_end >> 12); 2886 } 2887 } else { 2888 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2889 rdev->mc.vram_start >> 12); 2890 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2891 rdev->mc.vram_end >> 12); 2892 } 2893 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2894 /* llano/ontario only */ 2895 if ((rdev->family == CHIP_PALM) || 2896 (rdev->family == CHIP_SUMO) || 2897 (rdev->family == CHIP_SUMO2)) { 2898 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2899 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2900 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2901 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2902 } 2903 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2904 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2905 WREG32(MC_VM_FB_LOCATION, tmp); 2906 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2907 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2908 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2909 if (rdev->flags & RADEON_IS_AGP) { 2910 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2911 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2912 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 2913 } else { 2914 WREG32(MC_VM_AGP_BASE, 0); 2915 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2916 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2917 } 2918 if (evergreen_mc_wait_for_idle(rdev)) { 2919 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2920 } 2921 evergreen_mc_resume(rdev, &save); 2922 /* we need to own VRAM, so turn off the VGA renderer here 2923 * to stop it overwriting our objects */ 2924 rv515_vga_render_disable(rdev); 2925} 2926 2927/* 2928 * CP. 2929 */ 2930void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 2931{ 2932 struct radeon_ring *ring = &rdev->ring[ib->ring]; 2933 u32 next_rptr; 2934 2935 /* set to DX10/11 mode */ 2936 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 2937 radeon_ring_write(ring, 1); 2938 2939 if (ring->rptr_save_reg) { 2940 next_rptr = ring->wptr + 3 + 4; 2941 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2942 radeon_ring_write(ring, ((ring->rptr_save_reg - 2943 PACKET3_SET_CONFIG_REG_START) >> 2)); 2944 radeon_ring_write(ring, next_rptr); 2945 } else if (rdev->wb.enabled) { 2946 next_rptr = ring->wptr + 5 + 4; 2947 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 2948 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 2949 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 2950 radeon_ring_write(ring, next_rptr); 2951 radeon_ring_write(ring, 0); 2952 } 2953 2954 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 2955 radeon_ring_write(ring, 2956#ifdef __BIG_ENDIAN 2957 (2 << 0) | 2958#endif 2959 (ib->gpu_addr & 0xFFFFFFFC)); 2960 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 2961 radeon_ring_write(ring, ib->length_dw); 2962} 2963 2964 2965static int evergreen_cp_load_microcode(struct radeon_device *rdev) 2966{ 2967 const __be32 *fw_data; 2968 int i; 2969 2970 if (!rdev->me_fw || !rdev->pfp_fw) 2971 return -EINVAL; 2972 2973 r700_cp_stop(rdev); 2974 WREG32(CP_RB_CNTL, 2975#ifdef __BIG_ENDIAN 2976 BUF_SWAP_32BIT | 2977#endif 2978 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 2979 2980 fw_data = (const __be32 *)rdev->pfp_fw->data; 2981 WREG32(CP_PFP_UCODE_ADDR, 0); 2982 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 2983 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 2984 WREG32(CP_PFP_UCODE_ADDR, 0); 2985 2986 fw_data = (const __be32 *)rdev->me_fw->data; 2987 WREG32(CP_ME_RAM_WADDR, 0); 2988 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 2989 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 2990 2991 WREG32(CP_PFP_UCODE_ADDR, 0); 2992 WREG32(CP_ME_RAM_WADDR, 0); 2993 WREG32(CP_ME_RAM_RADDR, 0); 2994 return 0; 2995} 2996 2997static int evergreen_cp_start(struct radeon_device *rdev) 2998{ 2999 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3000 int r, i; 3001 uint32_t cp_me; 3002 3003 r = radeon_ring_lock(rdev, ring, 7); 3004 if (r) { 3005 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3006 return r; 3007 } 3008 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3009 radeon_ring_write(ring, 0x1); 3010 radeon_ring_write(ring, 0x0); 3011 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 3012 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3013 radeon_ring_write(ring, 0); 3014 radeon_ring_write(ring, 0); 3015 radeon_ring_unlock_commit(rdev, ring, false); 3016 3017 cp_me = 0xff; 3018 WREG32(CP_ME_CNTL, cp_me); 3019 3020 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 3021 if (r) { 3022 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3023 return r; 3024 } 3025 3026 /* setup clear context state */ 3027 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3028 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3029 3030 for (i = 0; i < evergreen_default_size; i++) 3031 radeon_ring_write(ring, evergreen_default_state[i]); 3032 3033 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3034 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3035 3036 /* set clear context state */ 3037 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3038 radeon_ring_write(ring, 0); 3039 3040 /* SQ_VTX_BASE_VTX_LOC */ 3041 radeon_ring_write(ring, 0xc0026f00); 3042 radeon_ring_write(ring, 0x00000000); 3043 radeon_ring_write(ring, 0x00000000); 3044 radeon_ring_write(ring, 0x00000000); 3045 3046 /* Clear consts */ 3047 radeon_ring_write(ring, 0xc0036f00); 3048 radeon_ring_write(ring, 0x00000bc4); 3049 radeon_ring_write(ring, 0xffffffff); 3050 radeon_ring_write(ring, 0xffffffff); 3051 radeon_ring_write(ring, 0xffffffff); 3052 3053 radeon_ring_write(ring, 0xc0026900); 3054 radeon_ring_write(ring, 0x00000316); 3055 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3056 radeon_ring_write(ring, 0x00000010); /* */ 3057 3058 radeon_ring_unlock_commit(rdev, ring, false); 3059 3060 return 0; 3061} 3062 3063static int evergreen_cp_resume(struct radeon_device *rdev) 3064{ 3065 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3066 u32 tmp; 3067 u32 rb_bufsz; 3068 int r; 3069 3070 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 3071 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 3072 SOFT_RESET_PA | 3073 SOFT_RESET_SH | 3074 SOFT_RESET_VGT | 3075 SOFT_RESET_SPI | 3076 SOFT_RESET_SX)); 3077 RREG32(GRBM_SOFT_RESET); 3078 mdelay(15); 3079 WREG32(GRBM_SOFT_RESET, 0); 3080 RREG32(GRBM_SOFT_RESET); 3081 3082 /* Set ring buffer size */ 3083 rb_bufsz = order_base_2(ring->ring_size / 8); 3084 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3085#ifdef __BIG_ENDIAN 3086 tmp |= BUF_SWAP_32BIT; 3087#endif 3088 WREG32(CP_RB_CNTL, tmp); 3089 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3090 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3091 3092 /* Set the write pointer delay */ 3093 WREG32(CP_RB_WPTR_DELAY, 0); 3094 3095 /* Initialize the ring buffer's read and write pointers */ 3096 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 3097 WREG32(CP_RB_RPTR_WR, 0); 3098 ring->wptr = 0; 3099 WREG32(CP_RB_WPTR, ring->wptr); 3100 3101 /* set the wb address whether it's enabled or not */ 3102 WREG32(CP_RB_RPTR_ADDR, 3103 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 3104 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3105 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3106 3107 if (rdev->wb.enabled) 3108 WREG32(SCRATCH_UMSK, 0xff); 3109 else { 3110 tmp |= RB_NO_UPDATE; 3111 WREG32(SCRATCH_UMSK, 0); 3112 } 3113 3114 mdelay(1); 3115 WREG32(CP_RB_CNTL, tmp); 3116 3117 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 3118 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 3119 3120 evergreen_cp_start(rdev); 3121 ring->ready = true; 3122 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 3123 if (r) { 3124 ring->ready = false; 3125 return r; 3126 } 3127 return 0; 3128} 3129 3130/* 3131 * Core functions 3132 */ 3133static void evergreen_gpu_init(struct radeon_device *rdev) 3134{ 3135 u32 gb_addr_config; 3136 u32 mc_shared_chmap, mc_arb_ramcfg; 3137 u32 sx_debug_1; 3138 u32 smx_dc_ctl0; 3139 u32 sq_config; 3140 u32 sq_lds_resource_mgmt; 3141 u32 sq_gpr_resource_mgmt_1; 3142 u32 sq_gpr_resource_mgmt_2; 3143 u32 sq_gpr_resource_mgmt_3; 3144 u32 sq_thread_resource_mgmt; 3145 u32 sq_thread_resource_mgmt_2; 3146 u32 sq_stack_resource_mgmt_1; 3147 u32 sq_stack_resource_mgmt_2; 3148 u32 sq_stack_resource_mgmt_3; 3149 u32 vgt_cache_invalidation; 3150 u32 hdp_host_path_cntl, tmp; 3151 u32 disabled_rb_mask; 3152 int i, j, ps_thread_count; 3153 3154 switch (rdev->family) { 3155 case CHIP_CYPRESS: 3156 case CHIP_HEMLOCK: 3157 rdev->config.evergreen.num_ses = 2; 3158 rdev->config.evergreen.max_pipes = 4; 3159 rdev->config.evergreen.max_tile_pipes = 8; 3160 rdev->config.evergreen.max_simds = 10; 3161 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3162 rdev->config.evergreen.max_gprs = 256; 3163 rdev->config.evergreen.max_threads = 248; 3164 rdev->config.evergreen.max_gs_threads = 32; 3165 rdev->config.evergreen.max_stack_entries = 512; 3166 rdev->config.evergreen.sx_num_of_sets = 4; 3167 rdev->config.evergreen.sx_max_export_size = 256; 3168 rdev->config.evergreen.sx_max_export_pos_size = 64; 3169 rdev->config.evergreen.sx_max_export_smx_size = 192; 3170 rdev->config.evergreen.max_hw_contexts = 8; 3171 rdev->config.evergreen.sq_num_cf_insts = 2; 3172 3173 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3174 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3175 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3176 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3177 break; 3178 case CHIP_JUNIPER: 3179 rdev->config.evergreen.num_ses = 1; 3180 rdev->config.evergreen.max_pipes = 4; 3181 rdev->config.evergreen.max_tile_pipes = 4; 3182 rdev->config.evergreen.max_simds = 10; 3183 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3184 rdev->config.evergreen.max_gprs = 256; 3185 rdev->config.evergreen.max_threads = 248; 3186 rdev->config.evergreen.max_gs_threads = 32; 3187 rdev->config.evergreen.max_stack_entries = 512; 3188 rdev->config.evergreen.sx_num_of_sets = 4; 3189 rdev->config.evergreen.sx_max_export_size = 256; 3190 rdev->config.evergreen.sx_max_export_pos_size = 64; 3191 rdev->config.evergreen.sx_max_export_smx_size = 192; 3192 rdev->config.evergreen.max_hw_contexts = 8; 3193 rdev->config.evergreen.sq_num_cf_insts = 2; 3194 3195 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3196 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3197 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3198 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3199 break; 3200 case CHIP_REDWOOD: 3201 rdev->config.evergreen.num_ses = 1; 3202 rdev->config.evergreen.max_pipes = 4; 3203 rdev->config.evergreen.max_tile_pipes = 4; 3204 rdev->config.evergreen.max_simds = 5; 3205 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3206 rdev->config.evergreen.max_gprs = 256; 3207 rdev->config.evergreen.max_threads = 248; 3208 rdev->config.evergreen.max_gs_threads = 32; 3209 rdev->config.evergreen.max_stack_entries = 256; 3210 rdev->config.evergreen.sx_num_of_sets = 4; 3211 rdev->config.evergreen.sx_max_export_size = 256; 3212 rdev->config.evergreen.sx_max_export_pos_size = 64; 3213 rdev->config.evergreen.sx_max_export_smx_size = 192; 3214 rdev->config.evergreen.max_hw_contexts = 8; 3215 rdev->config.evergreen.sq_num_cf_insts = 2; 3216 3217 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3218 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3219 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3220 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3221 break; 3222 case CHIP_CEDAR: 3223 default: 3224 rdev->config.evergreen.num_ses = 1; 3225 rdev->config.evergreen.max_pipes = 2; 3226 rdev->config.evergreen.max_tile_pipes = 2; 3227 rdev->config.evergreen.max_simds = 2; 3228 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3229 rdev->config.evergreen.max_gprs = 256; 3230 rdev->config.evergreen.max_threads = 192; 3231 rdev->config.evergreen.max_gs_threads = 16; 3232 rdev->config.evergreen.max_stack_entries = 256; 3233 rdev->config.evergreen.sx_num_of_sets = 4; 3234 rdev->config.evergreen.sx_max_export_size = 128; 3235 rdev->config.evergreen.sx_max_export_pos_size = 32; 3236 rdev->config.evergreen.sx_max_export_smx_size = 96; 3237 rdev->config.evergreen.max_hw_contexts = 4; 3238 rdev->config.evergreen.sq_num_cf_insts = 1; 3239 3240 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3241 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3242 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3243 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3244 break; 3245 case CHIP_PALM: 3246 rdev->config.evergreen.num_ses = 1; 3247 rdev->config.evergreen.max_pipes = 2; 3248 rdev->config.evergreen.max_tile_pipes = 2; 3249 rdev->config.evergreen.max_simds = 2; 3250 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3251 rdev->config.evergreen.max_gprs = 256; 3252 rdev->config.evergreen.max_threads = 192; 3253 rdev->config.evergreen.max_gs_threads = 16; 3254 rdev->config.evergreen.max_stack_entries = 256; 3255 rdev->config.evergreen.sx_num_of_sets = 4; 3256 rdev->config.evergreen.sx_max_export_size = 128; 3257 rdev->config.evergreen.sx_max_export_pos_size = 32; 3258 rdev->config.evergreen.sx_max_export_smx_size = 96; 3259 rdev->config.evergreen.max_hw_contexts = 4; 3260 rdev->config.evergreen.sq_num_cf_insts = 1; 3261 3262 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3263 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3264 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3265 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3266 break; 3267 case CHIP_SUMO: 3268 rdev->config.evergreen.num_ses = 1; 3269 rdev->config.evergreen.max_pipes = 4; 3270 rdev->config.evergreen.max_tile_pipes = 4; 3271 if (rdev->pdev->device == 0x9648) 3272 rdev->config.evergreen.max_simds = 3; 3273 else if ((rdev->pdev->device == 0x9647) || 3274 (rdev->pdev->device == 0x964a)) 3275 rdev->config.evergreen.max_simds = 4; 3276 else 3277 rdev->config.evergreen.max_simds = 5; 3278 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3279 rdev->config.evergreen.max_gprs = 256; 3280 rdev->config.evergreen.max_threads = 248; 3281 rdev->config.evergreen.max_gs_threads = 32; 3282 rdev->config.evergreen.max_stack_entries = 256; 3283 rdev->config.evergreen.sx_num_of_sets = 4; 3284 rdev->config.evergreen.sx_max_export_size = 256; 3285 rdev->config.evergreen.sx_max_export_pos_size = 64; 3286 rdev->config.evergreen.sx_max_export_smx_size = 192; 3287 rdev->config.evergreen.max_hw_contexts = 8; 3288 rdev->config.evergreen.sq_num_cf_insts = 2; 3289 3290 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3291 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3292 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3293 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3294 break; 3295 case CHIP_SUMO2: 3296 rdev->config.evergreen.num_ses = 1; 3297 rdev->config.evergreen.max_pipes = 4; 3298 rdev->config.evergreen.max_tile_pipes = 4; 3299 rdev->config.evergreen.max_simds = 2; 3300 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3301 rdev->config.evergreen.max_gprs = 256; 3302 rdev->config.evergreen.max_threads = 248; 3303 rdev->config.evergreen.max_gs_threads = 32; 3304 rdev->config.evergreen.max_stack_entries = 512; 3305 rdev->config.evergreen.sx_num_of_sets = 4; 3306 rdev->config.evergreen.sx_max_export_size = 256; 3307 rdev->config.evergreen.sx_max_export_pos_size = 64; 3308 rdev->config.evergreen.sx_max_export_smx_size = 192; 3309 rdev->config.evergreen.max_hw_contexts = 4; 3310 rdev->config.evergreen.sq_num_cf_insts = 2; 3311 3312 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3313 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3314 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3315 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3316 break; 3317 case CHIP_BARTS: 3318 rdev->config.evergreen.num_ses = 2; 3319 rdev->config.evergreen.max_pipes = 4; 3320 rdev->config.evergreen.max_tile_pipes = 8; 3321 rdev->config.evergreen.max_simds = 7; 3322 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3323 rdev->config.evergreen.max_gprs = 256; 3324 rdev->config.evergreen.max_threads = 248; 3325 rdev->config.evergreen.max_gs_threads = 32; 3326 rdev->config.evergreen.max_stack_entries = 512; 3327 rdev->config.evergreen.sx_num_of_sets = 4; 3328 rdev->config.evergreen.sx_max_export_size = 256; 3329 rdev->config.evergreen.sx_max_export_pos_size = 64; 3330 rdev->config.evergreen.sx_max_export_smx_size = 192; 3331 rdev->config.evergreen.max_hw_contexts = 8; 3332 rdev->config.evergreen.sq_num_cf_insts = 2; 3333 3334 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3335 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3336 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3337 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3338 break; 3339 case CHIP_TURKS: 3340 rdev->config.evergreen.num_ses = 1; 3341 rdev->config.evergreen.max_pipes = 4; 3342 rdev->config.evergreen.max_tile_pipes = 4; 3343 rdev->config.evergreen.max_simds = 6; 3344 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3345 rdev->config.evergreen.max_gprs = 256; 3346 rdev->config.evergreen.max_threads = 248; 3347 rdev->config.evergreen.max_gs_threads = 32; 3348 rdev->config.evergreen.max_stack_entries = 256; 3349 rdev->config.evergreen.sx_num_of_sets = 4; 3350 rdev->config.evergreen.sx_max_export_size = 256; 3351 rdev->config.evergreen.sx_max_export_pos_size = 64; 3352 rdev->config.evergreen.sx_max_export_smx_size = 192; 3353 rdev->config.evergreen.max_hw_contexts = 8; 3354 rdev->config.evergreen.sq_num_cf_insts = 2; 3355 3356 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3357 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3358 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3359 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3360 break; 3361 case CHIP_CAICOS: 3362 rdev->config.evergreen.num_ses = 1; 3363 rdev->config.evergreen.max_pipes = 2; 3364 rdev->config.evergreen.max_tile_pipes = 2; 3365 rdev->config.evergreen.max_simds = 2; 3366 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3367 rdev->config.evergreen.max_gprs = 256; 3368 rdev->config.evergreen.max_threads = 192; 3369 rdev->config.evergreen.max_gs_threads = 16; 3370 rdev->config.evergreen.max_stack_entries = 256; 3371 rdev->config.evergreen.sx_num_of_sets = 4; 3372 rdev->config.evergreen.sx_max_export_size = 128; 3373 rdev->config.evergreen.sx_max_export_pos_size = 32; 3374 rdev->config.evergreen.sx_max_export_smx_size = 96; 3375 rdev->config.evergreen.max_hw_contexts = 4; 3376 rdev->config.evergreen.sq_num_cf_insts = 1; 3377 3378 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3379 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3380 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3381 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3382 break; 3383 } 3384 3385 /* Initialize HDP */ 3386 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3387 WREG32((0x2c14 + j), 0x00000000); 3388 WREG32((0x2c18 + j), 0x00000000); 3389 WREG32((0x2c1c + j), 0x00000000); 3390 WREG32((0x2c20 + j), 0x00000000); 3391 WREG32((0x2c24 + j), 0x00000000); 3392 } 3393 3394 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3395 WREG32(SRBM_INT_CNTL, 0x1); 3396 WREG32(SRBM_INT_ACK, 0x1); 3397 3398 evergreen_fix_pci_max_read_req_size(rdev); 3399 3400 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3401 if ((rdev->family == CHIP_PALM) || 3402 (rdev->family == CHIP_SUMO) || 3403 (rdev->family == CHIP_SUMO2)) 3404 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3405 else 3406 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3407 3408 /* setup tiling info dword. gb_addr_config is not adequate since it does 3409 * not have bank info, so create a custom tiling dword. 3410 * bits 3:0 num_pipes 3411 * bits 7:4 num_banks 3412 * bits 11:8 group_size 3413 * bits 15:12 row_size 3414 */ 3415 rdev->config.evergreen.tile_config = 0; 3416 switch (rdev->config.evergreen.max_tile_pipes) { 3417 case 1: 3418 default: 3419 rdev->config.evergreen.tile_config |= (0 << 0); 3420 break; 3421 case 2: 3422 rdev->config.evergreen.tile_config |= (1 << 0); 3423 break; 3424 case 4: 3425 rdev->config.evergreen.tile_config |= (2 << 0); 3426 break; 3427 case 8: 3428 rdev->config.evergreen.tile_config |= (3 << 0); 3429 break; 3430 } 3431 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3432 if (rdev->flags & RADEON_IS_IGP) 3433 rdev->config.evergreen.tile_config |= 1 << 4; 3434 else { 3435 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3436 case 0: /* four banks */ 3437 rdev->config.evergreen.tile_config |= 0 << 4; 3438 break; 3439 case 1: /* eight banks */ 3440 rdev->config.evergreen.tile_config |= 1 << 4; 3441 break; 3442 case 2: /* sixteen banks */ 3443 default: 3444 rdev->config.evergreen.tile_config |= 2 << 4; 3445 break; 3446 } 3447 } 3448 rdev->config.evergreen.tile_config |= 0 << 8; 3449 rdev->config.evergreen.tile_config |= 3450 ((gb_addr_config & 0x30000000) >> 28) << 12; 3451 3452 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3453 u32 efuse_straps_4; 3454 u32 efuse_straps_3; 3455 3456 efuse_straps_4 = RREG32_RCU(0x204); 3457 efuse_straps_3 = RREG32_RCU(0x203); 3458 tmp = (((efuse_straps_4 & 0xf) << 4) | 3459 ((efuse_straps_3 & 0xf0000000) >> 28)); 3460 } else { 3461 tmp = 0; 3462 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3463 u32 rb_disable_bitmap; 3464 3465 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3466 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3467 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3468 tmp <<= 4; 3469 tmp |= rb_disable_bitmap; 3470 } 3471 } 3472 /* enabled rb are just the one not disabled :) */ 3473 disabled_rb_mask = tmp; 3474 tmp = 0; 3475 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3476 tmp |= (1 << i); 3477 /* if all the backends are disabled, fix it up here */ 3478 if ((disabled_rb_mask & tmp) == tmp) { 3479 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3480 disabled_rb_mask &= ~(1 << i); 3481 } 3482 3483 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3484 u32 simd_disable_bitmap; 3485 3486 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3487 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3488 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3489 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3490 tmp <<= 16; 3491 tmp |= simd_disable_bitmap; 3492 } 3493 rdev->config.evergreen.active_simds = hweight32(~tmp); 3494 3495 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3496 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3497 3498 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3499 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3500 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3501 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3502 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3503 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3504 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3505 3506 if ((rdev->config.evergreen.max_backends == 1) && 3507 (rdev->flags & RADEON_IS_IGP)) { 3508 if ((disabled_rb_mask & 3) == 1) { 3509 /* RB0 disabled, RB1 enabled */ 3510 tmp = 0x11111111; 3511 } else { 3512 /* RB1 disabled, RB0 enabled */ 3513 tmp = 0x00000000; 3514 } 3515 } else { 3516 tmp = gb_addr_config & NUM_PIPES_MASK; 3517 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3518 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3519 } 3520 rdev->config.evergreen.backend_map = tmp; 3521 WREG32(GB_BACKEND_MAP, tmp); 3522 3523 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3524 WREG32(CGTS_TCC_DISABLE, 0); 3525 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3526 WREG32(CGTS_USER_TCC_DISABLE, 0); 3527 3528 /* set HW defaults for 3D engine */ 3529 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3530 ROQ_IB2_START(0x2b))); 3531 3532 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3533 3534 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3535 SYNC_GRADIENT | 3536 SYNC_WALKER | 3537 SYNC_ALIGNER)); 3538 3539 sx_debug_1 = RREG32(SX_DEBUG_1); 3540 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3541 WREG32(SX_DEBUG_1, sx_debug_1); 3542 3543 3544 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3545 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3546 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3547 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3548 3549 if (rdev->family <= CHIP_SUMO2) 3550 WREG32(SMX_SAR_CTL0, 0x00010000); 3551 3552 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3553 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3554 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3555 3556 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3557 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3558 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3559 3560 WREG32(VGT_NUM_INSTANCES, 1); 3561 WREG32(SPI_CONFIG_CNTL, 0); 3562 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3563 WREG32(CP_PERFMON_CNTL, 0); 3564 3565 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3566 FETCH_FIFO_HIWATER(0x4) | 3567 DONE_FIFO_HIWATER(0xe0) | 3568 ALU_UPDATE_FIFO_HIWATER(0x8))); 3569 3570 sq_config = RREG32(SQ_CONFIG); 3571 sq_config &= ~(PS_PRIO(3) | 3572 VS_PRIO(3) | 3573 GS_PRIO(3) | 3574 ES_PRIO(3)); 3575 sq_config |= (VC_ENABLE | 3576 EXPORT_SRC_C | 3577 PS_PRIO(0) | 3578 VS_PRIO(1) | 3579 GS_PRIO(2) | 3580 ES_PRIO(3)); 3581 3582 switch (rdev->family) { 3583 case CHIP_CEDAR: 3584 case CHIP_PALM: 3585 case CHIP_SUMO: 3586 case CHIP_SUMO2: 3587 case CHIP_CAICOS: 3588 /* no vertex cache */ 3589 sq_config &= ~VC_ENABLE; 3590 break; 3591 default: 3592 break; 3593 } 3594 3595 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3596 3597 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3598 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3599 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3600 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3601 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3602 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3603 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3604 3605 switch (rdev->family) { 3606 case CHIP_CEDAR: 3607 case CHIP_PALM: 3608 case CHIP_SUMO: 3609 case CHIP_SUMO2: 3610 ps_thread_count = 96; 3611 break; 3612 default: 3613 ps_thread_count = 128; 3614 break; 3615 } 3616 3617 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3618 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3619 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3620 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3621 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3622 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3623 3624 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3625 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3626 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3627 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3628 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3629 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3630 3631 WREG32(SQ_CONFIG, sq_config); 3632 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3633 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3634 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3635 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3636 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3637 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3638 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3639 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3640 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3641 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3642 3643 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3644 FORCE_EOV_MAX_REZ_CNT(255))); 3645 3646 switch (rdev->family) { 3647 case CHIP_CEDAR: 3648 case CHIP_PALM: 3649 case CHIP_SUMO: 3650 case CHIP_SUMO2: 3651 case CHIP_CAICOS: 3652 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3653 break; 3654 default: 3655 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3656 break; 3657 } 3658 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3659 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3660 3661 WREG32(VGT_GS_VERTEX_REUSE, 16); 3662 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3663 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3664 3665 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3666 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3667 3668 WREG32(CB_PERF_CTR0_SEL_0, 0); 3669 WREG32(CB_PERF_CTR0_SEL_1, 0); 3670 WREG32(CB_PERF_CTR1_SEL_0, 0); 3671 WREG32(CB_PERF_CTR1_SEL_1, 0); 3672 WREG32(CB_PERF_CTR2_SEL_0, 0); 3673 WREG32(CB_PERF_CTR2_SEL_1, 0); 3674 WREG32(CB_PERF_CTR3_SEL_0, 0); 3675 WREG32(CB_PERF_CTR3_SEL_1, 0); 3676 3677 /* clear render buffer base addresses */ 3678 WREG32(CB_COLOR0_BASE, 0); 3679 WREG32(CB_COLOR1_BASE, 0); 3680 WREG32(CB_COLOR2_BASE, 0); 3681 WREG32(CB_COLOR3_BASE, 0); 3682 WREG32(CB_COLOR4_BASE, 0); 3683 WREG32(CB_COLOR5_BASE, 0); 3684 WREG32(CB_COLOR6_BASE, 0); 3685 WREG32(CB_COLOR7_BASE, 0); 3686 WREG32(CB_COLOR8_BASE, 0); 3687 WREG32(CB_COLOR9_BASE, 0); 3688 WREG32(CB_COLOR10_BASE, 0); 3689 WREG32(CB_COLOR11_BASE, 0); 3690 3691 /* set the shader const cache sizes to 0 */ 3692 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3693 WREG32(i, 0); 3694 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3695 WREG32(i, 0); 3696 3697 tmp = RREG32(HDP_MISC_CNTL); 3698 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3699 WREG32(HDP_MISC_CNTL, tmp); 3700 3701 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3702 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3703 3704 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3705 3706 udelay(50); 3707 3708} 3709 3710int evergreen_mc_init(struct radeon_device *rdev) 3711{ 3712 u32 tmp; 3713 int chansize, numchan; 3714 3715 /* Get VRAM informations */ 3716 rdev->mc.vram_is_ddr = true; 3717 if ((rdev->family == CHIP_PALM) || 3718 (rdev->family == CHIP_SUMO) || 3719 (rdev->family == CHIP_SUMO2)) 3720 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3721 else 3722 tmp = RREG32(MC_ARB_RAMCFG); 3723 if (tmp & CHANSIZE_OVERRIDE) { 3724 chansize = 16; 3725 } else if (tmp & CHANSIZE_MASK) { 3726 chansize = 64; 3727 } else { 3728 chansize = 32; 3729 } 3730 tmp = RREG32(MC_SHARED_CHMAP); 3731 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3732 case 0: 3733 default: 3734 numchan = 1; 3735 break; 3736 case 1: 3737 numchan = 2; 3738 break; 3739 case 2: 3740 numchan = 4; 3741 break; 3742 case 3: 3743 numchan = 8; 3744 break; 3745 } 3746 rdev->mc.vram_width = numchan * chansize; 3747 /* Could aper size report 0 ? */ 3748 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3749 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3750 /* Setup GPU memory space */ 3751 if ((rdev->family == CHIP_PALM) || 3752 (rdev->family == CHIP_SUMO) || 3753 (rdev->family == CHIP_SUMO2)) { 3754 /* size in bytes on fusion */ 3755 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3756 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3757 } else { 3758 /* size in MB on evergreen/cayman/tn */ 3759 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3760 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3761 } 3762 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3763 r700_vram_gtt_location(rdev, &rdev->mc); 3764 radeon_update_bandwidth_info(rdev); 3765 3766 return 0; 3767} 3768 3769void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3770{ 3771 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3772 RREG32(GRBM_STATUS)); 3773 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3774 RREG32(GRBM_STATUS_SE0)); 3775 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3776 RREG32(GRBM_STATUS_SE1)); 3777 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3778 RREG32(SRBM_STATUS)); 3779 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3780 RREG32(SRBM_STATUS2)); 3781 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3782 RREG32(CP_STALLED_STAT1)); 3783 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3784 RREG32(CP_STALLED_STAT2)); 3785 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3786 RREG32(CP_BUSY_STAT)); 3787 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3788 RREG32(CP_STAT)); 3789 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3790 RREG32(DMA_STATUS_REG)); 3791 if (rdev->family >= CHIP_CAYMAN) { 3792 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3793 RREG32(DMA_STATUS_REG + 0x800)); 3794 } 3795} 3796 3797bool evergreen_is_display_hung(struct radeon_device *rdev) 3798{ 3799 u32 crtc_hung = 0; 3800 u32 crtc_status[6]; 3801 u32 i, j, tmp; 3802 3803 for (i = 0; i < rdev->num_crtc; i++) { 3804 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3805 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3806 crtc_hung |= (1 << i); 3807 } 3808 } 3809 3810 for (j = 0; j < 10; j++) { 3811 for (i = 0; i < rdev->num_crtc; i++) { 3812 if (crtc_hung & (1 << i)) { 3813 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3814 if (tmp != crtc_status[i]) 3815 crtc_hung &= ~(1 << i); 3816 } 3817 } 3818 if (crtc_hung == 0) 3819 return false; 3820 udelay(100); 3821 } 3822 3823 return true; 3824} 3825 3826u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3827{ 3828 u32 reset_mask = 0; 3829 u32 tmp; 3830 3831 /* GRBM_STATUS */ 3832 tmp = RREG32(GRBM_STATUS); 3833 if (tmp & (PA_BUSY | SC_BUSY | 3834 SH_BUSY | SX_BUSY | 3835 TA_BUSY | VGT_BUSY | 3836 DB_BUSY | CB_BUSY | 3837 SPI_BUSY | VGT_BUSY_NO_DMA)) 3838 reset_mask |= RADEON_RESET_GFX; 3839 3840 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3841 CP_BUSY | CP_COHERENCY_BUSY)) 3842 reset_mask |= RADEON_RESET_CP; 3843 3844 if (tmp & GRBM_EE_BUSY) 3845 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3846 3847 /* DMA_STATUS_REG */ 3848 tmp = RREG32(DMA_STATUS_REG); 3849 if (!(tmp & DMA_IDLE)) 3850 reset_mask |= RADEON_RESET_DMA; 3851 3852 /* SRBM_STATUS2 */ 3853 tmp = RREG32(SRBM_STATUS2); 3854 if (tmp & DMA_BUSY) 3855 reset_mask |= RADEON_RESET_DMA; 3856 3857 /* SRBM_STATUS */ 3858 tmp = RREG32(SRBM_STATUS); 3859 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3860 reset_mask |= RADEON_RESET_RLC; 3861 3862 if (tmp & IH_BUSY) 3863 reset_mask |= RADEON_RESET_IH; 3864 3865 if (tmp & SEM_BUSY) 3866 reset_mask |= RADEON_RESET_SEM; 3867 3868 if (tmp & GRBM_RQ_PENDING) 3869 reset_mask |= RADEON_RESET_GRBM; 3870 3871 if (tmp & VMC_BUSY) 3872 reset_mask |= RADEON_RESET_VMC; 3873 3874 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3875 MCC_BUSY | MCD_BUSY)) 3876 reset_mask |= RADEON_RESET_MC; 3877 3878 if (evergreen_is_display_hung(rdev)) 3879 reset_mask |= RADEON_RESET_DISPLAY; 3880 3881 /* VM_L2_STATUS */ 3882 tmp = RREG32(VM_L2_STATUS); 3883 if (tmp & L2_BUSY) 3884 reset_mask |= RADEON_RESET_VMC; 3885 3886 /* Skip MC reset as it's mostly likely not hung, just busy */ 3887 if (reset_mask & RADEON_RESET_MC) { 3888 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3889 reset_mask &= ~RADEON_RESET_MC; 3890 } 3891 3892 return reset_mask; 3893} 3894 3895static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3896{ 3897 struct evergreen_mc_save save; 3898 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3899 u32 tmp; 3900 3901 if (reset_mask == 0) 3902 return; 3903 3904 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3905 3906 evergreen_print_gpu_status_regs(rdev); 3907 3908 /* Disable CP parsing/prefetching */ 3909 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3910 3911 if (reset_mask & RADEON_RESET_DMA) { 3912 /* Disable DMA */ 3913 tmp = RREG32(DMA_RB_CNTL); 3914 tmp &= ~DMA_RB_ENABLE; 3915 WREG32(DMA_RB_CNTL, tmp); 3916 } 3917 3918 udelay(50); 3919 3920 evergreen_mc_stop(rdev, &save); 3921 if (evergreen_mc_wait_for_idle(rdev)) { 3922 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3923 } 3924 3925 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 3926 grbm_soft_reset |= SOFT_RESET_DB | 3927 SOFT_RESET_CB | 3928 SOFT_RESET_PA | 3929 SOFT_RESET_SC | 3930 SOFT_RESET_SPI | 3931 SOFT_RESET_SX | 3932 SOFT_RESET_SH | 3933 SOFT_RESET_TC | 3934 SOFT_RESET_TA | 3935 SOFT_RESET_VC | 3936 SOFT_RESET_VGT; 3937 } 3938 3939 if (reset_mask & RADEON_RESET_CP) { 3940 grbm_soft_reset |= SOFT_RESET_CP | 3941 SOFT_RESET_VGT; 3942 3943 srbm_soft_reset |= SOFT_RESET_GRBM; 3944 } 3945 3946 if (reset_mask & RADEON_RESET_DMA) 3947 srbm_soft_reset |= SOFT_RESET_DMA; 3948 3949 if (reset_mask & RADEON_RESET_DISPLAY) 3950 srbm_soft_reset |= SOFT_RESET_DC; 3951 3952 if (reset_mask & RADEON_RESET_RLC) 3953 srbm_soft_reset |= SOFT_RESET_RLC; 3954 3955 if (reset_mask & RADEON_RESET_SEM) 3956 srbm_soft_reset |= SOFT_RESET_SEM; 3957 3958 if (reset_mask & RADEON_RESET_IH) 3959 srbm_soft_reset |= SOFT_RESET_IH; 3960 3961 if (reset_mask & RADEON_RESET_GRBM) 3962 srbm_soft_reset |= SOFT_RESET_GRBM; 3963 3964 if (reset_mask & RADEON_RESET_VMC) 3965 srbm_soft_reset |= SOFT_RESET_VMC; 3966 3967 if (!(rdev->flags & RADEON_IS_IGP)) { 3968 if (reset_mask & RADEON_RESET_MC) 3969 srbm_soft_reset |= SOFT_RESET_MC; 3970 } 3971 3972 if (grbm_soft_reset) { 3973 tmp = RREG32(GRBM_SOFT_RESET); 3974 tmp |= grbm_soft_reset; 3975 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3976 WREG32(GRBM_SOFT_RESET, tmp); 3977 tmp = RREG32(GRBM_SOFT_RESET); 3978 3979 udelay(50); 3980 3981 tmp &= ~grbm_soft_reset; 3982 WREG32(GRBM_SOFT_RESET, tmp); 3983 tmp = RREG32(GRBM_SOFT_RESET); 3984 } 3985 3986 if (srbm_soft_reset) { 3987 tmp = RREG32(SRBM_SOFT_RESET); 3988 tmp |= srbm_soft_reset; 3989 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3990 WREG32(SRBM_SOFT_RESET, tmp); 3991 tmp = RREG32(SRBM_SOFT_RESET); 3992 3993 udelay(50); 3994 3995 tmp &= ~srbm_soft_reset; 3996 WREG32(SRBM_SOFT_RESET, tmp); 3997 tmp = RREG32(SRBM_SOFT_RESET); 3998 } 3999 4000 /* Wait a little for things to settle down */ 4001 udelay(50); 4002 4003 evergreen_mc_resume(rdev, &save); 4004 udelay(50); 4005 4006 evergreen_print_gpu_status_regs(rdev); 4007} 4008 4009void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 4010{ 4011 struct evergreen_mc_save save; 4012 u32 tmp, i; 4013 4014 dev_info(rdev->dev, "GPU pci config reset\n"); 4015 4016 /* disable dpm? */ 4017 4018 /* Disable CP parsing/prefetching */ 4019 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 4020 udelay(50); 4021 /* Disable DMA */ 4022 tmp = RREG32(DMA_RB_CNTL); 4023 tmp &= ~DMA_RB_ENABLE; 4024 WREG32(DMA_RB_CNTL, tmp); 4025 /* XXX other engines? */ 4026 4027 /* halt the rlc */ 4028 r600_rlc_stop(rdev); 4029 4030 udelay(50); 4031 4032 /* set mclk/sclk to bypass */ 4033 rv770_set_clk_bypass_mode(rdev); 4034 /* disable BM */ 4035 pci_clear_master(rdev->pdev); 4036 /* disable mem access */ 4037 evergreen_mc_stop(rdev, &save); 4038 if (evergreen_mc_wait_for_idle(rdev)) { 4039 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 4040 } 4041 /* reset */ 4042 radeon_pci_config_reset(rdev); 4043 /* wait for asic to come out of reset */ 4044 for (i = 0; i < rdev->usec_timeout; i++) { 4045 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 4046 break; 4047 udelay(1); 4048 } 4049} 4050 4051int evergreen_asic_reset(struct radeon_device *rdev, bool hard) 4052{ 4053 u32 reset_mask; 4054 4055 if (hard) { 4056 evergreen_gpu_pci_config_reset(rdev); 4057 return 0; 4058 } 4059 4060 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4061 4062 if (reset_mask) 4063 r600_set_bios_scratch_engine_hung(rdev, true); 4064 4065 /* try soft reset */ 4066 evergreen_gpu_soft_reset(rdev, reset_mask); 4067 4068 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4069 4070 /* try pci config reset */ 4071 if (reset_mask && radeon_hard_reset) 4072 evergreen_gpu_pci_config_reset(rdev); 4073 4074 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4075 4076 if (!reset_mask) 4077 r600_set_bios_scratch_engine_hung(rdev, false); 4078 4079 return 0; 4080} 4081 4082/** 4083 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 4084 * 4085 * @rdev: radeon_device pointer 4086 * @ring: radeon_ring structure holding ring information 4087 * 4088 * Check if the GFX engine is locked up. 4089 * Returns true if the engine appears to be locked up, false if not. 4090 */ 4091bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 4092{ 4093 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4094 4095 if (!(reset_mask & (RADEON_RESET_GFX | 4096 RADEON_RESET_COMPUTE | 4097 RADEON_RESET_CP))) { 4098 radeon_ring_lockup_update(rdev, ring); 4099 return false; 4100 } 4101 return radeon_ring_test_lockup(rdev, ring); 4102} 4103 4104/* 4105 * RLC 4106 */ 4107#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 4108#define RLC_CLEAR_STATE_END_MARKER 0x00000001 4109 4110void sumo_rlc_fini(struct radeon_device *rdev) 4111{ 4112 int r; 4113 4114 /* save restore block */ 4115 if (rdev->rlc.save_restore_obj) { 4116 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4117 if (unlikely(r != 0)) 4118 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 4119 radeon_bo_unpin(rdev->rlc.save_restore_obj); 4120 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4121 4122 radeon_bo_unref(&rdev->rlc.save_restore_obj); 4123 rdev->rlc.save_restore_obj = NULL; 4124 } 4125 4126 /* clear state block */ 4127 if (rdev->rlc.clear_state_obj) { 4128 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4129 if (unlikely(r != 0)) 4130 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 4131 radeon_bo_unpin(rdev->rlc.clear_state_obj); 4132 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4133 4134 radeon_bo_unref(&rdev->rlc.clear_state_obj); 4135 rdev->rlc.clear_state_obj = NULL; 4136 } 4137 4138 /* clear state block */ 4139 if (rdev->rlc.cp_table_obj) { 4140 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4141 if (unlikely(r != 0)) 4142 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4143 radeon_bo_unpin(rdev->rlc.cp_table_obj); 4144 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4145 4146 radeon_bo_unref(&rdev->rlc.cp_table_obj); 4147 rdev->rlc.cp_table_obj = NULL; 4148 } 4149} 4150 4151#define CP_ME_TABLE_SIZE 96 4152 4153int sumo_rlc_init(struct radeon_device *rdev) 4154{ 4155 const u32 *src_ptr; 4156 volatile u32 *dst_ptr; 4157 u32 dws, data, i, j, k, reg_num; 4158 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 4159 u64 reg_list_mc_addr; 4160 const struct cs_section_def *cs_data; 4161 int r; 4162 4163 src_ptr = rdev->rlc.reg_list; 4164 dws = rdev->rlc.reg_list_size; 4165 if (rdev->family >= CHIP_BONAIRE) { 4166 dws += (5 * 16) + 48 + 48 + 64; 4167 } 4168 cs_data = rdev->rlc.cs_data; 4169 4170 if (src_ptr) { 4171 /* save restore block */ 4172 if (rdev->rlc.save_restore_obj == NULL) { 4173 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4174 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4175 NULL, &rdev->rlc.save_restore_obj); 4176 if (r) { 4177 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4178 return r; 4179 } 4180 } 4181 4182 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4183 if (unlikely(r != 0)) { 4184 sumo_rlc_fini(rdev); 4185 return r; 4186 } 4187 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4188 &rdev->rlc.save_restore_gpu_addr); 4189 if (r) { 4190 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4191 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4192 sumo_rlc_fini(rdev); 4193 return r; 4194 } 4195 4196 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr); 4197 if (r) { 4198 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4199 sumo_rlc_fini(rdev); 4200 return r; 4201 } 4202 /* write the sr buffer */ 4203 dst_ptr = rdev->rlc.sr_ptr; 4204 if (rdev->family >= CHIP_TAHITI) { 4205 /* SI */ 4206 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4207 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4208 } else { 4209 /* ON/LN/TN */ 4210 /* format: 4211 * dw0: (reg2 << 16) | reg1 4212 * dw1: reg1 save space 4213 * dw2: reg2 save space 4214 */ 4215 for (i = 0; i < dws; i++) { 4216 data = src_ptr[i] >> 2; 4217 i++; 4218 if (i < dws) 4219 data |= (src_ptr[i] >> 2) << 16; 4220 j = (((i - 1) * 3) / 2); 4221 dst_ptr[j] = cpu_to_le32(data); 4222 } 4223 j = ((i * 3) / 2); 4224 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4225 } 4226 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4227 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4228 } 4229 4230 if (cs_data) { 4231 /* clear state block */ 4232 if (rdev->family >= CHIP_BONAIRE) { 4233 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4234 } else if (rdev->family >= CHIP_TAHITI) { 4235 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4236 dws = rdev->rlc.clear_state_size + (256 / 4); 4237 } else { 4238 reg_list_num = 0; 4239 dws = 0; 4240 for (i = 0; cs_data[i].section != NULL; i++) { 4241 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4242 reg_list_num++; 4243 dws += cs_data[i].section[j].reg_count; 4244 } 4245 } 4246 reg_list_blk_index = (3 * reg_list_num + 2); 4247 dws += reg_list_blk_index; 4248 rdev->rlc.clear_state_size = dws; 4249 } 4250 4251 if (rdev->rlc.clear_state_obj == NULL) { 4252 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4253 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4254 NULL, &rdev->rlc.clear_state_obj); 4255 if (r) { 4256 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4257 sumo_rlc_fini(rdev); 4258 return r; 4259 } 4260 } 4261 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4262 if (unlikely(r != 0)) { 4263 sumo_rlc_fini(rdev); 4264 return r; 4265 } 4266 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4267 &rdev->rlc.clear_state_gpu_addr); 4268 if (r) { 4269 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4270 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4271 sumo_rlc_fini(rdev); 4272 return r; 4273 } 4274 4275 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr); 4276 if (r) { 4277 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4278 sumo_rlc_fini(rdev); 4279 return r; 4280 } 4281 /* set up the cs buffer */ 4282 dst_ptr = rdev->rlc.cs_ptr; 4283 if (rdev->family >= CHIP_BONAIRE) { 4284 cik_get_csb_buffer(rdev, dst_ptr); 4285 } else if (rdev->family >= CHIP_TAHITI) { 4286 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4287 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4288 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4289 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4290 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4291 } else { 4292 reg_list_hdr_blk_index = 0; 4293 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4294 data = upper_32_bits(reg_list_mc_addr); 4295 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4296 reg_list_hdr_blk_index++; 4297 for (i = 0; cs_data[i].section != NULL; i++) { 4298 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4299 reg_num = cs_data[i].section[j].reg_count; 4300 data = reg_list_mc_addr & 0xffffffff; 4301 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4302 reg_list_hdr_blk_index++; 4303 4304 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4305 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4306 reg_list_hdr_blk_index++; 4307 4308 data = 0x08000000 | (reg_num * 4); 4309 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4310 reg_list_hdr_blk_index++; 4311 4312 for (k = 0; k < reg_num; k++) { 4313 data = cs_data[i].section[j].extent[k]; 4314 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4315 } 4316 reg_list_mc_addr += reg_num * 4; 4317 reg_list_blk_index += reg_num; 4318 } 4319 } 4320 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4321 } 4322 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4323 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4324 } 4325 4326 if (rdev->rlc.cp_table_size) { 4327 if (rdev->rlc.cp_table_obj == NULL) { 4328 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4329 PAGE_SIZE, true, 4330 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4331 NULL, &rdev->rlc.cp_table_obj); 4332 if (r) { 4333 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4334 sumo_rlc_fini(rdev); 4335 return r; 4336 } 4337 } 4338 4339 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4340 if (unlikely(r != 0)) { 4341 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4342 sumo_rlc_fini(rdev); 4343 return r; 4344 } 4345 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4346 &rdev->rlc.cp_table_gpu_addr); 4347 if (r) { 4348 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4349 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4350 sumo_rlc_fini(rdev); 4351 return r; 4352 } 4353 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr); 4354 if (r) { 4355 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4356 sumo_rlc_fini(rdev); 4357 return r; 4358 } 4359 4360 cik_init_cp_pg_table(rdev); 4361 4362 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4363 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4364 4365 } 4366 4367 return 0; 4368} 4369 4370static void evergreen_rlc_start(struct radeon_device *rdev) 4371{ 4372 u32 mask = RLC_ENABLE; 4373 4374 if (rdev->flags & RADEON_IS_IGP) { 4375 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4376 } 4377 4378 WREG32(RLC_CNTL, mask); 4379} 4380 4381int evergreen_rlc_resume(struct radeon_device *rdev) 4382{ 4383 u32 i; 4384 const __be32 *fw_data; 4385 4386 if (!rdev->rlc_fw) 4387 return -EINVAL; 4388 4389 r600_rlc_stop(rdev); 4390 4391 WREG32(RLC_HB_CNTL, 0); 4392 4393 if (rdev->flags & RADEON_IS_IGP) { 4394 if (rdev->family == CHIP_ARUBA) { 4395 u32 always_on_bitmap = 4396 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4397 /* find out the number of active simds */ 4398 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4399 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4400 tmp = hweight32(~tmp); 4401 if (tmp == rdev->config.cayman.max_simds_per_se) { 4402 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4403 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4404 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4405 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4406 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4407 } 4408 } else { 4409 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4410 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4411 } 4412 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4413 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4414 } else { 4415 WREG32(RLC_HB_BASE, 0); 4416 WREG32(RLC_HB_RPTR, 0); 4417 WREG32(RLC_HB_WPTR, 0); 4418 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4419 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4420 } 4421 WREG32(RLC_MC_CNTL, 0); 4422 WREG32(RLC_UCODE_CNTL, 0); 4423 4424 fw_data = (const __be32 *)rdev->rlc_fw->data; 4425 if (rdev->family >= CHIP_ARUBA) { 4426 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4427 WREG32(RLC_UCODE_ADDR, i); 4428 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4429 } 4430 } else if (rdev->family >= CHIP_CAYMAN) { 4431 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4432 WREG32(RLC_UCODE_ADDR, i); 4433 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4434 } 4435 } else { 4436 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4437 WREG32(RLC_UCODE_ADDR, i); 4438 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4439 } 4440 } 4441 WREG32(RLC_UCODE_ADDR, 0); 4442 4443 evergreen_rlc_start(rdev); 4444 4445 return 0; 4446} 4447 4448/* Interrupts */ 4449 4450u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4451{ 4452 if (crtc >= rdev->num_crtc) 4453 return 0; 4454 else 4455 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4456} 4457 4458void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4459{ 4460 int i; 4461 u32 tmp; 4462 4463 if (rdev->family >= CHIP_CAYMAN) { 4464 cayman_cp_int_cntl_setup(rdev, 0, 4465 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4466 cayman_cp_int_cntl_setup(rdev, 1, 0); 4467 cayman_cp_int_cntl_setup(rdev, 2, 0); 4468 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4469 WREG32(CAYMAN_DMA1_CNTL, tmp); 4470 } else 4471 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4472 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4473 WREG32(DMA_CNTL, tmp); 4474 WREG32(GRBM_INT_CNTL, 0); 4475 WREG32(SRBM_INT_CNTL, 0); 4476 for (i = 0; i < rdev->num_crtc; i++) 4477 WREG32(INT_MASK + crtc_offsets[i], 0); 4478 for (i = 0; i < rdev->num_crtc; i++) 4479 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0); 4480 4481 /* only one DAC on DCE5 */ 4482 if (!ASIC_IS_DCE5(rdev)) 4483 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4484 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4485 4486 for (i = 0; i < 6; i++) 4487 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY); 4488} 4489 4490/* Note that the order we write back regs here is important */ 4491int evergreen_irq_set(struct radeon_device *rdev) 4492{ 4493 int i; 4494 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4495 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4496 u32 grbm_int_cntl = 0; 4497 u32 dma_cntl, dma_cntl1 = 0; 4498 u32 thermal_int = 0; 4499 4500 if (!rdev->irq.installed) { 4501 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4502 return -EINVAL; 4503 } 4504 /* don't enable anything if the ih is disabled */ 4505 if (!rdev->ih.enabled) { 4506 r600_disable_interrupts(rdev); 4507 /* force the active interrupt state to all disabled */ 4508 evergreen_disable_interrupt_state(rdev); 4509 return 0; 4510 } 4511 4512 if (rdev->family == CHIP_ARUBA) 4513 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4514 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4515 else 4516 thermal_int = RREG32(CG_THERMAL_INT) & 4517 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4518 4519 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4520 4521 if (rdev->family >= CHIP_CAYMAN) { 4522 /* enable CP interrupts on all rings */ 4523 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4524 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4525 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4526 } 4527 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4528 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4529 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4530 } 4531 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4532 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4533 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4534 } 4535 } else { 4536 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4537 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4538 cp_int_cntl |= RB_INT_ENABLE; 4539 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4540 } 4541 } 4542 4543 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4544 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4545 dma_cntl |= TRAP_ENABLE; 4546 } 4547 4548 if (rdev->family >= CHIP_CAYMAN) { 4549 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4550 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4551 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4552 dma_cntl1 |= TRAP_ENABLE; 4553 } 4554 } 4555 4556 if (rdev->irq.dpm_thermal) { 4557 DRM_DEBUG("dpm thermal\n"); 4558 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4559 } 4560 4561 if (rdev->family >= CHIP_CAYMAN) { 4562 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4563 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4564 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4565 } else 4566 WREG32(CP_INT_CNTL, cp_int_cntl); 4567 4568 WREG32(DMA_CNTL, dma_cntl); 4569 4570 if (rdev->family >= CHIP_CAYMAN) 4571 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4572 4573 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4574 4575 for (i = 0; i < rdev->num_crtc; i++) { 4576 radeon_irq_kms_set_irq_n_enabled( 4577 rdev, INT_MASK + crtc_offsets[i], 4578 VBLANK_INT_MASK, 4579 rdev->irq.crtc_vblank_int[i] || 4580 atomic_read(&rdev->irq.pflip[i]), "vblank", i); 4581 } 4582 4583 for (i = 0; i < rdev->num_crtc; i++) 4584 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK); 4585 4586 for (i = 0; i < 6; i++) { 4587 radeon_irq_kms_set_irq_n_enabled( 4588 rdev, DC_HPDx_INT_CONTROL(i), 4589 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN, 4590 rdev->irq.hpd[i], "HPD", i); 4591 } 4592 4593 if (rdev->family == CHIP_ARUBA) 4594 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4595 else 4596 WREG32(CG_THERMAL_INT, thermal_int); 4597 4598 for (i = 0; i < 6; i++) { 4599 radeon_irq_kms_set_irq_n_enabled( 4600 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i], 4601 AFMT_AZ_FORMAT_WTRIG_MASK, 4602 rdev->irq.afmt[i], "HDMI", i); 4603 } 4604 4605 /* posting read */ 4606 RREG32(SRBM_STATUS); 4607 4608 return 0; 4609} 4610 4611/* Note that the order we write back regs here is important */ 4612static void evergreen_irq_ack(struct radeon_device *rdev) 4613{ 4614 int i, j; 4615 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int; 4616 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; 4617 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status; 4618 4619 for (i = 0; i < 6; i++) { 4620 disp_int[i] = RREG32(evergreen_disp_int_status[i]); 4621 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]); 4622 if (i < rdev->num_crtc) 4623 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]); 4624 } 4625 4626 /* We write back each interrupt register in pairs of two */ 4627 for (i = 0; i < rdev->num_crtc; i += 2) { 4628 for (j = i; j < (i + 2); j++) { 4629 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED) 4630 WREG32(GRPH_INT_STATUS + crtc_offsets[j], 4631 GRPH_PFLIP_INT_CLEAR); 4632 } 4633 4634 for (j = i; j < (i + 2); j++) { 4635 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT) 4636 WREG32(VBLANK_STATUS + crtc_offsets[j], 4637 VBLANK_ACK); 4638 if (disp_int[j] & LB_D1_VLINE_INTERRUPT) 4639 WREG32(VLINE_STATUS + crtc_offsets[j], 4640 VLINE_ACK); 4641 } 4642 } 4643 4644 for (i = 0; i < 6; i++) { 4645 if (disp_int[i] & DC_HPD1_INTERRUPT) 4646 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK); 4647 } 4648 4649 for (i = 0; i < 6; i++) { 4650 if (disp_int[i] & DC_HPD1_RX_INTERRUPT) 4651 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK); 4652 } 4653 4654 for (i = 0; i < 6; i++) { 4655 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG) 4656 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i], 4657 AFMT_AZ_FORMAT_WTRIG_ACK); 4658 } 4659} 4660 4661static void evergreen_irq_disable(struct radeon_device *rdev) 4662{ 4663 r600_disable_interrupts(rdev); 4664 /* Wait and acknowledge irq */ 4665 mdelay(1); 4666 evergreen_irq_ack(rdev); 4667 evergreen_disable_interrupt_state(rdev); 4668} 4669 4670void evergreen_irq_suspend(struct radeon_device *rdev) 4671{ 4672 evergreen_irq_disable(rdev); 4673 r600_rlc_stop(rdev); 4674} 4675 4676static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 4677{ 4678 u32 wptr, tmp; 4679 4680 if (rdev->wb.enabled) 4681 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 4682 else 4683 wptr = RREG32(IH_RB_WPTR); 4684 4685 if (wptr & RB_OVERFLOW) { 4686 wptr &= ~RB_OVERFLOW; 4687 /* When a ring buffer overflow happen start parsing interrupt 4688 * from the last not overwritten vector (wptr + 16). Hopefully 4689 * this should allow us to catchup. 4690 */ 4691 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 4692 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 4693 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 4694 tmp = RREG32(IH_RB_CNTL); 4695 tmp |= IH_WPTR_OVERFLOW_CLEAR; 4696 WREG32(IH_RB_CNTL, tmp); 4697 } 4698 return (wptr & rdev->ih.ptr_mask); 4699} 4700 4701int evergreen_irq_process(struct radeon_device *rdev) 4702{ 4703 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; 4704 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status; 4705 u32 crtc_idx, hpd_idx, afmt_idx; 4706 u32 mask; 4707 u32 wptr; 4708 u32 rptr; 4709 u32 src_id, src_data; 4710 u32 ring_index; 4711 bool queue_hotplug = false; 4712 bool queue_hdmi = false; 4713 bool queue_dp = false; 4714 bool queue_thermal = false; 4715 u32 status, addr; 4716 const char *event_name; 4717 4718 if (!rdev->ih.enabled || rdev->shutdown) 4719 return IRQ_NONE; 4720 4721 wptr = evergreen_get_ih_wptr(rdev); 4722 4723restart_ih: 4724 /* is somebody else already processing irqs? */ 4725 if (atomic_xchg(&rdev->ih.lock, 1)) 4726 return IRQ_NONE; 4727 4728 rptr = rdev->ih.rptr; 4729 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 4730 4731 /* Order reading of wptr vs. reading of IH ring data */ 4732 rmb(); 4733 4734 /* display interrupts */ 4735 evergreen_irq_ack(rdev); 4736 4737 while (rptr != wptr) { 4738 /* wptr/rptr are in bytes! */ 4739 ring_index = rptr / 4; 4740 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 4741 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 4742 4743 switch (src_id) { 4744 case 1: /* D1 vblank/vline */ 4745 case 2: /* D2 vblank/vline */ 4746 case 3: /* D3 vblank/vline */ 4747 case 4: /* D4 vblank/vline */ 4748 case 5: /* D5 vblank/vline */ 4749 case 6: /* D6 vblank/vline */ 4750 crtc_idx = src_id - 1; 4751 4752 if (src_data == 0) { /* vblank */ 4753 mask = LB_D1_VBLANK_INTERRUPT; 4754 event_name = "vblank"; 4755 4756 if (rdev->irq.crtc_vblank_int[crtc_idx]) { 4757 drm_handle_vblank(rdev->ddev, crtc_idx); 4758 rdev->pm.vblank_sync = true; 4759 wake_up(&rdev->irq.vblank_queue); 4760 } 4761 if (atomic_read(&rdev->irq.pflip[crtc_idx])) { 4762 radeon_crtc_handle_vblank(rdev, 4763 crtc_idx); 4764 } 4765 4766 } else if (src_data == 1) { /* vline */ 4767 mask = LB_D1_VLINE_INTERRUPT; 4768 event_name = "vline"; 4769 } else { 4770 DRM_DEBUG("Unhandled interrupt: %d %d\n", 4771 src_id, src_data); 4772 break; 4773 } 4774 4775 if (!(disp_int[crtc_idx] & mask)) { 4776 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n", 4777 crtc_idx + 1, event_name); 4778 } 4779 4780 disp_int[crtc_idx] &= ~mask; 4781 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name); 4782 4783 break; 4784 case 8: /* D1 page flip */ 4785 case 10: /* D2 page flip */ 4786 case 12: /* D3 page flip */ 4787 case 14: /* D4 page flip */ 4788 case 16: /* D5 page flip */ 4789 case 18: /* D6 page flip */ 4790 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 4791 if (radeon_use_pflipirq > 0) 4792 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 4793 break; 4794 case 42: /* HPD hotplug */ 4795 if (src_data <= 5) { 4796 hpd_idx = src_data; 4797 mask = DC_HPD1_INTERRUPT; 4798 queue_hotplug = true; 4799 event_name = "HPD"; 4800 4801 } else if (src_data <= 11) { 4802 hpd_idx = src_data - 6; 4803 mask = DC_HPD1_RX_INTERRUPT; 4804 queue_dp = true; 4805 event_name = "HPD_RX"; 4806 4807 } else { 4808 DRM_DEBUG("Unhandled interrupt: %d %d\n", 4809 src_id, src_data); 4810 break; 4811 } 4812 4813 if (!(disp_int[hpd_idx] & mask)) 4814 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 4815 4816 disp_int[hpd_idx] &= ~mask; 4817 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1); 4818 4819 break; 4820 case 44: /* hdmi */ 4821 afmt_idx = src_data; 4822 if (afmt_idx > 5) { 4823 DRM_ERROR("Unhandled interrupt: %d %d\n", 4824 src_id, src_data); 4825 break; 4826 } 4827 4828 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG)) 4829 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 4830 4831 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG; 4832 queue_hdmi = true; 4833 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1); 4834 break; 4835 case 96: 4836 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 4837 WREG32(SRBM_INT_ACK, 0x1); 4838 break; 4839 case 124: /* UVD */ 4840 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 4841 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 4842 break; 4843 case 146: 4844 case 147: 4845 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 4846 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 4847 /* reset addr and status */ 4848 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 4849 if (addr == 0x0 && status == 0x0) 4850 break; 4851 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 4852 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 4853 addr); 4854 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 4855 status); 4856 cayman_vm_decode_fault(rdev, status, addr); 4857 break; 4858 case 176: /* CP_INT in ring buffer */ 4859 case 177: /* CP_INT in IB1 */ 4860 case 178: /* CP_INT in IB2 */ 4861 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 4862 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4863 break; 4864 case 181: /* CP EOP event */ 4865 DRM_DEBUG("IH: CP EOP\n"); 4866 if (rdev->family >= CHIP_CAYMAN) { 4867 switch (src_data) { 4868 case 0: 4869 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4870 break; 4871 case 1: 4872 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 4873 break; 4874 case 2: 4875 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 4876 break; 4877 } 4878 } else 4879 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4880 break; 4881 case 224: /* DMA trap event */ 4882 DRM_DEBUG("IH: DMA trap\n"); 4883 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 4884 break; 4885 case 230: /* thermal low to high */ 4886 DRM_DEBUG("IH: thermal low to high\n"); 4887 rdev->pm.dpm.thermal.high_to_low = false; 4888 queue_thermal = true; 4889 break; 4890 case 231: /* thermal high to low */ 4891 DRM_DEBUG("IH: thermal high to low\n"); 4892 rdev->pm.dpm.thermal.high_to_low = true; 4893 queue_thermal = true; 4894 break; 4895 case 233: /* GUI IDLE */ 4896 DRM_DEBUG("IH: GUI idle\n"); 4897 break; 4898 case 244: /* DMA trap event */ 4899 if (rdev->family >= CHIP_CAYMAN) { 4900 DRM_DEBUG("IH: DMA1 trap\n"); 4901 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 4902 } 4903 break; 4904 default: 4905 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4906 break; 4907 } 4908 4909 /* wptr/rptr are in bytes! */ 4910 rptr += 16; 4911 rptr &= rdev->ih.ptr_mask; 4912 WREG32(IH_RB_RPTR, rptr); 4913 } 4914 if (queue_dp) 4915 schedule_work(&rdev->dp_work); 4916 if (queue_hotplug) 4917 schedule_delayed_work(&rdev->hotplug_work, 0); 4918 if (queue_hdmi) 4919 schedule_work(&rdev->audio_work); 4920 if (queue_thermal && rdev->pm.dpm_enabled) 4921 schedule_work(&rdev->pm.dpm.thermal.work); 4922 rdev->ih.rptr = rptr; 4923 WREG32(IH_RB_RPTR, rptr); 4924 atomic_set(&rdev->ih.lock, 0); 4925 4926 /* make sure wptr hasn't changed while processing */ 4927 wptr = evergreen_get_ih_wptr(rdev); 4928 if (wptr != rptr) 4929 goto restart_ih; 4930 4931 return IRQ_HANDLED; 4932} 4933 4934static void evergreen_uvd_init(struct radeon_device *rdev) 4935{ 4936 int r; 4937 4938 if (!rdev->has_uvd) 4939 return; 4940 4941 r = radeon_uvd_init(rdev); 4942 if (r) { 4943 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 4944 /* 4945 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 4946 * to early fails uvd_v2_2_resume() and thus nothing happens 4947 * there. So it is pointless to try to go through that code 4948 * hence why we disable uvd here. 4949 */ 4950 rdev->has_uvd = false; 4951 return; 4952 } 4953 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 4954 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 4955} 4956 4957static void evergreen_uvd_start(struct radeon_device *rdev) 4958{ 4959 int r; 4960 4961 if (!rdev->has_uvd) 4962 return; 4963 4964 r = uvd_v2_2_resume(rdev); 4965 if (r) { 4966 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 4967 goto error; 4968 } 4969 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 4970 if (r) { 4971 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 4972 goto error; 4973 } 4974 return; 4975 4976error: 4977 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 4978} 4979 4980static void evergreen_uvd_resume(struct radeon_device *rdev) 4981{ 4982 struct radeon_ring *ring; 4983 int r; 4984 4985 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 4986 return; 4987 4988 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 4989 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); 4990 if (r) { 4991 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 4992 return; 4993 } 4994 r = uvd_v1_0_init(rdev); 4995 if (r) { 4996 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 4997 return; 4998 } 4999} 5000 5001static int evergreen_startup(struct radeon_device *rdev) 5002{ 5003 struct radeon_ring *ring; 5004 int r; 5005 5006 /* enable pcie gen2 link */ 5007 evergreen_pcie_gen2_enable(rdev); 5008 /* enable aspm */ 5009 evergreen_program_aspm(rdev); 5010 5011 /* scratch needs to be initialized before MC */ 5012 r = r600_vram_scratch_init(rdev); 5013 if (r) 5014 return r; 5015 5016 evergreen_mc_program(rdev); 5017 5018 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5019 r = ni_mc_load_microcode(rdev); 5020 if (r) { 5021 DRM_ERROR("Failed to load MC firmware!\n"); 5022 return r; 5023 } 5024 } 5025 5026 if (rdev->flags & RADEON_IS_AGP) { 5027 evergreen_agp_enable(rdev); 5028 } else { 5029 r = evergreen_pcie_gart_enable(rdev); 5030 if (r) 5031 return r; 5032 } 5033 evergreen_gpu_init(rdev); 5034 5035 /* allocate rlc buffers */ 5036 if (rdev->flags & RADEON_IS_IGP) { 5037 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5038 rdev->rlc.reg_list_size = 5039 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5040 rdev->rlc.cs_data = evergreen_cs_data; 5041 r = sumo_rlc_init(rdev); 5042 if (r) { 5043 DRM_ERROR("Failed to init rlc BOs!\n"); 5044 return r; 5045 } 5046 } 5047 5048 /* allocate wb buffer */ 5049 r = radeon_wb_init(rdev); 5050 if (r) 5051 return r; 5052 5053 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5054 if (r) { 5055 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5056 return r; 5057 } 5058 5059 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5060 if (r) { 5061 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5062 return r; 5063 } 5064 5065 evergreen_uvd_start(rdev); 5066 5067 /* Enable IRQ */ 5068 if (!rdev->irq.installed) { 5069 r = radeon_irq_kms_init(rdev); 5070 if (r) 5071 return r; 5072 } 5073 5074 r = r600_irq_init(rdev); 5075 if (r) { 5076 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5077 radeon_irq_kms_fini(rdev); 5078 return r; 5079 } 5080 evergreen_irq_set(rdev); 5081 5082 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5083 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5084 RADEON_CP_PACKET2); 5085 if (r) 5086 return r; 5087 5088 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5089 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5090 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5091 if (r) 5092 return r; 5093 5094 r = evergreen_cp_load_microcode(rdev); 5095 if (r) 5096 return r; 5097 r = evergreen_cp_resume(rdev); 5098 if (r) 5099 return r; 5100 r = r600_dma_resume(rdev); 5101 if (r) 5102 return r; 5103 5104 evergreen_uvd_resume(rdev); 5105 5106 r = radeon_ib_pool_init(rdev); 5107 if (r) { 5108 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5109 return r; 5110 } 5111 5112 r = radeon_audio_init(rdev); 5113 if (r) { 5114 DRM_ERROR("radeon: audio init failed\n"); 5115 return r; 5116 } 5117 5118 return 0; 5119} 5120 5121int evergreen_resume(struct radeon_device *rdev) 5122{ 5123 int r; 5124 5125 /* reset the asic, the gfx blocks are often in a bad state 5126 * after the driver is unloaded or after a resume 5127 */ 5128 if (radeon_asic_reset(rdev)) 5129 dev_warn(rdev->dev, "GPU reset failed !\n"); 5130 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5131 * posting will perform necessary task to bring back GPU into good 5132 * shape. 5133 */ 5134 /* post card */ 5135 atom_asic_init(rdev->mode_info.atom_context); 5136 5137 /* init golden registers */ 5138 evergreen_init_golden_registers(rdev); 5139 5140 if (rdev->pm.pm_method == PM_METHOD_DPM) 5141 radeon_pm_resume(rdev); 5142 5143 rdev->accel_working = true; 5144 r = evergreen_startup(rdev); 5145 if (r) { 5146 DRM_ERROR("evergreen startup failed on resume\n"); 5147 rdev->accel_working = false; 5148 return r; 5149 } 5150 5151 return r; 5152 5153} 5154 5155int evergreen_suspend(struct radeon_device *rdev) 5156{ 5157 radeon_pm_suspend(rdev); 5158 radeon_audio_fini(rdev); 5159 if (rdev->has_uvd) { 5160 uvd_v1_0_fini(rdev); 5161 radeon_uvd_suspend(rdev); 5162 } 5163 r700_cp_stop(rdev); 5164 r600_dma_stop(rdev); 5165 evergreen_irq_suspend(rdev); 5166 radeon_wb_disable(rdev); 5167 evergreen_pcie_gart_disable(rdev); 5168 5169 return 0; 5170} 5171 5172/* Plan is to move initialization in that function and use 5173 * helper function so that radeon_device_init pretty much 5174 * do nothing more than calling asic specific function. This 5175 * should also allow to remove a bunch of callback function 5176 * like vram_info. 5177 */ 5178int evergreen_init(struct radeon_device *rdev) 5179{ 5180 int r; 5181 5182 /* Read BIOS */ 5183 if (!radeon_get_bios(rdev)) { 5184 if (ASIC_IS_AVIVO(rdev)) 5185 return -EINVAL; 5186 } 5187 /* Must be an ATOMBIOS */ 5188 if (!rdev->is_atom_bios) { 5189 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5190 return -EINVAL; 5191 } 5192 r = radeon_atombios_init(rdev); 5193 if (r) 5194 return r; 5195 /* reset the asic, the gfx blocks are often in a bad state 5196 * after the driver is unloaded or after a resume 5197 */ 5198 if (radeon_asic_reset(rdev)) 5199 dev_warn(rdev->dev, "GPU reset failed !\n"); 5200 /* Post card if necessary */ 5201 if (!radeon_card_posted(rdev)) { 5202 if (!rdev->bios) { 5203 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5204 return -EINVAL; 5205 } 5206 DRM_INFO("GPU not posted. posting now...\n"); 5207 atom_asic_init(rdev->mode_info.atom_context); 5208 } 5209 /* init golden registers */ 5210 evergreen_init_golden_registers(rdev); 5211 /* Initialize scratch registers */ 5212 r600_scratch_init(rdev); 5213 /* Initialize surface registers */ 5214 radeon_surface_init(rdev); 5215 /* Initialize clocks */ 5216 radeon_get_clock_info(rdev->ddev); 5217 /* Fence driver */ 5218 r = radeon_fence_driver_init(rdev); 5219 if (r) 5220 return r; 5221 /* initialize AGP */ 5222 if (rdev->flags & RADEON_IS_AGP) { 5223 r = radeon_agp_init(rdev); 5224 if (r) 5225 radeon_agp_disable(rdev); 5226 } 5227 /* initialize memory controller */ 5228 r = evergreen_mc_init(rdev); 5229 if (r) 5230 return r; 5231 /* Memory manager */ 5232 r = radeon_bo_init(rdev); 5233 if (r) 5234 return r; 5235 5236 if (ASIC_IS_DCE5(rdev)) { 5237 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5238 r = ni_init_microcode(rdev); 5239 if (r) { 5240 DRM_ERROR("Failed to load firmware!\n"); 5241 return r; 5242 } 5243 } 5244 } else { 5245 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5246 r = r600_init_microcode(rdev); 5247 if (r) { 5248 DRM_ERROR("Failed to load firmware!\n"); 5249 return r; 5250 } 5251 } 5252 } 5253 5254 /* Initialize power management */ 5255 radeon_pm_init(rdev); 5256 5257 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5258 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5259 5260 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5261 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5262 5263 evergreen_uvd_init(rdev); 5264 5265 rdev->ih.ring_obj = NULL; 5266 r600_ih_ring_init(rdev, 64 * 1024); 5267 5268 r = r600_pcie_gart_init(rdev); 5269 if (r) 5270 return r; 5271 5272 rdev->accel_working = true; 5273 r = evergreen_startup(rdev); 5274 if (r) { 5275 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5276 r700_cp_fini(rdev); 5277 r600_dma_fini(rdev); 5278 r600_irq_fini(rdev); 5279 if (rdev->flags & RADEON_IS_IGP) 5280 sumo_rlc_fini(rdev); 5281 radeon_wb_fini(rdev); 5282 radeon_ib_pool_fini(rdev); 5283 radeon_irq_kms_fini(rdev); 5284 evergreen_pcie_gart_fini(rdev); 5285 rdev->accel_working = false; 5286 } 5287 5288 /* Don't start up if the MC ucode is missing on BTC parts. 5289 * The default clocks and voltages before the MC ucode 5290 * is loaded are not suffient for advanced operations. 5291 */ 5292 if (ASIC_IS_DCE5(rdev)) { 5293 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5294 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5295 return -EINVAL; 5296 } 5297 } 5298 5299 return 0; 5300} 5301 5302void evergreen_fini(struct radeon_device *rdev) 5303{ 5304 radeon_pm_fini(rdev); 5305 radeon_audio_fini(rdev); 5306 r700_cp_fini(rdev); 5307 r600_dma_fini(rdev); 5308 r600_irq_fini(rdev); 5309 if (rdev->flags & RADEON_IS_IGP) 5310 sumo_rlc_fini(rdev); 5311 radeon_wb_fini(rdev); 5312 radeon_ib_pool_fini(rdev); 5313 radeon_irq_kms_fini(rdev); 5314 uvd_v1_0_fini(rdev); 5315 radeon_uvd_fini(rdev); 5316 evergreen_pcie_gart_fini(rdev); 5317 r600_vram_scratch_fini(rdev); 5318 radeon_gem_fini(rdev); 5319 radeon_fence_driver_fini(rdev); 5320 radeon_agp_fini(rdev); 5321 radeon_bo_fini(rdev); 5322 radeon_atombios_fini(rdev); 5323 kfree(rdev->bios); 5324 rdev->bios = NULL; 5325} 5326 5327void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5328{ 5329 u32 link_width_cntl, speed_cntl; 5330 5331 if (radeon_pcie_gen2 == 0) 5332 return; 5333 5334 if (rdev->flags & RADEON_IS_IGP) 5335 return; 5336 5337 if (!(rdev->flags & RADEON_IS_PCIE)) 5338 return; 5339 5340 /* x2 cards have a special sequence */ 5341 if (ASIC_IS_X2(rdev)) 5342 return; 5343 5344 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) && 5345 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT)) 5346 return; 5347 5348 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5349 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5350 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5351 return; 5352 } 5353 5354 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5355 5356 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5357 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5358 5359 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5360 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5361 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5362 5363 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5364 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5365 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5366 5367 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5368 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5369 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5370 5371 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5372 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5373 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5374 5375 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5376 speed_cntl |= LC_GEN2_EN_STRAP; 5377 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5378 5379 } else { 5380 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5381 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5382 if (1) 5383 link_width_cntl |= LC_UPCONFIGURE_DIS; 5384 else 5385 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5386 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5387 } 5388} 5389 5390void evergreen_program_aspm(struct radeon_device *rdev) 5391{ 5392 u32 data, orig; 5393 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5394 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5395 /* fusion_platform = true 5396 * if the system is a fusion system 5397 * (APU or DGPU in a fusion system). 5398 * todo: check if the system is a fusion platform. 5399 */ 5400 bool fusion_platform = false; 5401 5402 if (radeon_aspm == 0) 5403 return; 5404 5405 if (!(rdev->flags & RADEON_IS_PCIE)) 5406 return; 5407 5408 switch (rdev->family) { 5409 case CHIP_CYPRESS: 5410 case CHIP_HEMLOCK: 5411 case CHIP_JUNIPER: 5412 case CHIP_REDWOOD: 5413 case CHIP_CEDAR: 5414 case CHIP_SUMO: 5415 case CHIP_SUMO2: 5416 case CHIP_PALM: 5417 case CHIP_ARUBA: 5418 disable_l0s = true; 5419 break; 5420 default: 5421 disable_l0s = false; 5422 break; 5423 } 5424 5425 if (rdev->flags & RADEON_IS_IGP) 5426 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 5427 5428 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 5429 if (fusion_platform) 5430 data &= ~MULTI_PIF; 5431 else 5432 data |= MULTI_PIF; 5433 if (data != orig) 5434 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 5435 5436 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 5437 if (fusion_platform) 5438 data &= ~MULTI_PIF; 5439 else 5440 data |= MULTI_PIF; 5441 if (data != orig) 5442 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 5443 5444 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 5445 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 5446 if (!disable_l0s) { 5447 if (rdev->family >= CHIP_BARTS) 5448 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 5449 else 5450 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 5451 } 5452 5453 if (!disable_l1) { 5454 if (rdev->family >= CHIP_BARTS) 5455 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 5456 else 5457 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 5458 5459 if (!disable_plloff_in_l1) { 5460 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5461 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5462 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5463 if (data != orig) 5464 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5465 5466 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5467 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5468 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5469 if (data != orig) 5470 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5471 5472 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5473 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5474 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5475 if (data != orig) 5476 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5477 5478 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5479 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5480 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5481 if (data != orig) 5482 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5483 5484 if (rdev->family >= CHIP_BARTS) { 5485 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5486 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5487 data |= PLL_RAMP_UP_TIME_0(4); 5488 if (data != orig) 5489 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5490 5491 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5492 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5493 data |= PLL_RAMP_UP_TIME_1(4); 5494 if (data != orig) 5495 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5496 5497 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5498 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5499 data |= PLL_RAMP_UP_TIME_0(4); 5500 if (data != orig) 5501 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5502 5503 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5504 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5505 data |= PLL_RAMP_UP_TIME_1(4); 5506 if (data != orig) 5507 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5508 } 5509 5510 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5511 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 5512 data |= LC_DYN_LANES_PWR_STATE(3); 5513 if (data != orig) 5514 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 5515 5516 if (rdev->family >= CHIP_BARTS) { 5517 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 5518 data &= ~LS2_EXIT_TIME_MASK; 5519 data |= LS2_EXIT_TIME(1); 5520 if (data != orig) 5521 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 5522 5523 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 5524 data &= ~LS2_EXIT_TIME_MASK; 5525 data |= LS2_EXIT_TIME(1); 5526 if (data != orig) 5527 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 5528 } 5529 } 5530 } 5531 5532 /* evergreen parts only */ 5533 if (rdev->family < CHIP_BARTS) 5534 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 5535 5536 if (pcie_lc_cntl != pcie_lc_cntl_old) 5537 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 5538} 5539