Lines Matching defs:rdev
128 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
129 static void si_pcie_gen3_enable(struct radeon_device *rdev);
130 static void si_program_aspm(struct radeon_device *rdev);
131 extern void sumo_rlc_fini(struct radeon_device *rdev);
132 extern int sumo_rlc_init(struct radeon_device *rdev);
133 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
135 static void si_init_pg(struct radeon_device *rdev);
136 static void si_init_cg(struct radeon_device *rdev);
137 static void si_fini_pg(struct radeon_device *rdev);
138 static void si_fini_cg(struct radeon_device *rdev);
139 static void si_rlc_stop(struct radeon_device *rdev);
1228 static void si_init_golden_registers(struct radeon_device *rdev)
1230 switch (rdev->family) {
1232 radeon_program_register_sequence(rdev,
1235 radeon_program_register_sequence(rdev,
1238 radeon_program_register_sequence(rdev,
1241 radeon_program_register_sequence(rdev,
1246 radeon_program_register_sequence(rdev,
1249 radeon_program_register_sequence(rdev,
1252 radeon_program_register_sequence(rdev,
1257 radeon_program_register_sequence(rdev,
1260 radeon_program_register_sequence(rdev,
1263 radeon_program_register_sequence(rdev,
1266 radeon_program_register_sequence(rdev,
1271 radeon_program_register_sequence(rdev,
1274 radeon_program_register_sequence(rdev,
1277 radeon_program_register_sequence(rdev,
1282 radeon_program_register_sequence(rdev,
1285 radeon_program_register_sequence(rdev,
1288 radeon_program_register_sequence(rdev,
1300 * @rdev: radeon_device pointer
1307 int si_get_allowed_info_register(struct radeon_device *rdev,
1333 * @rdev: radeon_device pointer
1338 u32 si_get_xclk(struct radeon_device *rdev)
1340 u32 reference_clock = rdev->clock.spll.reference_freq;
1355 int si_get_temp(struct radeon_device *rdev)
1571 int si_mc_load_microcode(struct radeon_device *rdev)
1580 if (!rdev->mc_fw)
1583 if (rdev->new_fw) {
1585 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1590 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1593 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1595 ucode_size = rdev->mc_fw->size / 4;
1597 switch (rdev->family) {
1620 fw_data = (const __be32 *)rdev->mc_fw->data;
1632 if (rdev->new_fw) {
1642 if (rdev->new_fw)
1654 for (i = 0; i < rdev->usec_timeout; i++) {
1659 for (i = 0; i < rdev->usec_timeout; i++) {
1669 static int si_init_microcode(struct radeon_device *rdev)
1684 switch (rdev->family) {
1698 if ((rdev->pdev->revision == 0x81) &&
1699 ((rdev->pdev->device == 0x6810) ||
1700 (rdev->pdev->device == 0x6811)))
1713 if (((rdev->pdev->device == 0x6820) &&
1714 ((rdev->pdev->revision == 0x81) ||
1715 (rdev->pdev->revision == 0x83))) ||
1716 ((rdev->pdev->device == 0x6821) &&
1717 ((rdev->pdev->revision == 0x83) ||
1718 (rdev->pdev->revision == 0x87))) ||
1719 ((rdev->pdev->revision == 0x87) &&
1720 ((rdev->pdev->device == 0x6823) ||
1721 (rdev->pdev->device == 0x682b))))
1734 if (((rdev->pdev->revision == 0x81) &&
1735 ((rdev->pdev->device == 0x6600) ||
1736 (rdev->pdev->device == 0x6604) ||
1737 (rdev->pdev->device == 0x6605) ||
1738 (rdev->pdev->device == 0x6610))) ||
1739 ((rdev->pdev->revision == 0x83) &&
1740 (rdev->pdev->device == 0x6610)))
1752 if (((rdev->pdev->revision == 0x81) &&
1753 (rdev->pdev->device == 0x6660)) ||
1754 ((rdev->pdev->revision == 0x83) &&
1755 ((rdev->pdev->device == 0x6660) ||
1756 (rdev->pdev->device == 0x6663) ||
1757 (rdev->pdev->device == 0x6665) ||
1758 (rdev->pdev->device == 0x6667))))
1760 else if ((rdev->pdev->revision == 0xc3) &&
1761 (rdev->pdev->device == 0x6665))
1781 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1784 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1787 if (rdev->pfp_fw->size != pfp_req_size) {
1789 rdev->pfp_fw->size, fw_name);
1794 err = radeon_ucode_validate(rdev->pfp_fw);
1805 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1808 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1811 if (rdev->me_fw->size != me_req_size) {
1813 rdev->me_fw->size, fw_name);
1817 err = radeon_ucode_validate(rdev->me_fw);
1828 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1831 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1834 if (rdev->ce_fw->size != ce_req_size) {
1836 rdev->ce_fw->size, fw_name);
1840 err = radeon_ucode_validate(rdev->ce_fw);
1851 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1854 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1857 if (rdev->rlc_fw->size != rlc_req_size) {
1859 rdev->rlc_fw->size, fw_name);
1863 err = radeon_ucode_validate(rdev->rlc_fw);
1877 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1880 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1883 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1887 if ((rdev->mc_fw->size != mc_req_size) &&
1888 (rdev->mc_fw->size != mc2_req_size)) {
1890 rdev->mc_fw->size, fw_name);
1893 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1895 err = radeon_ucode_validate(rdev->mc_fw);
1911 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1914 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1917 release_firmware(rdev->smc_fw);
1918 rdev->smc_fw = NULL;
1920 } else if (rdev->smc_fw->size != smc_req_size) {
1922 rdev->smc_fw->size, fw_name);
1926 err = radeon_ucode_validate(rdev->smc_fw);
1937 rdev->new_fw = false;
1942 rdev->new_fw = true;
1949 release_firmware(rdev->pfp_fw);
1950 rdev->pfp_fw = NULL;
1951 release_firmware(rdev->me_fw);
1952 rdev->me_fw = NULL;
1953 release_firmware(rdev->ce_fw);
1954 rdev->ce_fw = NULL;
1955 release_firmware(rdev->rlc_fw);
1956 rdev->rlc_fw = NULL;
1957 release_firmware(rdev->mc_fw);
1958 rdev->mc_fw = NULL;
1959 release_firmware(rdev->smc_fw);
1960 rdev->smc_fw = NULL;
1966 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
2004 for (i = 0; i < rdev->usec_timeout; i++) {
2025 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
2293 static void dce6_program_watermarks(struct radeon_device *rdev,
2318 if (rdev->family == CHIP_ARUBA)
2319 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2321 dram_channels = si_get_number_of_dram_channels(rdev);
2324 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2326 radeon_dpm_get_mclk(rdev, false) * 10;
2328 radeon_dpm_get_sclk(rdev, false) * 10;
2330 wm_high.yclk = rdev->pm.current_mclk * 10;
2331 wm_high.sclk = rdev->pm.current_sclk * 10;
2351 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2353 radeon_dpm_get_mclk(rdev, true) * 10;
2355 radeon_dpm_get_sclk(rdev, true) * 10;
2357 wm_low.yclk = rdev->pm.current_mclk * 10;
2358 wm_low.sclk = rdev->pm.current_sclk * 10;
2387 (rdev->disp_priority == 2)) {
2395 (rdev->disp_priority == 2)) {
2459 void dce6_bandwidth_update(struct radeon_device *rdev)
2466 if (!rdev->mode_info.mode_config_initialized)
2469 radeon_update_display_priority(rdev);
2471 for (i = 0; i < rdev->num_crtc; i++) {
2472 if (rdev->mode_info.crtcs[i]->base.enabled)
2475 for (i = 0; i < rdev->num_crtc; i += 2) {
2476 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2477 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2478 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2479 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2480 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2481 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2488 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2490 u32 *tile = rdev->config.si.tile_mode_array;
2492 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2495 switch (rdev->config.si.mem_row_size_in_kb) {
2511 switch(rdev->family) {
2942 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2946 static void si_select_se_sh(struct radeon_device *rdev,
2973 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2991 static void si_setup_spi(struct radeon_device *rdev,
3000 si_select_se_sh(rdev, i, j);
3002 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3015 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3018 static u32 si_get_rb_disabled(struct radeon_device *rdev,
3038 static void si_setup_rb(struct radeon_device *rdev,
3049 si_select_se_sh(rdev, i, j);
3050 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3054 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3063 rdev->config.si.backend_enable_mask = enabled_rbs;
3066 si_select_se_sh(rdev, i, 0xffffffff);
3085 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3088 static void si_gpu_init(struct radeon_device *rdev)
3097 switch (rdev->family) {
3099 rdev->config.si.max_shader_engines = 2;
3100 rdev->config.si.max_tile_pipes = 12;
3101 rdev->config.si.max_cu_per_sh = 8;
3102 rdev->config.si.max_sh_per_se = 2;
3103 rdev->config.si.max_backends_per_se = 4;
3104 rdev->config.si.max_texture_channel_caches = 12;
3105 rdev->config.si.max_gprs = 256;
3106 rdev->config.si.max_gs_threads = 32;
3107 rdev->config.si.max_hw_contexts = 8;
3109 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3110 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3111 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3112 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3116 rdev->config.si.max_shader_engines = 2;
3117 rdev->config.si.max_tile_pipes = 8;
3118 rdev->config.si.max_cu_per_sh = 5;
3119 rdev->config.si.max_sh_per_se = 2;
3120 rdev->config.si.max_backends_per_se = 4;
3121 rdev->config.si.max_texture_channel_caches = 8;
3122 rdev->config.si.max_gprs = 256;
3123 rdev->config.si.max_gs_threads = 32;
3124 rdev->config.si.max_hw_contexts = 8;
3126 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3127 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3128 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3129 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3134 rdev->config.si.max_shader_engines = 1;
3135 rdev->config.si.max_tile_pipes = 4;
3136 rdev->config.si.max_cu_per_sh = 5;
3137 rdev->config.si.max_sh_per_se = 2;
3138 rdev->config.si.max_backends_per_se = 4;
3139 rdev->config.si.max_texture_channel_caches = 4;
3140 rdev->config.si.max_gprs = 256;
3141 rdev->config.si.max_gs_threads = 32;
3142 rdev->config.si.max_hw_contexts = 8;
3144 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3145 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3146 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3147 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3151 rdev->config.si.max_shader_engines = 1;
3152 rdev->config.si.max_tile_pipes = 4;
3153 rdev->config.si.max_cu_per_sh = 6;
3154 rdev->config.si.max_sh_per_se = 1;
3155 rdev->config.si.max_backends_per_se = 2;
3156 rdev->config.si.max_texture_channel_caches = 4;
3157 rdev->config.si.max_gprs = 256;
3158 rdev->config.si.max_gs_threads = 16;
3159 rdev->config.si.max_hw_contexts = 8;
3161 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3162 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3163 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3164 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3168 rdev->config.si.max_shader_engines = 1;
3169 rdev->config.si.max_tile_pipes = 4;
3170 rdev->config.si.max_cu_per_sh = 5;
3171 rdev->config.si.max_sh_per_se = 1;
3172 rdev->config.si.max_backends_per_se = 1;
3173 rdev->config.si.max_texture_channel_caches = 2;
3174 rdev->config.si.max_gprs = 256;
3175 rdev->config.si.max_gs_threads = 16;
3176 rdev->config.si.max_hw_contexts = 8;
3178 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3179 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3180 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3181 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3199 evergreen_fix_pci_max_read_req_size(rdev);
3206 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3207 rdev->config.si.mem_max_burst_length_bytes = 256;
3209 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3210 if (rdev->config.si.mem_row_size_in_kb > 4)
3211 rdev->config.si.mem_row_size_in_kb = 4;
3213 rdev->config.si.shader_engine_tile_size = 32;
3214 rdev->config.si.num_gpus = 1;
3215 rdev->config.si.multi_gpu_tile_size = 64;
3219 switch (rdev->config.si.mem_row_size_in_kb) {
3239 rdev->config.si.tile_config = 0;
3240 switch (rdev->config.si.num_tile_pipes) {
3242 rdev->config.si.tile_config |= (0 << 0);
3245 rdev->config.si.tile_config |= (1 << 0);
3248 rdev->config.si.tile_config |= (2 << 0);
3253 rdev->config.si.tile_config |= (3 << 0);
3258 rdev->config.si.tile_config |= 0 << 4;
3261 rdev->config.si.tile_config |= 1 << 4;
3265 rdev->config.si.tile_config |= 2 << 4;
3268 rdev->config.si.tile_config |=
3270 rdev->config.si.tile_config |=
3279 if (rdev->has_uvd) {
3285 si_tiling_mode_table_init(rdev);
3287 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3288 rdev->config.si.max_sh_per_se,
3289 rdev->config.si.max_backends_per_se);
3291 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3292 rdev->config.si.max_sh_per_se,
3293 rdev->config.si.max_cu_per_sh);
3295 rdev->config.si.active_cus = 0;
3296 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3297 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3298 rdev->config.si.active_cus +=
3299 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3313 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3314 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3315 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3316 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3357 static void si_scratch_init(struct radeon_device *rdev)
3361 rdev->scratch.num_reg = 7;
3362 rdev->scratch.reg_base = SCRATCH_REG0;
3363 for (i = 0; i < rdev->scratch.num_reg; i++) {
3364 rdev->scratch.free[i] = true;
3365 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3369 void si_fence_ring_emit(struct radeon_device *rdev,
3372 struct radeon_ring *ring = &rdev->ring[fence->ring];
3373 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3399 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3401 struct radeon_ring *ring = &rdev->ring[ib->ring];
3419 } else if (rdev->wb.enabled) {
3459 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3464 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3465 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3468 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3469 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3470 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3475 static int si_cp_load_microcode(struct radeon_device *rdev)
3479 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3482 si_cp_enable(rdev, false);
3484 if (rdev->new_fw) {
3486 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3488 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3490 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3500 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3509 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3518 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3528 fw_data = (const __be32 *)rdev->pfp_fw->data;
3535 fw_data = (const __be32 *)rdev->ce_fw->data;
3542 fw_data = (const __be32 *)rdev->me_fw->data;
3556 static int si_cp_start(struct radeon_device *rdev)
3558 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3561 r = radeon_ring_lock(rdev, ring, 7 + 4);
3570 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3580 radeon_ring_unlock_commit(rdev, ring, false);
3582 si_cp_enable(rdev, true);
3584 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3609 radeon_ring_unlock_commit(rdev, ring, false);
3612 ring = &rdev->ring[i];
3613 r = radeon_ring_lock(rdev, ring, 2);
3623 radeon_ring_unlock_commit(rdev, ring, false);
3629 static void si_cp_fini(struct radeon_device *rdev)
3632 si_cp_enable(rdev, false);
3634 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3635 radeon_ring_fini(rdev, ring);
3636 radeon_scratch_free(rdev, ring->rptr_save_reg);
3638 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3639 radeon_ring_fini(rdev, ring);
3640 radeon_scratch_free(rdev, ring->rptr_save_reg);
3642 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3643 radeon_ring_fini(rdev, ring);
3644 radeon_scratch_free(rdev, ring->rptr_save_reg);
3647 static int si_cp_resume(struct radeon_device *rdev)
3654 si_enable_gui_idle_interrupt(rdev, false);
3663 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3667 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3681 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3682 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3684 if (rdev->wb.enabled)
3698 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3712 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3713 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3722 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3736 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3737 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3745 si_cp_start(rdev);
3746 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3747 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3748 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3749 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3751 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3752 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3753 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3756 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3758 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3760 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3762 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3765 si_enable_gui_idle_interrupt(rdev, true);
3767 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3768 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3773 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3837 if (evergreen_is_display_hung(rdev))
3854 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3863 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3865 evergreen_print_gpu_status_regs(rdev);
3866 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3868 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3872 si_fini_pg(rdev);
3873 si_fini_cg(rdev);
3876 si_rlc_stop(rdev);
3896 evergreen_mc_stop(rdev, &save);
3897 if (evergreen_mc_wait_for_idle(rdev)) {
3898 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3952 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3966 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3980 evergreen_mc_resume(rdev, &save);
3983 evergreen_print_gpu_status_regs(rdev);
3986 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3998 for (i = 0; i < rdev->usec_timeout; i++) {
4013 static void si_spll_powerdown(struct radeon_device *rdev)
4034 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4039 dev_info(rdev->dev, "GPU pci config reset\n");
4044 si_fini_pg(rdev);
4045 si_fini_cg(rdev);
4060 si_rlc_stop(rdev);
4065 evergreen_mc_stop(rdev, &save);
4066 if (evergreen_mc_wait_for_idle(rdev)) {
4067 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4071 si_set_clk_bypass_mode(rdev);
4073 si_spll_powerdown(rdev);
4075 pci_clear_master(rdev->pdev);
4077 radeon_pci_config_reset(rdev);
4079 for (i = 0; i < rdev->usec_timeout; i++) {
4086 int si_asic_reset(struct radeon_device *rdev, bool hard)
4091 si_gpu_pci_config_reset(rdev);
4095 reset_mask = si_gpu_check_soft_reset(rdev);
4098 r600_set_bios_scratch_engine_hung(rdev, true);
4101 si_gpu_soft_reset(rdev, reset_mask);
4103 reset_mask = si_gpu_check_soft_reset(rdev);
4107 si_gpu_pci_config_reset(rdev);
4109 reset_mask = si_gpu_check_soft_reset(rdev);
4112 r600_set_bios_scratch_engine_hung(rdev, false);
4120 * @rdev: radeon_device pointer
4126 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4128 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4133 radeon_ring_lockup_update(rdev, ring);
4136 return radeon_ring_test_lockup(rdev, ring);
4140 static void si_mc_program(struct radeon_device *rdev)
4156 evergreen_mc_stop(rdev, &save);
4157 if (radeon_mc_wait_for_idle(rdev)) {
4158 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4160 if (!ASIC_IS_NODCE(rdev))
4165 rdev->mc.vram_start >> 12);
4167 rdev->mc.vram_end >> 12);
4169 rdev->vram_scratch.gpu_addr >> 12);
4170 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4171 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4174 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4180 if (radeon_mc_wait_for_idle(rdev)) {
4181 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4183 evergreen_mc_resume(rdev, &save);
4184 if (!ASIC_IS_NODCE(rdev)) {
4187 rv515_vga_render_disable(rdev);
4191 void si_vram_gtt_location(struct radeon_device *rdev,
4196 dev_warn(rdev->dev, "limiting VRAM\n");
4200 radeon_vram_location(rdev, &rdev->mc, 0);
4201 rdev->mc.gtt_base_align = 0;
4202 radeon_gtt_location(rdev, mc);
4205 static int si_mc_init(struct radeon_device *rdev)
4211 rdev->mc.vram_is_ddr = true;
4251 rdev->mc.vram_width = numchan * chansize;
4253 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4254 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4263 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4264 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4265 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4266 si_vram_gtt_location(rdev, &rdev->mc);
4267 radeon_update_bandwidth_info(rdev);
4275 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4284 static int si_pcie_gart_enable(struct radeon_device *rdev)
4288 if (rdev->gart.robj == NULL) {
4289 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4292 r = radeon_gart_table_vram_pin(rdev);
4315 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4316 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4317 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4319 (u32)(rdev->dummy_page.addr >> 12));
4331 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4339 rdev->vm_manager.saved_table_addr[i]);
4342 rdev->vm_manager.saved_table_addr[i]);
4347 (u32)(rdev->dummy_page.addr >> 12));
4364 si_pcie_gart_tlb_flush(rdev);
4366 (unsigned)(rdev->mc.gtt_size >> 20),
4367 (unsigned long long)rdev->gart.table_addr);
4368 rdev->gart.ready = true;
4372 static void si_pcie_gart_disable(struct radeon_device *rdev)
4382 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4399 radeon_gart_table_vram_unpin(rdev);
4402 static void si_pcie_gart_fini(struct radeon_device *rdev)
4404 si_pcie_gart_disable(rdev);
4405 radeon_gart_table_vram_free(rdev);
4406 radeon_gart_fini(rdev);
4456 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4529 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4647 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4735 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4748 dev_err(rdev->dev, "Packet0 not allowed!\n");
4757 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4761 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4765 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4768 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4776 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4797 int si_vm_init(struct radeon_device *rdev)
4800 rdev->vm_manager.nvm = 16;
4802 rdev->vm_manager.vram_base_offset = 0;
4807 void si_vm_fini(struct radeon_device *rdev)
4814 * @rdev: radeon_device pointer
4820 static void si_vm_decode_fault(struct radeon_device *rdev,
4828 if (rdev->family == CHIP_TAHITI) {
5075 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5127 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5131 for (i = 0; i < rdev->usec_timeout; i++) {
5137 for (i = 0; i < rdev->usec_timeout; i++) {
5144 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5162 for (i = 0; i < rdev->usec_timeout; i++) {
5170 static void si_set_uvd_dcm(struct radeon_device *rdev,
5191 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5196 si_set_uvd_dcm(rdev, false);
5204 static u32 si_halt_rlc(struct radeon_device *rdev)
5214 si_wait_for_rlc_serdes(rdev);
5220 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5229 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5234 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5242 static void si_init_dma_pg(struct radeon_device *rdev)
5253 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5258 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5278 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5282 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5288 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5298 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5303 si_select_se_sh(rdev, se, sh);
5306 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5313 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5321 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5327 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5328 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5332 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5333 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5354 static void si_enable_cgcg(struct radeon_device *rdev,
5361 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5362 si_enable_gui_idle_interrupt(rdev, true);
5366 tmp = si_halt_rlc(rdev);
5372 si_wait_for_rlc_serdes(rdev);
5374 si_update_rlc(rdev, tmp);
5380 si_enable_gui_idle_interrupt(rdev, false);
5394 static void si_enable_mgcg(struct radeon_device *rdev,
5399 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5405 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5417 tmp = si_halt_rlc(rdev);
5423 si_update_rlc(rdev, tmp);
5440 tmp = si_halt_rlc(rdev);
5446 si_update_rlc(rdev, tmp);
5450 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5455 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5495 static void si_enable_mc_ls(struct radeon_device *rdev,
5503 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5512 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5520 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5529 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5535 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5566 static void si_enable_bif_mgls(struct radeon_device *rdev,
5573 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5584 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5591 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5600 static void si_enable_hdp_ls(struct radeon_device *rdev,
5607 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5616 static void si_update_cg(struct radeon_device *rdev,
5620 si_enable_gui_idle_interrupt(rdev, false);
5623 si_enable_mgcg(rdev, true);
5624 si_enable_cgcg(rdev, true);
5626 si_enable_cgcg(rdev, false);
5627 si_enable_mgcg(rdev, false);
5629 si_enable_gui_idle_interrupt(rdev, true);
5633 si_enable_mc_mgcg(rdev, enable);
5634 si_enable_mc_ls(rdev, enable);
5638 si_enable_dma_mgcg(rdev, enable);
5642 si_enable_bif_mgls(rdev, enable);
5646 if (rdev->has_uvd) {
5647 si_enable_uvd_mgcg(rdev, enable);
5652 si_enable_hdp_mgcg(rdev, enable);
5653 si_enable_hdp_ls(rdev, enable);
5657 static void si_init_cg(struct radeon_device *rdev)
5659 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5664 if (rdev->has_uvd) {
5665 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5666 si_init_uvd_internal_cg(rdev);
5670 static void si_fini_cg(struct radeon_device *rdev)
5672 if (rdev->has_uvd) {
5673 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5675 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5682 u32 si_get_csb_size(struct radeon_device *rdev)
5688 if (rdev->rlc.cs_data == NULL)
5696 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5714 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5720 if (rdev->rlc.cs_data == NULL)
5732 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5748 switch (rdev->family) {
5774 static void si_init_pg(struct radeon_device *rdev)
5776 if (rdev->pg_flags) {
5777 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5778 si_init_dma_pg(rdev);
5780 si_init_ao_cu_mask(rdev);
5781 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5782 si_init_gfx_cgpg(rdev);
5784 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5785 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5787 si_enable_dma_pg(rdev, true);
5788 si_enable_gfx_cgpg(rdev, true);
5790 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5791 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5795 static void si_fini_pg(struct radeon_device *rdev)
5797 if (rdev->pg_flags) {
5798 si_enable_dma_pg(rdev, false);
5799 si_enable_gfx_cgpg(rdev, false);
5806 void si_rlc_reset(struct radeon_device *rdev)
5818 static void si_rlc_stop(struct radeon_device *rdev)
5822 si_enable_gui_idle_interrupt(rdev, false);
5824 si_wait_for_rlc_serdes(rdev);
5827 static void si_rlc_start(struct radeon_device *rdev)
5831 si_enable_gui_idle_interrupt(rdev, true);
5836 static bool si_lbpw_supported(struct radeon_device *rdev)
5847 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5859 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5864 static int si_rlc_resume(struct radeon_device *rdev)
5868 if (!rdev->rlc_fw)
5871 si_rlc_stop(rdev);
5873 si_rlc_reset(rdev);
5875 si_init_pg(rdev);
5877 si_init_cg(rdev);
5889 if (rdev->new_fw) {
5891 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5894 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5904 (const __be32 *)rdev->rlc_fw->data;
5912 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5914 si_rlc_start(rdev);
5919 static void si_enable_interrupts(struct radeon_device *rdev)
5928 rdev->ih.enabled = true;
5931 static void si_disable_interrupts(struct radeon_device *rdev)
5943 rdev->ih.enabled = false;
5944 rdev->ih.rptr = 0;
5947 static void si_disable_interrupt_state(struct radeon_device *rdev)
5963 for (i = 0; i < rdev->num_crtc; i++)
5965 for (i = 0; i < rdev->num_crtc; i++)
5968 if (!ASIC_IS_NODCE(rdev)) {
5977 static int si_irq_init(struct radeon_device *rdev)
5984 ret = r600_ih_ring_alloc(rdev);
5989 si_disable_interrupts(rdev);
5992 ret = si_rlc_resume(rdev);
5994 r600_ih_ring_fini(rdev);
6000 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
6010 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6011 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6017 if (rdev->wb.enabled)
6021 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6022 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6033 if (rdev->msi_enabled)
6038 si_disable_interrupt_state(rdev);
6040 pci_set_master(rdev->pdev);
6043 si_enable_interrupts(rdev);
6049 int si_irq_set(struct radeon_device *rdev)
6058 if (!rdev->irq.installed) {
6063 if (!rdev->ih.enabled) {
6064 si_disable_interrupts(rdev);
6066 si_disable_interrupt_state(rdev);
6080 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6084 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6088 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6092 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6097 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6111 if (rdev->irq.dpm_thermal) {
6116 for (i = 0; i < rdev->num_crtc; i++) {
6118 rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6119 rdev->irq.crtc_vblank_int[i] ||
6120 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
6123 for (i = 0; i < rdev->num_crtc; i++)
6126 if (!ASIC_IS_NODCE(rdev)) {
6129 rdev, DC_HPDx_INT_CONTROL(i),
6131 rdev->irq.hpd[i], "HPD", i);
6144 static inline void si_irq_ack(struct radeon_device *rdev)
6147 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6148 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
6150 if (ASIC_IS_NODCE(rdev))
6155 if (i < rdev->num_crtc)
6160 for (i = 0; i < rdev->num_crtc; i += 2) {
6188 static void si_irq_disable(struct radeon_device *rdev)
6190 si_disable_interrupts(rdev);
6193 si_irq_ack(rdev);
6194 si_disable_interrupt_state(rdev);
6197 static void si_irq_suspend(struct radeon_device *rdev)
6199 si_irq_disable(rdev);
6200 si_rlc_stop(rdev);
6203 static void si_irq_fini(struct radeon_device *rdev)
6205 si_irq_suspend(rdev);
6206 r600_ih_ring_fini(rdev);
6209 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6213 if (rdev->wb.enabled)
6214 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6224 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6225 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6226 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6231 return (wptr & rdev->ih.ptr_mask);
6244 int si_irq_process(struct radeon_device *rdev)
6246 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6259 if (!rdev->ih.enabled || rdev->shutdown)
6262 wptr = si_get_ih_wptr(rdev);
6266 if (atomic_xchg(&rdev->ih.lock, 1))
6269 rptr = rdev->ih.rptr;
6276 si_irq_ack(rdev);
6281 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6282 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6283 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6298 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6299 drm_handle_vblank(rdev->ddev, crtc_idx);
6300 rdev->pm.vblank_sync = true;
6301 wake_up(&rdev->irq.vblank_queue);
6303 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6304 radeon_crtc_handle_vblank(rdev,
6334 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6367 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6377 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6378 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6380 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6382 si_vm_decode_fault(rdev, status, addr);
6385 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6388 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6391 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6397 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6400 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6403 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6409 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6413 rdev->pm.dpm.thermal.high_to_low = false;
6418 rdev->pm.dpm.thermal.high_to_low = true;
6426 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6435 rptr &= rdev->ih.ptr_mask;
6439 schedule_work(&rdev->dp_work);
6441 schedule_delayed_work(&rdev->hotplug_work, 0);
6442 if (queue_thermal && rdev->pm.dpm_enabled)
6443 schedule_work(&rdev->pm.dpm.thermal.work);
6444 rdev->ih.rptr = rptr;
6445 atomic_set(&rdev->ih.lock, 0);
6448 wptr = si_get_ih_wptr(rdev);
6458 static void si_uvd_init(struct radeon_device *rdev)
6462 if (!rdev->has_uvd)
6465 r = radeon_uvd_init(rdev);
6467 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6469 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6474 rdev->has_uvd = false;
6477 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6478 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6481 static void si_uvd_start(struct radeon_device *rdev)
6485 if (!rdev->has_uvd)
6488 r = uvd_v2_2_resume(rdev);
6490 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6493 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6495 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6501 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6504 static void si_uvd_resume(struct radeon_device *rdev)
6509 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6512 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6513 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6515 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6518 r = uvd_v1_0_init(rdev);
6520 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6525 static void si_vce_init(struct radeon_device *rdev)
6529 if (!rdev->has_vce)
6532 r = radeon_vce_init(rdev);
6534 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6536 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6541 rdev->has_vce = false;
6544 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6545 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6546 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6547 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6550 static void si_vce_start(struct radeon_device *rdev)
6554 if (!rdev->has_vce)
6557 r = radeon_vce_resume(rdev);
6559 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6562 r = vce_v1_0_resume(rdev);
6564 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6567 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6569 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6572 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6574 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6580 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6581 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6584 static void si_vce_resume(struct radeon_device *rdev)
6589 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6592 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6593 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6595 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6598 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6599 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6601 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6604 r = vce_v1_0_init(rdev);
6606 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6611 static int si_startup(struct radeon_device *rdev)
6617 si_pcie_gen3_enable(rdev);
6619 si_program_aspm(rdev);
6622 r = r600_vram_scratch_init(rdev);
6626 si_mc_program(rdev);
6628 if (!rdev->pm.dpm_enabled) {
6629 r = si_mc_load_microcode(rdev);
6636 r = si_pcie_gart_enable(rdev);
6639 si_gpu_init(rdev);
6642 if (rdev->family == CHIP_VERDE) {
6643 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6644 rdev->rlc.reg_list_size =
6647 rdev->rlc.cs_data = si_cs_data;
6648 r = sumo_rlc_init(rdev);
6655 r = radeon_wb_init(rdev);
6659 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6661 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6665 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6667 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6671 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6673 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6677 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6679 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6683 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6685 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6689 si_uvd_start(rdev);
6690 si_vce_start(rdev);
6693 if (!rdev->irq.installed) {
6694 r = radeon_irq_kms_init(rdev);
6699 r = si_irq_init(rdev);
6702 radeon_irq_kms_fini(rdev);
6705 si_irq_set(rdev);
6707 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6708 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6713 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6714 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6719 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6720 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6725 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6726 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6731 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6732 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6737 r = si_cp_load_microcode(rdev);
6740 r = si_cp_resume(rdev);
6744 r = cayman_dma_resume(rdev);
6748 si_uvd_resume(rdev);
6749 si_vce_resume(rdev);
6751 r = radeon_ib_pool_init(rdev);
6753 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6757 r = radeon_vm_manager_init(rdev);
6759 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6763 r = radeon_audio_init(rdev);
6770 int si_resume(struct radeon_device *rdev)
6779 atom_asic_init(rdev->mode_info.atom_context);
6782 si_init_golden_registers(rdev);
6784 if (rdev->pm.pm_method == PM_METHOD_DPM)
6785 radeon_pm_resume(rdev);
6787 rdev->accel_working = true;
6788 r = si_startup(rdev);
6791 rdev->accel_working = false;
6799 int si_suspend(struct radeon_device *rdev)
6801 radeon_pm_suspend(rdev);
6802 radeon_audio_fini(rdev);
6803 radeon_vm_manager_fini(rdev);
6804 si_cp_enable(rdev, false);
6805 cayman_dma_stop(rdev);
6806 if (rdev->has_uvd) {
6807 radeon_uvd_suspend(rdev);
6808 uvd_v1_0_fini(rdev);
6810 if (rdev->has_vce)
6811 radeon_vce_suspend(rdev);
6812 si_fini_pg(rdev);
6813 si_fini_cg(rdev);
6814 si_irq_suspend(rdev);
6815 radeon_wb_disable(rdev);
6816 si_pcie_gart_disable(rdev);
6826 int si_init(struct radeon_device *rdev)
6828 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6832 if (!radeon_get_bios(rdev)) {
6833 if (ASIC_IS_AVIVO(rdev))
6837 if (!rdev->is_atom_bios) {
6838 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6841 r = radeon_atombios_init(rdev);
6846 if (!radeon_card_posted(rdev)) {
6847 if (!rdev->bios) {
6848 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6852 atom_asic_init(rdev->mode_info.atom_context);
6855 si_init_golden_registers(rdev);
6857 si_scratch_init(rdev);
6859 radeon_surface_init(rdev);
6861 radeon_get_clock_info(rdev->ddev);
6864 radeon_fence_driver_init(rdev);
6867 r = si_mc_init(rdev);
6871 r = radeon_bo_init(rdev);
6875 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6876 !rdev->rlc_fw || !rdev->mc_fw) {
6877 r = si_init_microcode(rdev);
6885 radeon_pm_init(rdev);
6887 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6889 r600_ring_init(rdev, ring, 1024 * 1024);
6891 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6893 r600_ring_init(rdev, ring, 1024 * 1024);
6895 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6897 r600_ring_init(rdev, ring, 1024 * 1024);
6899 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6901 r600_ring_init(rdev, ring, 64 * 1024);
6903 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6905 r600_ring_init(rdev, ring, 64 * 1024);
6907 si_uvd_init(rdev);
6908 si_vce_init(rdev);
6910 rdev->ih.ring_obj = NULL;
6911 r600_ih_ring_init(rdev, 64 * 1024);
6913 r = r600_pcie_gart_init(rdev);
6917 rdev->accel_working = true;
6918 r = si_startup(rdev);
6920 dev_err(rdev->dev, "disabling GPU acceleration\n");
6921 si_cp_fini(rdev);
6922 cayman_dma_fini(rdev);
6923 si_irq_fini(rdev);
6924 sumo_rlc_fini(rdev);
6925 radeon_wb_fini(rdev);
6926 radeon_ib_pool_fini(rdev);
6927 radeon_vm_manager_fini(rdev);
6928 radeon_irq_kms_fini(rdev);
6929 si_pcie_gart_fini(rdev);
6930 rdev->accel_working = false;
6937 if (!rdev->mc_fw) {
6945 void si_fini(struct radeon_device *rdev)
6947 radeon_pm_fini(rdev);
6948 si_cp_fini(rdev);
6949 cayman_dma_fini(rdev);
6950 si_fini_pg(rdev);
6951 si_fini_cg(rdev);
6952 si_irq_fini(rdev);
6953 sumo_rlc_fini(rdev);
6954 radeon_wb_fini(rdev);
6955 radeon_vm_manager_fini(rdev);
6956 radeon_ib_pool_fini(rdev);
6957 radeon_irq_kms_fini(rdev);
6958 if (rdev->has_uvd) {
6959 uvd_v1_0_fini(rdev);
6960 radeon_uvd_fini(rdev);
6962 if (rdev->has_vce)
6963 radeon_vce_fini(rdev);
6964 si_pcie_gart_fini(rdev);
6965 r600_vram_scratch_fini(rdev);
6966 radeon_gem_fini(rdev);
6967 radeon_fence_driver_fini(rdev);
6968 radeon_bo_fini(rdev);
6969 radeon_atombios_fini(rdev);
6970 kfree(rdev->bios);
6971 rdev->bios = NULL;
6977 * @rdev: radeon_device pointer
6982 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6986 mutex_lock(&rdev->gpu_clock_mutex);
6990 mutex_unlock(&rdev->gpu_clock_mutex);
6994 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7012 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7032 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7069 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7083 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7085 struct pci_dev *root = rdev->pdev->bus->self;
7091 if (pci_is_root_bus(rdev->pdev->bus))
7097 if (rdev->flags & RADEON_IS_IGP)
7100 if (!(rdev->flags & RADEON_IS_PCIE))
7128 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
7139 pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7157 pcie_capability_read_word(rdev->pdev,
7165 pcie_capability_read_word(rdev->pdev,
7171 pcie_capability_read_word(rdev->pdev,
7190 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,
7207 pcie_capability_read_word(rdev->pdev,
7215 pcie_capability_write_word(rdev->pdev,
7231 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
7239 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
7245 for (i = 0; i < rdev->usec_timeout; i++) {
7253 static void si_program_aspm(struct radeon_device *rdev)
7262 if (!(rdev->flags & RADEON_IS_PCIE))
7320 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7369 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7376 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7382 !pci_is_root_bus(rdev->pdev->bus)) {
7383 struct pci_dev *root = rdev->pdev->bus->self;
7458 static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7489 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7510 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7533 r = si_vce_send_vcepll_ctlreq(rdev);
7565 r = si_vce_send_vcepll_ctlreq(rdev);