1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6#include "ivpu_drv.h" 7#include "ivpu_fw.h" 8#include "ivpu_hw.h" 9#include "ivpu_hw_40xx_reg.h" 10#include "ivpu_hw_reg_io.h" 11#include "ivpu_ipc.h" 12#include "ivpu_mmu.h" 13#include "ivpu_pm.h" 14 15#include <linux/dmi.h> 16 17#define TILE_MAX_NUM 6 18#define TILE_MAX_MASK 0x3f 19 20#define LNL_HW_ID 0x4040 21 22#define SKU_TILE_SHIFT 0u 23#define SKU_TILE_MASK 0x0000ffffu 24#define SKU_HW_ID_SHIFT 16u 25#define SKU_HW_ID_MASK 0xffff0000u 26 27#define PLL_CONFIG_DEFAULT 0x0 28#define PLL_CDYN_DEFAULT 0x80 29#define PLL_EPP_DEFAULT 0x80 30#define PLL_REF_CLK_FREQ (50 * 1000000) 31#define PLL_RATIO_TO_FREQ(x) ((x) * PLL_REF_CLK_FREQ) 32 33#define PLL_PROFILING_FREQ_DEFAULT 38400000 34#define PLL_PROFILING_FREQ_HIGH 400000000 35 36#define TIM_SAFE_ENABLE 0xf1d0dead 37#define TIM_WATCHDOG_RESET_VALUE 0xffffffff 38 39#define TIMEOUT_US (150 * USEC_PER_MSEC) 40#define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC) 41#define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC) 42 43#define WEIGHTS_DEFAULT 0xf711f711u 44#define WEIGHTS_ATS_DEFAULT 0x0000f711u 45 46#define ICB_0_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ 47 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ 48 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ 49 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ 50 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ 51 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ 52 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) 53 54#define ICB_1_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ 55 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ 56 (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) 57 58#define ICB_0_1_IRQ_MASK ((((u64)ICB_1_IRQ_MASK) << 32) | ICB_0_IRQ_MASK) 59 60#define BUTTRESS_IRQ_MASK ((REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR)) | \ 61 (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR)) | \ 62 (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR)) | \ 63 (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR)) | \ 64 (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR)) | \ 65 (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR))) 66 67#define BUTTRESS_IRQ_ENABLE_MASK ((u32)~BUTTRESS_IRQ_MASK) 68#define BUTTRESS_IRQ_DISABLE_MASK ((u32)-1) 69 70#define ITF_FIREWALL_VIOLATION_MASK ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ 71 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ 72 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ 73 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ 74 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ 75 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ 76 (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) 77 78static char *ivpu_platform_to_str(u32 platform) 79{ 80 switch (platform) { 81 case IVPU_PLATFORM_SILICON: 82 return "IVPU_PLATFORM_SILICON"; 83 case IVPU_PLATFORM_SIMICS: 84 return "IVPU_PLATFORM_SIMICS"; 85 case IVPU_PLATFORM_FPGA: 86 return "IVPU_PLATFORM_FPGA"; 87 default: 88 return "Invalid platform"; 89 } 90} 91 92static const struct dmi_system_id ivpu_dmi_platform_simulation[] = { 93 { 94 .ident = "Intel Simics", 95 .matches = { 96 DMI_MATCH(DMI_BOARD_NAME, "lnlrvp"), 97 DMI_MATCH(DMI_BOARD_VERSION, "1.0"), 98 DMI_MATCH(DMI_BOARD_SERIAL, "123456789"), 99 }, 100 }, 101 { 102 .ident = "Intel Simics", 103 .matches = { 104 DMI_MATCH(DMI_BOARD_NAME, "Simics"), 105 }, 106 }, 107 { } 108}; 109 110static void ivpu_hw_read_platform(struct ivpu_device *vdev) 111{ 112 if (dmi_check_system(ivpu_dmi_platform_simulation)) 113 vdev->platform = IVPU_PLATFORM_SIMICS; 114 else 115 vdev->platform = IVPU_PLATFORM_SILICON; 116 117 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", 118 ivpu_platform_to_str(vdev->platform), vdev->platform); 119} 120 121static void ivpu_hw_wa_init(struct ivpu_device *vdev) 122{ 123 vdev->wa.punit_disabled = ivpu_is_fpga(vdev); 124 vdev->wa.clear_runtime_mem = false; 125 126 if (ivpu_hw_gen(vdev) == IVPU_HW_40XX) 127 vdev->wa.disable_clock_relinquish = true; 128 129 IVPU_PRINT_WA(punit_disabled); 130 IVPU_PRINT_WA(clear_runtime_mem); 131 IVPU_PRINT_WA(disable_clock_relinquish); 132} 133 134static void ivpu_hw_timeouts_init(struct ivpu_device *vdev) 135{ 136 if (ivpu_is_fpga(vdev)) { 137 vdev->timeout.boot = 100000; 138 vdev->timeout.jsm = 50000; 139 vdev->timeout.tdr = 2000000; 140 vdev->timeout.reschedule_suspend = 1000; 141 } else if (ivpu_is_simics(vdev)) { 142 vdev->timeout.boot = 50; 143 vdev->timeout.jsm = 500; 144 vdev->timeout.tdr = 10000; 145 vdev->timeout.reschedule_suspend = 10; 146 } else { 147 vdev->timeout.boot = 1000; 148 vdev->timeout.jsm = 500; 149 vdev->timeout.tdr = 2000; 150 vdev->timeout.reschedule_suspend = 10; 151 } 152} 153 154static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev) 155{ 156 return REGB_POLL_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); 157} 158 159static int ivpu_pll_cmd_send(struct ivpu_device *vdev, u16 min_ratio, u16 max_ratio, 160 u16 target_ratio, u16 epp, u16 config, u16 cdyn) 161{ 162 int ret; 163 u32 val; 164 165 ret = ivpu_pll_wait_for_cmd_send(vdev); 166 if (ret) { 167 ivpu_err(vdev, "Failed to sync before WP request: %d\n", ret); 168 return ret; 169 } 170 171 val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0); 172 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MIN_RATIO, min_ratio, val); 173 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MAX_RATIO, max_ratio, val); 174 REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, val); 175 176 val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1); 177 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, TARGET_RATIO, target_ratio, val); 178 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, EPP, epp, val); 179 REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, val); 180 181 val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2); 182 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CONFIG, config, val); 183 val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CDYN, cdyn, val); 184 REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, val); 185 186 val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_CMD); 187 val = REG_SET_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, val); 188 REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_CMD, val); 189 190 ret = ivpu_pll_wait_for_cmd_send(vdev); 191 if (ret) 192 ivpu_err(vdev, "Failed to sync after WP request: %d\n", ret); 193 194 return ret; 195} 196 197static int ivpu_pll_wait_for_status_ready(struct ivpu_device *vdev) 198{ 199 return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, 1, PLL_TIMEOUT_US); 200} 201 202static int ivpu_wait_for_clock_own_resource_ack(struct ivpu_device *vdev) 203{ 204 if (ivpu_is_simics(vdev)) 205 return 0; 206 207 return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US); 208} 209 210static void ivpu_pll_init_frequency_ratios(struct ivpu_device *vdev) 211{ 212 struct ivpu_hw_info *hw = vdev->hw; 213 u8 fuse_min_ratio, fuse_pn_ratio, fuse_max_ratio; 214 u32 fmin_fuse, fmax_fuse; 215 216 fmin_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMIN_FUSE); 217 fuse_min_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, MIN_RATIO, fmin_fuse); 218 fuse_pn_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, PN_RATIO, fmin_fuse); 219 220 fmax_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMAX_FUSE); 221 fuse_max_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMAX_FUSE, MAX_RATIO, fmax_fuse); 222 223 hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, fuse_min_ratio, fuse_max_ratio); 224 hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, fuse_max_ratio); 225 hw->pll.pn_ratio = clamp_t(u8, fuse_pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio); 226} 227 228static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable) 229{ 230 u16 config = enable ? PLL_CONFIG_DEFAULT : 0; 231 u16 cdyn = enable ? PLL_CDYN_DEFAULT : 0; 232 u16 epp = enable ? PLL_EPP_DEFAULT : 0; 233 struct ivpu_hw_info *hw = vdev->hw; 234 u16 target_ratio = hw->pll.pn_ratio; 235 int ret; 236 237 ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, epp: 0x%x, config: 0x%x, cdyn: 0x%x\n", 238 PLL_RATIO_TO_FREQ(target_ratio), epp, config, cdyn); 239 240 ret = ivpu_pll_cmd_send(vdev, hw->pll.min_ratio, hw->pll.max_ratio, 241 target_ratio, epp, config, cdyn); 242 if (ret) { 243 ivpu_err(vdev, "Failed to send PLL workpoint request: %d\n", ret); 244 return ret; 245 } 246 247 if (enable) { 248 ret = ivpu_pll_wait_for_status_ready(vdev); 249 if (ret) { 250 ivpu_err(vdev, "Timed out waiting for PLL ready status\n"); 251 return ret; 252 } 253 } 254 255 return 0; 256} 257 258static int ivpu_pll_enable(struct ivpu_device *vdev) 259{ 260 return ivpu_pll_drive(vdev, true); 261} 262 263static int ivpu_pll_disable(struct ivpu_device *vdev) 264{ 265 return ivpu_pll_drive(vdev, false); 266} 267 268static void ivpu_boot_host_ss_rst_drive(struct ivpu_device *vdev, bool enable) 269{ 270 u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN); 271 272 if (enable) { 273 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); 274 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); 275 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); 276 } else { 277 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); 278 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); 279 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); 280 } 281 282 REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val); 283} 284 285static void ivpu_boot_host_ss_clk_drive(struct ivpu_device *vdev, bool enable) 286{ 287 u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN); 288 289 if (enable) { 290 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); 291 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); 292 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); 293 } else { 294 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); 295 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); 296 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); 297 } 298 299 REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val); 300} 301 302static int ivpu_boot_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) 303{ 304 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); 305 306 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) 307 return -EIO; 308 309 return 0; 310} 311 312static int ivpu_boot_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) 313{ 314 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN); 315 316 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) 317 return -EIO; 318 319 return 0; 320} 321 322static int ivpu_boot_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) 323{ 324 u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY); 325 326 if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) 327 return -EIO; 328 329 return 0; 330} 331 332static int ivpu_boot_top_noc_qrenqn_check(struct ivpu_device *vdev, u32 exp_val) 333{ 334 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); 335 336 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || 337 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) 338 return -EIO; 339 340 return 0; 341} 342 343static int ivpu_boot_top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) 344{ 345 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN); 346 347 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || 348 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) 349 return -EIO; 350 351 return 0; 352} 353 354static int ivpu_boot_top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) 355{ 356 u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY); 357 358 if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || 359 !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) 360 return -EIO; 361 362 return 0; 363} 364 365static void ivpu_boot_idle_gen_drive(struct ivpu_device *vdev, bool enable) 366{ 367 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN); 368 369 if (enable) 370 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); 371 else 372 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); 373 374 REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val); 375} 376 377static int ivpu_boot_host_ss_check(struct ivpu_device *vdev) 378{ 379 int ret; 380 381 ret = ivpu_boot_noc_qreqn_check(vdev, 0x0); 382 if (ret) { 383 ivpu_err(vdev, "Failed qreqn check: %d\n", ret); 384 return ret; 385 } 386 387 ret = ivpu_boot_noc_qacceptn_check(vdev, 0x0); 388 if (ret) { 389 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 390 return ret; 391 } 392 393 ret = ivpu_boot_noc_qdeny_check(vdev, 0x0); 394 if (ret) 395 ivpu_err(vdev, "Failed qdeny check %d\n", ret); 396 397 return ret; 398} 399 400static int ivpu_boot_host_ss_axi_drive(struct ivpu_device *vdev, bool enable) 401{ 402 int ret; 403 u32 val; 404 405 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); 406 if (enable) 407 val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 408 else 409 val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); 410 REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val); 411 412 ret = ivpu_boot_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); 413 if (ret) { 414 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 415 return ret; 416 } 417 418 ret = ivpu_boot_noc_qdeny_check(vdev, 0x0); 419 if (ret) { 420 ivpu_err(vdev, "Failed qdeny check: %d\n", ret); 421 return ret; 422 } 423 424 if (enable) { 425 REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT); 426 REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT); 427 } 428 429 return ret; 430} 431 432static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev) 433{ 434 return ivpu_boot_host_ss_axi_drive(vdev, true); 435} 436 437static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable) 438{ 439 int ret; 440 u32 val; 441 442 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); 443 if (enable) { 444 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); 445 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 446 } else { 447 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); 448 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); 449 } 450 REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val); 451 452 ret = ivpu_boot_top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); 453 if (ret) { 454 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 455 return ret; 456 } 457 458 ret = ivpu_boot_top_noc_qdeny_check(vdev, 0x0); 459 if (ret) 460 ivpu_err(vdev, "Failed qdeny check: %d\n", ret); 461 462 return ret; 463} 464 465static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev) 466{ 467 return ivpu_boot_host_ss_top_noc_drive(vdev, true); 468} 469 470static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable) 471{ 472 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); 473 474 if (enable) 475 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); 476 else 477 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); 478 479 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); 480 481 if (enable) 482 ndelay(500); 483} 484 485static void ivpu_boot_pwr_island_drive(struct ivpu_device *vdev, bool enable) 486{ 487 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0); 488 489 if (enable) 490 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); 491 else 492 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); 493 494 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val); 495 496 if (!enable) 497 ndelay(500); 498} 499 500static int ivpu_boot_wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val) 501{ 502 if (ivpu_is_fpga(vdev)) 503 return 0; 504 505 return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, 506 exp_val, PWR_ISLAND_STATUS_TIMEOUT_US); 507} 508 509static void ivpu_boot_pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable) 510{ 511 u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0); 512 513 if (enable) 514 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); 515 else 516 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); 517 518 REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val); 519} 520 521static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev) 522{ 523 u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES); 524 525 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val); 526 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val); 527 val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); 528 529 REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val); 530} 531 532static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev) 533{ 534 u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV); 535 536 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); 537 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); 538 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val); 539 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val); 540 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); 541 val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); 542 543 REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val); 544} 545 546static int ivpu_boot_cpu_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) 547{ 548 u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN); 549 550 if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val)) 551 return -EIO; 552 553 return 0; 554} 555 556static int ivpu_boot_cpu_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) 557{ 558 u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY); 559 560 if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val)) 561 return -EIO; 562 563 return 0; 564} 565 566static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev) 567{ 568 int ret; 569 570 ret = ivpu_wait_for_clock_own_resource_ack(vdev); 571 if (ret) { 572 ivpu_err(vdev, "Timed out waiting for clock own resource ACK\n"); 573 return ret; 574 } 575 576 ivpu_boot_pwr_island_trickle_drive(vdev, true); 577 ivpu_boot_pwr_island_drive(vdev, true); 578 579 ret = ivpu_boot_wait_for_pwr_island_status(vdev, 0x1); 580 if (ret) { 581 ivpu_err(vdev, "Timed out waiting for power island status\n"); 582 return ret; 583 } 584 585 ret = ivpu_boot_top_noc_qrenqn_check(vdev, 0x0); 586 if (ret) { 587 ivpu_err(vdev, "Failed qrenqn check %d\n", ret); 588 return ret; 589 } 590 591 ivpu_boot_host_ss_clk_drive(vdev, true); 592 ivpu_boot_host_ss_rst_drive(vdev, true); 593 ivpu_boot_pwr_island_isolation_drive(vdev, false); 594 595 return ret; 596} 597 598static int ivpu_boot_soc_cpu_drive(struct ivpu_device *vdev, bool enable) 599{ 600 int ret; 601 u32 val; 602 603 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN); 604 if (enable) 605 val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); 606 else 607 val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); 608 REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val); 609 610 ret = ivpu_boot_cpu_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0); 611 if (ret) { 612 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret); 613 return ret; 614 } 615 616 ret = ivpu_boot_cpu_noc_qdeny_check(vdev, 0x0); 617 if (ret) 618 ivpu_err(vdev, "Failed qdeny check: %d\n", ret); 619 620 return ret; 621} 622 623static int ivpu_boot_soc_cpu_enable(struct ivpu_device *vdev) 624{ 625 return ivpu_boot_soc_cpu_drive(vdev, true); 626} 627 628static int ivpu_boot_soc_cpu_boot(struct ivpu_device *vdev) 629{ 630 int ret; 631 u32 val; 632 u64 val64; 633 634 ret = ivpu_boot_soc_cpu_enable(vdev); 635 if (ret) { 636 ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret); 637 return ret; 638 } 639 640 val64 = vdev->fw->entry_point; 641 val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1; 642 REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64); 643 644 val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO); 645 val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val); 646 REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val); 647 648 ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n", 649 ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume"); 650 651 return 0; 652} 653 654static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable) 655{ 656 int ret; 657 u32 val; 658 659 ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 660 if (ret) { 661 ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n", ret); 662 return ret; 663 } 664 665 val = REGB_RD32(VPU_40XX_BUTTRESS_D0I3_CONTROL); 666 if (enable) 667 val = REG_SET_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val); 668 else 669 val = REG_CLR_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val); 670 REGB_WR32(VPU_40XX_BUTTRESS_D0I3_CONTROL, val); 671 672 ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); 673 if (ret) { 674 ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n", ret); 675 return ret; 676 } 677 678 return 0; 679} 680 681static bool ivpu_tile_disable_check(u32 config) 682{ 683 /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */ 684 if (config == 0) 685 return true; 686 687 if (config > BIT(TILE_MAX_NUM - 1)) 688 return false; 689 690 if ((config & (config - 1)) == 0) 691 return true; 692 693 return false; 694} 695 696static int ivpu_hw_40xx_info_init(struct ivpu_device *vdev) 697{ 698 struct ivpu_hw_info *hw = vdev->hw; 699 u32 tile_disable; 700 u32 fuse; 701 702 fuse = REGB_RD32(VPU_40XX_BUTTRESS_TILE_FUSE); 703 if (!REG_TEST_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, VALID, fuse)) { 704 ivpu_err(vdev, "Fuse: invalid (0x%x)\n", fuse); 705 return -EIO; 706 } 707 708 tile_disable = REG_GET_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, CONFIG, fuse); 709 if (!ivpu_tile_disable_check(tile_disable)) { 710 ivpu_err(vdev, "Fuse: Invalid tile disable config (0x%x)\n", tile_disable); 711 return -EIO; 712 } 713 714 if (tile_disable) 715 ivpu_dbg(vdev, MISC, "Fuse: %d tiles enabled. Tile number %d disabled\n", 716 TILE_MAX_NUM - 1, ffs(tile_disable) - 1); 717 else 718 ivpu_dbg(vdev, MISC, "Fuse: All %d tiles enabled\n", TILE_MAX_NUM); 719 720 hw->tile_fuse = tile_disable; 721 hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; 722 723 ivpu_pll_init_frequency_ratios(vdev); 724 725 ivpu_hw_init_range(&vdev->hw->ranges.global, 0x80000000, SZ_512M); 726 ivpu_hw_init_range(&vdev->hw->ranges.user, 0x80000000, SZ_256M); 727 ivpu_hw_init_range(&vdev->hw->ranges.shave, 0x80000000 + SZ_256M, SZ_2G - SZ_256M); 728 ivpu_hw_init_range(&vdev->hw->ranges.dma, 0x200000000, SZ_8G); 729 730 ivpu_hw_read_platform(vdev); 731 ivpu_hw_wa_init(vdev); 732 ivpu_hw_timeouts_init(vdev); 733 734 return 0; 735} 736 737static int ivpu_hw_40xx_reset(struct ivpu_device *vdev) 738{ 739 int ret; 740 u32 val; 741 742 ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US); 743 if (ret) { 744 ivpu_err(vdev, "Wait for *_TRIGGER timed out\n"); 745 return ret; 746 } 747 748 val = REGB_RD32(VPU_40XX_BUTTRESS_IP_RESET); 749 val = REG_SET_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, val); 750 REGB_WR32(VPU_40XX_BUTTRESS_IP_RESET, val); 751 752 ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US); 753 if (ret) 754 ivpu_err(vdev, "Timed out waiting for RESET completion\n"); 755 756 return ret; 757} 758 759static int ivpu_hw_40xx_d0i3_enable(struct ivpu_device *vdev) 760{ 761 int ret; 762 763 if (IVPU_WA(punit_disabled)) 764 return 0; 765 766 ret = ivpu_boot_d0i3_drive(vdev, true); 767 if (ret) 768 ivpu_err(vdev, "Failed to enable D0i3: %d\n", ret); 769 770 udelay(5); /* VPU requires 5 us to complete the transition */ 771 772 return ret; 773} 774 775static int ivpu_hw_40xx_d0i3_disable(struct ivpu_device *vdev) 776{ 777 int ret; 778 779 if (IVPU_WA(punit_disabled)) 780 return 0; 781 782 ret = ivpu_boot_d0i3_drive(vdev, false); 783 if (ret) 784 ivpu_err(vdev, "Failed to disable D0i3: %d\n", ret); 785 786 return ret; 787} 788 789static void ivpu_hw_40xx_profiling_freq_reg_set(struct ivpu_device *vdev) 790{ 791 u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); 792 793 if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT) 794 val = REG_CLR_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val); 795 else 796 val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val); 797 798 REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val); 799} 800 801static void ivpu_hw_40xx_ats_print(struct ivpu_device *vdev) 802{ 803 ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n", 804 REGB_RD32(VPU_40XX_BUTTRESS_HM_ATS) ? "Enable" : "Disable"); 805} 806 807static void ivpu_hw_40xx_clock_relinquish_disable(struct ivpu_device *vdev) 808{ 809 u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); 810 811 val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, DISABLE_CLK_RELINQUISH, val); 812 REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val); 813} 814 815static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev) 816{ 817 int ret; 818 819 ret = ivpu_hw_40xx_reset(vdev); 820 if (ret) { 821 ivpu_err(vdev, "Failed to reset HW: %d\n", ret); 822 return ret; 823 } 824 825 ret = ivpu_hw_40xx_d0i3_disable(vdev); 826 if (ret) 827 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret); 828 829 ret = ivpu_pll_enable(vdev); 830 if (ret) { 831 ivpu_err(vdev, "Failed to enable PLL: %d\n", ret); 832 return ret; 833 } 834 835 if (IVPU_WA(disable_clock_relinquish)) 836 ivpu_hw_40xx_clock_relinquish_disable(vdev); 837 ivpu_hw_40xx_profiling_freq_reg_set(vdev); 838 ivpu_hw_40xx_ats_print(vdev); 839 840 ret = ivpu_boot_host_ss_check(vdev); 841 if (ret) { 842 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret); 843 return ret; 844 } 845 846 ivpu_boot_idle_gen_drive(vdev, false); 847 848 ret = ivpu_boot_pwr_domain_enable(vdev); 849 if (ret) { 850 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret); 851 return ret; 852 } 853 854 ret = ivpu_boot_host_ss_axi_enable(vdev); 855 if (ret) { 856 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret); 857 return ret; 858 } 859 860 ret = ivpu_boot_host_ss_top_noc_enable(vdev); 861 if (ret) 862 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret); 863 864 return ret; 865} 866 867static int ivpu_hw_40xx_boot_fw(struct ivpu_device *vdev) 868{ 869 int ret; 870 871 ivpu_boot_no_snoop_enable(vdev); 872 ivpu_boot_tbu_mmu_enable(vdev); 873 874 ret = ivpu_boot_soc_cpu_boot(vdev); 875 if (ret) 876 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret); 877 878 return ret; 879} 880 881static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev) 882{ 883 u32 val; 884 885 if (IVPU_WA(punit_disabled)) 886 return true; 887 888 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); 889 return REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, val) && 890 REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val); 891} 892 893static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev) 894{ 895 int ret = 0; 896 897 if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_reset(vdev)) 898 ivpu_warn(vdev, "Failed to reset the VPU\n"); 899 900 if (ivpu_pll_disable(vdev)) { 901 ivpu_err(vdev, "Failed to disable PLL\n"); 902 ret = -EIO; 903 } 904 905 if (ivpu_hw_40xx_d0i3_enable(vdev)) { 906 ivpu_err(vdev, "Failed to enter D0I3\n"); 907 ret = -EIO; 908 } 909 910 return ret; 911} 912 913static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev) 914{ 915 u32 val; 916 917 REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 918 REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); 919 920 REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); 921 REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0); 922 923 val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG); 924 val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); 925 REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val); 926} 927 928/* Register indirect accesses */ 929static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev) 930{ 931 u32 pll_curr_ratio; 932 933 pll_curr_ratio = REGB_RD32(VPU_40XX_BUTTRESS_PLL_FREQ); 934 pll_curr_ratio &= VPU_40XX_BUTTRESS_PLL_FREQ_RATIO_MASK; 935 936 return PLL_RATIO_TO_FREQ(pll_curr_ratio); 937} 938 939static u32 ivpu_hw_40xx_reg_telemetry_offset_get(struct ivpu_device *vdev) 940{ 941 return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_OFFSET); 942} 943 944static u32 ivpu_hw_40xx_reg_telemetry_size_get(struct ivpu_device *vdev) 945{ 946 return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_SIZE); 947} 948 949static u32 ivpu_hw_40xx_reg_telemetry_enable_get(struct ivpu_device *vdev) 950{ 951 return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_ENABLE); 952} 953 954static void ivpu_hw_40xx_reg_db_set(struct ivpu_device *vdev, u32 db_id) 955{ 956 u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0; 957 u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET); 958 959 REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); 960} 961 962static u32 ivpu_hw_40xx_reg_ipc_rx_addr_get(struct ivpu_device *vdev) 963{ 964 return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM); 965} 966 967static u32 ivpu_hw_40xx_reg_ipc_rx_count_get(struct ivpu_device *vdev) 968{ 969 u32 count = REGV_RD32_SILENT(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT); 970 971 return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); 972} 973 974static void ivpu_hw_40xx_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr) 975{ 976 REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); 977} 978 979static void ivpu_hw_40xx_irq_clear(struct ivpu_device *vdev) 980{ 981 REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK); 982} 983 984static void ivpu_hw_40xx_irq_enable(struct ivpu_device *vdev) 985{ 986 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK); 987 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK); 988 REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_ENABLE_MASK); 989 REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0); 990} 991 992static void ivpu_hw_40xx_irq_disable(struct ivpu_device *vdev) 993{ 994 REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1); 995 REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_DISABLE_MASK); 996 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull); 997 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul); 998} 999 1000static void ivpu_hw_40xx_irq_wdt_nce_handler(struct ivpu_device *vdev) 1001{ 1002 /* TODO: For LNN hang consider engine reset instead of full recovery */ 1003 ivpu_pm_schedule_recovery(vdev); 1004} 1005 1006static void ivpu_hw_40xx_irq_wdt_mss_handler(struct ivpu_device *vdev) 1007{ 1008 ivpu_hw_wdt_disable(vdev); 1009 ivpu_pm_schedule_recovery(vdev); 1010} 1011 1012static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev) 1013{ 1014 ivpu_pm_schedule_recovery(vdev); 1015} 1016 1017/* Handler for IRQs from VPU core (irqV) */ 1018static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq) 1019{ 1020 u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK; 1021 irqreturn_t ret = IRQ_NONE; 1022 1023 if (!status) 1024 return IRQ_NONE; 1025 1026 REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status); 1027 1028 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) 1029 ivpu_mmu_irq_evtq_handler(vdev); 1030 1031 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) 1032 ret |= ivpu_ipc_irq_handler(vdev); 1033 1034 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) 1035 ivpu_dbg(vdev, IRQ, "MMU sync complete\n"); 1036 1037 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) 1038 ivpu_mmu_irq_gerr_handler(vdev); 1039 1040 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) 1041 ivpu_hw_40xx_irq_wdt_mss_handler(vdev); 1042 1043 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) 1044 ivpu_hw_40xx_irq_wdt_nce_handler(vdev); 1045 1046 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) 1047 ivpu_hw_40xx_irq_noc_firewall_handler(vdev); 1048 1049 return ret; 1050} 1051 1052/* Handler for IRQs from Buttress core (irqB) */ 1053static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq) 1054{ 1055 bool schedule_recovery = false; 1056 u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK; 1057 1058 if (status == 0) 1059 return IRQ_NONE; 1060 1061 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status)) 1062 ivpu_dbg(vdev, IRQ, "FREQ_CHANGE"); 1063 1064 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) { 1065 ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 1066 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1), 1067 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2)); 1068 REGB_WR32(VPU_40XX_BUTTRESS_ATS_ERR_CLEAR, 0x1); 1069 schedule_recovery = true; 1070 } 1071 1072 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, status)) { 1073 ivpu_err(vdev, "CFI0_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG)); 1074 REGB_WR32(VPU_40XX_BUTTRESS_CFI0_ERR_CLEAR, 0x1); 1075 schedule_recovery = true; 1076 } 1077 1078 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, status)) { 1079 ivpu_err(vdev, "CFI1_ERR 0x%08x", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG)); 1080 REGB_WR32(VPU_40XX_BUTTRESS_CFI1_ERR_CLEAR, 0x1); 1081 schedule_recovery = true; 1082 } 1083 1084 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, status)) { 1085 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x", 1086 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW), 1087 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH)); 1088 REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_CLEAR, 0x1); 1089 schedule_recovery = true; 1090 } 1091 1092 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, status)) { 1093 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x", 1094 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW), 1095 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH)); 1096 REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_CLEAR, 0x1); 1097 schedule_recovery = true; 1098 } 1099 1100 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, status)) { 1101 ivpu_err(vdev, "Survivability error detected\n"); 1102 schedule_recovery = true; 1103 } 1104 1105 /* This must be done after interrupts are cleared at the source. */ 1106 REGB_WR32(VPU_40XX_BUTTRESS_INTERRUPT_STAT, status); 1107 1108 if (schedule_recovery) 1109 ivpu_pm_schedule_recovery(vdev); 1110 1111 return IRQ_HANDLED; 1112} 1113 1114static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr) 1115{ 1116 struct ivpu_device *vdev = ptr; 1117 irqreturn_t ret = IRQ_NONE; 1118 1119 REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1); 1120 1121 ret |= ivpu_hw_40xx_irqv_handler(vdev, irq); 1122 ret |= ivpu_hw_40xx_irqb_handler(vdev, irq); 1123 1124 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */ 1125 REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0); 1126 1127 if (ret & IRQ_WAKE_THREAD) 1128 return IRQ_WAKE_THREAD; 1129 1130 return ret; 1131} 1132 1133static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev) 1134{ 1135 u32 irqv = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK; 1136 u32 irqb = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK; 1137 1138 if (ivpu_hw_40xx_reg_ipc_rx_count_get(vdev)) 1139 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ"); 1140 1141 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, irqv)) 1142 ivpu_err(vdev, "WDT MSS timeout detected\n"); 1143 1144 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, irqv)) 1145 ivpu_err(vdev, "WDT NCE timeout detected\n"); 1146 1147 if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, irqv)) 1148 ivpu_err(vdev, "NOC Firewall irq detected\n"); 1149 1150 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, irqb)) { 1151 ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", 1152 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1), 1153 REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2)); 1154 } 1155 1156 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, irqb)) 1157 ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG)); 1158 1159 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, irqb)) 1160 ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n", REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG)); 1161 1162 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, irqb)) 1163 ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n", 1164 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW), 1165 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH)); 1166 1167 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, irqb)) 1168 ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n", 1169 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW), 1170 REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH)); 1171 1172 if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, irqb)) 1173 ivpu_err(vdev, "Survivability error detected\n"); 1174} 1175 1176const struct ivpu_hw_ops ivpu_hw_40xx_ops = { 1177 .info_init = ivpu_hw_40xx_info_init, 1178 .power_up = ivpu_hw_40xx_power_up, 1179 .is_idle = ivpu_hw_40xx_is_idle, 1180 .power_down = ivpu_hw_40xx_power_down, 1181 .reset = ivpu_hw_40xx_reset, 1182 .boot_fw = ivpu_hw_40xx_boot_fw, 1183 .wdt_disable = ivpu_hw_40xx_wdt_disable, 1184 .diagnose_failure = ivpu_hw_40xx_diagnose_failure, 1185 .reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get, 1186 .reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get, 1187 .reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get, 1188 .reg_telemetry_enable_get = ivpu_hw_40xx_reg_telemetry_enable_get, 1189 .reg_db_set = ivpu_hw_40xx_reg_db_set, 1190 .reg_ipc_rx_addr_get = ivpu_hw_40xx_reg_ipc_rx_addr_get, 1191 .reg_ipc_rx_count_get = ivpu_hw_40xx_reg_ipc_rx_count_get, 1192 .reg_ipc_tx_set = ivpu_hw_40xx_reg_ipc_tx_set, 1193 .irq_clear = ivpu_hw_40xx_irq_clear, 1194 .irq_enable = ivpu_hw_40xx_irq_enable, 1195 .irq_disable = ivpu_hw_40xx_irq_disable, 1196 .irq_handler = ivpu_hw_40xx_irq_handler, 1197}; 1198