Lines Matching defs:emc
14 #include "tegra210-emc.h"
36 #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__)
108 emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \
116 static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type)
119 struct tegra210_emc_timing *last = emc->last;
120 struct tegra210_emc_timing *next = emc->next;
134 value = tegra210_emc_mrr_read(emc, 2, 19);
136 for (i = 0; i < emc->num_channels; i++) {
145 value = tegra210_emc_mrr_read(emc, 2, 18);
147 for (i = 0; i < emc->num_channels; i++) {
206 if (emc->num_channels > 1) {
262 if (emc->num_devices < 2)
269 value = tegra210_emc_mrr_read(emc, 1, 19);
271 for (i = 0; i < emc->num_channels; i++) {
280 value = tegra210_emc_mrr_read(emc, 1, 18);
282 for (i = 0; i < emc->num_channels; i++) {
343 if (emc->num_channels > 1) {
403 static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type,
450 tegra210_emc_start_periodic_compensation(emc);
456 adel = update_clock_tree_delay(emc, DVFS_PT1);
465 adel = update_clock_tree_delay(emc, DVFS_UPDATE);
469 tegra210_emc_start_periodic_compensation(emc);
472 adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE);
478 static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc)
493 struct tegra210_emc_timing *last = emc->last;
498 emc_dbg(emc, PER_TRAIN, "Periodic training starting\n");
500 value = emc_readl(emc, EMC_DBG);
501 emc_cfg_o = emc_readl(emc, EMC_CFG);
510 emc_writel(emc, emc_cfg, EMC_CFG);
513 tegra210_emc_dll_disable(emc);
515 for (i = 0; i < emc->num_channels; i++)
516 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
520 for (i = 0; i < emc->num_channels; i++)
521 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
525 emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE);
528 emc_writel(emc, value, EMC_CFG_UPDATE);
534 tegra210_emc_start_periodic_compensation(emc);
548 del = periodic_compensation_handler(emc,
559 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
561 emc_writel(emc, value, list[i]);
565 emc_writel(emc, emc_cfg_o, EMC_CFG);
570 tegra210_emc_timing_update(emc);
573 emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
576 tegra210_emc_dll_enable(emc);
585 static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc)
607 struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next;
624 emc_dbg(emc, INFO, "Running clock change.\n");
627 fake = tegra210_emc_find_timing(emc, last->rate * 1000UL);
630 value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK;
648 emc_readl(emc, EMC_CFG);
649 emc_readl(emc, EMC_AUTO_CAL_CONFIG);
661 emc_dbg = emc_readl(emc, EMC_DBG);
662 emc_pin = emc_readl(emc, EMC_PIN);
663 emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK);
675 emc_dbg(emc, INFO, "Clock change version: %d\n",
677 emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type);
678 emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices);
679 emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
680 emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src);
681 emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate,
683 emc_dbg(emc, INFO, "last period: %u, next period: %u\n",
685 emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor);
686 emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels);
687 emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode);
693 emc_dbg(emc, STEPS, "Step 1\n");
694 emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n");
696 value = emc_readl(emc, EMC_CFG_DIG_DLL);
698 emc_writel(emc, value, EMC_CFG_DIG_DLL);
700 tegra210_emc_timing_update(emc);
702 for (i = 0; i < emc->num_channels; i++)
703 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
706 emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
714 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
715 emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
717 emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n");
719 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
720 emc_writel(emc, emc_cfg, EMC_CFG);
721 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
722 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
727 for (i = 0; i < emc->num_channels; i++)
728 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
732 for (i = 0; i < emc->num_channels; i++)
733 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
737 tegra210_emc_start_periodic_compensation(emc);
742 value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake,
750 emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
751 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
752 emc_writel(emc, emc_cfg, EMC_CFG);
753 emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
754 emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
756 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp &
778 emc_writel(emc, last->burst_regs
784 emc_writel(emc, last->burst_regs
813 emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL);
819 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
825 emc_dbg(emc, STEPS, "Step 2\n");
829 emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n");
830 value = tegra210_emc_dll_prelock(emc, clksrc);
831 emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value);
833 emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n");
834 tegra210_emc_dll_disable(emc);
841 emc_dbg(emc, STEPS, "Step 3\n");
843 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
844 emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
845 emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
846 emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
847 emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
848 emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
849 emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
850 emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
851 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
855 emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
861 emc_dbg(emc, STEPS, "Step 4\n");
864 ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
866 emc_writel(emc, next->emc_cfg_2, EMC_CFG_2);
872 emc_dbg(emc, STEPS, "Step 5\n");
889 emc_dbg(emc, STEPS, "Step 6\n");
895 emc_dbg(emc, STEPS, "Step 7\n");
896 emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
929 emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
972 emc_writel(emc, RP_war, EMC_RP);
973 emc_writel(emc, R2P_war, EMC_R2P);
974 emc_writel(emc, W2P_war, EMC_W2P);
975 emc_writel(emc, TRPab_war, EMC_TRPAB);
978 tegra210_emc_timing_update(emc);
980 emc_dbg(emc, INFO, "Skipped WAR\n");
993 emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
994 emc_writel(emc, next->emc_mrw, EMC_MRW);
995 emc_writel(emc, next->emc_mrw2, EMC_MRW2);
1002 emc_dbg(emc, STEPS, "Step 8\n");
1003 emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n");
1006 const u16 *offsets = emc->offsets->burst;
1069 emc_writel(emc, value, offset);
1073 tegra210_emc_adjust_timing(emc, next);
1078 emc_writel(emc, value, EMC_MRW);
1082 emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n");
1086 emc->offsets->burst_per_channel;
1105 if (emc->num_channels < 2 && burst[i].bank >= 1)
1108 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1110 emc_channel_writel(emc, burst[i].bank,
1116 emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n");
1120 emc->offsets->vref_per_channel;
1125 if (emc->num_channels < 2 && vref[i].bank >= 1)
1128 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1130 emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i],
1135 emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n");
1138 const u16 *offsets = emc->offsets->trim;
1155 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1157 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
1159 emc_writel(emc, value, offsets[i]);
1161 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1163 emc_writel(emc, next->trim_regs[i], offsets[i]);
1168 emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n");
1172 &emc->offsets->trim_per_channel[0];
1178 if (emc->num_channels < 2 && trim[i].bank >= 1)
1195 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1197 emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset,
1199 emc_channel_writel(emc, trim[i].bank, value, offset);
1201 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1203 emc_channel_writel(emc, trim[i].bank,
1208 emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n");
1211 const u16 *offsets = emc->offsets->burst_mc;
1214 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1216 mc_writel(emc->mc, values[i], offsets[i]);
1221 const u16 *la = emc->offsets->la_scale;
1223 emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n");
1226 emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
1228 mc_writel(emc->mc, next->la_scale_regs[i], la[i]);
1233 mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1239 emc_dbg(emc, STEPS, "Step 9\n");
1245 emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
1246 emc_writel(emc, value, EMC_ZCAL_WAIT_CNT);
1251 emc_writel(emc, value, EMC_DBG);
1252 emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
1253 emc_writel(emc, emc_dbg, EMC_DBG);
1260 emc_dbg(emc, STEPS, "Step 10\n");
1264 ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0);
1266 ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0);
1270 ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0);
1271 ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] &
1275 ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] &
1280 if (emc->num_devices > 1) {
1281 ccfifo_writel(emc,
1286 ccfifo_writel(emc,
1294 if (emc->num_devices < 2)
1295 ccfifo_writel(emc,
1300 ccfifo_writel(emc,
1305 ccfifo_writel(emc,
1314 ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value);
1315 ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period);
1327 ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV |
1357 emc_dbg(emc, STEPS, "Step 11\n");
1359 ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay);
1362 ccfifo_writel(emc, value, EMC_DBG, 0);
1364 ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period,
1371 emc_dbg(emc, STEPS, "Step 12\n");
1373 ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0);
1375 ccfifo_writel(emc, value, EMC_DBG, 0);
1381 emc_dbg(emc, STEPS, "Step 13\n");
1383 ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0);
1384 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
1390 emc_dbg(emc, STEPS, "Step 14\n");
1395 if (emc->num_devices <= 1)
1400 ccfifo_writel(emc, value, EMC_PIN, 0);
1407 emc_dbg(emc, STEPS, "Step 15\n");
1419 emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj);
1420 emc_dbg(emc, INFO, "dst_clk_period = %u\n",
1422 emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n",
1424 emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n",
1431 if (emc->num_devices < 2) {
1433 ccfifo_writel(emc,
1439 ccfifo_writel(emc, value, EMC_MRW3, delay);
1440 ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1441 ccfifo_writel(emc, 0, EMC_REF, 0);
1442 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1448 ccfifo_writel(emc,
1453 ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1457 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1462 ccfifo_writel(emc, value, EMC_MRW3, 0);
1463 ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1464 ccfifo_writel(emc, 0, EMC_REF, 0);
1466 ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1471 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD,
1475 ccfifo_writel(emc, value, EMC_MRW3, delay);
1476 ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1477 ccfifo_writel(emc, 0, EMC_REF, 0);
1479 ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
1485 ccfifo_writel(emc, 0, 0, 10);
1496 emc_dbg(emc, STEPS, "Step 17\n");
1499 ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
1505 emc_dbg(emc, STEPS, "Step 18\n");
1508 ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0);
1509 ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0);
1511 ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0);
1514 ccfifo_writel(emc, next->emc_emrs &
1516 ccfifo_writel(emc, next->emc_emrs2 &
1518 ccfifo_writel(emc, next->emc_mrs |
1526 emc_dbg(emc, STEPS, "Step 19\n");
1536 ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0);
1539 ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
1545 if (emc->num_devices > 1) {
1550 ccfifo_writel(emc, value, EMC_MRW, 0);
1555 ccfifo_writel(emc, value |
1560 if (emc->num_devices > 1) {
1563 ccfifo_writel(emc, value, EMC_ZQ_CAL, 0);
1569 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1576 ccfifo_writel(emc,
1579 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1586 emc_dbg(emc, STEPS, "Step 20\n");
1589 ccfifo_writel(emc, 0, EMC_REF, 0);
1592 ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0);
1593 ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2);
1600 emc_dbg(emc, STEPS, "Step 21\n");
1603 ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE,
1606 ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
1610 ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] &
1613 ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
1620 emc_dbg(emc, STEPS, "Step 22\n");
1622 ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0);
1626 emc_writel(emc,
1631 emc_writel(emc,
1640 emc_dbg(emc, STEPS, "Step 23\n");
1642 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1649 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1651 tegra210_emc_do_clock_change(emc, clksrc);
1662 emc_dbg(emc, STEPS, "Step 25\n");
1666 mc_writel(emc->mc, next->la_scale_regs[i],
1667 emc->offsets->la_scale[i]);
1669 tegra210_emc_timing_update(emc);
1676 emc_dbg(emc, STEPS, "Step 26\n");
1679 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1680 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
1682 emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
1684 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1691 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1693 emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX],
1696 emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
1698 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1705 emc_dbg(emc, STEPS, "Step 27\n");
1707 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1708 emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG);
1709 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1710 emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp,
1712 emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
1718 emc_dbg(emc, STEPS, "Step 28\n");
1720 tegra210_emc_set_shadow_bypass(emc, ACTIVE);
1721 emc_writel(emc,
1724 tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
1730 emc_dbg(emc, STEPS, "Step 29\n");
1732 emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
1741 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR,
1743 emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR,
1745 emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0);
1751 emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n");
1754 value = emc_readl(emc, EMC_CFG_DIG_DLL);
1761 emc_writel(emc, value, EMC_CFG_DIG_DLL);
1762 tegra210_emc_timing_update(emc);
1765 emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);