1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (c) 2019 Samsung Electronics Co., Ltd. 4 * Author: Lukasz Luba <l.luba@partner.samsung.com> 5 */ 6 7#include <linux/clk.h> 8#include <linux/devfreq.h> 9#include <linux/devfreq-event.h> 10#include <linux/device.h> 11#include <linux/interrupt.h> 12#include <linux/io.h> 13#include <linux/mfd/syscon.h> 14#include <linux/module.h> 15#include <linux/moduleparam.h> 16#include <linux/of.h> 17#include <linux/pm_opp.h> 18#include <linux/platform_device.h> 19#include <linux/regmap.h> 20#include <linux/regulator/consumer.h> 21#include <linux/slab.h> 22#include "../jedec_ddr.h" 23#include "../of_memory.h" 24 25static int irqmode; 26module_param(irqmode, int, 0644); 27MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)"); 28 29#define EXYNOS5_DREXI_TIMINGAREF (0x0030) 30#define EXYNOS5_DREXI_TIMINGROW0 (0x0034) 31#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038) 32#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C) 33#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4) 34#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8) 35#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC) 36#define CDREX_PAUSE (0x2091c) 37#define CDREX_LPDDR3PHY_CON3 (0x20a20) 38#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700) 39#define EXYNOS5_TIMING_SET_SWI BIT(28) 40#define USE_MX_MSPLL_TIMINGS (1) 41#define USE_BPLL_TIMINGS (0) 42#define EXYNOS5_AREF_NORMAL (0x2e) 43 44#define DREX_PPCCLKCON (0x0130) 45#define DREX_PEREV2CONFIG (0x013c) 46#define DREX_PMNC_PPC (0xE000) 47#define DREX_CNTENS_PPC (0xE010) 48#define DREX_CNTENC_PPC (0xE020) 49#define DREX_INTENS_PPC (0xE030) 50#define DREX_INTENC_PPC (0xE040) 51#define DREX_FLAG_PPC (0xE050) 52#define DREX_PMCNT2_PPC (0xE130) 53 54/* 55 * A value for register DREX_PMNC_PPC which should be written to reset 56 * the cycle counter CCNT (a reference wall clock). It sets zero to the 57 * CCNT counter. 58 */ 59#define CC_RESET BIT(2) 60 61/* 62 * A value for register DREX_PMNC_PPC which does the reset of all performance 63 * counters to zero. 64 */ 65#define PPC_COUNTER_RESET BIT(1) 66 67/* 68 * Enables all configured counters (including cycle counter). The value should 69 * be written to the register DREX_PMNC_PPC. 70 */ 71#define PPC_ENABLE BIT(0) 72 73/* A value for register DREX_PPCCLKCON which enables performance events clock. 74 * Must be written before first access to the performance counters register 75 * set, otherwise it could crash. 76 */ 77#define PEREV_CLK_EN BIT(0) 78 79/* 80 * Values which are used to enable counters, interrupts or configure flags of 81 * the performance counters. They configure counter 2 and cycle counter. 82 */ 83#define PERF_CNT2 BIT(2) 84#define PERF_CCNT BIT(31) 85 86/* 87 * Performance event types which are used for setting the preferred event 88 * to track in the counters. 89 * There is a set of different types, the values are from range 0 to 0x6f. 90 * These settings should be written to the configuration register which manages 91 * the type of the event (register DREX_PEREV2CONFIG). 92 */ 93#define READ_TRANSFER_CH0 (0x6d) 94#define READ_TRANSFER_CH1 (0x6f) 95 96#define PERF_COUNTER_START_VALUE 0xff000000 97#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL 98 99/** 100 * struct dmc_opp_table - Operating level desciption 101 * @freq_hz: target frequency in Hz 102 * @volt_uv: target voltage in uV 103 * 104 * Covers frequency and voltage settings of the DMC operating mode. 105 */ 106struct dmc_opp_table { 107 u32 freq_hz; 108 u32 volt_uv; 109}; 110 111/** 112 * struct exynos5_dmc - main structure describing DMC device 113 * @dev: DMC device 114 * @df: devfreq device structure returned by devfreq framework 115 * @gov_data: configuration of devfreq governor 116 * @base_drexi0: DREX0 registers mapping 117 * @base_drexi1: DREX1 registers mapping 118 * @clk_regmap: regmap for clock controller registers 119 * @lock: protects curr_rate and frequency/voltage setting section 120 * @curr_rate: current frequency 121 * @curr_volt: current voltage 122 * @opp: OPP table 123 * @opp_count: number of 'opp' elements 124 * @timings_arr_size: number of 'timings' elements 125 * @timing_row: values for timing row register, for each OPP 126 * @timing_data: values for timing data register, for each OPP 127 * @timing_power: balues for timing power register, for each OPP 128 * @timings: DDR memory timings, from device tree 129 * @min_tck: DDR memory minimum timing values, from device tree 130 * @bypass_timing_row: value for timing row register for bypass timings 131 * @bypass_timing_data: value for timing data register for bypass timings 132 * @bypass_timing_power: value for timing power register for bypass 133 * timings 134 * @vdd_mif: Memory interface regulator 135 * @fout_spll: clock: SPLL 136 * @fout_bpll: clock: BPLL 137 * @mout_spll: clock: mux SPLL 138 * @mout_bpll: clock: mux BPLL 139 * @mout_mclk_cdrex: clock: mux mclk_cdrex 140 * @mout_mx_mspll_ccore: clock: mux mx_mspll_ccore 141 * @counter: devfreq events 142 * @num_counters: number of 'counter' elements 143 * @last_overflow_ts: time (in ns) of last overflow of each DREX 144 * @load: utilization in percents 145 * @total: total time between devfreq events 146 * @in_irq_mode: whether running in interrupt mode (true) 147 * or polling (false) 148 * 149 * The main structure for the Dynamic Memory Controller which covers clocks, 150 * memory regions, HW information, parameters and current operating mode. 151 */ 152struct exynos5_dmc { 153 struct device *dev; 154 struct devfreq *df; 155 struct devfreq_simple_ondemand_data gov_data; 156 void __iomem *base_drexi0; 157 void __iomem *base_drexi1; 158 struct regmap *clk_regmap; 159 /* Protects curr_rate and frequency/voltage setting section */ 160 struct mutex lock; 161 unsigned long curr_rate; 162 unsigned long curr_volt; 163 struct dmc_opp_table *opp; 164 int opp_count; 165 u32 timings_arr_size; 166 u32 *timing_row; 167 u32 *timing_data; 168 u32 *timing_power; 169 const struct lpddr3_timings *timings; 170 const struct lpddr3_min_tck *min_tck; 171 u32 bypass_timing_row; 172 u32 bypass_timing_data; 173 u32 bypass_timing_power; 174 struct regulator *vdd_mif; 175 struct clk *fout_spll; 176 struct clk *fout_bpll; 177 struct clk *mout_spll; 178 struct clk *mout_bpll; 179 struct clk *mout_mclk_cdrex; 180 struct clk *mout_mx_mspll_ccore; 181 struct devfreq_event_dev **counter; 182 int num_counters; 183 u64 last_overflow_ts[2]; 184 unsigned long load; 185 unsigned long total; 186 bool in_irq_mode; 187}; 188 189#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \ 190 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end } 191 192#define TIMING_VAL2REG(timing, t_val) \ 193({ \ 194 u32 __val; \ 195 __val = (t_val) << (timing)->bit_beg; \ 196 __val; \ 197}) 198 199struct timing_reg { 200 char *name; 201 int bit_beg; 202 int bit_end; 203 unsigned int val; 204}; 205 206static const struct timing_reg timing_row_reg_fields[] = { 207 TIMING_FIELD("tRFC", 24, 31), 208 TIMING_FIELD("tRRD", 20, 23), 209 TIMING_FIELD("tRP", 16, 19), 210 TIMING_FIELD("tRCD", 12, 15), 211 TIMING_FIELD("tRC", 6, 11), 212 TIMING_FIELD("tRAS", 0, 5), 213}; 214 215static const struct timing_reg timing_data_reg_fields[] = { 216 TIMING_FIELD("tWTR", 28, 31), 217 TIMING_FIELD("tWR", 24, 27), 218 TIMING_FIELD("tRTP", 20, 23), 219 TIMING_FIELD("tW2W-C2C", 14, 14), 220 TIMING_FIELD("tR2R-C2C", 12, 12), 221 TIMING_FIELD("WL", 8, 11), 222 TIMING_FIELD("tDQSCK", 4, 7), 223 TIMING_FIELD("RL", 0, 3), 224}; 225 226static const struct timing_reg timing_power_reg_fields[] = { 227 TIMING_FIELD("tFAW", 26, 31), 228 TIMING_FIELD("tXSR", 16, 25), 229 TIMING_FIELD("tXP", 8, 15), 230 TIMING_FIELD("tCKE", 4, 7), 231 TIMING_FIELD("tMRD", 0, 3), 232}; 233 234#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \ 235 ARRAY_SIZE(timing_data_reg_fields) + \ 236 ARRAY_SIZE(timing_power_reg_fields)) 237 238static int exynos5_counters_set_event(struct exynos5_dmc *dmc) 239{ 240 int i, ret; 241 242 for (i = 0; i < dmc->num_counters; i++) { 243 if (!dmc->counter[i]) 244 continue; 245 ret = devfreq_event_set_event(dmc->counter[i]); 246 if (ret < 0) 247 return ret; 248 } 249 return 0; 250} 251 252static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc) 253{ 254 int i, ret; 255 256 for (i = 0; i < dmc->num_counters; i++) { 257 if (!dmc->counter[i]) 258 continue; 259 ret = devfreq_event_enable_edev(dmc->counter[i]); 260 if (ret < 0) 261 return ret; 262 } 263 return 0; 264} 265 266static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc) 267{ 268 int i, ret; 269 270 for (i = 0; i < dmc->num_counters; i++) { 271 if (!dmc->counter[i]) 272 continue; 273 ret = devfreq_event_disable_edev(dmc->counter[i]); 274 if (ret < 0) 275 return ret; 276 } 277 return 0; 278} 279 280/** 281 * find_target_freq_idx() - Finds requested frequency in local DMC configuration 282 * @dmc: device for which the information is checked 283 * @target_rate: requested frequency in KHz 284 * 285 * Seeks in the local DMC driver structure for the requested frequency value 286 * and returns index or error value. 287 */ 288static int find_target_freq_idx(struct exynos5_dmc *dmc, 289 unsigned long target_rate) 290{ 291 int i; 292 293 for (i = dmc->opp_count - 1; i >= 0; i--) 294 if (dmc->opp[i].freq_hz <= target_rate) 295 return i; 296 297 return -EINVAL; 298} 299 300/** 301 * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings 302 * @dmc: device for which the new settings is going to be applied 303 * @set: boolean variable passing set value 304 * 305 * Changes the register set, which holds timing parameters. 306 * There is two register sets: 0 and 1. The register set 0 307 * is used in normal operation when the clock is provided from main PLL. 308 * The bank register set 1 is used when the main PLL frequency is going to be 309 * changed and the clock is taken from alternative, stable source. 310 * This function switches between these banks according to the 311 * currently used clock source. 312 */ 313static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set) 314{ 315 unsigned int reg; 316 int ret; 317 318 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®); 319 if (ret) 320 return ret; 321 322 if (set) 323 reg |= EXYNOS5_TIMING_SET_SWI; 324 else 325 reg &= ~EXYNOS5_TIMING_SET_SWI; 326 327 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg); 328 329 return 0; 330} 331 332/** 333 * exynos5_init_freq_table() - Initialized PM OPP framework 334 * @dmc: DMC device for which the frequencies are used for OPP init 335 * @profile: devfreq device's profile 336 * 337 * Populate the devfreq device's OPP table based on current frequency, voltage. 338 */ 339static int exynos5_init_freq_table(struct exynos5_dmc *dmc, 340 struct devfreq_dev_profile *profile) 341{ 342 int i, ret; 343 int idx; 344 unsigned long freq; 345 346 ret = devm_pm_opp_of_add_table(dmc->dev); 347 if (ret < 0) { 348 dev_err(dmc->dev, "Failed to get OPP table\n"); 349 return ret; 350 } 351 352 dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev); 353 354 dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count, 355 sizeof(struct dmc_opp_table), GFP_KERNEL); 356 if (!dmc->opp) 357 return -ENOMEM; 358 359 idx = dmc->opp_count - 1; 360 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) { 361 struct dev_pm_opp *opp; 362 363 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq); 364 if (IS_ERR(opp)) 365 return PTR_ERR(opp); 366 367 dmc->opp[idx - i].freq_hz = freq; 368 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp); 369 370 dev_pm_opp_put(opp); 371 } 372 373 return 0; 374} 375 376/** 377 * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings 378 * @dmc: device for which the new settings is going to be applied 379 * 380 * Low-level function for changing timings for DRAM memory clocking from 381 * 'bypass' clock source (fixed frequency @400MHz). 382 * It uses timing bank registers set 1. 383 */ 384static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc) 385{ 386 writel(EXYNOS5_AREF_NORMAL, 387 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF); 388 389 writel(dmc->bypass_timing_row, 390 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1); 391 writel(dmc->bypass_timing_row, 392 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1); 393 writel(dmc->bypass_timing_data, 394 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1); 395 writel(dmc->bypass_timing_data, 396 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1); 397 writel(dmc->bypass_timing_power, 398 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1); 399 writel(dmc->bypass_timing_power, 400 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1); 401} 402 403/** 404 * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings 405 * @dmc: device for which the new settings is going to be applied 406 * @target_rate: target frequency of the DMC 407 * 408 * Low-level function for changing timings for DRAM memory operating from main 409 * clock source (BPLL), which can have different frequencies. Thus, each 410 * frequency must have corresponding timings register values in order to keep 411 * the needed delays. 412 * It uses timing bank registers set 0. 413 */ 414static int exynos5_dram_change_timings(struct exynos5_dmc *dmc, 415 unsigned long target_rate) 416{ 417 int idx; 418 419 for (idx = dmc->opp_count - 1; idx >= 0; idx--) 420 if (dmc->opp[idx].freq_hz <= target_rate) 421 break; 422 423 if (idx < 0) 424 return -EINVAL; 425 426 writel(EXYNOS5_AREF_NORMAL, 427 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF); 428 429 writel(dmc->timing_row[idx], 430 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0); 431 writel(dmc->timing_row[idx], 432 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0); 433 writel(dmc->timing_data[idx], 434 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0); 435 writel(dmc->timing_data[idx], 436 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0); 437 writel(dmc->timing_power[idx], 438 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0); 439 writel(dmc->timing_power[idx], 440 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0); 441 442 return 0; 443} 444 445/** 446 * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC 447 * @dmc: device for which it is going to be set 448 * @target_volt: new voltage which is chosen to be final 449 * 450 * Function tries to align voltage to the safe level for 'normal' mode. 451 * It checks the need of higher voltage and changes the value. The target 452 * voltage might be lower that currently set and still the system will be 453 * stable. 454 */ 455static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc, 456 unsigned long target_volt) 457{ 458 int ret = 0; 459 460 if (dmc->curr_volt <= target_volt) 461 return 0; 462 463 ret = regulator_set_voltage(dmc->vdd_mif, target_volt, 464 target_volt); 465 if (!ret) 466 dmc->curr_volt = target_volt; 467 468 return ret; 469} 470 471/** 472 * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC 473 * @dmc: device for which it is going to be set 474 * @target_volt: new voltage which is chosen to be final 475 * 476 * Function tries to align voltage to the safe level for the 'bypass' mode. 477 * It checks the need of higher voltage and changes the value. 478 * The target voltage must not be less than currently needed, because 479 * for current frequency the device might become unstable. 480 */ 481static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc, 482 unsigned long target_volt) 483{ 484 int ret = 0; 485 486 if (dmc->curr_volt >= target_volt) 487 return 0; 488 489 ret = regulator_set_voltage(dmc->vdd_mif, target_volt, 490 target_volt); 491 if (!ret) 492 dmc->curr_volt = target_volt; 493 494 return ret; 495} 496 497/** 498 * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings 499 * @dmc: device for which it is going to be set 500 * @target_rate: new frequency which is chosen to be final 501 * 502 * Function changes the DRAM timings for the temporary 'bypass' mode. 503 */ 504static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc, 505 unsigned long target_rate) 506{ 507 int idx = find_target_freq_idx(dmc, target_rate); 508 509 if (idx < 0) 510 return -EINVAL; 511 512 exynos5_set_bypass_dram_timings(dmc); 513 514 return 0; 515} 516 517/** 518 * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock 519 * @dmc: DMC device for which the switching is going to happen 520 * @target_rate: new frequency which is going to be set as a final 521 * @target_volt: new voltage which is going to be set as a final 522 * 523 * Function configures DMC and clocks for operating in temporary 'bypass' mode. 524 * This mode is used only temporary but if required, changes voltage and timings 525 * for DRAM chips. It switches the main clock to stable clock source for the 526 * period of the main PLL reconfiguration. 527 */ 528static int 529exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc, 530 unsigned long target_rate, 531 unsigned long target_volt) 532{ 533 int ret; 534 535 /* 536 * Having higher voltage for a particular frequency does not harm 537 * the chip. Use it for the temporary frequency change when one 538 * voltage manipulation might be avoided. 539 */ 540 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt); 541 if (ret) 542 return ret; 543 544 /* 545 * Longer delays for DRAM does not cause crash, the opposite does. 546 */ 547 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate); 548 if (ret) 549 return ret; 550 551 /* 552 * Delays are long enough, so use them for the new coming clock. 553 */ 554 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS); 555 556 return ret; 557} 558 559/** 560 * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC 561 * using safe procedure 562 * @dmc: device for which the frequency is going to be changed 563 * @target_rate: requested new frequency 564 * @target_volt: requested voltage which corresponds to the new frequency 565 * 566 * The DMC frequency change procedure requires a few steps. 567 * The main requirement is to change the clock source in the clk mux 568 * for the time of main clock PLL locking. The assumption is that the 569 * alternative clock source set as parent is stable. 570 * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass' 571 * clock. This requires alignment in DRAM timing parameters for the new 572 * T-period. There is two bank sets for keeping DRAM 573 * timings: set 0 and set 1. The set 0 is used when main clock source is 574 * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between 575 * the two bank sets is part of the process. 576 * The voltage must also be aligned to the minimum required level. There is 577 * this intermediate step with switching to 'bypass' parent clock source. 578 * if the old voltage is lower, it requires an increase of the voltage level. 579 * The complexity of the voltage manipulation is hidden in low level function. 580 * In this function there is last alignment of the voltage level at the end. 581 */ 582static int 583exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc, 584 unsigned long target_rate, 585 unsigned long target_volt) 586{ 587 int ret; 588 589 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate, 590 target_volt); 591 if (ret) 592 return ret; 593 594 /* 595 * Voltage is set at least to a level needed for this frequency, 596 * so switching clock source is safe now. 597 */ 598 clk_prepare_enable(dmc->fout_spll); 599 clk_prepare_enable(dmc->mout_spll); 600 clk_prepare_enable(dmc->mout_mx_mspll_ccore); 601 602 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore); 603 if (ret) 604 goto disable_clocks; 605 606 /* 607 * We are safe to increase the timings for current bypass frequency. 608 * Thanks to this the settings will be ready for the upcoming clock 609 * source change. 610 */ 611 exynos5_dram_change_timings(dmc, target_rate); 612 613 clk_set_rate(dmc->fout_bpll, target_rate); 614 615 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS); 616 if (ret) 617 goto disable_clocks; 618 619 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll); 620 if (ret) 621 goto disable_clocks; 622 623 /* 624 * Make sure if the voltage is not from 'bypass' settings and align to 625 * the right level for power efficiency. 626 */ 627 ret = exynos5_dmc_align_target_voltage(dmc, target_volt); 628 629disable_clocks: 630 clk_disable_unprepare(dmc->mout_mx_mspll_ccore); 631 clk_disable_unprepare(dmc->mout_spll); 632 clk_disable_unprepare(dmc->fout_spll); 633 634 return ret; 635} 636 637/** 638 * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP 639 * table. 640 * @dmc: device for which the frequency is going to be changed 641 * @freq: requested frequency in KHz 642 * @target_rate: returned frequency which is the same or lower than 643 * requested 644 * @target_volt: returned voltage which corresponds to the returned 645 * frequency 646 * @flags: devfreq flags provided for this frequency change request 647 * 648 * Function gets requested frequency and checks OPP framework for needed 649 * frequency and voltage. It populates the values 'target_rate' and 650 * 'target_volt' or returns error value when OPP framework fails. 651 */ 652static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc, 653 unsigned long *freq, 654 unsigned long *target_rate, 655 unsigned long *target_volt, u32 flags) 656{ 657 struct dev_pm_opp *opp; 658 659 opp = devfreq_recommended_opp(dmc->dev, freq, flags); 660 if (IS_ERR(opp)) 661 return PTR_ERR(opp); 662 663 *target_rate = dev_pm_opp_get_freq(opp); 664 *target_volt = dev_pm_opp_get_voltage(opp); 665 dev_pm_opp_put(opp); 666 667 return 0; 668} 669 670/** 671 * exynos5_dmc_target() - Function responsible for changing frequency of DMC 672 * @dev: device for which the frequency is going to be changed 673 * @freq: requested frequency in KHz 674 * @flags: flags provided for this frequency change request 675 * 676 * An entry function provided to the devfreq framework which provides frequency 677 * change of the DMC. The function gets the possible rate from OPP table based 678 * on requested frequency. It calls the next function responsible for the 679 * frequency and voltage change. In case of failure, does not set 'curr_rate' 680 * and returns error value to the framework. 681 */ 682static int exynos5_dmc_target(struct device *dev, unsigned long *freq, 683 u32 flags) 684{ 685 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 686 unsigned long target_rate = 0; 687 unsigned long target_volt = 0; 688 int ret; 689 690 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt, 691 flags); 692 693 if (ret) 694 return ret; 695 696 if (target_rate == dmc->curr_rate) 697 return 0; 698 699 mutex_lock(&dmc->lock); 700 701 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt); 702 703 if (ret) { 704 mutex_unlock(&dmc->lock); 705 return ret; 706 } 707 708 dmc->curr_rate = target_rate; 709 710 mutex_unlock(&dmc->lock); 711 return 0; 712} 713 714/** 715 * exynos5_counters_get() - Gets the performance counters values. 716 * @dmc: device for which the counters are going to be checked 717 * @load_count: variable which is populated with counter value 718 * @total_count: variable which is used as 'wall clock' reference 719 * 720 * Function which provides performance counters values. It sums up counters for 721 * two DMC channels. The 'total_count' is used as a reference and max value. 722 * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%]. 723 */ 724static int exynos5_counters_get(struct exynos5_dmc *dmc, 725 unsigned long *load_count, 726 unsigned long *total_count) 727{ 728 unsigned long total = 0; 729 struct devfreq_event_data event; 730 int ret, i; 731 732 *load_count = 0; 733 734 /* Take into account only read+write counters, but stop all */ 735 for (i = 0; i < dmc->num_counters; i++) { 736 if (!dmc->counter[i]) 737 continue; 738 739 ret = devfreq_event_get_event(dmc->counter[i], &event); 740 if (ret < 0) 741 return ret; 742 743 *load_count += event.load_count; 744 745 if (total < event.total_count) 746 total = event.total_count; 747 } 748 749 *total_count = total; 750 751 return 0; 752} 753 754/** 755 * exynos5_dmc_start_perf_events() - Setup and start performance event counters 756 * @dmc: device for which the counters are going to be checked 757 * @beg_value: initial value for the counter 758 * 759 * Function which enables needed counters, interrupts and sets initial values 760 * then starts the counters. 761 */ 762static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc, 763 u32 beg_value) 764{ 765 /* Enable interrupts for counter 2 */ 766 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC); 767 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC); 768 769 /* Enable counter 2 and CCNT */ 770 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC); 771 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC); 772 773 /* Clear overflow flag for all counters */ 774 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC); 775 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC); 776 777 /* Reset all counters */ 778 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC); 779 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC); 780 781 /* 782 * Set start value for the counters, the number of samples that 783 * will be gathered is calculated as: 0xffffffff - beg_value 784 */ 785 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC); 786 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC); 787 788 /* Start all counters */ 789 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC); 790 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC); 791} 792 793/** 794 * exynos5_dmc_perf_events_calc() - Calculate utilization 795 * @dmc: device for which the counters are going to be checked 796 * @diff_ts: time between last interrupt and current one 797 * 798 * Function which calculates needed utilization for the devfreq governor. 799 * It prepares values for 'busy_time' and 'total_time' based on elapsed time 800 * between interrupts, which approximates utilization. 801 */ 802static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts) 803{ 804 /* 805 * This is a simple algorithm for managing traffic on DMC. 806 * When there is almost no load the counters overflow every 4s, 807 * no mater the DMC frequency. 808 * The high load might be approximated using linear function. 809 * Knowing that, simple calculation can provide 'busy_time' and 810 * 'total_time' to the devfreq governor which picks up target 811 * frequency. 812 * We want a fast ramp up and slow decay in frequency change function. 813 */ 814 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) { 815 /* 816 * Set higher utilization for the simple_ondemand governor. 817 * The governor should increase the frequency of the DMC. 818 */ 819 dmc->load = 70; 820 dmc->total = 100; 821 } else { 822 /* 823 * Set low utilization for the simple_ondemand governor. 824 * The governor should decrease the frequency of the DMC. 825 */ 826 dmc->load = 35; 827 dmc->total = 100; 828 } 829 830 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts); 831} 832 833/** 834 * exynos5_dmc_perf_events_check() - Checks the status of the counters 835 * @dmc: device for which the counters are going to be checked 836 * 837 * Function which is called from threaded IRQ to check the counters state 838 * and to call approximation for the needed utilization. 839 */ 840static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc) 841{ 842 u32 val; 843 u64 diff_ts, ts; 844 845 ts = ktime_get_ns(); 846 847 /* Stop all counters */ 848 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC); 849 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC); 850 851 /* Check the source in interrupt flag registers (which channel) */ 852 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC); 853 if (val) { 854 diff_ts = ts - dmc->last_overflow_ts[0]; 855 dmc->last_overflow_ts[0] = ts; 856 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val); 857 } else { 858 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC); 859 diff_ts = ts - dmc->last_overflow_ts[1]; 860 dmc->last_overflow_ts[1] = ts; 861 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val); 862 } 863 864 exynos5_dmc_perf_events_calc(dmc, diff_ts); 865 866 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE); 867} 868 869/** 870 * exynos5_dmc_enable_perf_events() - Enable performance events 871 * @dmc: device for which the counters are going to be checked 872 * 873 * Function which is setup needed environment and enables counters. 874 */ 875static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc) 876{ 877 u64 ts; 878 879 /* Enable Performance Event Clock */ 880 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON); 881 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON); 882 883 /* Select read transfers as performance event2 */ 884 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG); 885 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG); 886 887 ts = ktime_get_ns(); 888 dmc->last_overflow_ts[0] = ts; 889 dmc->last_overflow_ts[1] = ts; 890 891 /* Devfreq shouldn't be faster than initialization, play safe though. */ 892 dmc->load = 99; 893 dmc->total = 100; 894} 895 896/** 897 * exynos5_dmc_disable_perf_events() - Disable performance events 898 * @dmc: device for which the counters are going to be checked 899 * 900 * Function which stops, disables performance event counters and interrupts. 901 */ 902static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc) 903{ 904 /* Stop all counters */ 905 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC); 906 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC); 907 908 /* Disable interrupts for counter 2 */ 909 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC); 910 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC); 911 912 /* Disable counter 2 and CCNT */ 913 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC); 914 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC); 915 916 /* Clear overflow flag for all counters */ 917 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC); 918 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC); 919} 920 921/** 922 * exynos5_dmc_get_status() - Read current DMC performance statistics. 923 * @dev: device for which the statistics are requested 924 * @stat: structure which has statistic fields 925 * 926 * Function reads the DMC performance counters and calculates 'busy_time' 927 * and 'total_time'. To protect from overflow, the values are shifted right 928 * by 10. After read out the counters are setup to count again. 929 */ 930static int exynos5_dmc_get_status(struct device *dev, 931 struct devfreq_dev_status *stat) 932{ 933 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 934 unsigned long load, total; 935 int ret; 936 937 if (dmc->in_irq_mode) { 938 mutex_lock(&dmc->lock); 939 stat->current_frequency = dmc->curr_rate; 940 mutex_unlock(&dmc->lock); 941 942 stat->busy_time = dmc->load; 943 stat->total_time = dmc->total; 944 } else { 945 ret = exynos5_counters_get(dmc, &load, &total); 946 if (ret < 0) 947 return -EINVAL; 948 949 /* To protect from overflow, divide by 1024 */ 950 stat->busy_time = load >> 10; 951 stat->total_time = total >> 10; 952 953 ret = exynos5_counters_set_event(dmc); 954 if (ret < 0) { 955 dev_err(dev, "could not set event counter\n"); 956 return ret; 957 } 958 } 959 960 return 0; 961} 962 963/** 964 * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency 965 * @dev: device for which the framework checks operating frequency 966 * @freq: returned frequency value 967 * 968 * It returns the currently used frequency of the DMC. The real operating 969 * frequency might be lower when the clock source value could not be divided 970 * to the requested value. 971 */ 972static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq) 973{ 974 struct exynos5_dmc *dmc = dev_get_drvdata(dev); 975 976 mutex_lock(&dmc->lock); 977 *freq = dmc->curr_rate; 978 mutex_unlock(&dmc->lock); 979 980 return 0; 981} 982 983/* 984 * exynos5_dmc_df_profile - Devfreq governor's profile structure 985 * 986 * It provides to the devfreq framework needed functions and polling period. 987 */ 988static struct devfreq_dev_profile exynos5_dmc_df_profile = { 989 .timer = DEVFREQ_TIMER_DELAYED, 990 .target = exynos5_dmc_target, 991 .get_dev_status = exynos5_dmc_get_status, 992 .get_cur_freq = exynos5_dmc_get_cur_freq, 993}; 994 995/** 996 * exynos5_dmc_align_init_freq() - Align initial frequency value 997 * @dmc: device for which the frequency is going to be set 998 * @bootloader_init_freq: initial frequency set by the bootloader in KHz 999 * 1000 * The initial bootloader frequency, which is present during boot, might be 1001 * different that supported frequency values in the driver. It is possible 1002 * due to different PLL settings or used PLL as a source. 1003 * This function provides the 'initial_freq' for the devfreq framework 1004 * statistics engine which supports only registered values. Thus, some alignment 1005 * must be made. 1006 */ 1007static unsigned long 1008exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc, 1009 unsigned long bootloader_init_freq) 1010{ 1011 unsigned long aligned_freq; 1012 int idx; 1013 1014 idx = find_target_freq_idx(dmc, bootloader_init_freq); 1015 if (idx >= 0) 1016 aligned_freq = dmc->opp[idx].freq_hz; 1017 else 1018 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz; 1019 1020 return aligned_freq; 1021} 1022 1023/** 1024 * create_timings_aligned() - Create register values and align with standard 1025 * @dmc: device for which the frequency is going to be set 1026 * @reg_timing_row: array to fill with values for timing row register 1027 * @reg_timing_data: array to fill with values for timing data register 1028 * @reg_timing_power: array to fill with values for timing power register 1029 * @clk_period_ps: the period of the clock, known as tCK 1030 * 1031 * The function calculates timings and creates a register value ready for 1032 * a frequency transition. The register contains a few timings. They are 1033 * shifted by a known offset. The timing value is calculated based on memory 1034 * specyfication: minimal time required and minimal cycles required. 1035 */ 1036static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row, 1037 u32 *reg_timing_data, u32 *reg_timing_power, 1038 u32 clk_period_ps) 1039{ 1040 u32 val; 1041 const struct timing_reg *reg; 1042 1043 if (clk_period_ps == 0) 1044 return -EINVAL; 1045 1046 *reg_timing_row = 0; 1047 *reg_timing_data = 0; 1048 *reg_timing_power = 0; 1049 1050 val = dmc->timings->tRFC / clk_period_ps; 1051 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0; 1052 val = max(val, dmc->min_tck->tRFC); 1053 reg = &timing_row_reg_fields[0]; 1054 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1055 1056 val = dmc->timings->tRRD / clk_period_ps; 1057 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0; 1058 val = max(val, dmc->min_tck->tRRD); 1059 reg = &timing_row_reg_fields[1]; 1060 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1061 1062 val = dmc->timings->tRPab / clk_period_ps; 1063 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0; 1064 val = max(val, dmc->min_tck->tRPab); 1065 reg = &timing_row_reg_fields[2]; 1066 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1067 1068 val = dmc->timings->tRCD / clk_period_ps; 1069 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0; 1070 val = max(val, dmc->min_tck->tRCD); 1071 reg = &timing_row_reg_fields[3]; 1072 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1073 1074 val = dmc->timings->tRC / clk_period_ps; 1075 val += dmc->timings->tRC % clk_period_ps ? 1 : 0; 1076 val = max(val, dmc->min_tck->tRC); 1077 reg = &timing_row_reg_fields[4]; 1078 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1079 1080 val = dmc->timings->tRAS / clk_period_ps; 1081 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0; 1082 val = max(val, dmc->min_tck->tRAS); 1083 reg = &timing_row_reg_fields[5]; 1084 *reg_timing_row |= TIMING_VAL2REG(reg, val); 1085 1086 /* data related timings */ 1087 val = dmc->timings->tWTR / clk_period_ps; 1088 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0; 1089 val = max(val, dmc->min_tck->tWTR); 1090 reg = &timing_data_reg_fields[0]; 1091 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1092 1093 val = dmc->timings->tWR / clk_period_ps; 1094 val += dmc->timings->tWR % clk_period_ps ? 1 : 0; 1095 val = max(val, dmc->min_tck->tWR); 1096 reg = &timing_data_reg_fields[1]; 1097 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1098 1099 val = dmc->timings->tRTP / clk_period_ps; 1100 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0; 1101 val = max(val, dmc->min_tck->tRTP); 1102 reg = &timing_data_reg_fields[2]; 1103 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1104 1105 val = dmc->timings->tW2W_C2C / clk_period_ps; 1106 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0; 1107 val = max(val, dmc->min_tck->tW2W_C2C); 1108 reg = &timing_data_reg_fields[3]; 1109 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1110 1111 val = dmc->timings->tR2R_C2C / clk_period_ps; 1112 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0; 1113 val = max(val, dmc->min_tck->tR2R_C2C); 1114 reg = &timing_data_reg_fields[4]; 1115 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1116 1117 val = dmc->timings->tWL / clk_period_ps; 1118 val += dmc->timings->tWL % clk_period_ps ? 1 : 0; 1119 val = max(val, dmc->min_tck->tWL); 1120 reg = &timing_data_reg_fields[5]; 1121 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1122 1123 val = dmc->timings->tDQSCK / clk_period_ps; 1124 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0; 1125 val = max(val, dmc->min_tck->tDQSCK); 1126 reg = &timing_data_reg_fields[6]; 1127 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1128 1129 val = dmc->timings->tRL / clk_period_ps; 1130 val += dmc->timings->tRL % clk_period_ps ? 1 : 0; 1131 val = max(val, dmc->min_tck->tRL); 1132 reg = &timing_data_reg_fields[7]; 1133 *reg_timing_data |= TIMING_VAL2REG(reg, val); 1134 1135 /* power related timings */ 1136 val = dmc->timings->tFAW / clk_period_ps; 1137 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0; 1138 val = max(val, dmc->min_tck->tFAW); 1139 reg = &timing_power_reg_fields[0]; 1140 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1141 1142 val = dmc->timings->tXSR / clk_period_ps; 1143 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0; 1144 val = max(val, dmc->min_tck->tXSR); 1145 reg = &timing_power_reg_fields[1]; 1146 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1147 1148 val = dmc->timings->tXP / clk_period_ps; 1149 val += dmc->timings->tXP % clk_period_ps ? 1 : 0; 1150 val = max(val, dmc->min_tck->tXP); 1151 reg = &timing_power_reg_fields[2]; 1152 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1153 1154 val = dmc->timings->tCKE / clk_period_ps; 1155 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0; 1156 val = max(val, dmc->min_tck->tCKE); 1157 reg = &timing_power_reg_fields[3]; 1158 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1159 1160 val = dmc->timings->tMRD / clk_period_ps; 1161 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0; 1162 val = max(val, dmc->min_tck->tMRD); 1163 reg = &timing_power_reg_fields[4]; 1164 *reg_timing_power |= TIMING_VAL2REG(reg, val); 1165 1166 return 0; 1167} 1168 1169/** 1170 * of_get_dram_timings() - helper function for parsing DT settings for DRAM 1171 * @dmc: device for which the frequency is going to be set 1172 * 1173 * The function parses DT entries with DRAM information. 1174 */ 1175static int of_get_dram_timings(struct exynos5_dmc *dmc) 1176{ 1177 int ret = 0; 1178 int idx; 1179 struct device_node *np_ddr; 1180 u32 freq_mhz, clk_period_ps; 1181 1182 np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0); 1183 if (!np_ddr) { 1184 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n"); 1185 return -EINVAL; 1186 } 1187 1188 dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1189 sizeof(u32), GFP_KERNEL); 1190 if (!dmc->timing_row) { 1191 ret = -ENOMEM; 1192 goto put_node; 1193 } 1194 1195 dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1196 sizeof(u32), GFP_KERNEL); 1197 if (!dmc->timing_data) { 1198 ret = -ENOMEM; 1199 goto put_node; 1200 } 1201 1202 dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT, 1203 sizeof(u32), GFP_KERNEL); 1204 if (!dmc->timing_power) { 1205 ret = -ENOMEM; 1206 goto put_node; 1207 } 1208 1209 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev, 1210 DDR_TYPE_LPDDR3, 1211 &dmc->timings_arr_size); 1212 if (!dmc->timings) { 1213 dev_warn(dmc->dev, "could not get timings from DT\n"); 1214 ret = -EINVAL; 1215 goto put_node; 1216 } 1217 1218 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev); 1219 if (!dmc->min_tck) { 1220 dev_warn(dmc->dev, "could not get tck from DT\n"); 1221 ret = -EINVAL; 1222 goto put_node; 1223 } 1224 1225 /* Sorted array of OPPs with frequency ascending */ 1226 for (idx = 0; idx < dmc->opp_count; idx++) { 1227 freq_mhz = dmc->opp[idx].freq_hz / 1000000; 1228 clk_period_ps = 1000000 / freq_mhz; 1229 1230 ret = create_timings_aligned(dmc, &dmc->timing_row[idx], 1231 &dmc->timing_data[idx], 1232 &dmc->timing_power[idx], 1233 clk_period_ps); 1234 } 1235 1236 1237 /* Take the highest frequency's timings as 'bypass' */ 1238 dmc->bypass_timing_row = dmc->timing_row[idx - 1]; 1239 dmc->bypass_timing_data = dmc->timing_data[idx - 1]; 1240 dmc->bypass_timing_power = dmc->timing_power[idx - 1]; 1241 1242put_node: 1243 of_node_put(np_ddr); 1244 return ret; 1245} 1246 1247/** 1248 * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation. 1249 * @dmc: DMC structure containing needed fields 1250 * 1251 * Get the needed clocks defined in DT device, enable and set the right parents. 1252 * Read current frequency and initialize the initial rate for governor. 1253 */ 1254static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc) 1255{ 1256 int ret; 1257 unsigned long target_volt = 0; 1258 unsigned long target_rate = 0; 1259 unsigned int tmp; 1260 1261 dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll"); 1262 if (IS_ERR(dmc->fout_spll)) 1263 return PTR_ERR(dmc->fout_spll); 1264 1265 dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll"); 1266 if (IS_ERR(dmc->fout_bpll)) 1267 return PTR_ERR(dmc->fout_bpll); 1268 1269 dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex"); 1270 if (IS_ERR(dmc->mout_mclk_cdrex)) 1271 return PTR_ERR(dmc->mout_mclk_cdrex); 1272 1273 dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll"); 1274 if (IS_ERR(dmc->mout_bpll)) 1275 return PTR_ERR(dmc->mout_bpll); 1276 1277 dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev, 1278 "mout_mx_mspll_ccore"); 1279 if (IS_ERR(dmc->mout_mx_mspll_ccore)) 1280 return PTR_ERR(dmc->mout_mx_mspll_ccore); 1281 1282 dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2"); 1283 if (IS_ERR(dmc->mout_spll)) { 1284 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll"); 1285 if (IS_ERR(dmc->mout_spll)) 1286 return PTR_ERR(dmc->mout_spll); 1287 } 1288 1289 /* 1290 * Convert frequency to KHz values and set it for the governor. 1291 */ 1292 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex); 1293 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate); 1294 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate; 1295 1296 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate, 1297 &target_volt, 0); 1298 if (ret) 1299 return ret; 1300 1301 dmc->curr_volt = target_volt; 1302 1303 ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll); 1304 if (ret) 1305 return ret; 1306 1307 clk_prepare_enable(dmc->fout_bpll); 1308 clk_prepare_enable(dmc->mout_bpll); 1309 1310 /* 1311 * Some bootloaders do not set clock routes correctly. 1312 * Stop one path in clocks to PHY. 1313 */ 1314 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp); 1315 tmp &= ~(BIT(1) | BIT(0)); 1316 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp); 1317 1318 return 0; 1319} 1320 1321/** 1322 * exynos5_performance_counters_init() - Initializes performance DMC's counters 1323 * @dmc: DMC for which it does the setup 1324 * 1325 * Initialization of performance counters in DMC for estimating usage. 1326 * The counter's values are used for calculation of a memory bandwidth and based 1327 * on that the governor changes the frequency. 1328 * The counters are not used when the governor is GOVERNOR_USERSPACE. 1329 */ 1330static int exynos5_performance_counters_init(struct exynos5_dmc *dmc) 1331{ 1332 int ret, i; 1333 1334 dmc->num_counters = devfreq_event_get_edev_count(dmc->dev, 1335 "devfreq-events"); 1336 if (dmc->num_counters < 0) { 1337 dev_err(dmc->dev, "could not get devfreq-event counters\n"); 1338 return dmc->num_counters; 1339 } 1340 1341 dmc->counter = devm_kcalloc(dmc->dev, dmc->num_counters, 1342 sizeof(*dmc->counter), GFP_KERNEL); 1343 if (!dmc->counter) 1344 return -ENOMEM; 1345 1346 for (i = 0; i < dmc->num_counters; i++) { 1347 dmc->counter[i] = 1348 devfreq_event_get_edev_by_phandle(dmc->dev, 1349 "devfreq-events", i); 1350 if (IS_ERR_OR_NULL(dmc->counter[i])) 1351 return -EPROBE_DEFER; 1352 } 1353 1354 ret = exynos5_counters_enable_edev(dmc); 1355 if (ret < 0) { 1356 dev_err(dmc->dev, "could not enable event counter\n"); 1357 return ret; 1358 } 1359 1360 ret = exynos5_counters_set_event(dmc); 1361 if (ret < 0) { 1362 exynos5_counters_disable_edev(dmc); 1363 dev_err(dmc->dev, "could not set event counter\n"); 1364 return ret; 1365 } 1366 1367 return 0; 1368} 1369 1370/** 1371 * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC 1372 * @dmc: device which is used for changing this feature 1373 * 1374 * There is a need of pausing DREX DMC when divider or MUX in clock tree 1375 * changes its configuration. In such situation access to the memory is blocked 1376 * in DMC automatically. This feature is used when clock frequency change 1377 * request appears and touches clock tree. 1378 */ 1379static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc) 1380{ 1381 unsigned int val; 1382 int ret; 1383 1384 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val); 1385 if (ret) 1386 return ret; 1387 1388 val |= 1UL; 1389 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val); 1390 1391 return 0; 1392} 1393 1394static irqreturn_t dmc_irq_thread(int irq, void *priv) 1395{ 1396 int res; 1397 struct exynos5_dmc *dmc = priv; 1398 1399 mutex_lock(&dmc->df->lock); 1400 exynos5_dmc_perf_events_check(dmc); 1401 res = update_devfreq(dmc->df); 1402 mutex_unlock(&dmc->df->lock); 1403 1404 if (res) 1405 dev_warn(dmc->dev, "devfreq failed with %d\n", res); 1406 1407 return IRQ_HANDLED; 1408} 1409 1410/** 1411 * exynos5_dmc_probe() - Probe function for the DMC driver 1412 * @pdev: platform device for which the driver is going to be initialized 1413 * 1414 * Initialize basic components: clocks, regulators, performance counters, etc. 1415 * Read out product version and based on the information setup 1416 * internal structures for the controller (frequency and voltage) and for DRAM 1417 * memory parameters: timings for each operating frequency. 1418 * Register new devfreq device for controlling DVFS of the DMC. 1419 */ 1420static int exynos5_dmc_probe(struct platform_device *pdev) 1421{ 1422 int ret = 0; 1423 struct device *dev = &pdev->dev; 1424 struct device_node *np = dev->of_node; 1425 struct exynos5_dmc *dmc; 1426 int irq[2]; 1427 1428 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL); 1429 if (!dmc) 1430 return -ENOMEM; 1431 1432 mutex_init(&dmc->lock); 1433 1434 dmc->dev = dev; 1435 platform_set_drvdata(pdev, dmc); 1436 1437 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0); 1438 if (IS_ERR(dmc->base_drexi0)) 1439 return PTR_ERR(dmc->base_drexi0); 1440 1441 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1); 1442 if (IS_ERR(dmc->base_drexi1)) 1443 return PTR_ERR(dmc->base_drexi1); 1444 1445 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np, 1446 "samsung,syscon-clk"); 1447 if (IS_ERR(dmc->clk_regmap)) 1448 return PTR_ERR(dmc->clk_regmap); 1449 1450 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile); 1451 if (ret) { 1452 dev_warn(dev, "couldn't initialize frequency settings\n"); 1453 return ret; 1454 } 1455 1456 dmc->vdd_mif = devm_regulator_get(dev, "vdd"); 1457 if (IS_ERR(dmc->vdd_mif)) { 1458 ret = PTR_ERR(dmc->vdd_mif); 1459 return ret; 1460 } 1461 1462 ret = exynos5_dmc_init_clks(dmc); 1463 if (ret) 1464 return ret; 1465 1466 ret = of_get_dram_timings(dmc); 1467 if (ret) { 1468 dev_warn(dev, "couldn't initialize timings settings\n"); 1469 goto remove_clocks; 1470 } 1471 1472 ret = exynos5_dmc_set_pause_on_switching(dmc); 1473 if (ret) { 1474 dev_warn(dev, "couldn't get access to PAUSE register\n"); 1475 goto remove_clocks; 1476 } 1477 1478 /* There is two modes in which the driver works: polling or IRQ */ 1479 irq[0] = platform_get_irq_byname(pdev, "drex_0"); 1480 irq[1] = platform_get_irq_byname(pdev, "drex_1"); 1481 if (irq[0] > 0 && irq[1] > 0 && irqmode) { 1482 ret = devm_request_threaded_irq(dev, irq[0], NULL, 1483 dmc_irq_thread, IRQF_ONESHOT, 1484 dev_name(dev), dmc); 1485 if (ret) { 1486 dev_err(dev, "couldn't grab IRQ\n"); 1487 goto remove_clocks; 1488 } 1489 1490 ret = devm_request_threaded_irq(dev, irq[1], NULL, 1491 dmc_irq_thread, IRQF_ONESHOT, 1492 dev_name(dev), dmc); 1493 if (ret) { 1494 dev_err(dev, "couldn't grab IRQ\n"); 1495 goto remove_clocks; 1496 } 1497 1498 /* 1499 * Setup default thresholds for the devfreq governor. 1500 * The values are chosen based on experiments. 1501 */ 1502 dmc->gov_data.upthreshold = 55; 1503 dmc->gov_data.downdifferential = 5; 1504 1505 exynos5_dmc_enable_perf_events(dmc); 1506 1507 dmc->in_irq_mode = 1; 1508 } else { 1509 ret = exynos5_performance_counters_init(dmc); 1510 if (ret) { 1511 dev_warn(dev, "couldn't probe performance counters\n"); 1512 goto remove_clocks; 1513 } 1514 1515 /* 1516 * Setup default thresholds for the devfreq governor. 1517 * The values are chosen based on experiments. 1518 */ 1519 dmc->gov_data.upthreshold = 10; 1520 dmc->gov_data.downdifferential = 5; 1521 1522 exynos5_dmc_df_profile.polling_ms = 100; 1523 } 1524 1525 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile, 1526 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1527 &dmc->gov_data); 1528 1529 if (IS_ERR(dmc->df)) { 1530 ret = PTR_ERR(dmc->df); 1531 goto err_devfreq_add; 1532 } 1533 1534 if (dmc->in_irq_mode) 1535 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE); 1536 1537 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode); 1538 1539 return 0; 1540 1541err_devfreq_add: 1542 if (dmc->in_irq_mode) 1543 exynos5_dmc_disable_perf_events(dmc); 1544 else 1545 exynos5_counters_disable_edev(dmc); 1546remove_clocks: 1547 clk_disable_unprepare(dmc->mout_bpll); 1548 clk_disable_unprepare(dmc->fout_bpll); 1549 1550 return ret; 1551} 1552 1553/** 1554 * exynos5_dmc_remove() - Remove function for the platform device 1555 * @pdev: platform device which is going to be removed 1556 * 1557 * The function relies on 'devm' framework function which automatically 1558 * clean the device's resources. It just calls explicitly disable function for 1559 * the performance counters. 1560 */ 1561static int exynos5_dmc_remove(struct platform_device *pdev) 1562{ 1563 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev); 1564 1565 if (dmc->in_irq_mode) 1566 exynos5_dmc_disable_perf_events(dmc); 1567 else 1568 exynos5_counters_disable_edev(dmc); 1569 1570 clk_disable_unprepare(dmc->mout_bpll); 1571 clk_disable_unprepare(dmc->fout_bpll); 1572 1573 return 0; 1574} 1575 1576static const struct of_device_id exynos5_dmc_of_match[] = { 1577 { .compatible = "samsung,exynos5422-dmc", }, 1578 { }, 1579}; 1580MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match); 1581 1582static struct platform_driver exynos5_dmc_platdrv = { 1583 .probe = exynos5_dmc_probe, 1584 .remove = exynos5_dmc_remove, 1585 .driver = { 1586 .name = "exynos5-dmc", 1587 .of_match_table = exynos5_dmc_of_match, 1588 }, 1589}; 1590module_platform_driver(exynos5_dmc_platdrv); 1591MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change"); 1592MODULE_LICENSE("GPL v2"); 1593MODULE_AUTHOR("Lukasz Luba"); 1594