xref: /device/soc/rockchip/common/vendor/drivers/devfreq/rockchip_dmc.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Rockchip Generic dmc support.
 *
 * Copyright (c) 2022 Rockchip Electronics Co. Ltd.
 * Author: Finley Xiao <finley.xiao@rock-chips.com>
 */

#include <dt-bindings/clock/rockchip-ddr.h>
#include <dt-bindings/soc/rockchip-system-status.h>
#include <drm/drm_modeset_lock.h>
#include <linux/arm-smccc.h>
#include <linux/clk.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/devfreq.h>
#include <linux/devfreq_cooling.h>
#include <linux/devfreq-event.h>
#include <linux/input.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/mfd/syscon.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/platform_device.h>
#include <linux/pm_opp.h>
#include <linux/pm_qos.h>
#include <linux/regmap.h>
#include <linux/regulator/consumer.h>
#include <linux/rockchip/rockchip_sip.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/suspend.h>
#include <linux/thermal.h>

#include <soc/rockchip/pm_domains.h>
#include <soc/rockchip/rkfb_dmc.h>
#include <soc/rockchip/rockchip_dmc.h>
#include <soc/rockchip/rockchip_sip.h>
#include <soc/rockchip/rockchip_system_monitor.h>
#include <soc/rockchip/rockchip-system-status.h>
#include <soc/rockchip/rockchip_opp_select.h>
#include <soc/rockchip/scpi.h>
#include <uapi/drm/drm_mode.h>

#include "governor.h"
#include "rockchip_dmc_timing.h"
#include "../clk/rockchip/clk.h"
#include "../gpu/drm/rockchip/rockchip_drm_drv.h"

#define system_status_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, status_nb)
#define reboot_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, reboot_nb)
#define boost_to_dmcfreq(work) container_of(work, struct rockchip_dmcfreq, boost_work)
#define input_hd_to_dmcfreq(hd) container_of(hd, struct rockchip_dmcfreq, input_handler)

#define VIDEO_1080P_SIZE (1920 * 1080)
#define FIQ_INIT_HANDLER (0x1)
#define FIQ_CPU_TGT_BOOT (0x0) /* to booting cpu */
#define FIQ_NUM_FOR_DCF (143)  /* NA irq map to fiq for dcf */
#define DTS_PAR_OFFSET (4096)

#define FALLBACK_STATIC_TEMPERATURE 55000

#define ROCKCHIP_DE_SKEW_TWO 2
#define ROCKCHIP_DE_SKEW_FOUR 4
#define ROCKCHIP_DE_SKEW_ELEVEN 11
#define ROCKCHIP_DE_SKEW_TWENTY 20
#define ROCKCHIP_DE_SKEW_TWENTYONE 21

#define DMCFREQ_WAIT_CTRL_T_DCF_EN_TWO 2

struct dmc_freq_table {
    unsigned long freq;
    unsigned long volt;
};

struct share_params {
    u32 hz;
    u32 lcdc_type;
    u32 vop;
    u32 vop_dclk_mode;
    u32 sr_idle_en;
    u32 addr_mcu_el3;
    /*
     * 1: need to wait flag1
     * 0: never wait flag1
     */
    u32 wait_flag1;
    /*
     * 1: need to wait flag1
     * 0: never wait flag1
     */
    u32 wait_flag0;
    u32 complt_hwirq;
    u32 update_drv_odt_cfg;
    u32 update_deskew_cfg;

    u32 freq_count;
    u32 freq_info_mhz[6];
    /* if need, add parameter after */
};

static struct share_params *ddr_psci_param;

struct rockchip_dmcfreq_ondemand_data {
    unsigned int upthreshold;
    unsigned int downdifferential;
};

struct rockchip_dmcfreq {
    struct device *dev;
    struct dmcfreq_common_info info;
    struct rockchip_dmcfreq_ondemand_data ondemand_data;
    struct clk *dmc_clk;
    struct devfreq_event_dev **edev;
    struct mutex lock; /* serializes access to video_info_list */
    struct dram_timing *timing;
    struct regulator *vdd_center;
    struct notifier_block status_nb;
    struct list_head video_info_list;
    struct freq_map_table *cpu_bw_tbl;
    struct work_struct boost_work;
    struct input_handler input_handler;
    struct monitor_dev_info *mdev_info;
    struct share_params *set_rate_params;

    unsigned long *nocp_bw;
    unsigned long rate, target_rate;
    unsigned long volt, target_volt;
    unsigned long auto_min_rate;
    unsigned long status_rate;
    unsigned long normal_rate;
    unsigned long video_1080p_rate;
    unsigned long video_4k_rate;
    unsigned long video_4k_10b_rate;
    unsigned long performance_rate;
    unsigned long hdmi_rate;
    unsigned long idle_rate;
    unsigned long suspend_rate;
    unsigned long reboot_rate;
    unsigned long boost_rate;
    unsigned long fixed_rate;
    unsigned long low_power_rate;

    unsigned long freq_count;
    unsigned long freq_info_rate[6];
    unsigned long rate_low;
    unsigned long rate_mid_low;
    unsigned long rate_mid_high;
    unsigned long rate_high;

    unsigned int min_cpu_freq;
    unsigned int system_status_en;
    unsigned int refresh;
    int edev_count;
    int dfi_id;
    int nocp_cpu_id;

    bool is_fixed;
    bool is_set_rate_direct;

    struct thermal_cooling_device *devfreq_cooling;
    u32 static_coefficient;
    s32 ts[4];
    struct thermal_zone_device *ddr_tz;

    unsigned int touchboostpulse_duration_val;
    u64 touchboostpulse_endtime;

    int (*set_auto_self_refresh)(u32 en);
};

static struct pm_qos_request pm_qos;

static inline unsigned long is_dualview(unsigned long status)
{
    return (status & SYS_STATUS_LCDC0) && (status & SYS_STATUS_LCDC1);
}

static inline unsigned long is_isp(unsigned long status)
{
    return (status & SYS_STATUS_ISP) || (status & SYS_STATUS_CIF0) || (status & SYS_STATUS_CIF1);
}

/*
 * function: packaging de-skew setting to px30_ddr_dts_config_timing,
 *           px30_ddr_dts_config_timing will pass to trust firmware, and
 *           used direct to set register.
 * input: de_skew
 * output: tim
 */
static void px30_de_skew_set_2_reg(struct rk3328_ddr_de_skew_setting *de_skew, struct px30_ddr_dts_config_timing *tim)
{
    u32 n;
    u32 offset;
    u32 shift;

    memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew));
    memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew));
    memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew));

    /* CA de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) {
        offset = n / ROCKCHIP_DE_SKEW_TWO;
        shift = n % ROCKCHIP_DE_SKEW_TWO;
        /* 0 => 4; 1 => 0 */
        shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        tim->ca_skew[offset] &= ~(0xf << shift);
        tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift);
    }

    /* CS0 data de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) {
        offset = ((n / ROCKCHIP_DE_SKEW_TWENTYONE) * ROCKCHIP_DE_SKEW_ELEVEN) +
                 ((n % ROCKCHIP_DE_SKEW_TWENTYONE) / ROCKCHIP_DE_SKEW_TWO);
        shift = ((n % ROCKCHIP_DE_SKEW_TWENTYONE) % ROCKCHIP_DE_SKEW_TWO);
        if ((n % ROCKCHIP_DE_SKEW_TWENTYONE) == ROCKCHIP_DE_SKEW_TWENTY) {
            shift = 0;
        } else {
            /* 0 => 4; 1 => 0 */
            shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        }
        tim->cs0_skew[offset] &= ~(0xf << shift);
        tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift);
    }

    /* CS1 data de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) {
        offset = ((n / ROCKCHIP_DE_SKEW_TWENTYONE) * ROCKCHIP_DE_SKEW_ELEVEN) +
                 ((n % ROCKCHIP_DE_SKEW_TWENTYONE) / ROCKCHIP_DE_SKEW_TWO);
        shift = ((n % ROCKCHIP_DE_SKEW_TWENTYONE) % ROCKCHIP_DE_SKEW_TWO);
        if ((n % ROCKCHIP_DE_SKEW_TWENTYONE) == ROCKCHIP_DE_SKEW_TWENTY) {
            shift = 0;
        } else {
            /* 0 => 4; 1 => 0 */
            shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        }
        tim->cs1_skew[offset] &= ~(0xf << shift);
        tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift);
    }
}

/*
 * function: packaging de-skew setting to rk3328_ddr_dts_config_timing,
 *           rk3328_ddr_dts_config_timing will pass to trust firmware, and
 *           used direct to set register.
 * input: de_skew
 * output: tim
 */
static void rk3328_de_skew_setting_2_register(struct rk3328_ddr_de_skew_setting *de_skew,
                                              struct rk3328_ddr_dts_config_timing *tim)
{
    u32 n;
    u32 offset;
    u32 shift;

    memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew));
    memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew));
    memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew));

    /* CA de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) {
        offset = n / ROCKCHIP_DE_SKEW_TWO;
        shift = n % ROCKCHIP_DE_SKEW_TWO;
        /* 0 => 4; 1 => 0 */
        shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        tim->ca_skew[offset] &= ~(0xf << shift);
        tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift);
    }

    /* CS0 data de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) {
        offset = ((n / ROCKCHIP_DE_SKEW_TWENTYONE) * ROCKCHIP_DE_SKEW_ELEVEN) +
                 ((n % ROCKCHIP_DE_SKEW_TWENTYONE) / ROCKCHIP_DE_SKEW_TWO);
        shift = ((n % ROCKCHIP_DE_SKEW_TWENTYONE) % ROCKCHIP_DE_SKEW_TWO);
        if ((n % ROCKCHIP_DE_SKEW_TWENTYONE) == ROCKCHIP_DE_SKEW_TWENTY) {
            shift = 0;
        } else {
            /* 0 => 4; 1 => 0 */
            shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        }
        tim->cs0_skew[offset] &= ~(0xf << shift);
        tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift);
    }

    /* CS1 data de-skew */
    for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) {
        offset = ((n / ROCKCHIP_DE_SKEW_TWENTYONE) * ROCKCHIP_DE_SKEW_ELEVEN) +
                 ((n % ROCKCHIP_DE_SKEW_TWENTYONE) / ROCKCHIP_DE_SKEW_TWO);
        shift = ((n % ROCKCHIP_DE_SKEW_TWENTYONE) % ROCKCHIP_DE_SKEW_TWO);
        if ((n % ROCKCHIP_DE_SKEW_TWENTYONE) == ROCKCHIP_DE_SKEW_TWENTY) {
            shift = 0;
        } else {
            /* 0 => 4; 1 => 0 */
            shift = (shift == 0) ? ROCKCHIP_DE_SKEW_FOUR : 0;
        }
        tim->cs1_skew[offset] &= ~(0xf << shift);
        tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift);
    }
}

static int rk_drm_get_lcdc_type(void)
{
    u32 lcdc_type = rockchip_drm_get_sub_dev_type();

    switch (lcdc_type) {
        case DRM_MODE_CONNECTOR_DPI:
        case DRM_MODE_CONNECTOR_LVDS:
            lcdc_type = SCREEN_LVDS;
            break;
        case DRM_MODE_CONNECTOR_DisplayPort:
            lcdc_type = SCREEN_DP;
            break;
        case DRM_MODE_CONNECTOR_HDMIA:
        case DRM_MODE_CONNECTOR_HDMIB:
            lcdc_type = SCREEN_HDMI;
            break;
        case DRM_MODE_CONNECTOR_TV:
            lcdc_type = SCREEN_TVOUT;
            break;
        case DRM_MODE_CONNECTOR_eDP:
            lcdc_type = SCREEN_EDP;
            break;
        case DRM_MODE_CONNECTOR_DSI:
            lcdc_type = SCREEN_MIPI;
            break;
        default:
            lcdc_type = SCREEN_NULL;
            break;
    }

    return lcdc_type;
}

static int rockchip_ddr_set_rate(unsigned long target_rate)
{
    struct arm_smccc_res res;

    ddr_psci_param->hz = target_rate;
    ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type();
    ddr_psci_param->wait_flag1 = 1;
    ddr_psci_param->wait_flag0 = 1;

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE);
    if ((int)res.a1 == SIP_RET_SET_RATE_TIMEOUT) {
        rockchip_dmcfreq_wait_complete();
    }

    return res.a0;
}

static int rockchip_dmcfreq_target(struct device *dev, unsigned long *freq, u32 flags)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);
    struct dev_pm_opp *opp;
    struct cpufreq_policy *policy;
    unsigned long old_clk_rate = dmcfreq->rate;
    unsigned long target_volt, target_rate;
    unsigned int cpu_cur, cpufreq_cur;
    bool is_cpufreq_changed = false;
    int err = 0;

    opp = devfreq_recommended_opp(dev, freq, flags);
    if (IS_ERR(opp)) {
        dev_err(dev, "Failed to find opp for %lu Hz\n", *freq);
        return PTR_ERR(opp);
    }
    target_volt = dev_pm_opp_get_voltage(opp);
    dev_pm_opp_put(opp);

    if (dmcfreq->is_set_rate_direct) {
        target_rate = *freq;
    } else {
        target_rate = clk_round_rate(dmcfreq->dmc_clk, *freq);
        if ((long)target_rate <= 0) {
            target_rate = *freq;
        }
    }

    if (dmcfreq->rate == target_rate) {
        if (dmcfreq->volt == target_volt) {
            return 0;
        }
        err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, INT_MAX);
        if (err) {
            dev_err(dev, "Cannot set voltage %lu uV\n", target_volt);
            return err;
        }
        dmcfreq->volt = target_volt;
        return 0;
    } else if (!dmcfreq->volt) {
        dmcfreq->volt = regulator_get_voltage(dmcfreq->vdd_center);
    }

    /*
     * We need to prevent cpu hotplug from happening while a dmc freq rate
     * change is happening.
     *
     * Do this before taking the policy rwsem to avoid deadlocks between the
     * mutex that is locked/unlocked in cpu_hotplug_disable/enable. And it
     * can also avoid deadlocks between the mutex that is locked/unlocked
     * in get/put_online_cpus (such as store_scaling_max_freq()).
     */
    get_online_cpus();

    /*
     * Go to specified cpufreq and block other cpufreq changes since
     * set_rate needs to complete during vblank.
     */
    cpu_cur = raw_smp_processor_id();
    policy = cpufreq_cpu_get(cpu_cur);
    if (!policy) {
        dev_err(dev, "cpu%d policy NULL\n", cpu_cur);
        goto cpufreq;
    }
    down_write(&policy->rwsem);
    cpufreq_cur = cpufreq_quick_get(cpu_cur);
    if (dmcfreq->min_cpu_freq && cpufreq_cur < dmcfreq->min_cpu_freq) {
        if (policy->max >= dmcfreq->min_cpu_freq) {
            __cpufreq_driver_target(policy, dmcfreq->min_cpu_freq, CPUFREQ_RELATION_L);
            is_cpufreq_changed = true;
        } else {
            dev_dbg(dev, "CPU may too slow for DMC (%d MHz)\n", policy->max);
        }
    }

    /*
     * If frequency scaling from low to high, adjust voltage first.
     * If frequency scaling from high to low, adjust frequency first.
     */
    if (old_clk_rate < target_rate) {
        err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, INT_MAX);
        if (err) {
            dev_err(dev, "Cannot set voltage %lu uV\n", target_volt);
            goto out;
        }
    }

    /*
     * Writer in rwsem may block readers even during its waiting in queue,
     * and this may lead to a deadlock when the code path takes read sem
     * twice (e.g. one in vop_lock() and another in rockchip_pmu_lock()).
     * As a (suboptimal) workaround, let writer to spin until it gets the
     * lock.
     */
    while (!rockchip_dmcfreq_write_trylock()) {
        cond_resched();
    }
    dev_dbg(dev, "%lu-->%lu\n", old_clk_rate, target_rate);

    if (dmcfreq->set_rate_params) {
        dmcfreq->set_rate_params->lcdc_type = rk_drm_get_lcdc_type();
        dmcfreq->set_rate_params->wait_flag1 = 1;
        dmcfreq->set_rate_params->wait_flag0 = 1;
    }

    if (dmcfreq->is_set_rate_direct) {
        err = rockchip_ddr_set_rate(target_rate);
    } else {
        err = clk_set_rate(dmcfreq->dmc_clk, target_rate);
    }

    rockchip_dmcfreq_write_unlock();
    if (err) {
        dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate, err);
        regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, INT_MAX);
        goto out;
    }

    /*
     * Check the dpll rate,
     * There only two result we will get,
     * 1. Ddr frequency scaling fail, we still get the old rate.
     * 2. Ddr frequency scaling sucessful, we get the rate we set.
     */
    dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk);

    /* If get the incorrect rate, set voltage to old value. */
    if (dmcfreq->rate != target_rate) {
        dev_err(dev, "Get wrong frequency, Request %lu, Current %lu\n", target_rate, dmcfreq->rate);
        regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, INT_MAX);
        goto out;
    } else if (old_clk_rate > target_rate) {
        err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, INT_MAX);
        if (err) {
            dev_err(dev, "Cannot set vol %lu uV\n", target_volt);
            goto out;
        }
    }

    if (dmcfreq->info.devfreq) {
        struct devfreq *devfreq = dmcfreq->info.devfreq;

        devfreq->last_status.current_frequency = *freq;
    }

    dmcfreq->volt = target_volt;
out:
    if (is_cpufreq_changed) {
        __cpufreq_driver_target(policy, cpufreq_cur, CPUFREQ_RELATION_L);
    }
    up_write(&policy->rwsem);
    cpufreq_cpu_put(policy);
cpufreq:
    put_online_cpus();
    return err;
}

static int rockchip_dmcfreq_get_dev_status(struct device *dev, struct devfreq_dev_status *stat)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);
    struct devfreq_event_data edata;
    int i, ret = 0;

    if (!dmcfreq->info.auto_freq_en) {
        return -EINVAL;
    }

    for (i = 0; i < dmcfreq->edev_count; i++) {
        ret = devfreq_event_get_event(dmcfreq->edev[i], &edata);
        if (ret < 0) {
            dev_err(dev, "failed to get event %s\n", dmcfreq->edev[i]->desc->name);
            return ret;
        }
        if (i == dmcfreq->dfi_id) {
            stat->busy_time = edata.load_count;
            stat->total_time = edata.total_count;
        } else {
            dmcfreq->nocp_bw[i] = edata.load_count;
        }
    }

    return 0;
}

static int rockchip_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);

    *freq = dmcfreq->rate;

    return 0;
}

static struct devfreq_dev_profile rockchip_devfreq_dmc_profile = {
    .polling_ms = 50,
    .target = rockchip_dmcfreq_target,
    .get_dev_status = rockchip_dmcfreq_get_dev_status,
    .get_cur_freq = rockchip_dmcfreq_get_cur_freq,
};

static inline void reset_last_status(struct devfreq *devfreq)
{
    devfreq->last_status.total_time = 1;
    devfreq->last_status.busy_time = 1;
}

static void of_get_px30_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct px30_ddr_dts_config_timing *dts_timing;
    struct rk3328_ddr_de_skew_setting *de_skew;
    int ret = 0;
    u32 i;

    dts_timing = (struct px30_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }
    de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL);
    if (!de_skew) {
        ret = -ENOMEM;
        goto end;
    }
    p = (u32 *)dts_timing;
    for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, px30_dts_timing[i], p + i);
    }
    p = (u32 *)de_skew->ca_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], p + i);
    }
    p = (u32 *)de_skew->cs0_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], p + i);
    }
    p = (u32 *)de_skew->cs1_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], p + i);
    }
    if (!ret) {
        px30_de_skew_set_2_reg(de_skew, dts_timing);
    }
    kfree(de_skew);
end:
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static void of_get_rk1808_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct rk1808_ddr_dts_config_timing *dts_timing;
    int ret = 0;
    u32 i;

    dts_timing = (struct rk1808_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }

    p = (u32 *)dts_timing;
    for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, px30_dts_timing[i], p + i);
    }
    p = (u32 *)dts_timing->ca_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk1808_dts_ca_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk1808_dts_ca_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs0_a_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_a_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_a_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs0_b_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_b_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_b_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs1_a_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_a_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_a_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs1_b_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_b_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_b_timing[i], p + i);
    }

end:
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static void of_get_rk3128_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct rk3128_ddr_dts_config_timing *dts_timing;
    struct share_params *init_timing;
    int ret = 0;
    u32 i;

    init_timing = (struct share_params *)timing;

    if (of_property_read_u32(np, "vop-dclk-mode", &init_timing->vop_dclk_mode)) {
        init_timing->vop_dclk_mode = 0;
    }

    p = timing + DTS_PAR_OFFSET / 0x4;
    np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }
    for (i = 0; i < ARRAY_SIZE(rk3128_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3128_dts_timing[i], p + i);
    }
end:
    dts_timing = (struct rk3128_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static uint32_t of_get_rk3228_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    int ret = 0;
    u32 i;

    p = timing + DTS_PAR_OFFSET / 0x4;
    np_tim = of_parse_phandle(np, "rockchip,dram_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }
    for (i = 0; i < ARRAY_SIZE(rk3228_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3228_dts_timing[i], p + i);
    }
end:
    if (ret) {
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
    return ret;
}

static void of_get_rk3288_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct rk3288_ddr_dts_config_timing *dts_timing;
    struct share_params *init_timing;
    int ret = 0;
    u32 i;

    init_timing = (struct share_params *)timing;

    if (of_property_read_u32(np, "vop-dclk-mode", &init_timing->vop_dclk_mode)) {
        init_timing->vop_dclk_mode = 0;
    }

    p = timing + DTS_PAR_OFFSET / 0x4;
    np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }
    for (i = 0; i < ARRAY_SIZE(rk3288_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3288_dts_timing[i], p + i);
    }
end:
    dts_timing = (struct rk3288_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static void of_get_rk3328_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct rk3328_ddr_dts_config_timing *dts_timing;
    struct rk3328_ddr_de_skew_setting *de_skew;
    int ret = 0;
    u32 i;

    dts_timing = (struct rk3328_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }
    de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL);
    if (!de_skew) {
        ret = -ENOMEM;
        goto end;
    }
    p = (u32 *)dts_timing;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_timing[i], p + i);
    }
    p = (u32 *)de_skew->ca_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], p + i);
    }
    p = (u32 *)de_skew->cs0_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], p + i);
    }
    p = (u32 *)de_skew->cs1_de_skew;
    for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) {
        ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], p + i);
    }
    if (!ret) {
        rk3328_de_skew_setting_2_register(de_skew, dts_timing);
    }
    kfree(de_skew);
end:
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static void of_get_rv1126_timings(struct device *dev, struct device_node *np, uint32_t *timing)
{
    struct device_node *np_tim;
    u32 *p;
    struct rk1808_ddr_dts_config_timing *dts_timing;
    int ret = 0;
    u32 i;

    dts_timing = (struct rk1808_ddr_dts_config_timing *)(timing + DTS_PAR_OFFSET / 0x4);

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (!np_tim) {
        ret = -EINVAL;
        goto end;
    }

    p = (u32 *)dts_timing;
    for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) {
        ret |= of_property_read_u32(np_tim, px30_dts_timing[i], p + i);
    }
    p = (u32 *)dts_timing->ca_de_skew;
    for (i = 0; i < ARRAY_SIZE(rv1126_dts_ca_timing); i++) {
        ret |= of_property_read_u32(np_tim, rv1126_dts_ca_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs0_a_de_skew;
    for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_a_timing); i++) {
        ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_a_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs0_b_de_skew;
    for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_b_timing); i++) {
        ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_b_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs1_a_de_skew;
    for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_a_timing); i++) {
        ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_a_timing[i], p + i);
    }
    p = (u32 *)dts_timing->cs1_b_de_skew;
    for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_b_timing); i++) {
        ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_b_timing[i], p + i);
    }

end:
    if (!ret) {
        dts_timing->available = 1;
    } else {
        dts_timing->available = 0;
        dev_err(dev, "of_get_ddr_timings: fail\n");
    }

    of_node_put(np_tim);
}

static struct rk3368_dram_timing *of_get_rk3368_timings(struct device *dev, struct device_node *np)
{
    struct rk3368_dram_timing *timing = NULL;
    struct device_node *np_tim;
    int ret = 0;

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (np_tim) {
        timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL);
        if (!timing) {
            goto err;
        }

        ret |= of_property_read_u32(np_tim, "dram_spd_bin", &timing->dram_spd_bin);
        ret |= of_property_read_u32(np_tim, "sr_idle", &timing->sr_idle);
        ret |= of_property_read_u32(np_tim, "pd_idle", &timing->pd_idle);
        ret |= of_property_read_u32(np_tim, "dram_dll_disb_freq", &timing->dram_dll_dis_freq);
        ret |= of_property_read_u32(np_tim, "phy_dll_disb_freq", &timing->phy_dll_dis_freq);
        ret |= of_property_read_u32(np_tim, "dram_odt_disb_freq", &timing->dram_odt_dis_freq);
        ret |= of_property_read_u32(np_tim, "phy_odt_disb_freq", &timing->phy_odt_dis_freq);
        ret |= of_property_read_u32(np_tim, "ddr3_drv", &timing->ddr3_drv);
        ret |= of_property_read_u32(np_tim, "ddr3_odt", &timing->ddr3_odt);
        ret |= of_property_read_u32(np_tim, "lpddr3_drv", &timing->lpddr3_drv);
        ret |= of_property_read_u32(np_tim, "lpddr3_odt", &timing->lpddr3_odt);
        ret |= of_property_read_u32(np_tim, "lpddr2_drv", &timing->lpddr2_drv);
        ret |= of_property_read_u32(np_tim, "phy_clk_drv", &timing->phy_clk_drv);
        ret |= of_property_read_u32(np_tim, "phy_cmd_drv", &timing->phy_cmd_drv);
        ret |= of_property_read_u32(np_tim, "phy_dqs_drv", &timing->phy_dqs_drv);
        ret |= of_property_read_u32(np_tim, "phy_odt", &timing->phy_odt);
        ret |= of_property_read_u32(np_tim, "ddr_2t", &timing->ddr_2t);
        if (ret) {
            devm_kfree(dev, timing);
            goto err;
        }
        of_node_put(np_tim);
        return timing;
    }

err:
    if (timing) {
        devm_kfree(dev, timing);
        timing = NULL;
    }
    of_node_put(np_tim);
    return timing;
}

static struct rk3399_dram_timing *of_get_rk3399_timings(struct device *dev, struct device_node *np)
{
    struct rk3399_dram_timing *timing = NULL;
    struct device_node *np_tim;
    int ret;

    np_tim = of_parse_phandle(np, "ddr_timing", 0);
    if (np_tim) {
        timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL);
        if (!timing) {
            goto err;
        }

        ret = of_property_read_u32(np_tim, "ddr3_speed_bin", &timing->ddr3_speed_bin);
        ret |= of_property_read_u32(np_tim, "pd_idle", &timing->pd_idle);
        ret |= of_property_read_u32(np_tim, "sr_idle", &timing->sr_idle);
        ret |= of_property_read_u32(np_tim, "sr_mc_gate_idle", &timing->sr_mc_gate_idle);
        ret |= of_property_read_u32(np_tim, "srpd_lite_idle", &timing->srpd_lite_idle);
        ret |= of_property_read_u32(np_tim, "standby_idle", &timing->standby_idle);
        ret |= of_property_read_u32(np_tim, "auto_lp_dis_freq", &timing->auto_lp_dis_freq);
        ret |= of_property_read_u32(np_tim, "ddr3_dll_dis_freq", &timing->ddr3_dll_dis_freq);
        ret |= of_property_read_u32(np_tim, "phy_dll_dis_freq", &timing->phy_dll_dis_freq);
        ret |= of_property_read_u32(np_tim, "ddr3_odt_dis_freq", &timing->ddr3_odt_dis_freq);
        ret |= of_property_read_u32(np_tim, "ddr3_drv", &timing->ddr3_drv);
        ret |= of_property_read_u32(np_tim, "ddr3_odt", &timing->ddr3_odt);
        ret |= of_property_read_u32(np_tim, "phy_ddr3_ca_drv", &timing->phy_ddr3_ca_drv);
        ret |= of_property_read_u32(np_tim, "phy_ddr3_dq_drv", &timing->phy_ddr3_dq_drv);
        ret |= of_property_read_u32(np_tim, "phy_ddr3_odt", &timing->phy_ddr3_odt);
        ret |= of_property_read_u32(np_tim, "lpddr3_odt_dis_freq", &timing->lpddr3_odt_dis_freq);
        ret |= of_property_read_u32(np_tim, "lpddr3_drv", &timing->lpddr3_drv);
        ret |= of_property_read_u32(np_tim, "lpddr3_odt", &timing->lpddr3_odt);
        ret |= of_property_read_u32(np_tim, "phy_lpddr3_ca_drv", &timing->phy_lpddr3_ca_drv);
        ret |= of_property_read_u32(np_tim, "phy_lpddr3_dq_drv", &timing->phy_lpddr3_dq_drv);
        ret |= of_property_read_u32(np_tim, "phy_lpddr3_odt", &timing->phy_lpddr3_odt);
        ret |= of_property_read_u32(np_tim, "lpddr4_odt_dis_freq", &timing->lpddr4_odt_dis_freq);
        ret |= of_property_read_u32(np_tim, "lpddr4_drv", &timing->lpddr4_drv);
        ret |= of_property_read_u32(np_tim, "lpddr4_dq_odt", &timing->lpddr4_dq_odt);
        ret |= of_property_read_u32(np_tim, "lpddr4_ca_odt", &timing->lpddr4_ca_odt);
        ret |= of_property_read_u32(np_tim, "phy_lpddr4_ca_drv", &timing->phy_lpddr4_ca_drv);
        ret |= of_property_read_u32(np_tim, "phy_lpddr4_ck_cs_drv", &timing->phy_lpddr4_ck_cs_drv);
        ret |= of_property_read_u32(np_tim, "phy_lpddr4_dq_drv", &timing->phy_lpddr4_dq_drv);
        ret |= of_property_read_u32(np_tim, "phy_lpddr4_odt", &timing->phy_lpddr4_odt);
        if (ret) {
            devm_kfree(dev, timing);
            goto err;
        }
        of_node_put(np_tim);
        return timing;
    }

err:
    if (timing) {
        devm_kfree(dev, timing);
        timing = NULL;
    }
    of_node_put(np_tim);
    return timing;
}

static int rockchip_ddr_set_auto_self_refresh(uint32_t en)
{
    struct arm_smccc_res res;

    ddr_psci_param->sr_idle_en = en;
    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_SET_AT_SR);

    return res.a0;
}

struct dmcfreq_wait_ctrl_t {
    wait_queue_head_t wait_wq;
    int complt_irq;
    int wait_flag;
    int wait_en;
    int wait_time_out_ms;
    int dcf_en;
    struct regmap *regmap_dcf;
};

static struct dmcfreq_wait_ctrl_t wait_ctrl;

static irqreturn_t wait_complete_irq(int irqno, void *dev_id)
{
    struct dmcfreq_wait_ctrl_t *ctrl = dev_id;

    ctrl->wait_flag = 0;
    wake_up(&ctrl->wait_wq);
    return IRQ_HANDLED;
}

static irqreturn_t wait_dcf_complete_irq(int irqno, void *dev_id)
{
    struct arm_smccc_res res;
    struct dmcfreq_wait_ctrl_t *ctrl = dev_id;

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_POST_SET_RATE);
    if (res.a0) {
        pr_err("%s: dram post set rate error:%lx\n", __func__, res.a0);
    }

    ctrl->wait_flag = 0;
    wake_up(&ctrl->wait_wq);
    return IRQ_HANDLED;
}

int rockchip_dmcfreq_wait_complete(void)
{
    struct arm_smccc_res res;

    if (!wait_ctrl.wait_en) {
        pr_err("%s: Do not support time out!\n", __func__);
        return 0;
    }
    wait_ctrl.wait_flag = -1;

    enable_irq(wait_ctrl.complt_irq);
    /*
     * CPUs only enter WFI when idle to make sure that
     * FIQn can quick response.
     */
    cpu_latency_qos_update_request(&pm_qos, 0);

    if (wait_ctrl.dcf_en == 1) {
        /* start dcf */
        regmap_update_bits(wait_ctrl.regmap_dcf, 0x0, 0x1, 0x1);
    } else if (wait_ctrl.dcf_en == DMCFREQ_WAIT_CTRL_T_DCF_EN_TWO) {
        res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_MCU_START);
        if (res.a0) {
            pr_err("rockchip_sip_config_mcu_start error:%lx\n", res.a0);
            return -ENOMEM;
        }
    }

    wait_event_timeout(wait_ctrl.wait_wq, (wait_ctrl.wait_flag == 0), msecs_to_jiffies(wait_ctrl.wait_time_out_ms));

    cpu_latency_qos_update_request(&pm_qos, PM_QOS_DEFAULT_VALUE);
    disable_irq(wait_ctrl.complt_irq);

    return 0;
}

static __maybe_unused int rockchip_get_freq_info(struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    struct dev_pm_opp *opp;
    struct dmc_freq_table *freq_table;
    unsigned long rate;
    int i, j, count, ret = 0;

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_FREQ_INFO);
    if (res.a0) {
        dev_err(dmcfreq->dev, "rockchip_sip_config_dram_get_freq_info error:%lx\n", res.a0);
        return -ENOMEM;
    }

    if (ddr_psci_param->freq_count == 0 || ddr_psci_param->freq_count > 6) {
        dev_err(dmcfreq->dev, "it is no available frequencies!\n");
        return -EPERM;
    }

    for (i = 0; i < ddr_psci_param->freq_count; i++) {
        dmcfreq->freq_info_rate[i] = ddr_psci_param->freq_info_mhz[i] * 1000000;
    }
    dmcfreq->freq_count = ddr_psci_param->freq_count;

    /* update dmc_opp_table */
    count = dev_pm_opp_get_opp_count(dmcfreq->dev);
    if (count <= 0) {
        ret = count ? count : -ENODATA;
        return ret;
    }

    freq_table = kmalloc(sizeof(struct dmc_freq_table) * count, GFP_KERNEL);
    for (i = 0, rate = 0; i < count; i++, rate++) {
        /* find next rate */
        opp = dev_pm_opp_find_freq_ceil(dmcfreq->dev, &rate);
        if (IS_ERR(opp)) {
            ret = PTR_ERR(opp);
            dev_err(dmcfreq->dev, "failed to find OPP for freq %lu.\n", rate);
            goto out;
        }
        freq_table[i].freq = rate;
        freq_table[i].volt = dev_pm_opp_get_voltage(opp);
        dev_pm_opp_put(opp);

        for (j = 0; j < dmcfreq->freq_count; j++) {
            if (rate == dmcfreq->freq_info_rate[j]) {
                break;
            }
        }
        if (j == dmcfreq->freq_count) {
            dev_pm_opp_remove(dmcfreq->dev, rate);
        }
    }

    for (i = 0; i < dmcfreq->freq_count; i++) {
        for (j = 0; j < count; j++) {
            if (dmcfreq->freq_info_rate[i] == freq_table[j].freq) {
                break;
            } else if (dmcfreq->freq_info_rate[i] < freq_table[j].freq) {
                dev_pm_opp_add(dmcfreq->dev, dmcfreq->freq_info_rate[i], freq_table[j].volt);
                break;
            }
        }
        if (j == count) {
            dev_err(dmcfreq->dev, "failed to match dmc_opp_table for %ld\n", dmcfreq->freq_info_rate[i]);
            if (i == 0) {
                ret = -EPERM;
            } else {
                dmcfreq->freq_count = i;
            }
            goto out;
        }
    }

out:
    kfree(freq_table);
    return ret;
}

static __maybe_unused int px30_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    u32 size;
    int ret;
    int complt_irq;
    u32 complt_hwirq;
    struct irq_data *complt_irq_data;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1);
    if (res.a0 || res.a1 < 0x103) {
        dev_err(&pdev->dev, "trusted firmware need to update or is invalid!\n");
        return -ENXIO;
    }

    dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1);

    /*
     * first 4KB is used for interface parameters
     * after 4KB * N is dts parameters
     */
    size = sizeof(struct px30_ddr_dts_config_timing);
    res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 0x1000) + 1, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    of_get_px30_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    init_waitqueue_head(&wait_ctrl.wait_wq);
    wait_ctrl.wait_en = 1;
    wait_ctrl.wait_time_out_ms = 0x55;

    complt_irq = platform_get_irq_byname(pdev, "complete_irq");
    if (complt_irq < 0) {
        dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", complt_irq);
        return complt_irq;
    }
    wait_ctrl.complt_irq = complt_irq;

    ret = devm_request_irq(&pdev->dev, complt_irq, wait_complete_irq, 0, dev_name(&pdev->dev), &wait_ctrl);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot request complete_irq\n");
        return ret;
    }
    disable_irq(complt_irq);

    complt_irq_data = irq_get_irq_data(complt_irq);
    complt_hwirq = irqd_to_hwirq(complt_irq_data);
    ddr_psci_param->complt_hwirq = complt_hwirq;

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);
    rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk1808_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    u32 size;
    int ret;
    int complt_irq;
    struct device_node *node;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1);
    if (res.a0 || res.a1 < 0x101) {
        dev_err(&pdev->dev, "trusted firmware need to update or is invalid!\n");
        return -ENXIO;
    }

    /*
     * first 4KB is used for interface parameters
     * after 4KB * N is dts parameters
     */
    size = sizeof(struct rk1808_ddr_dts_config_timing);
    res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 0x1000) + 1, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    of_get_rk1808_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    /* enable start dcf in kernel after dcf ready */
    node = of_parse_phandle(pdev->dev.of_node, "dcf_reg", 0);
    wait_ctrl.regmap_dcf = syscon_node_to_regmap(node);
    if (IS_ERR(wait_ctrl.regmap_dcf)) {
        return PTR_ERR(wait_ctrl.regmap_dcf);
    }
    wait_ctrl.dcf_en = 1;

    init_waitqueue_head(&wait_ctrl.wait_wq);
    wait_ctrl.wait_en = 1;
    wait_ctrl.wait_time_out_ms = 0x55;

    complt_irq = platform_get_irq_byname(pdev, "complete_irq");
    if (complt_irq < 0) {
        dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", complt_irq);
        return complt_irq;
    }
    wait_ctrl.complt_irq = complt_irq;

    ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, 0, dev_name(&pdev->dev), &wait_ctrl);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot request complete_irq\n");
        return ret;
    }
    disable_irq(complt_irq);

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);
    rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3128_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;

    res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof(struct rk3128_ddr_dts_config_timing), 0x1000) + 1,
                                    SHARE_PAGE_TYPE_DDR);
    if (res.a0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    of_get_rk3128_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    ddr_psci_param->hz = 0;
    ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type();

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);

    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3228_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;

    res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof(struct rk3228_ddr_dts_config_timing), 0x1000) + 1,
                                    SHARE_PAGE_TYPE_DDR);
    if (res.a0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }

    ddr_psci_param = (struct share_params *)res.a1;
    if (of_get_rk3228_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param)) {
        return -ENOMEM;
    }

    ddr_psci_param->hz = 0;

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);

    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3288_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = &pdev->dev;
    struct clk *pclk_phy, *pclk_upctl, *dmc_clk;
    struct arm_smccc_res res;
    int ret;

    dmc_clk = devm_clk_get(dev, "dmc_clk");
    if (IS_ERR(dmc_clk)) {
        dev_err(dev, "Cannot get the clk dmc_clk\n");
        return PTR_ERR(dmc_clk);
    }
    ret = clk_prepare_enable(dmc_clk);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable dmc_clk\n");
        return ret;
    }

    pclk_phy = devm_clk_get(dev, "pclk_phy0");
    if (IS_ERR(pclk_phy)) {
        dev_err(dev, "Cannot get the clk pclk_phy0\n");
        return PTR_ERR(pclk_phy);
    }
    ret = clk_prepare_enable(pclk_phy);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_phy0\n");
        return ret;
    }
    pclk_upctl = devm_clk_get(dev, "pclk_upctl0");
    if (IS_ERR(pclk_upctl)) {
        dev_err(dev, "Cannot get the clk pclk_upctl0\n");
        return PTR_ERR(pclk_upctl);
    }
    ret = clk_prepare_enable(pclk_upctl);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_upctl1\n");
        return ret;
    }

    pclk_phy = devm_clk_get(dev, "pclk_phy1");
    if (IS_ERR(pclk_phy)) {
        dev_err(dev, "Cannot get the clk pclk_phy1\n");
        return PTR_ERR(pclk_phy);
    }
    ret = clk_prepare_enable(pclk_phy);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_phy1\n");
        return ret;
    }
    pclk_upctl = devm_clk_get(dev, "pclk_upctl1");
    if (IS_ERR(pclk_upctl)) {
        dev_err(dev, "Cannot get the clk pclk_upctl1\n");
        return PTR_ERR(pclk_upctl);
    }
    ret = clk_prepare_enable(pclk_upctl);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_upctl1\n");
        return ret;
    }

    res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof(struct rk3288_ddr_dts_config_timing), 0x1000) + 1,
                                    SHARE_PAGE_TYPE_DDR);
    if (res.a0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }

    ddr_psci_param = (struct share_params *)res.a1;
    of_get_rk3288_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    ddr_psci_param->hz = 0;
    ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type();

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);

    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3328_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    u32 size;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1);
    if (res.a0 || (res.a1 < 0x101)) {
        dev_err(&pdev->dev, "trusted firmware need to update or is invalid!\n");
        return -ENXIO;
    }

    dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1);

    /*
     * first 4KB is used for interface parameters
     * after 4KB * N is dts parameters
     */
    size = sizeof(struct rk3328_ddr_dts_config_timing);
    res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 0x1000) + 1, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    of_get_rk3328_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3368_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = &pdev->dev;
    struct device_node *np = pdev->dev.of_node;
    struct arm_smccc_res res;
    struct rk3368_dram_timing *dram_timing;
    struct clk *pclk_phy, *pclk_upctl;
    int ret;
    u32 dram_spd_bin;
    u32 addr_mcu_el3;
    u32 dclk_mode;
    u32 lcdc_type;

    pclk_phy = devm_clk_get(dev, "pclk_phy");
    if (IS_ERR(pclk_phy)) {
        dev_err(dev, "Cannot get the clk pclk_phy\n");
        return PTR_ERR(pclk_phy);
    }
    ret = clk_prepare_enable(pclk_phy);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_phy\n");
        return ret;
    }
    pclk_upctl = devm_clk_get(dev, "pclk_upctl");
    if (IS_ERR(pclk_upctl)) {
        dev_err(dev, "Cannot get the clk pclk_upctl\n");
        return PTR_ERR(pclk_upctl);
    }
    ret = clk_prepare_enable(pclk_upctl);
    if (ret < 0) {
        dev_err(dev, "failed to prepare/enable pclk_upctl\n");
        return ret;
    }

    /*
     * Get dram timing and pass it to arm trust firmware,
     * the dram drvier in arm trust firmware will get these
     * timing and to do dram initial.
     */
    dram_timing = of_get_rk3368_timings(dev, np);
    if (dram_timing) {
        dram_spd_bin = dram_timing->dram_spd_bin;
        if (scpi_ddr_send_timing((u32 *)dram_timing, sizeof(struct rk3368_dram_timing))) {
            dev_err(dev, "send ddr timing timeout\n");
        }
    } else {
        dev_err(dev, "get ddr timing from dts error\n");
        dram_spd_bin = DDR3_DEFAULT;
    }

    res = sip_smc_mcu_el3fiq(FIQ_INIT_HANDLER, FIQ_NUM_FOR_DCF, FIQ_CPU_TGT_BOOT);
    if ((res.a0) || (res.a1 == 0) || (res.a1 > 0x80000)) {
        dev_err(dev, "Trust version error, pls check trust version\n");
    }
    addr_mcu_el3 = res.a1;

    if (of_property_read_u32(np, "vop-dclk-mode", &dclk_mode) == 0) {
        scpi_ddr_dclk_mode(dclk_mode);
    }

    dmcfreq->set_rate_params = devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL);
    if (!dmcfreq->set_rate_params) {
        return -ENOMEM;
    }
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    lcdc_type = rk_drm_get_lcdc_type();

    if (scpi_ddr_init(dram_spd_bin, 0, lcdc_type, addr_mcu_el3)) {
        dev_err(dev, "ddr init error\n");
    } else {
        dev_dbg(dev, ("%s out\n"), __func__);
    }

    dmcfreq->set_auto_self_refresh = scpi_ddr_set_auto_self_refresh;

    return 0;
}

static int rk3399_set_msch_readlatency(unsigned int readlatency)
{
    struct arm_smccc_res res;

    arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, readlatency, 0, ROCKCHIP_SIP_CONFIG_DRAM_SET_MSCH_RL, 0, 0, 0, 0, &res);

    return res.a0;
}

static __maybe_unused int rk3399_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = &pdev->dev;
    struct device_node *np = pdev->dev.of_node;
    struct arm_smccc_res res;
    struct rk3399_dram_timing *dram_timing;
    int index, size;
    u32 *timing;

    /*
     * Get dram timing and pass it to arm trust firmware,
     * the dram drvier in arm trust firmware will get these
     * timing and to do dram initial.
     */
    dram_timing = of_get_rk3399_timings(dev, np);
    if (dram_timing) {
        timing = (u32 *)dram_timing;
        size = sizeof(struct rk3399_dram_timing) / 0x4;
        for (index = 0; index < size; index++) {
            arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index, ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM, 0, 0, 0, 0,
                          &res);
            if (res.a0) {
                dev_err(dev, "Failed to set dram param: %ld\n", res.a0);
                return -EINVAL;
            }
        }
    }

    dmcfreq->set_rate_params = devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL);
    if (!dmcfreq->set_rate_params) {
        return -ENOMEM;
    }
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);

    arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT, 0, 0, 0, 0, &res);

    dmcfreq->info.set_msch_readlatency = rk3399_set_msch_readlatency;

    return 0;
}

static __maybe_unused int rk3568_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    int ret;
    int complt_irq;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1);
    if (res.a0 || res.a1 < 0x101) {
        dev_err(&pdev->dev, "trusted firmware need update to V1.01 and above.\n");
        return -ENXIO;
    }

    /*
     * first 4KB is used for interface parameters
     * after 4KB is dts parameters
     * request share memory size 4KB * 2
     */
    res = sip_smc_request_share_mem(2, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    /* Clear ddr_psci_param, size is 4KB * 2 */
    memset_io(ddr_psci_param, 0x0, 0x1000 * 0x2);

    /* start mcu with sip_smc_dram */
    wait_ctrl.dcf_en = 0x2;

    init_waitqueue_head(&wait_ctrl.wait_wq);
    wait_ctrl.wait_en = 1;
    wait_ctrl.wait_time_out_ms = 0x55;

    complt_irq = platform_get_irq_byname(pdev, "complete");
    if (complt_irq < 0) {
        dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", complt_irq);
        return complt_irq;
    }
    wait_ctrl.complt_irq = complt_irq;

    ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, 0, dev_name(&pdev->dev), &wait_ctrl);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot request complt_irq\n");
        return ret;
    }
    disable_irq(complt_irq);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    ret = rockchip_get_freq_info(dmcfreq);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot get frequency info\n");
        return ret;
    }
    dmcfreq->is_set_rate_direct = true;

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rk3588_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    int ret;
    int complt_irq;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1);
    if (res.a0) {
        dev_err(&pdev->dev, "trusted firmware unsupported, please update.\n");
        return -ENXIO;
    }

    /*
     * first 4KB is used for interface parameters
     * after 4KB is dts parameters
     * request share memory size 4KB * 2
     */
    res = sip_smc_request_share_mem(0x2, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    /* Clear ddr_psci_param, size is 4KB * 2 */
    memset_io(ddr_psci_param, 0x0, 0x1000 * 0x2);

    /* start mcu with sip_smc_dram */
    wait_ctrl.dcf_en = 0x2;

    init_waitqueue_head(&wait_ctrl.wait_wq);
    wait_ctrl.wait_en = 1;
    wait_ctrl.wait_time_out_ms = 0x55;

    complt_irq = platform_get_irq_byname(pdev, "complete");
    if (complt_irq < 0) {
        dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", complt_irq);
        return complt_irq;
    }
    wait_ctrl.complt_irq = complt_irq;

    ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, 0, dev_name(&pdev->dev), &wait_ctrl);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot request complt_irq\n");
        return ret;
    }
    disable_irq(complt_irq);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    ret = rockchip_get_freq_info(dmcfreq);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot get frequency info\n");
        return ret;
    }
    dmcfreq->is_set_rate_direct = true;

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static __maybe_unused int rv1126_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    struct arm_smccc_res res;
    u32 size;
    int ret;
    int complt_irq;
    struct device_node *node;

    res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION);
    dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1);
    if (res.a0 || res.a1 < 0x100) {
        dev_err(&pdev->dev, "trusted firmware need to update or is invalid!\n");
        return -ENXIO;
    }

    /*
     * first 4KB is used for interface parameters
     * after 4KB * N is dts parameters
     */
    size = sizeof(struct rk1808_ddr_dts_config_timing);
    res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 0x1000) + 1, SHARE_PAGE_TYPE_DDR);
    if (res.a0 != 0) {
        dev_err(&pdev->dev, "no ATF memory for init\n");
        return -ENOMEM;
    }
    ddr_psci_param = (struct share_params *)res.a1;
    of_get_rv1126_timings(&pdev->dev, pdev->dev.of_node, (uint32_t *)ddr_psci_param);

    /* enable start dcf in kernel after dcf ready */
    node = of_parse_phandle(pdev->dev.of_node, "dcf", 0);
    wait_ctrl.regmap_dcf = syscon_node_to_regmap(node);
    if (IS_ERR(wait_ctrl.regmap_dcf)) {
        return PTR_ERR(wait_ctrl.regmap_dcf);
    }
    wait_ctrl.dcf_en = 1;

    init_waitqueue_head(&wait_ctrl.wait_wq);
    wait_ctrl.wait_en = 1;
    wait_ctrl.wait_time_out_ms = 0x55;

    complt_irq = platform_get_irq_byname(pdev, "complete");
    if (complt_irq < 0) {
        dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", complt_irq);
        return complt_irq;
    }
    wait_ctrl.complt_irq = complt_irq;

    ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, 0, dev_name(&pdev->dev), &wait_ctrl);
    if (ret < 0) {
        dev_err(&pdev->dev, "cannot request complt_irq\n");
        return ret;
    }
    disable_irq(complt_irq);

    if (of_property_read_u32(pdev->dev.of_node, "update_drv_odt_cfg", &ddr_psci_param->update_drv_odt_cfg)) {
        ddr_psci_param->update_drv_odt_cfg = 0;
    }

    if (of_property_read_u32(pdev->dev.of_node, "update_deskew_cfg", &ddr_psci_param->update_deskew_cfg)) {
        ddr_psci_param->update_deskew_cfg = 0;
    }

    dmcfreq->set_rate_params = ddr_psci_param;
    rockchip_set_ddrclk_params(dmcfreq->set_rate_params);
    rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete);

    res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT);
    if (res.a0) {
        dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0);
        return -ENOMEM;
    }

    dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh;

    return 0;
}

static const struct of_device_id rockchip_dmcfreq_of_match[] = {
#if IS_ENABLED(CONFIG_CPU_PX30)
    {.compatible = "rockchip,px30-dmc", .data = px30_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK1808)
    {.compatible = "rockchip,rk1808-dmc", .data = rk1808_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK312X)
    {.compatible = "rockchip,rk3128-dmc", .data = rk3128_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK322X)
    {.compatible = "rockchip,rk3228-dmc", .data = rk3228_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3288)
    {.compatible = "rockchip,rk3288-dmc", .data = rk3288_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3308)
    {.compatible = "rockchip,rk3308-dmc", .data = NULL},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3328)
    {.compatible = "rockchip,rk3328-dmc", .data = rk3328_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3368)
    {.compatible = "rockchip,rk3368-dmc", .data = rk3368_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3399)
    {.compatible = "rockchip,rk3399-dmc", .data = rk3399_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3568)
    {.compatible = "rockchip,rk3568-dmc", .data = rk3568_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RK3588)
    {.compatible = "rockchip,rk3588-dmc", .data = rk3588_dmc_init},
#endif
#if IS_ENABLED(CONFIG_CPU_RV1126)
    {.compatible = "rockchip,rv1126-dmc", .data = rv1126_dmc_init},
#endif
    {},
};
MODULE_DEVICE_TABLE(of, rockchip_dmcfreq_of_match);

static int rockchip_get_freq_map_talbe(struct device_node *np, char *porp_name, struct freq_map_table **table)
{
    struct freq_map_table *tbl;
    const struct property *prop;
    unsigned int temp_freq = 0;
    int count, i;

    prop = of_find_property(np, porp_name, NULL);
    if (!prop) {
        return -EINVAL;
    }

    if (!prop->value) {
        return -ENODATA;
    }

    count = of_property_count_u32_elems(np, porp_name);
    if (count < 0) {
        return -EINVAL;
    }

    if (count % 0x3) {
        return -EINVAL;
    }

    tbl = kzalloc(sizeof(*tbl) * (count / 0x3 + 1), GFP_KERNEL);
    if (!tbl) {
        return -ENOMEM;
    }

    for (i = 0; i < count / 0x3; i++) {
        of_property_read_u32_index(np, porp_name, 0x3 * i, &tbl[i].min);
        of_property_read_u32_index(np, porp_name, 0x3 * i + 0x1, &tbl[i].max);
        of_property_read_u32_index(np, porp_name, 0x3 * i + 0x2, &temp_freq);
        tbl[i].freq = temp_freq * 0x3e8;
    }

    tbl[i].min = 0;
    tbl[i].max = 0;
    tbl[i].freq = DMCFREQ_TABLE_END;

    *table = tbl;

    return 0;
}

static int rockchip_get_rl_map_talbe(struct device_node *np, char *porp_name, struct rl_map_table **table)
{
    struct rl_map_table *tbl;
    const struct property *prop;
    int count, i;

    prop = of_find_property(np, porp_name, NULL);
    if (!prop) {
        return -EINVAL;
    }

    if (!prop->value) {
        return -ENODATA;
    }

    count = of_property_count_u32_elems(np, porp_name);
    if (count < 0) {
        return -EINVAL;
    }

    if (count % 0x2) {
        return -EINVAL;
    }

    tbl = kzalloc(sizeof(*tbl) * (count / 0x2 + 1), GFP_KERNEL);
    if (!tbl) {
        return -ENOMEM;
    }

    for (i = 0; i < count / 0x2; i++) {
        of_property_read_u32_index(np, porp_name, 0x2 * i, &tbl[i].pn);
        of_property_read_u32_index(np, porp_name, 0x2 * i + 1, &tbl[i].rl);
    }

    tbl[i].pn = 0;
    tbl[i].rl = DMCFREQ_TABLE_END;

    *table = tbl;

    return 0;
}

static int rockchip_get_system_status_rate(struct device_node *np, char *porp_name, struct rockchip_dmcfreq *dmcfreq)
{
    const struct property *prop;
    unsigned int status = 0, freq = 0;
    unsigned long temp_rate = 0;
    int count, i;

    prop = of_find_property(np, porp_name, NULL);
    if (!prop) {
        return -ENODEV;
    }

    if (!prop->value) {
        return -ENODATA;
    }

    count = of_property_count_u32_elems(np, porp_name);
    if (count < 0) {
        return -EINVAL;
    }

    if (count % 0x2) {
        return -EINVAL;
    }

    for (i = 0; i < count / 0x2; i++) {
        of_property_read_u32_index(np, porp_name, 0x2 * i, &status);
        of_property_read_u32_index(np, porp_name, 0x2 * i + 1, &freq);
        switch (status) {
            case SYS_STATUS_NORMAL:
                dmcfreq->normal_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_SUSPEND:
                dmcfreq->suspend_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_VIDEO_1080P:
                dmcfreq->video_1080p_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_VIDEO_4K:
                dmcfreq->video_4k_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_VIDEO_4K_10B:
                dmcfreq->video_4k_10b_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_PERFORMANCE:
                dmcfreq->performance_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_HDMI:
                dmcfreq->hdmi_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_IDLE:
                dmcfreq->idle_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_REBOOT:
                dmcfreq->reboot_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_BOOST:
                dmcfreq->boost_rate = (unsigned long)freq * 0x3e8;
                break;
            case SYS_STATUS_ISP:
            case SYS_STATUS_CIF0:
            case SYS_STATUS_CIF1:
            case SYS_STATUS_DUALVIEW:
                temp_rate = (unsigned long)freq * 0x3e8;
                if (dmcfreq->fixed_rate < temp_rate) {
                    dmcfreq->fixed_rate = temp_rate;
                }
                break;
            case SYS_STATUS_LOW_POWER:
                dmcfreq->low_power_rate = (unsigned long)freq * 0x3e8;
                break;
            default:
                break;
        }
    }

    return 0;
}

static unsigned long rockchip_freq_level_2_rate(struct rockchip_dmcfreq *dmcfreq, unsigned int level)
{
    unsigned long rate = 0;

    switch (level) {
        case DMC_FREQ_LEVEL_LOW:
            rate = dmcfreq->rate_low;
            break;
        case DMC_FREQ_LEVEL_MID_LOW:
            rate = dmcfreq->rate_mid_low;
            break;
        case DMC_FREQ_LEVEL_MID_HIGH:
            rate = dmcfreq->rate_mid_high;
            break;
        case DMC_FREQ_LEVEL_HIGH:
            rate = dmcfreq->rate_high;
            break;
        default:
            break;
    }

    return rate;
}

static int rockchip_get_system_status_level(struct device_node *np, char *porp_name, struct rockchip_dmcfreq *dmcfreq)
{
    const struct property *prop;
    unsigned int status = 0, level = 0;
    unsigned long temp_rate = 0;
    int count, i;

    prop = of_find_property(np, porp_name, NULL);
    if (!prop) {
        return -ENODEV;
    }

    if (!prop->value) {
        return -ENODATA;
    }

    count = of_property_count_u32_elems(np, porp_name);
    if (count < 0) {
        return -EINVAL;
    }

    if (count % 0x2) {
        return -EINVAL;
    }

    if (dmcfreq->freq_count == 0x1) {
        dmcfreq->rate_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_high = dmcfreq->freq_info_rate[0];
    } else if (dmcfreq->freq_count == 0x2) {
        dmcfreq->rate_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1];
        dmcfreq->rate_high = dmcfreq->freq_info_rate[1];
    } else if (dmcfreq->freq_count == 0x3) {
        dmcfreq->rate_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1];
        dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1];
        dmcfreq->rate_high = dmcfreq->freq_info_rate[0x2];
    } else if (dmcfreq->freq_count == 0x4) {
        dmcfreq->rate_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0x1];
        dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[0x2];
        dmcfreq->rate_high = dmcfreq->freq_info_rate[0x3];
    } else if (dmcfreq->freq_count == 0x5 || dmcfreq->freq_count == 0x6) {
        dmcfreq->rate_low = dmcfreq->freq_info_rate[0];
        dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1];
        dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 0x2];
        dmcfreq->rate_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 1];
    } else {
        return -EINVAL;
    }

    dmcfreq->auto_min_rate = dmcfreq->rate_low;

    for (i = 0; i < count / 0x2; i++) {
        of_property_read_u32_index(np, porp_name, 0x2 * i, &status);
        of_property_read_u32_index(np, porp_name, 0x2 * i + 1, &level);
        switch (status) {
            case SYS_STATUS_NORMAL:
                dmcfreq->normal_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "normal_rate = %ld\n", dmcfreq->normal_rate);
                break;
            case SYS_STATUS_SUSPEND:
                dmcfreq->suspend_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "suspend_rate = %ld\n", dmcfreq->suspend_rate);
                break;
            case SYS_STATUS_VIDEO_1080P:
                dmcfreq->video_1080p_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "video_1080p_rate = %ld\n", dmcfreq->video_1080p_rate);
                break;
            case SYS_STATUS_VIDEO_4K:
                dmcfreq->video_4k_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "video_4k_rate = %ld\n", dmcfreq->video_4k_rate);
                break;
            case SYS_STATUS_VIDEO_4K_10B:
                dmcfreq->video_4k_10b_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "video_4k_10b_rate = %ld\n", dmcfreq->video_4k_10b_rate);
                break;
            case SYS_STATUS_PERFORMANCE:
                dmcfreq->performance_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "performance_rate = %ld\n", dmcfreq->performance_rate);
                break;
            case SYS_STATUS_HDMI:
                dmcfreq->hdmi_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "hdmi_rate = %ld\n", dmcfreq->hdmi_rate);
                break;
            case SYS_STATUS_IDLE:
                dmcfreq->idle_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "idle_rate = %ld\n", dmcfreq->idle_rate);
                break;
            case SYS_STATUS_REBOOT:
                dmcfreq->reboot_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "reboot_rate = %ld\n", dmcfreq->reboot_rate);
                break;
            case SYS_STATUS_BOOST:
                dmcfreq->boost_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "boost_rate = %ld\n", dmcfreq->boost_rate);
                break;
            case SYS_STATUS_ISP:
            case SYS_STATUS_CIF0:
            case SYS_STATUS_CIF1:
            case SYS_STATUS_DUALVIEW:
                temp_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                if (dmcfreq->fixed_rate < temp_rate) {
                    dmcfreq->fixed_rate = temp_rate;
                    dev_info(dmcfreq->dev, "fixed_rate(isp|cif0|cif1|dualview) = %ld\n", dmcfreq->fixed_rate);
                }
                break;
            case SYS_STATUS_LOW_POWER:
                dmcfreq->low_power_rate = rockchip_freq_level_2_rate(dmcfreq, level);
                dev_info(dmcfreq->dev, "low_power_rate = %ld\n", dmcfreq->low_power_rate);
                break;
            default:
                break;
        }
    }

    return 0;
}

static void rockchip_dmcfreq_update_target(struct rockchip_dmcfreq *dmcfreq)
{
    struct devfreq *devfreq = dmcfreq->info.devfreq;

    mutex_lock(&devfreq->lock);
    update_devfreq(devfreq);
    mutex_unlock(&devfreq->lock);
}

static int rockchip_dmcfreq_system_status_notifier(struct notifier_block *nb, unsigned long status, void *ptr)
{
    struct rockchip_dmcfreq *dmcfreq = system_status_to_dmcfreq(nb);
    unsigned long target_rate = 0;
    unsigned int refresh = false;
    bool is_fixed = false;

    if (dmcfreq->fixed_rate && (is_dualview(status) || is_isp(status))) {
        if (dmcfreq->is_fixed) {
            return NOTIFY_OK;
        }
        is_fixed = true;
        target_rate = dmcfreq->fixed_rate;
        goto next;
    }

    if (dmcfreq->reboot_rate && (status & SYS_STATUS_REBOOT)) {
        if (dmcfreq->info.auto_freq_en) {
            devfreq_monitor_stop(dmcfreq->info.devfreq);
        }
        target_rate = dmcfreq->reboot_rate;
        goto next;
    }

    if (dmcfreq->suspend_rate && (status & SYS_STATUS_SUSPEND)) {
        target_rate = dmcfreq->suspend_rate;
        refresh = true;
        goto next;
    }

    if (dmcfreq->low_power_rate && (status & SYS_STATUS_LOW_POWER)) {
        target_rate = dmcfreq->low_power_rate;
        goto next;
    }

    if (dmcfreq->performance_rate && (status & SYS_STATUS_PERFORMANCE)) {
        if (dmcfreq->performance_rate > target_rate) {
            target_rate = dmcfreq->performance_rate;
        }
    }

    if (dmcfreq->hdmi_rate && (status & SYS_STATUS_HDMI)) {
        if (dmcfreq->hdmi_rate > target_rate) {
            target_rate = dmcfreq->hdmi_rate;
        }
    }

    if (dmcfreq->video_4k_rate && (status & SYS_STATUS_VIDEO_4K)) {
        if (dmcfreq->video_4k_rate > target_rate) {
            target_rate = dmcfreq->video_4k_rate;
        }
    }

    if (dmcfreq->video_4k_10b_rate && (status & SYS_STATUS_VIDEO_4K_10B)) {
        if (dmcfreq->video_4k_10b_rate > target_rate) {
            target_rate = dmcfreq->video_4k_10b_rate;
        }
    }

    if (dmcfreq->video_1080p_rate && (status & SYS_STATUS_VIDEO_1080P)) {
        if (dmcfreq->video_1080p_rate > target_rate) {
            target_rate = dmcfreq->video_1080p_rate;
        }
    }

next:

    dev_dbg(dmcfreq->dev, "status=0x%x\n", (unsigned int)status);
    dmcfreq->is_fixed = is_fixed;
    dmcfreq->status_rate = target_rate;
    if (dmcfreq->refresh != refresh) {
        if (dmcfreq->set_auto_self_refresh) {
            dmcfreq->set_auto_self_refresh(refresh);
        }
        dmcfreq->refresh = refresh;
    }
    rockchip_dmcfreq_update_target(dmcfreq);

    return NOTIFY_OK;
}

static ssize_t rockchip_dmcfreq_status_show(struct device *dev, struct device_attribute *attr, char *buf)
{
    unsigned int status = rockchip_get_system_status();

    return sprintf(buf, "0x%x\n", status);
}

static ssize_t rockchip_dmcfreq_status_store(struct device *dev, struct device_attribute *attr, const char *buf,
                                             size_t count)
{
    if (!count) {
        return -EINVAL;
    }

    rockchip_update_system_status(buf);

    return count;
}

static DEVICE_ATTR(system_status, 0x1a4, rockchip_dmcfreq_status_show, rockchip_dmcfreq_status_store);

static ssize_t upthreshold_show(struct device *dev, struct device_attribute *attr, char *buf)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent);
    struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data;

    return sprintf(buf, "%d\n", data->upthreshold);
}

static ssize_t upthreshold_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent);
    struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data;
    unsigned int value;

    if (kstrtouint(buf, 0xa, &value)) {
        return -EINVAL;
    }

    data->upthreshold = value;

    return count;
}

static DEVICE_ATTR_RW(upthreshold);

static ssize_t downdifferential_show(struct device *dev, struct device_attribute *attr, char *buf)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent);
    struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data;

    return sprintf(buf, "%d\n", data->downdifferential);
}

static ssize_t downdifferential_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent);
    struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data;
    unsigned int value;

    if (kstrtouint(buf, 0xa, &value)) {
        return -EINVAL;
    }

    data->downdifferential = value;

    return count;
}

static DEVICE_ATTR_RW(downdifferential);

static unsigned long get_nocp_req_rate(struct rockchip_dmcfreq *dmcfreq)
{
    unsigned long target = 0, cpu_bw = 0;
    int i;

    if (!dmcfreq->cpu_bw_tbl || dmcfreq->nocp_cpu_id < 0) {
        goto out;
    }

    cpu_bw = dmcfreq->nocp_bw[dmcfreq->nocp_cpu_id];

    for (i = 0; dmcfreq->cpu_bw_tbl[i].freq != CPUFREQ_TABLE_END; i++) {
        if (cpu_bw >= dmcfreq->cpu_bw_tbl[i].min) {
            target = dmcfreq->cpu_bw_tbl[i].freq;
        }
    }

out:
    return target;
}

static int devfreq_dmc_ondemand_func(struct devfreq *df, unsigned long *freq)
{
    int err;
    struct devfreq_dev_status *stat;
    unsigned long long a, b;
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(df->dev.parent);
    struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data;
    unsigned int upthreshold = data->upthreshold;
    unsigned int downdifferential = data->downdifferential;
    unsigned long target_freq = 0, nocp_req_rate = 0;
    u64 now;

    if (dmcfreq->info.auto_freq_en && !dmcfreq->is_fixed) {
        if (dmcfreq->status_rate) {
            target_freq = dmcfreq->status_rate;
        } else if (dmcfreq->auto_min_rate) {
            target_freq = dmcfreq->auto_min_rate;
        }
        nocp_req_rate = get_nocp_req_rate(dmcfreq);
        target_freq = max3(target_freq, nocp_req_rate, dmcfreq->info.vop_req_rate);
        now = ktime_to_us(ktime_get());
        if (now < dmcfreq->touchboostpulse_endtime) {
            target_freq = max(target_freq, dmcfreq->boost_rate);
        }
    } else {
        if (dmcfreq->status_rate) {
            target_freq = dmcfreq->status_rate;
        } else if (dmcfreq->normal_rate) {
            target_freq = dmcfreq->normal_rate;
        }
        if (target_freq) {
            *freq = target_freq;
        }
        if (dmcfreq->info.auto_freq_en && !devfreq_update_stats(df)) {
            return 0;
        }
        goto reset_last_status;
    }

    if (!upthreshold || !downdifferential) {
        goto reset_last_status;
    }

    if (upthreshold > 0x64 || upthreshold < downdifferential) {
        goto reset_last_status;
    }

    err = devfreq_update_stats(df);
    if (err) {
        goto reset_last_status;
    }

    stat = &df->last_status;

    /* Assume MAX if it is going to be divided by zero */
    if (stat->total_time == 0) {
        *freq = DEVFREQ_MAX_FREQ;
        return 0;
    }

    /* Prevent overflow */
    if (stat->busy_time >= (1 << 0x18) || stat->total_time >= (1 << 0x18)) {
        stat->busy_time >>= 0x7;
        stat->total_time >>= 0x7;
    }

    /* Set MAX if it's busy enough */
    if (stat->busy_time * 0x64 > stat->total_time * upthreshold) {
        *freq = DEVFREQ_MAX_FREQ;
        return 0;
    }

    /* Set MAX if we do not know the initial frequency */
    if (stat->current_frequency == 0) {
        *freq = DEVFREQ_MAX_FREQ;
        return 0;
    }

    /* Keep the current frequency */
    if (stat->busy_time * 0x64 > stat->total_time * (upthreshold - downdifferential)) {
        *freq = max(target_freq, stat->current_frequency);
        return 0;
    }

    /* Set the desired frequency based on the load */
    a = stat->busy_time;
    a *= stat->current_frequency;
    b = div_u64(a, stat->total_time);
    b *= 0x64;
    b = div_u64(b, (upthreshold - downdifferential / 0x2));
    *freq = max_t(unsigned long, target_freq, b);

    return 0;

reset_last_status:
    reset_last_status(df);

    return 0;
}

static int devfreq_dmc_ondemand_handler(struct devfreq *devfreq, unsigned int event, void *data)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(devfreq->dev.parent);

    if (!dmcfreq->info.auto_freq_en) {
        return 0;
    }

    switch (event) {
        case DEVFREQ_GOV_START:
            devfreq_monitor_start(devfreq);
            break;

        case DEVFREQ_GOV_STOP:
            devfreq_monitor_stop(devfreq);
            break;

        case DEVFREQ_GOV_UPDATE_INTERVAL:
            devfreq_update_interval(devfreq, (unsigned int *)data);
            break;

        case DEVFREQ_GOV_SUSPEND:
            devfreq_monitor_suspend(devfreq);
            break;

        case DEVFREQ_GOV_RESUME:
            devfreq_monitor_resume(devfreq);
            break;

        default:
            break;
    }

    return 0;
}

static struct devfreq_governor devfreq_dmc_ondemand = {
    .name = "dmc_ondemand",
    .get_target_freq = devfreq_dmc_ondemand_func,
    .event_handler = devfreq_dmc_ondemand_handler,
};

static int rockchip_dmcfreq_enable_event(struct rockchip_dmcfreq *dmcfreq)
{
    int i, ret;

    if (!dmcfreq->info.auto_freq_en) {
        return 0;
    }

    for (i = 0; i < dmcfreq->edev_count; i++) {
        ret = devfreq_event_enable_edev(dmcfreq->edev[i]);
        if (ret < 0) {
            dev_err(dmcfreq->dev, "failed to enable devfreq-event\n");
            return ret;
        }
    }

    return 0;
}

static int rockchip_dmcfreq_disable_event(struct rockchip_dmcfreq *dmcfreq)
{
    int i, ret;

    if (!dmcfreq->info.auto_freq_en) {
        return 0;
    }

    for (i = 0; i < dmcfreq->edev_count; i++) {
        ret = devfreq_event_disable_edev(dmcfreq->edev[i]);
        if (ret < 0) {
            dev_err(dmcfreq->dev, "failed to disable devfreq-event\n");
            return ret;
        }
    }

    return 0;
}

static int rockchip_get_edev_id(struct rockchip_dmcfreq *dmcfreq, const char *name)
{
    struct devfreq_event_dev *edev;
    int i;

    for (i = 0; i < dmcfreq->edev_count; i++) {
        edev = dmcfreq->edev[i];
        if (!strcmp(edev->desc->name, name)) {
            return i;
        }
    }

    return -EINVAL;
}

static int rockchip_dmcfreq_get_event(struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = dmcfreq->dev;
    struct device_node *events_np, *np = dev->of_node;
    int i, j, count, available_count = 0;

    count = devfreq_event_get_edev_count(dev, "devfreq-events");
    if (count < 0) {
        dev_dbg(dev, "failed to get count of devfreq-event dev\n");
        return 0;
    }
    for (i = 0; i < count; i++) {
        events_np = of_parse_phandle(np, "devfreq-events", i);
        if (!events_np) {
            continue;
        }
        if (of_device_is_available(events_np)) {
            available_count++;
        }
        of_node_put(events_np);
    }
    if (!available_count) {
        dev_dbg(dev, "failed to get available devfreq-event\n");
        return 0;
    }
    dmcfreq->edev_count = available_count;
    dmcfreq->edev = devm_kzalloc(dev, sizeof(*dmcfreq->edev) * available_count, GFP_KERNEL);
    if (!dmcfreq->edev) {
        return -ENOMEM;
    }

    for (i = 0, j = 0; i < count; i++) {
        events_np = of_parse_phandle(np, "devfreq-events", i);
        if (!events_np) {
            continue;
        }
        if (of_device_is_available(events_np)) {
            of_node_put(events_np);
            if (j >= available_count) {
                dev_err(dev, "invalid event conut\n");
                return -EINVAL;
            }
            dmcfreq->edev[j] = devfreq_event_get_edev_by_phandle(dev, "devfreq-events", i);
            if (IS_ERR(dmcfreq->edev[j])) {
                return -EPROBE_DEFER;
            }
            j++;
        } else {
            of_node_put(events_np);
        }
    }
    dmcfreq->info.auto_freq_en = true;
    dmcfreq->dfi_id = rockchip_get_edev_id(dmcfreq, "dfi");
    dmcfreq->nocp_cpu_id = rockchip_get_edev_id(dmcfreq, "nocp-cpu");
    dmcfreq->nocp_bw = devm_kzalloc(dev, sizeof(*dmcfreq->nocp_bw) * available_count, GFP_KERNEL);
    if (!dmcfreq->nocp_bw) {
        return -ENOMEM;
    }

    return 0;
}

static int rockchip_dmcfreq_power_control(struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = dmcfreq->dev;

    dmcfreq->vdd_center = devm_regulator_get_optional(dev, "center");
    if (IS_ERR(dmcfreq->vdd_center)) {
        dev_err(dev, "Cannot get the regulator \"center\"\n");
        return PTR_ERR(dmcfreq->vdd_center);
    }

    dmcfreq->dmc_clk = devm_clk_get(dev, "dmc_clk");
    if (IS_ERR(dmcfreq->dmc_clk)) {
        dev_err(dev, "Cannot get the clk dmc_clk. If using SCMI, trusted firmware need update to V1.01 and above.\n");
        return PTR_ERR(dmcfreq->dmc_clk);
    }
    dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk);

    return 0;
}

static int rockchip_dmcfreq_dmc_init(struct platform_device *pdev, struct rockchip_dmcfreq *dmcfreq)
{
    const struct of_device_id *match;
    int (*init)(struct platform_device * pdev, struct rockchip_dmcfreq * data);
    int ret;

    match = of_match_node(rockchip_dmcfreq_of_match, pdev->dev.of_node);
    if (match) {
        init = match->data;
        if (init) {
            ret = init(pdev, dmcfreq);
            if (ret) {
                return ret;
            }
        }
    }

    return 0;
}

static void rockchip_dmcfreq_parse_dt(struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = dmcfreq->dev;
    struct device_node *np = dev->of_node;

    if (!rockchip_get_system_status_rate(np, "system-status-freq", dmcfreq)) {
        dmcfreq->system_status_en = true;
    } else if (!rockchip_get_system_status_level(np, "system-status-level", dmcfreq)) {
        dmcfreq->system_status_en = true;
    }

    of_property_read_u32(np, "min-cpu-freq", &dmcfreq->min_cpu_freq);

    of_property_read_u32(np, "upthreshold", &dmcfreq->ondemand_data.upthreshold);
    of_property_read_u32(np, "downdifferential", &dmcfreq->ondemand_data.downdifferential);
    if (dmcfreq->info.auto_freq_en) {
        of_property_read_u32(np, "auto-freq-en", &dmcfreq->info.auto_freq_en);
    }
    if (!dmcfreq->auto_min_rate) {
        of_property_read_u32(np, "auto-min-freq", (u32 *)&dmcfreq->auto_min_rate);
        dmcfreq->auto_min_rate *= 0x3e8;
    }

    if (rockchip_get_freq_map_talbe(np, "cpu-bw-dmc-freq", &dmcfreq->cpu_bw_tbl)) {
        dev_dbg(dev, "failed to get cpu bandwidth to dmc rate\n");
    }
    if (rockchip_get_freq_map_talbe(np, "vop-frame-bw-dmc-freq", &dmcfreq->info.vop_frame_bw_tbl)) {
        dev_dbg(dev, "failed to get vop frame bandwidth to dmc rate\n");
    }
    if (rockchip_get_freq_map_talbe(np, "vop-bw-dmc-freq", &dmcfreq->info.vop_bw_tbl)) {
        dev_err(dev, "failed to get vop bandwidth to dmc rate\n");
    }
    if (rockchip_get_rl_map_talbe(np, "vop-pn-msch-readlatency", &dmcfreq->info.vop_pn_rl_tbl)) {
        dev_err(dev, "failed to get vop pn to msch rl\n");
    }

    of_property_read_u32(np, "touchboost_duration", (u32 *)&dmcfreq->touchboostpulse_duration_val);
    if (dmcfreq->touchboostpulse_duration_val) {
        dmcfreq->touchboostpulse_duration_val *= USEC_PER_MSEC;
    } else {
        dmcfreq->touchboostpulse_duration_val = 0x1f4 * USEC_PER_MSEC;
    }
}

static int rockchip_dmcfreq_set_volt_only(struct rockchip_dmcfreq *dmcfreq)
{
    struct device *dev = dmcfreq->dev;
    struct dev_pm_opp *opp;
    unsigned long opp_volt, opp_rate = dmcfreq->rate;
    int ret;

    opp = devfreq_recommended_opp(dev, &opp_rate, 0);
    if (IS_ERR(opp)) {
        dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate);
        return PTR_ERR(opp);
    }
    opp_volt = dev_pm_opp_get_voltage(opp);
    dev_pm_opp_put(opp);

    ret = regulator_set_voltage(dmcfreq->vdd_center, opp_volt, INT_MAX);
    if (ret) {
        dev_err(dev, "Cannot set voltage %lu uV\n", opp_volt);
        return ret;
    }

    return 0;
}

static int rockchip_dmcfreq_add_devfreq(struct rockchip_dmcfreq *dmcfreq)
{
    struct devfreq_dev_profile *devp = &rockchip_devfreq_dmc_profile;
    struct device *dev = dmcfreq->dev;
    struct dev_pm_opp *opp;
    struct devfreq *devfreq;
    unsigned long opp_rate = dmcfreq->rate;

    opp = devfreq_recommended_opp(dev, &opp_rate, 0);
    if (IS_ERR(opp)) {
        dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate);
        return PTR_ERR(opp);
    }
    dev_pm_opp_put(opp);

    devp->initial_freq = dmcfreq->rate;
    devfreq = devm_devfreq_add_device(dev, devp, "dmc_ondemand", &dmcfreq->ondemand_data);
    if (IS_ERR(devfreq)) {
        dev_err(dev, "failed to add devfreq\n");
        return PTR_ERR(devfreq);
    }

    devm_devfreq_register_opp_notifier(dev, devfreq);

    devfreq->last_status.current_frequency = opp_rate;

    reset_last_status(devfreq);

    dmcfreq->info.devfreq = devfreq;

    return 0;
}

static struct monitor_dev_profile dmc_mdevp = {
    .type = MONITOR_TPYE_DEV,
    .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
    .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
};

static void rockchip_dmcfreq_register_notifier(struct rockchip_dmcfreq *dmcfreq)
{
    int ret;

    if (vop_register_dmc()) {
        dev_err(dmcfreq->dev, "fail to register notify to vop.\n");
    }

    dmcfreq->status_nb.notifier_call = rockchip_dmcfreq_system_status_notifier;
    ret = rockchip_register_system_status_notifier(&dmcfreq->status_nb);
    if (ret) {
        dev_err(dmcfreq->dev, "failed to register system_status nb\n");
    }

    dmc_mdevp.data = dmcfreq->info.devfreq;
    dmcfreq->mdev_info = rockchip_system_monitor_register(dmcfreq->dev, &dmc_mdevp);
    if (IS_ERR(dmcfreq->mdev_info)) {
        dev_dbg(dmcfreq->dev, "without without system monitor\n");
        dmcfreq->mdev_info = NULL;
    }
}

static void rockchip_dmcfreq_add_interface(struct rockchip_dmcfreq *dmcfreq)
{
    struct devfreq *devfreq = dmcfreq->info.devfreq;

    if (sysfs_create_file(&devfreq->dev.kobj, &dev_attr_upthreshold.attr)) {
        dev_err(dmcfreq->dev, "failed to register upthreshold sysfs file\n");
    }
    if (sysfs_create_file(&devfreq->dev.kobj, &dev_attr_downdifferential.attr)) {
        dev_err(dmcfreq->dev, "failed to register downdifferential sysfs file\n");
    }

    if (!rockchip_add_system_status_interface(&devfreq->dev)) {
        return;
    }
    if (sysfs_create_file(&devfreq->dev.kobj, &dev_attr_system_status.attr)) {
        dev_err(dmcfreq->dev, "failed to register system_status sysfs file\n");
    }
}

static void rockchip_dmcfreq_boost_work(struct work_struct *work)
{
    struct rockchip_dmcfreq *dmcfreq = boost_to_dmcfreq(work);

    rockchip_dmcfreq_update_target(dmcfreq);
}

static void rockchip_dmcfreq_input_event(struct input_handle *handle, unsigned int type, unsigned int code, int value)
{
    struct rockchip_dmcfreq *dmcfreq = handle->private;
    u64 now, endtime;

    if (type != EV_ABS && type != EV_KEY) {
        return;
    }

    now = ktime_to_us(ktime_get());
    endtime = now + dmcfreq->touchboostpulse_duration_val;
    if (endtime < (dmcfreq->touchboostpulse_endtime + 0xa * USEC_PER_MSEC)) {
        return;
    }
    dmcfreq->touchboostpulse_endtime = endtime;

    schedule_work(&dmcfreq->boost_work);
}

static int rockchip_dmcfreq_input_connect(struct input_handler *handler, struct input_dev *dev,
                                          const struct input_device_id *id)
{
    int error;
    struct input_handle *handle;
    struct rockchip_dmcfreq *dmcfreq = input_hd_to_dmcfreq(handler);

    handle = kzalloc(sizeof(*handle), GFP_KERNEL);
    if (!handle) {
        return -ENOMEM;
    }

    handle->dev = dev;
    handle->handler = handler;
    handle->name = "dmcfreq";
    handle->private = dmcfreq;

    error = input_register_handle(handle);
    if (error) {
        goto err2;
    }

    error = input_open_device(handle);
    if (error) {
        goto err1;
    }

    return 0;
err1:
    input_unregister_handle(handle);
err2:
    kfree(handle);
    return error;
}

static void rockchip_dmcfreq_input_disconnect(struct input_handle *handle)
{
    input_close_device(handle);
    input_unregister_handle(handle);
    kfree(handle);
}

static const struct input_device_id rockchip_dmcfreq_input_ids[] = {
    {
        .flags = INPUT_DEVICE_ID_MATCH_EVBIT | INPUT_DEVICE_ID_MATCH_ABSBIT,
        .evbit = {BIT_MASK(EV_ABS)},
        .absbit = {[BIT_WORD(ABS_MT_POSITION_X)] = BIT_MASK(ABS_MT_POSITION_X) | BIT_MASK(ABS_MT_POSITION_Y)},
    },
    {
        .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | INPUT_DEVICE_ID_MATCH_ABSBIT,
        .keybit = {[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH)},
        .absbit = {[BIT_WORD(ABS_X)] = BIT_MASK(ABS_X) | BIT_MASK(ABS_Y)},
    },
    {
        .flags = INPUT_DEVICE_ID_MATCH_EVBIT,
        .evbit = {BIT_MASK(EV_KEY)},
    },
    {},
};

static void rockchip_dmcfreq_boost_init(struct rockchip_dmcfreq *dmcfreq)
{
    if (!dmcfreq->boost_rate) {
        return;
    }
    INIT_WORK(&dmcfreq->boost_work, rockchip_dmcfreq_boost_work);
    dmcfreq->input_handler.event = rockchip_dmcfreq_input_event;
    dmcfreq->input_handler.connect = rockchip_dmcfreq_input_connect;
    dmcfreq->input_handler.disconnect = rockchip_dmcfreq_input_disconnect;
    dmcfreq->input_handler.name = "dmcfreq";
    dmcfreq->input_handler.id_table = rockchip_dmcfreq_input_ids;
    if (input_register_handler(&dmcfreq->input_handler)) {
        dev_err(dmcfreq->dev, "failed to register input handler\n");
    }
}

static unsigned long model_static_power(struct devfreq *devfreq, unsigned long voltage)
{
    struct device *dev = devfreq->dev.parent;
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);

    int temperature;
    unsigned long temp;
    unsigned long temp_squared, temp_cubed, temp_scaling_factor;
    const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 0xa;

    if (!IS_ERR_OR_NULL(dmcfreq->ddr_tz) && dmcfreq->ddr_tz->ops->get_temp) {
        int ret;

        ret = dmcfreq->ddr_tz->ops->get_temp(dmcfreq->ddr_tz, &temperature);
        if (ret) {
            dev_warn_ratelimited(dev, "failed to read temp for ddr thermal zone: %d\n", ret);
            temperature = FALLBACK_STATIC_TEMPERATURE;
        }
    } else {
        temperature = FALLBACK_STATIC_TEMPERATURE;
    }

    /*
     * Calculate the temperature scaling factor. To be applied to the
     * voltage scaled power.
     */
    temp = temperature / 0x3e8;
    temp_squared = temp * temp;
    temp_cubed = temp_squared * temp;
    temp_scaling_factor = (dmcfreq->ts[0x3] * temp_cubed) + (dmcfreq->ts[0x2] * temp_squared) +
                          (dmcfreq->ts[0x1] * temp) + dmcfreq->ts[0x0];

    return (((dmcfreq->static_coefficient * voltage_cubed) >> 0x14) * temp_scaling_factor) / 0xf4240;
}

static struct devfreq_cooling_power ddr_cooling_power_data = {
    .get_static_power = model_static_power,
    .dyn_power_coeff = 0x78,
};

static int ddr_power_model_simple_init(struct rockchip_dmcfreq *dmcfreq)
{
    struct device_node *power_model_node;
    const char *tz_name;
    u32 temp;

    power_model_node = of_get_child_by_name(dmcfreq->dev->of_node, "ddr_power_model");
    if (!power_model_node) {
        dev_err(dmcfreq->dev, "could not find power_model node\n");
        return -ENODEV;
    }

    if (of_property_read_string(power_model_node, "thermal-zone", &tz_name)) {
        dev_err(dmcfreq->dev, "ts in power_model not available\n");
        return -EINVAL;
    }

    dmcfreq->ddr_tz = thermal_zone_get_zone_by_name(tz_name);
    if (IS_ERR(dmcfreq->ddr_tz)) {
        pr_warn_ratelimited("Error getting ddr thermal zone (%ld), not yet ready?\n", PTR_ERR(dmcfreq->ddr_tz));
        dmcfreq->ddr_tz = NULL;

        return -EPROBE_DEFER;
    }

    if (of_property_read_u32(power_model_node, "static-power-coefficient", &dmcfreq->static_coefficient)) {
        dev_err(dmcfreq->dev, "static-power-coefficient not available\n");
        return -EINVAL;
    }
    if (of_property_read_u32(power_model_node, "dynamic-power-coefficient", &temp)) {
        dev_err(dmcfreq->dev, "dynamic-power-coefficient not available\n");
        return -EINVAL;
    }
    ddr_cooling_power_data.dyn_power_coeff = (unsigned long)temp;

    if (of_property_read_u32_array(power_model_node, "ts", (u32 *)dmcfreq->ts, 0x4)) {
        dev_err(dmcfreq->dev, "ts in power_model not available\n");
        return -EINVAL;
    }

    return 0;
}

static void rockchip_dmcfreq_register_cooling_device(struct rockchip_dmcfreq *dmcfreq)
{
    int ret;

    ret = ddr_power_model_simple_init(dmcfreq);
    if (ret) {
        return;
    }
    dmcfreq->devfreq_cooling =
        of_devfreq_cooling_register_power(dmcfreq->dev->of_node, dmcfreq->info.devfreq, &ddr_cooling_power_data);
    if (IS_ERR(dmcfreq->devfreq_cooling)) {
        ret = PTR_ERR(dmcfreq->devfreq_cooling);
        dev_err(dmcfreq->dev, "Failed to register cooling device (%d)\n", ret);
    }
}

static int rockchip_dmcfreq_probe(struct platform_device *pdev)
{
    struct device *dev = &pdev->dev;
    struct rockchip_dmcfreq *data;
    int ret;

    data = devm_kzalloc(dev, sizeof(struct rockchip_dmcfreq), GFP_KERNEL);
    if (!data) {
        return -ENOMEM;
    }

    data->dev = dev;
    data->info.dev = dev;
    mutex_init(&data->lock);
    INIT_LIST_HEAD(&data->video_info_list);

    ret = rockchip_dmcfreq_get_event(data);
    if (ret) {
        return ret;
    }

    ret = rockchip_dmcfreq_power_control(data);
    if (ret) {
        return ret;
    }

    ret = rockchip_init_opp_table(dev, NULL, "ddr_leakage", "center");
    if (ret) {
        return ret;
    }

    ret = rockchip_dmcfreq_dmc_init(pdev, data);
    if (ret) {
        return ret;
    }

    rockchip_dmcfreq_parse_dt(data);
    if (!data->system_status_en && !data->info.auto_freq_en) {
        dev_info(dev, "don't add devfreq feature\n");
        return rockchip_dmcfreq_set_volt_only(data);
    }

    cpu_latency_qos_add_request(&pm_qos, PM_QOS_DEFAULT_VALUE);
    platform_set_drvdata(pdev, data);

    ret = devfreq_add_governor(&devfreq_dmc_ondemand);
    if (ret) {
        return ret;
    }
    ret = rockchip_dmcfreq_enable_event(data);
    if (ret) {
        return ret;
    }
    ret = rockchip_dmcfreq_add_devfreq(data);
    if (ret) {
        rockchip_dmcfreq_disable_event(data);
        return ret;
    }

    rockchip_dmcfreq_register_notifier(data);
    rockchip_dmcfreq_add_interface(data);
    rockchip_dmcfreq_boost_init(data);
    rockchip_dmcfreq_vop_bandwidth_init(&data->info);
    rockchip_dmcfreq_register_cooling_device(data);

    rockchip_set_system_status(SYS_STATUS_NORMAL);

    return 0;
}

static __maybe_unused int rockchip_dmcfreq_suspend(struct device *dev)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);
    int ret = 0;

    if (!dmcfreq) {
        return 0;
    }

    ret = rockchip_dmcfreq_disable_event(dmcfreq);
    if (ret) {
        return ret;
    }

    ret = devfreq_suspend_device(dmcfreq->info.devfreq);
    if (ret < 0) {
        dev_err(dev, "failed to suspend the devfreq devices\n");
        return ret;
    }

    return 0;
}

static __maybe_unused int rockchip_dmcfreq_resume(struct device *dev)
{
    struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev);
    int ret = 0;

    if (!dmcfreq) {
        return 0;
    }

    ret = rockchip_dmcfreq_enable_event(dmcfreq);
    if (ret) {
        return ret;
    }

    ret = devfreq_resume_device(dmcfreq->info.devfreq);
    if (ret < 0) {
        dev_err(dev, "failed to resume the devfreq devices\n");
        return ret;
    }
    return ret;
}

static SIMPLE_DEV_PM_OPS(rockchip_dmcfreq_pm, rockchip_dmcfreq_suspend, rockchip_dmcfreq_resume);
static struct platform_driver rockchip_dmcfreq_driver = {
    .probe = rockchip_dmcfreq_probe,
    .driver =
        {
            .name = "rockchip-dmc",
            .pm = &rockchip_dmcfreq_pm,
            .of_match_table = rockchip_dmcfreq_of_match,
        },
};
module_platform_driver(rockchip_dmcfreq_driver);

MODULE_AUTHOR("Finley Xiao <finley.xiao@rock-chips.com>");
MODULE_DESCRIPTION("rockchip dmcfreq driver with devfreq framework");
MODULE_LICENSE("GPL v2");