drivers/ram/rockchip/sdram_rk3399.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * (C) Copyright 2016-2017 Rockchip Inc.
   4  *
   5  * Adapted from coreboot.
   6  */
   7
   8 #include <common.h>
   9 #include <clk.h>
  10 #include <dm.h>
  11 #include <dt-structs.h>
  12 #include <ram.h>
  13 #include <regmap.h>
  14 #include <syscon.h>
  15 #include <asm/io.h>
  16 #include <asm/arch-rockchip/clock.h>
  17 #include <asm/arch-rockchip/cru_rk3399.h>
  18 #include <asm/arch-rockchip/grf_rk3399.h>
  19 #include <asm/arch-rockchip/pmu_rk3399.h>
  20 #include <asm/arch-rockchip/hardware.h>
  21 #include <asm/arch-rockchip/sdram_common.h>
  22 #include <asm/arch-rockchip/sdram_rk3399.h>
  23 #include <linux/err.h>
  24 #include <time.h>
  25
  26 #define PRESET_SGRF_HOLD(n)     ((0x1 << (6 + 16)) | ((n) << 6))
  27 #define PRESET_GPIO0_HOLD(n)    ((0x1 << (7 + 16)) | ((n) << 7))
  28 #define PRESET_GPIO1_HOLD(n)    ((0x1 << (8 + 16)) | ((n) << 8))
  29
  30 #define PHY_DRV_ODT_HI_Z        0x0
  31 #define PHY_DRV_ODT_240         0x1
  32 #define PHY_DRV_ODT_120         0x8
  33 #define PHY_DRV_ODT_80          0x9
  34 #define PHY_DRV_ODT_60          0xc
  35 #define PHY_DRV_ODT_48          0xd
  36 #define PHY_DRV_ODT_40          0xe
  37 #define PHY_DRV_ODT_34_3        0xf
  38
  39 #define PHY_BOOSTP_EN           0x1
  40 #define PHY_BOOSTN_EN           0x1
  41 #define PHY_SLEWP_EN            0x1
  42 #define PHY_SLEWN_EN            0x1
  43 #define PHY_RX_CM_INPUT         0x1
  44 #define CS0_MR22_VAL            0
  45 #define CS1_MR22_VAL            3
  46
  47 #define CRU_SFTRST_DDR_CTRL(ch, n)      ((0x1 << (8 + 16 + (ch) * 4)) | \
  48                                         ((n) << (8 + (ch) * 4)))
  49 #define CRU_SFTRST_DDR_PHY(ch, n)       ((0x1 << (9 + 16 + (ch) * 4)) | \
  50                                         ((n) << (9 + (ch) * 4)))
  51 struct chan_info {
  52         struct rk3399_ddr_pctl_regs *pctl;
  53         struct rk3399_ddr_pi_regs *pi;
  54         struct rk3399_ddr_publ_regs *publ;
  55         struct rk3399_msch_regs *msch;
  56 };
  57
  58 struct dram_info {
  59 #if defined(CONFIG_TPL_BUILD) || \
  60         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  61         u32 pwrup_srefresh_exit[2];
  62         struct chan_info chan[2];
  63         struct clk ddr_clk;
  64         struct rk3399_cru *cru;
  65         struct rk3399_grf_regs *grf;
  66         struct rk3399_pmu_regs *pmu;
  67         struct rk3399_pmucru *pmucru;
  68         struct rk3399_pmusgrf_regs *pmusgrf;
  69         struct rk3399_ddr_cic_regs *cic;
  70         const struct sdram_rk3399_ops *ops;
  71 #endif
  72         struct ram_info info;
  73         struct rk3399_pmugrf_regs *pmugrf;
  74 };
  75
  76 struct sdram_rk3399_ops {
  77         int (*data_training)(struct dram_info *dram, u32 channel, u8 rank,
  78                              struct rk3399_sdram_params *sdram);
  79         int (*set_rate)(struct dram_info *dram,
  80                         struct rk3399_sdram_params *params);
  81 };
  82
  83 #if defined(CONFIG_TPL_BUILD) || \
  84         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  85
  86 struct rockchip_dmc_plat {
  87 #if CONFIG_IS_ENABLED(OF_PLATDATA)
  88         struct dtd_rockchip_rk3399_dmc dtplat;
  89 #else
  90         struct rk3399_sdram_params sdram_params;
  91 #endif
  92         struct regmap *map;
  93 };
  94
  95 struct io_setting {
  96         u32 mhz;
  97         u32 mr5;
  98         /* dram side */
  99         u32 dq_odt;
 100         u32 ca_odt;
 101         u32 pdds;
 102         u32 dq_vref;
 103         u32 ca_vref;
 104         /* phy side */
 105         u32 rd_odt;
 106         u32 wr_dq_drv;
 107         u32 wr_ca_drv;
 108         u32 wr_ckcs_drv;
 109         u32 rd_odt_en;
 110         u32 rd_vref;
 111 } lpddr4_io_setting[] = {
 112         {
 113                 50 * MHz,
 114                 0,
 115                 /* dram side */
 116                 0,      /* dq_odt; */
 117                 0,      /* ca_odt; */
 118                 6,      /* pdds; */
 119                 0x72,   /* dq_vref; */
 120                 0x72,   /* ca_vref; */
 121                 /* phy side */
 122                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 123                 PHY_DRV_ODT_40, /* wr_dq_drv; */
 124                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 125                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 126                 0,      /* rd_odt_en;*/
 127                 41,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 128         },
 129         {
 130                 600 * MHz,
 131                 0,
 132                 /* dram side */
 133                 1,      /* dq_odt; */
 134                 0,      /* ca_odt; */
 135                 6,      /* pdds; */
 136                 0x72,   /* dq_vref; */
 137                 0x72,   /* ca_vref; */
 138                 /* phy side */
 139                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 140                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 141                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 142                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 143                 0,      /* rd_odt_en; */
 144                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 145         },
 146         {
 147                 800 * MHz,
 148                 0,
 149                 /* dram side */
 150                 1,      /* dq_odt; */
 151                 0,      /* ca_odt; */
 152                 1,      /* pdds; */
 153                 0x72,   /* dq_vref; */
 154                 0x72,   /* ca_vref; */
 155                 /* phy side */
 156                 PHY_DRV_ODT_40, /* rd_odt; */
 157                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 158                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 159                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 160                 1,      /* rd_odt_en; */
 161                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 162         },
 163         {
 164                 933 * MHz,
 165                 0,
 166                 /* dram side */
 167                 3,      /* dq_odt; */
 168                 0,      /* ca_odt; */
 169                 6,      /* pdds; */
 170                 0x59,   /* dq_vref; 32% */
 171                 0x72,   /* ca_vref; */
 172                 /* phy side */
 173                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 174                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 175                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 176                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 177                 0,      /* rd_odt_en; */
 178                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 179         },
 180         {
 181                 1066 * MHz,
 182                 0,
 183                 /* dram side */
 184                 6,      /* dq_odt; */
 185                 0,      /* ca_odt; */
 186                 1,      /* pdds; */
 187                 0x10,   /* dq_vref; */
 188                 0x72,   /* ca_vref; */
 189                 /* phy side */
 190                 PHY_DRV_ODT_40, /* rd_odt; */
 191                 PHY_DRV_ODT_60, /* wr_dq_drv; */
 192                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 193                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 194                 1,      /* rd_odt_en; */
 195                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 196         },
 197 };
 198
 199 /**
 200  * phy = 0, PHY boot freq
 201  * phy = 1, PHY index 0
 202  * phy = 2, PHY index 1
 203  */
 204 static struct io_setting *
 205 lpddr4_get_io_settings(const struct rk3399_sdram_params *params, u32 mr5)
 206 {
 207         struct io_setting *io = NULL;
 208         u32 n;
 209
 210         for (n = 0; n < ARRAY_SIZE(lpddr4_io_setting); n++) {
 211                 io = &lpddr4_io_setting[n];
 212
 213                 if (io->mr5 != 0) {
 214                         if (io->mhz >= params->base.ddr_freq &&
 215                             io->mr5 == mr5)
 216                                 break;
 217                 } else {
 218                         if (io->mhz >= params->base.ddr_freq)
 219                                 break;
 220                 }
 221         }
 222
 223         return io;
 224 }
 225
 226 static void *get_denali_phy(const struct chan_info *chan,
 227                             struct rk3399_sdram_params *params, bool reg)
 228 {
 229         return reg ? &chan->publ->denali_phy : &params->phy_regs.denali_phy;
 230 }
 231
 232 static void *get_denali_ctl(const struct chan_info *chan,
 233                             struct rk3399_sdram_params *params, bool reg)
 234 {
 235         return reg ? &chan->pctl->denali_ctl : &params->pctl_regs.denali_ctl;
 236 }
 237
 238 static void *get_ddrc0_con(struct dram_info *dram, u8 channel)
 239 {
 240         return (channel == 0) ? &dram->grf->ddrc0_con0 : &dram->grf->ddrc0_con1;
 241 }
 242
 243 static void copy_to_reg(u32 *dest, const u32 *src, u32 n)
 244 {
 245         int i;
 246
 247         for (i = 0; i < n / sizeof(u32); i++) {
 248                 writel(*src, dest);
 249                 src++;
 250                 dest++;
 251         }
 252 }
 253
 254 static void rkclk_ddr_reset(struct rk3399_cru *cru, u32 channel, u32 ctl,
 255                             u32 phy)
 256 {
 257         channel &= 0x1;
 258         ctl &= 0x1;
 259         phy &= 0x1;
 260         writel(CRU_SFTRST_DDR_CTRL(channel, ctl) |
 261                                    CRU_SFTRST_DDR_PHY(channel, phy),
 262                                    &cru->softrst_con[4]);
 263 }
 264
 265 static void phy_pctrl_reset(struct rk3399_cru *cru,  u32 channel)
 266 {
 267         rkclk_ddr_reset(cru, channel, 1, 1);
 268         udelay(10);
 269
 270         rkclk_ddr_reset(cru, channel, 1, 0);
 271         udelay(10);
 272
 273         rkclk_ddr_reset(cru, channel, 0, 0);
 274         udelay(10);
 275 }
 276
 277 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs,
 278                                u32 freq)
 279 {
 280         u32 *denali_phy = ddr_publ_regs->denali_phy;
 281
 282         /* From IP spec, only freq small than 125 can enter dll bypass mode */
 283         if (freq <= 125) {
 284                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 285                 setbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 286                 setbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 287                 setbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 288                 setbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 289
 290                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 291                 setbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 292                 setbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 293                 setbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 294         } else {
 295                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 296                 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 297                 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 298                 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 299                 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 300
 301                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 302                 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 303                 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 304                 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 305         }
 306 }
 307
 308 static void set_memory_map(const struct chan_info *chan, u32 channel,
 309                            const struct rk3399_sdram_params *params)
 310 {
 311         const struct rk3399_sdram_channel *sdram_ch = &params->ch[channel];
 312         u32 *denali_ctl = chan->pctl->denali_ctl;
 313         u32 *denali_pi = chan->pi->denali_pi;
 314         u32 cs_map;
 315         u32 reduc;
 316         u32 row;
 317
 318         /* Get row number from ddrconfig setting */
 319         if (sdram_ch->cap_info.ddrconfig < 2 ||
 320             sdram_ch->cap_info.ddrconfig == 4)
 321                 row = 16;
 322         else if (sdram_ch->cap_info.ddrconfig == 3)
 323                 row = 14;
 324         else
 325                 row = 15;
 326
 327         cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1;
 328         reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1;
 329
 330         /* Set the dram configuration to ctrl */
 331         clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col));
 332         clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24),
 333                         ((3 - sdram_ch->cap_info.bk) << 16) |
 334                         ((16 - row) << 24));
 335
 336         clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16),
 337                         cs_map | (reduc << 16));
 338
 339         /* PI_199 PI_COL_DIFF:RW:0:4 */
 340         clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col));
 341
 342         /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */
 343         clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24),
 344                         ((3 - sdram_ch->cap_info.bk) << 16) |
 345                         ((16 - row) << 24));
 346
 347         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 348                 if (cs_map == 1)
 349                         cs_map = 0x5;
 350                 else if (cs_map == 2)
 351                         cs_map = 0xa;
 352                 else
 353                         cs_map = 0xF;
 354         }
 355
 356         /* PI_41 PI_CS_MAP:RW:24:4 */
 357         clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24);
 358         if (sdram_ch->cap_info.rank == 1 && params->base.dramtype == DDR3)
 359                 writel(0x2EC7FFFF, &denali_pi[34]);
 360 }
 361
 362 static int phy_io_config(const struct chan_info *chan,
 363                          const struct rk3399_sdram_params *params, u32 mr5)
 364 {
 365         u32 *denali_phy = chan->publ->denali_phy;
 366         u32 vref_mode_dq, vref_value_dq, vref_mode_ac, vref_value_ac;
 367         u32 mode_sel;
 368         u32 reg_value;
 369         u32 drv_value, odt_value;
 370         u32 speed;
 371
 372         /* vref setting & mode setting */
 373         if (params->base.dramtype == LPDDR4) {
 374                 struct io_setting *io = lpddr4_get_io_settings(params, mr5);
 375                 u32 rd_vref = io->rd_vref * 1000;
 376
 377                 if (rd_vref < 36700) {
 378                         /* MODE_LV[2:0] = LPDDR4 (Range 2)*/
 379                         vref_mode_dq = 0x7;
 380                         /* MODE[2:0]= LPDDR4 Range 2(0.4*VDDQ) */
 381                         mode_sel = 0x5;
 382                         vref_value_dq = (rd_vref - 3300) / 521;
 383                 } else {
 384                         /* MODE_LV[2:0] = LPDDR4 (Range 1)*/
 385                         vref_mode_dq = 0x6;
 386                         /* MODE[2:0]= LPDDR4 Range 1(0.33*VDDQ) */
 387                         mode_sel = 0x4;
 388                         vref_value_dq = (rd_vref - 15300) / 521;
 389                 }
 390                 vref_mode_ac = 0x6;
 391                 /* VDDQ/3/2=16.8% */
 392                 vref_value_ac = 0x3;
 393         } else if (params->base.dramtype == LPDDR3) {
 394                 if (params->base.odt == 1) {
 395                         vref_mode_dq = 0x5;  /* LPDDR3 ODT */
 396                         drv_value = (readl(&denali_phy[6]) >> 12) & 0xf;
 397                         odt_value = (readl(&denali_phy[6]) >> 4) & 0xf;
 398                         if (drv_value == PHY_DRV_ODT_48) {
 399                                 switch (odt_value) {
 400                                 case PHY_DRV_ODT_240:
 401                                         vref_value_dq = 0x16;
 402                                         break;
 403                                 case PHY_DRV_ODT_120:
 404                                         vref_value_dq = 0x26;
 405                                         break;
 406                                 case PHY_DRV_ODT_60:
 407                                         vref_value_dq = 0x36;
 408                                         break;
 409                                 default:
 410                                         debug("Invalid ODT value.\n");
 411                                         return -EINVAL;
 412                                 }
 413                         } else if (drv_value == PHY_DRV_ODT_40) {
 414                                 switch (odt_value) {
 415                                 case PHY_DRV_ODT_240:
 416                                         vref_value_dq = 0x19;
 417                                         break;
 418                                 case PHY_DRV_ODT_120:
 419                                         vref_value_dq = 0x23;
 420                                         break;
 421                                 case PHY_DRV_ODT_60:
 422                                         vref_value_dq = 0x31;
 423                                         break;
 424                                 default:
 425                                         debug("Invalid ODT value.\n");
 426                                         return -EINVAL;
 427                                 }
 428                         } else if (drv_value == PHY_DRV_ODT_34_3) {
 429                                 switch (odt_value) {
 430                                 case PHY_DRV_ODT_240:
 431                                         vref_value_dq = 0x17;
 432                                         break;
 433                                 case PHY_DRV_ODT_120:
 434                                         vref_value_dq = 0x20;
 435                                         break;
 436                                 case PHY_DRV_ODT_60:
 437                                         vref_value_dq = 0x2e;
 438                                         break;
 439                                 default:
 440                                         debug("Invalid ODT value.\n");
 441                                         return -EINVAL;
 442                                 }
 443                         } else {
 444                                 debug("Invalid DRV value.\n");
 445                                 return -EINVAL;
 446                         }
 447                 } else {
 448                         vref_mode_dq = 0x2;  /* LPDDR3 */
 449                         vref_value_dq = 0x1f;
 450                 }
 451                 vref_mode_ac = 0x2;
 452                 vref_value_ac = 0x1f;
 453                 mode_sel = 0x0;
 454         } else if (params->base.dramtype == DDR3) {
 455                 /* DDR3L */
 456                 vref_mode_dq = 0x1;
 457                 vref_value_dq = 0x1f;
 458                 vref_mode_ac = 0x1;
 459                 vref_value_ac = 0x1f;
 460                 mode_sel = 0x1;
 461         } else {
 462                 debug("Unknown DRAM type.\n");
 463                 return -EINVAL;
 464         }
 465
 466         reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq;
 467
 468         /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */
 469         clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8);
 470         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */
 471         clrsetbits_le32(&denali_phy[914], 0xfff, reg_value);
 472         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */
 473         clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16);
 474         /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */
 475         clrsetbits_le32(&denali_phy[915], 0xfff, reg_value);
 476
 477         reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac;
 478
 479         /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */
 480         clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16);
 481
 482         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 483         clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15);
 484         /* PHY_926 PHY_PAD_DATA_DRIVE */
 485         clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6);
 486         /* PHY_927 PHY_PAD_DQS_DRIVE */
 487         clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6);
 488         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 489         clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14);
 490         /* PHY_929 PHY_PAD_CLK_DRIVE */
 491         clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14);
 492         /* PHY_935 PHY_PAD_CKE_DRIVE */
 493         clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14);
 494         /* PHY_937 PHY_PAD_RST_DRIVE */
 495         clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14);
 496         /* PHY_939 PHY_PAD_CS_DRIVE */
 497         clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14);
 498
 499         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 500                 /* BOOSTP_EN & BOOSTN_EN */
 501                 reg_value = ((PHY_BOOSTP_EN << 4) | PHY_BOOSTN_EN);
 502                 /* PHY_925 PHY_PAD_FDBK_DRIVE2 */
 503                 clrsetbits_le32(&denali_phy[925], 0xff << 8, reg_value << 8);
 504                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 505                 clrsetbits_le32(&denali_phy[926], 0xff << 12, reg_value << 12);
 506                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 507                 clrsetbits_le32(&denali_phy[927], 0xff << 14, reg_value << 14);
 508                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 509                 clrsetbits_le32(&denali_phy[928], 0xff << 20, reg_value << 20);
 510                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 511                 clrsetbits_le32(&denali_phy[929], 0xff << 22, reg_value << 22);
 512                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 513                 clrsetbits_le32(&denali_phy[935], 0xff << 20, reg_value << 20);
 514                 /* PHY_937 PHY_PAD_RST_DRIVE */
 515                 clrsetbits_le32(&denali_phy[937], 0xff << 20, reg_value << 20);
 516                 /* PHY_939 PHY_PAD_CS_DRIVE */
 517                 clrsetbits_le32(&denali_phy[939], 0xff << 20, reg_value << 20);
 518
 519                 /* SLEWP_EN & SLEWN_EN */
 520                 reg_value = ((PHY_SLEWP_EN << 3) | PHY_SLEWN_EN);
 521                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 522                 clrsetbits_le32(&denali_phy[924], 0x3f << 8, reg_value << 8);
 523                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 524                 clrsetbits_le32(&denali_phy[926], 0x3f, reg_value);
 525                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 526                 clrsetbits_le32(&denali_phy[927], 0x3f, reg_value);
 527                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 528                 clrsetbits_le32(&denali_phy[928], 0x3f << 8, reg_value << 8);
 529                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 530                 clrsetbits_le32(&denali_phy[929], 0x3f << 8, reg_value << 8);
 531                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 532                 clrsetbits_le32(&denali_phy[935], 0x3f << 8, reg_value << 8);
 533                 /* PHY_937 PHY_PAD_RST_DRIVE */
 534                 clrsetbits_le32(&denali_phy[937], 0x3f << 8, reg_value << 8);
 535                 /* PHY_939 PHY_PAD_CS_DRIVE */
 536                 clrsetbits_le32(&denali_phy[939], 0x3f << 8, reg_value << 8);
 537         }
 538
 539         /* speed setting */
 540         if (params->base.ddr_freq < 400)
 541                 speed = 0x0;
 542         else if (params->base.ddr_freq < 800)
 543                 speed = 0x1;
 544         else if (params->base.ddr_freq < 1200)
 545                 speed = 0x2;
 546         else
 547                 speed = 0x3;
 548
 549         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 550         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
 551         /* PHY_926 PHY_PAD_DATA_DRIVE */
 552         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
 553         /* PHY_927 PHY_PAD_DQS_DRIVE */
 554         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
 555         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 556         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
 557         /* PHY_929 PHY_PAD_CLK_DRIVE */
 558         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
 559         /* PHY_935 PHY_PAD_CKE_DRIVE */
 560         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
 561         /* PHY_937 PHY_PAD_RST_DRIVE */
 562         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
 563         /* PHY_939 PHY_PAD_CS_DRIVE */
 564         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
 565
 566         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 567                 /* RX_CM_INPUT */
 568                 reg_value = PHY_RX_CM_INPUT;
 569                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 570                 clrsetbits_le32(&denali_phy[924], 0x1 << 14, reg_value << 14);
 571                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 572                 clrsetbits_le32(&denali_phy[926], 0x1 << 11, reg_value << 11);
 573                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 574                 clrsetbits_le32(&denali_phy[927], 0x1 << 13, reg_value << 13);
 575                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 576                 clrsetbits_le32(&denali_phy[928], 0x1 << 19, reg_value << 19);
 577                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 578                 clrsetbits_le32(&denali_phy[929], 0x1 << 21, reg_value << 21);
 579                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 580                 clrsetbits_le32(&denali_phy[935], 0x1 << 19, reg_value << 19);
 581                 /* PHY_937 PHY_PAD_RST_DRIVE */
 582                 clrsetbits_le32(&denali_phy[937], 0x1 << 19, reg_value << 19);
 583                 /* PHY_939 PHY_PAD_CS_DRIVE */
 584                 clrsetbits_le32(&denali_phy[939], 0x1 << 19, reg_value << 19);
 585         }
 586
 587         return 0;
 588 }
 589
 590 static void set_ds_odt(const struct chan_info *chan,
 591                        struct rk3399_sdram_params *params,
 592                        bool ctl_phy_reg, u32 mr5)
 593 {
 594         u32 *denali_phy = get_denali_phy(chan, params, ctl_phy_reg);
 595         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
 596         u32 tsel_idle_en, tsel_wr_en, tsel_rd_en;
 597         u32 tsel_idle_select_p, tsel_rd_select_p;
 598         u32 tsel_idle_select_n, tsel_rd_select_n;
 599         u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p;
 600         u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n;
 601         u32 tsel_ckcs_select_p, tsel_ckcs_select_n;
 602         struct io_setting *io = NULL;
 603         u32 soc_odt = 0;
 604         u32 reg_value;
 605
 606         if (params->base.dramtype == LPDDR4) {
 607                 io = lpddr4_get_io_settings(params, mr5);
 608
 609                 tsel_rd_select_p = PHY_DRV_ODT_HI_Z;
 610                 tsel_rd_select_n = io->rd_odt;
 611
 612                 tsel_idle_select_p = PHY_DRV_ODT_HI_Z;
 613                 tsel_idle_select_n = PHY_DRV_ODT_240;
 614
 615                 tsel_wr_select_dq_p = io->wr_dq_drv;
 616                 tsel_wr_select_dq_n = PHY_DRV_ODT_40;
 617
 618                 tsel_wr_select_ca_p = io->wr_ca_drv;
 619                 tsel_wr_select_ca_n = PHY_DRV_ODT_40;
 620
 621                 tsel_ckcs_select_p = io->wr_ckcs_drv;
 622                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 623                 switch (tsel_rd_select_n) {
 624                 case PHY_DRV_ODT_240:
 625                         soc_odt = 1;
 626                         break;
 627                 case PHY_DRV_ODT_120:
 628                         soc_odt = 2;
 629                         break;
 630                 case PHY_DRV_ODT_80:
 631                         soc_odt = 3;
 632                         break;
 633                 case PHY_DRV_ODT_60:
 634                         soc_odt = 4;
 635                         break;
 636                 case PHY_DRV_ODT_48:
 637                         soc_odt = 5;
 638                         break;
 639                 case PHY_DRV_ODT_40:
 640                         soc_odt = 6;
 641                         break;
 642                 case PHY_DRV_ODT_34_3:
 643                         soc_odt = 6;
 644                         printf("%s: Unable to support LPDDR4 MR22 Soc ODT\n",
 645                                __func__);
 646                         break;
 647                 case PHY_DRV_ODT_HI_Z:
 648                 default:
 649                         soc_odt = 0;
 650                         break;
 651                 }
 652         } else if (params->base.dramtype == LPDDR3) {
 653                 tsel_rd_select_p = PHY_DRV_ODT_240;
 654                 tsel_rd_select_n = PHY_DRV_ODT_HI_Z;
 655
 656                 tsel_idle_select_p = PHY_DRV_ODT_240;
 657                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 658
 659                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 660                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 661
 662                 tsel_wr_select_ca_p = PHY_DRV_ODT_48;
 663                 tsel_wr_select_ca_n = PHY_DRV_ODT_48;
 664
 665                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 666                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 667         } else {
 668                 tsel_rd_select_p = PHY_DRV_ODT_240;
 669                 tsel_rd_select_n = PHY_DRV_ODT_240;
 670
 671                 tsel_idle_select_p = PHY_DRV_ODT_240;
 672                 tsel_idle_select_n = PHY_DRV_ODT_240;
 673
 674                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 675                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 676
 677                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 678                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 679
 680                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 681                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 682         }
 683
 684         if (params->base.odt == 1) {
 685                 tsel_rd_en = 1;
 686
 687                 if (params->base.dramtype == LPDDR4)
 688                         tsel_rd_en = io->rd_odt_en;
 689         } else {
 690                 tsel_rd_en = 0;
 691         }
 692
 693         tsel_wr_en = 0;
 694         tsel_idle_en = 0;
 695
 696         /* F0_0 */
 697         clrsetbits_le32(&denali_ctl[145], 0xFF << 16,
 698                         (soc_odt | (CS0_MR22_VAL << 3)) << 16);
 699         /* F2_0, F1_0 */
 700         clrsetbits_le32(&denali_ctl[146], 0xFF00FF,
 701                         ((soc_odt | (CS0_MR22_VAL << 3)) << 16) |
 702                         (soc_odt | (CS0_MR22_VAL << 3)));
 703         /* F0_1 */
 704         clrsetbits_le32(&denali_ctl[159], 0xFF << 16,
 705                         (soc_odt | (CS1_MR22_VAL << 3)) << 16);
 706         /* F2_1, F1_1 */
 707         clrsetbits_le32(&denali_ctl[160], 0xFF00FF,
 708                         ((soc_odt | (CS1_MR22_VAL << 3)) << 16) |
 709                         (soc_odt | (CS1_MR22_VAL << 3)));
 710
 711         /*
 712          * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0
 713          * sets termination values for read/idle cycles and drive strength
 714          * for write cycles for DQ/DM
 715          */
 716         reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) |
 717                     (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) |
 718                     (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20);
 719         clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value);
 720         clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value);
 721         clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value);
 722         clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value);
 723
 724         /*
 725          * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0
 726          * sets termination values for read/idle cycles and drive strength
 727          * for write cycles for DQS
 728          */
 729         clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value);
 730         clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value);
 731         clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value);
 732         clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value);
 733
 734         /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */
 735         reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4);
 736         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 737                 /* LPDDR4 these register read always return 0, so
 738                  * can not use clrsetbits_le32(), need to write32
 739                  */
 740                 writel((0x300 << 8) | reg_value, &denali_phy[544]);
 741                 writel((0x300 << 8) | reg_value, &denali_phy[672]);
 742                 writel((0x300 << 8) | reg_value, &denali_phy[800]);
 743         } else {
 744                 clrsetbits_le32(&denali_phy[544], 0xff, reg_value);
 745                 clrsetbits_le32(&denali_phy[672], 0xff, reg_value);
 746                 clrsetbits_le32(&denali_phy[800], 0xff, reg_value);
 747         }
 748
 749         /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */
 750         clrsetbits_le32(&denali_phy[928], 0xff, reg_value);
 751
 752         /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */
 753         if (!ctl_phy_reg)
 754                 clrsetbits_le32(&denali_phy[937], 0xff, reg_value);
 755
 756         /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */
 757         clrsetbits_le32(&denali_phy[935], 0xff, reg_value);
 758
 759         /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */
 760         clrsetbits_le32(&denali_phy[939], 0xff,
 761                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 762
 763         /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */
 764         clrsetbits_le32(&denali_phy[929], 0xff,
 765                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 766
 767         /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */
 768         clrsetbits_le32(&denali_phy[924], 0xff,
 769                         tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4));
 770         clrsetbits_le32(&denali_phy[925], 0xff,
 771                         tsel_rd_select_n | (tsel_rd_select_p << 4));
 772
 773         /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */
 774         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 775                 << 16;
 776         clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value);
 777         clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value);
 778         clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value);
 779         clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value);
 780
 781         /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */
 782         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 783                 << 24;
 784         clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value);
 785         clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value);
 786         clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value);
 787         clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value);
 788
 789         /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */
 790         reg_value = tsel_wr_en << 8;
 791         clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value);
 792         clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value);
 793         clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value);
 794
 795         /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */
 796         reg_value = tsel_wr_en << 17;
 797         clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value);
 798         /*
 799          * pad_rst/cke/cs/clk_term tsel 1bits
 800          * DENALI_PHY_938/936/940/934 offset_17
 801          */
 802         clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value);
 803         clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value);
 804         clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value);
 805         clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value);
 806
 807         /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */
 808         clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value);
 809
 810         phy_io_config(chan, params, mr5);
 811 }
 812
 813 static void pctl_start(struct dram_info *dram, u8 channel)
 814 {
 815         const struct chan_info *chan = &dram->chan[channel];
 816         u32 *denali_ctl = chan->pctl->denali_ctl;
 817         u32 *denali_phy = chan->publ->denali_phy;
 818         u32 *ddrc0_con = get_ddrc0_con(dram, channel);
 819         u32 count = 0;
 820         u32 byte, tmp;
 821
 822         writel(0x01000000, &ddrc0_con);
 823
 824         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 0x2 << 24);
 825
 826         while (!(readl(&denali_ctl[203]) & (1 << 3))) {
 827                 if (count > 1000) {
 828                         printf("%s: Failed to init pctl for channel %d\n",
 829                                __func__, channel);
 830                         while (1)
 831                                 ;
 832                 }
 833
 834                 udelay(1);
 835                 count++;
 836         }
 837
 838         writel(0x01000100, &ddrc0_con);
 839
 840         for (byte = 0; byte < 4; byte++) {
 841                 tmp = 0x820;
 842                 writel((tmp << 16) | tmp, &denali_phy[53 + (128 * byte)]);
 843                 writel((tmp << 16) | tmp, &denali_phy[54 + (128 * byte)]);
 844                 writel((tmp << 16) | tmp, &denali_phy[55 + (128 * byte)]);
 845                 writel((tmp << 16) | tmp, &denali_phy[56 + (128 * byte)]);
 846                 writel((tmp << 16) | tmp, &denali_phy[57 + (128 * byte)]);
 847
 848                 clrsetbits_le32(&denali_phy[58 + (128 * byte)], 0xffff, tmp);
 849         }
 850
 851         clrsetbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT,
 852                         dram->pwrup_srefresh_exit[channel]);
 853 }
 854
 855 static int pctl_cfg(struct dram_info *dram, const struct chan_info *chan,
 856                     u32 channel, struct rk3399_sdram_params *params)
 857 {
 858         u32 *denali_ctl = chan->pctl->denali_ctl;
 859         u32 *denali_pi = chan->pi->denali_pi;
 860         u32 *denali_phy = chan->publ->denali_phy;
 861         const u32 *params_ctl = params->pctl_regs.denali_ctl;
 862         const u32 *params_phy = params->phy_regs.denali_phy;
 863         u32 tmp, tmp1, tmp2;
 864
 865         /*
 866          * work around controller bug:
 867          * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed
 868          */
 869         copy_to_reg(&denali_ctl[1], &params_ctl[1],
 870                     sizeof(struct rk3399_ddr_pctl_regs) - 4);
 871         writel(params_ctl[0], &denali_ctl[0]);
 872
 873         /*
 874          * two channel init at the same time, then ZQ Cal Start
 875          * at the same time, it will use the same RZQ, but cannot
 876          * start at the same time.
 877          *
 878          * So, increase tINIT3 for channel 1, will avoid two
 879          * channel ZQ Cal Start at the same time
 880          */
 881         if (params->base.dramtype == LPDDR4 && channel == 1) {
 882                 tmp = ((params->base.ddr_freq * MHz + 999) / 1000);
 883                 tmp1 = readl(&denali_ctl[14]);
 884                 writel(tmp + tmp1, &denali_ctl[14]);
 885         }
 886
 887         copy_to_reg(denali_pi, &params->pi_regs.denali_pi[0],
 888                     sizeof(struct rk3399_ddr_pi_regs));
 889
 890         /* rank count need to set for init */
 891         set_memory_map(chan, channel, params);
 892
 893         writel(params->phy_regs.denali_phy[910], &denali_phy[910]);
 894         writel(params->phy_regs.denali_phy[911], &denali_phy[911]);
 895         writel(params->phy_regs.denali_phy[912], &denali_phy[912]);
 896
 897         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 898                 writel(params->phy_regs.denali_phy[898], &denali_phy[898]);
 899                 writel(params->phy_regs.denali_phy[919], &denali_phy[919]);
 900         }
 901
 902         dram->pwrup_srefresh_exit[channel] = readl(&denali_ctl[68]) &
 903                                              PWRUP_SREFRESH_EXIT;
 904         clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT);
 905
 906         /* PHY_DLL_RST_EN */
 907         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24);
 908
 909         setbits_le32(&denali_pi[0], START);
 910         setbits_le32(&denali_ctl[0], START);
 911
 912         /**
 913          * LPDDR4 use PLL bypass mode for init
 914          * not need to wait for the PLL to lock
 915          */
 916         if (params->base.dramtype != LPDDR4) {
 917                 /* Waiting for phy DLL lock */
 918                 while (1) {
 919                         tmp = readl(&denali_phy[920]);
 920                         tmp1 = readl(&denali_phy[921]);
 921                         tmp2 = readl(&denali_phy[922]);
 922                         if ((((tmp >> 16) & 0x1) == 0x1) &&
 923                             (((tmp1 >> 16) & 0x1) == 0x1) &&
 924                             (((tmp1 >> 0) & 0x1) == 0x1) &&
 925                             (((tmp2 >> 0) & 0x1) == 0x1))
 926                                 break;
 927                 }
 928         }
 929
 930         copy_to_reg(&denali_phy[896], &params_phy[896], (958 - 895) * 4);
 931         copy_to_reg(&denali_phy[0], &params_phy[0], (90 - 0 + 1) * 4);
 932         copy_to_reg(&denali_phy[128], &params_phy[128], (218 - 128 + 1) * 4);
 933         copy_to_reg(&denali_phy[256], &params_phy[256], (346 - 256 + 1) * 4);
 934         copy_to_reg(&denali_phy[384], &params_phy[384], (474 - 384 + 1) * 4);
 935         copy_to_reg(&denali_phy[512], &params_phy[512], (549 - 512 + 1) * 4);
 936         copy_to_reg(&denali_phy[640], &params_phy[640], (677 - 640 + 1) * 4);
 937         copy_to_reg(&denali_phy[768], &params_phy[768], (805 - 768 + 1) * 4);
 938         set_ds_odt(chan, params, true, 0);
 939
 940         /*
 941          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_84/212/340/468 offset_8
 942          * dqs_tsel_wr_end[7:4] add Half cycle
 943          */
 944         tmp = (readl(&denali_phy[84]) >> 8) & 0xff;
 945         clrsetbits_le32(&denali_phy[84], 0xff << 8, (tmp + 0x10) << 8);
 946         tmp = (readl(&denali_phy[212]) >> 8) & 0xff;
 947         clrsetbits_le32(&denali_phy[212], 0xff << 8, (tmp + 0x10) << 8);
 948         tmp = (readl(&denali_phy[340]) >> 8) & 0xff;
 949         clrsetbits_le32(&denali_phy[340], 0xff << 8, (tmp + 0x10) << 8);
 950         tmp = (readl(&denali_phy[468]) >> 8) & 0xff;
 951         clrsetbits_le32(&denali_phy[468], 0xff << 8, (tmp + 0x10) << 8);
 952
 953         /*
 954          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_83/211/339/467 offset_8
 955          * dq_tsel_wr_end[7:4] add Half cycle
 956          */
 957         tmp = (readl(&denali_phy[83]) >> 16) & 0xff;
 958         clrsetbits_le32(&denali_phy[83], 0xff << 16, (tmp + 0x10) << 16);
 959         tmp = (readl(&denali_phy[211]) >> 16) & 0xff;
 960         clrsetbits_le32(&denali_phy[211], 0xff << 16, (tmp + 0x10) << 16);
 961         tmp = (readl(&denali_phy[339]) >> 16) & 0xff;
 962         clrsetbits_le32(&denali_phy[339], 0xff << 16, (tmp + 0x10) << 16);
 963         tmp = (readl(&denali_phy[467]) >> 16) & 0xff;
 964         clrsetbits_le32(&denali_phy[467], 0xff << 16, (tmp + 0x10) << 16);
 965
 966         return 0;
 967 }
 968
 969 static void select_per_cs_training_index(const struct chan_info *chan,
 970                                          u32 rank)
 971 {
 972         u32 *denali_phy = chan->publ->denali_phy;
 973
 974         /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */
 975         if ((readl(&denali_phy[84]) >> 16) & 1) {
 976                 /*
 977                  * PHY_8/136/264/392
 978                  * phy_per_cs_training_index_X 1bit offset_24
 979                  */
 980                 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24);
 981                 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24);
 982                 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24);
 983                 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24);
 984         }
 985 }
 986
 987 static void override_write_leveling_value(const struct chan_info *chan)
 988 {
 989         u32 *denali_ctl = chan->pctl->denali_ctl;
 990         u32 *denali_phy = chan->publ->denali_phy;
 991         u32 byte;
 992
 993         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 994         setbits_le32(&denali_phy[896], 1);
 995
 996         /*
 997          * PHY_8/136/264/392
 998          * phy_per_cs_training_multicast_en_X 1bit offset_16
 999          */
1000         clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16);
1001         clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16);
1002         clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16);
1003         clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16);
1004
1005         for (byte = 0; byte < 4; byte++)
1006                 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16,
1007                                 0x200 << 16);
1008
1009         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
1010         clrbits_le32(&denali_phy[896], 1);
1011
1012         /* CTL_200 ctrlupd_req 1bit offset_8 */
1013         clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8);
1014 }
1015
1016 static int data_training_ca(const struct chan_info *chan, u32 channel,
1017                             const struct rk3399_sdram_params *params)
1018 {
1019         u32 *denali_pi = chan->pi->denali_pi;
1020         u32 *denali_phy = chan->publ->denali_phy;
1021         u32 i, tmp;
1022         u32 obs_0, obs_1, obs_2, obs_err = 0;
1023         u32 rank = params->ch[channel].cap_info.rank;
1024         u32 rank_mask;
1025
1026         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1027         writel(0x00003f7c, (&denali_pi[175]));
1028
1029         if (params->base.dramtype == LPDDR4)
1030                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1031         else
1032                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1033
1034         for (i = 0; i < 4; i++) {
1035                 if (!(rank_mask & (1 << i)))
1036                         continue;
1037
1038                 select_per_cs_training_index(chan, i);
1039
1040                 /* PI_100 PI_CALVL_EN:RW:8:2 */
1041                 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8);
1042
1043                 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */
1044                 clrsetbits_le32(&denali_pi[92],
1045                                 (0x1 << 16) | (0x3 << 24),
1046                                 (0x1 << 16) | (i << 24));
1047
1048                 /* Waiting for training complete */
1049                 while (1) {
1050                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1051                         tmp = readl(&denali_pi[174]) >> 8;
1052                         /*
1053                          * check status obs
1054                          * PHY_532/660/789 phy_adr_calvl_obs1_:0:32
1055                          */
1056                         obs_0 = readl(&denali_phy[532]);
1057                         obs_1 = readl(&denali_phy[660]);
1058                         obs_2 = readl(&denali_phy[788]);
1059                         if (((obs_0 >> 30) & 0x3) ||
1060                             ((obs_1 >> 30) & 0x3) ||
1061                             ((obs_2 >> 30) & 0x3))
1062                                 obs_err = 1;
1063                         if ((((tmp >> 11) & 0x1) == 0x1) &&
1064                             (((tmp >> 13) & 0x1) == 0x1) &&
1065                             (((tmp >> 5) & 0x1) == 0x0) &&
1066                             obs_err == 0)
1067                                 break;
1068                         else if ((((tmp >> 5) & 0x1) == 0x1) ||
1069                                  (obs_err == 1))
1070                                 return -EIO;
1071                 }
1072
1073                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1074                 writel(0x00003f7c, (&denali_pi[175]));
1075         }
1076
1077         clrbits_le32(&denali_pi[100], 0x3 << 8);
1078
1079         return 0;
1080 }
1081
1082 static int data_training_wl(const struct chan_info *chan, u32 channel,
1083                             const struct rk3399_sdram_params *params)
1084 {
1085         u32 *denali_pi = chan->pi->denali_pi;
1086         u32 *denali_phy = chan->publ->denali_phy;
1087         u32 i, tmp;
1088         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1089         u32 rank = params->ch[channel].cap_info.rank;
1090
1091         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1092         writel(0x00003f7c, (&denali_pi[175]));
1093
1094         for (i = 0; i < rank; i++) {
1095                 select_per_cs_training_index(chan, i);
1096
1097                 /* PI_60 PI_WRLVL_EN:RW:8:2 */
1098                 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8);
1099
1100                 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */
1101                 clrsetbits_le32(&denali_pi[59],
1102                                 (0x1 << 8) | (0x3 << 16),
1103                                 (0x1 << 8) | (i << 16));
1104
1105                 /* Waiting for training complete */
1106                 while (1) {
1107                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1108                         tmp = readl(&denali_pi[174]) >> 8;
1109
1110                         /*
1111                          * check status obs, if error maybe can not
1112                          * get leveling done PHY_40/168/296/424
1113                          * phy_wrlvl_status_obs_X:0:13
1114                          */
1115                         obs_0 = readl(&denali_phy[40]);
1116                         obs_1 = readl(&denali_phy[168]);
1117                         obs_2 = readl(&denali_phy[296]);
1118                         obs_3 = readl(&denali_phy[424]);
1119                         if (((obs_0 >> 12) & 0x1) ||
1120                             ((obs_1 >> 12) & 0x1) ||
1121                             ((obs_2 >> 12) & 0x1) ||
1122                             ((obs_3 >> 12) & 0x1))
1123                                 obs_err = 1;
1124                         if ((((tmp >> 10) & 0x1) == 0x1) &&
1125                             (((tmp >> 13) & 0x1) == 0x1) &&
1126                             (((tmp >> 4) & 0x1) == 0x0) &&
1127                             obs_err == 0)
1128                                 break;
1129                         else if ((((tmp >> 4) & 0x1) == 0x1) ||
1130                                  (obs_err == 1))
1131                                 return -EIO;
1132                 }
1133
1134                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1135                 writel(0x00003f7c, (&denali_pi[175]));
1136         }
1137
1138         override_write_leveling_value(chan);
1139         clrbits_le32(&denali_pi[60], 0x3 << 8);
1140
1141         return 0;
1142 }
1143
1144 static int data_training_rg(const struct chan_info *chan, u32 channel,
1145                             const struct rk3399_sdram_params *params)
1146 {
1147         u32 *denali_pi = chan->pi->denali_pi;
1148         u32 *denali_phy = chan->publ->denali_phy;
1149         u32 i, tmp;
1150         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1151         u32 rank = params->ch[channel].cap_info.rank;
1152
1153         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1154         writel(0x00003f7c, (&denali_pi[175]));
1155
1156         for (i = 0; i < rank; i++) {
1157                 select_per_cs_training_index(chan, i);
1158
1159                 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */
1160                 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24);
1161
1162                 /*
1163                  * PI_74 PI_RDLVL_GATE_REQ:WR:16:1
1164                  * PI_RDLVL_CS:RW:24:2
1165                  */
1166                 clrsetbits_le32(&denali_pi[74],
1167                                 (0x1 << 16) | (0x3 << 24),
1168                                 (0x1 << 16) | (i << 24));
1169
1170                 /* Waiting for training complete */
1171                 while (1) {
1172                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1173                         tmp = readl(&denali_pi[174]) >> 8;
1174
1175                         /*
1176                          * check status obs
1177                          * PHY_43/171/299/427
1178                          *     PHY_GTLVL_STATUS_OBS_x:16:8
1179                          */
1180                         obs_0 = readl(&denali_phy[43]);
1181                         obs_1 = readl(&denali_phy[171]);
1182                         obs_2 = readl(&denali_phy[299]);
1183                         obs_3 = readl(&denali_phy[427]);
1184                         if (((obs_0 >> (16 + 6)) & 0x3) ||
1185                             ((obs_1 >> (16 + 6)) & 0x3) ||
1186                             ((obs_2 >> (16 + 6)) & 0x3) ||
1187                             ((obs_3 >> (16 + 6)) & 0x3))
1188                                 obs_err = 1;
1189                         if ((((tmp >> 9) & 0x1) == 0x1) &&
1190                             (((tmp >> 13) & 0x1) == 0x1) &&
1191                             (((tmp >> 3) & 0x1) == 0x0) &&
1192                             obs_err == 0)
1193                                 break;
1194                         else if ((((tmp >> 3) & 0x1) == 0x1) ||
1195                                  (obs_err == 1))
1196                                 return -EIO;
1197                 }
1198
1199                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1200                 writel(0x00003f7c, (&denali_pi[175]));
1201         }
1202
1203         clrbits_le32(&denali_pi[80], 0x3 << 24);
1204
1205         return 0;
1206 }
1207
1208 static int data_training_rl(const struct chan_info *chan, u32 channel,
1209                             const struct rk3399_sdram_params *params)
1210 {
1211         u32 *denali_pi = chan->pi->denali_pi;
1212         u32 i, tmp;
1213         u32 rank = params->ch[channel].cap_info.rank;
1214
1215         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1216         writel(0x00003f7c, (&denali_pi[175]));
1217
1218         for (i = 0; i < rank; i++) {
1219                 select_per_cs_training_index(chan, i);
1220
1221                 /* PI_80 PI_RDLVL_EN:RW:16:2 */
1222                 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16);
1223
1224                 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */
1225                 clrsetbits_le32(&denali_pi[74],
1226                                 (0x1 << 8) | (0x3 << 24),
1227                                 (0x1 << 8) | (i << 24));
1228
1229                 /* Waiting for training complete */
1230                 while (1) {
1231                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1232                         tmp = readl(&denali_pi[174]) >> 8;
1233
1234                         /*
1235                          * make sure status obs not report error bit
1236                          * PHY_46/174/302/430
1237                          *     phy_rdlvl_status_obs_X:16:8
1238                          */
1239                         if ((((tmp >> 8) & 0x1) == 0x1) &&
1240                             (((tmp >> 13) & 0x1) == 0x1) &&
1241                             (((tmp >> 2) & 0x1) == 0x0))
1242                                 break;
1243                         else if (((tmp >> 2) & 0x1) == 0x1)
1244                                 return -EIO;
1245                 }
1246
1247                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1248                 writel(0x00003f7c, (&denali_pi[175]));
1249         }
1250
1251         clrbits_le32(&denali_pi[80], 0x3 << 16);
1252
1253         return 0;
1254 }
1255
1256 static int data_training_wdql(const struct chan_info *chan, u32 channel,
1257                               const struct rk3399_sdram_params *params)
1258 {
1259         u32 *denali_pi = chan->pi->denali_pi;
1260         u32 i, tmp;
1261         u32 rank = params->ch[channel].cap_info.rank;
1262         u32 rank_mask;
1263
1264         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1265         writel(0x00003f7c, (&denali_pi[175]));
1266
1267         if (params->base.dramtype == LPDDR4)
1268                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1269         else
1270                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1271
1272         for (i = 0; i < 4; i++) {
1273                 if (!(rank_mask & (1 << i)))
1274                         continue;
1275
1276                 select_per_cs_training_index(chan, i);
1277
1278                 /*
1279                  * disable PI_WDQLVL_VREF_EN before wdq leveling?
1280                  * PI_181 PI_WDQLVL_VREF_EN:RW:8:1
1281                  */
1282                 clrbits_le32(&denali_pi[181], 0x1 << 8);
1283
1284                 /* PI_124 PI_WDQLVL_EN:RW:16:2 */
1285                 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16);
1286
1287                 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */
1288                 clrsetbits_le32(&denali_pi[121],
1289                                 (0x1 << 8) | (0x3 << 16),
1290                                 (0x1 << 8) | (i << 16));
1291
1292                 /* Waiting for training complete */
1293                 while (1) {
1294                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1295                         tmp = readl(&denali_pi[174]) >> 8;
1296                         if ((((tmp >> 12) & 0x1) == 0x1) &&
1297                             (((tmp >> 13) & 0x1) == 0x1) &&
1298                             (((tmp >> 6) & 0x1) == 0x0))
1299                                 break;
1300                         else if (((tmp >> 6) & 0x1) == 0x1)
1301                                 return -EIO;
1302                 }
1303
1304                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1305                 writel(0x00003f7c, (&denali_pi[175]));
1306         }
1307
1308         clrbits_le32(&denali_pi[124], 0x3 << 16);
1309
1310         return 0;
1311 }
1312
1313 static int data_training(struct dram_info *dram, u32 channel,
1314                          const struct rk3399_sdram_params *params,
1315                          u32 training_flag)
1316 {
1317         struct chan_info *chan = &dram->chan[channel];
1318         u32 *denali_phy = chan->publ->denali_phy;
1319         int ret;
1320
1321         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1322         setbits_le32(&denali_phy[927], (1 << 22));
1323
1324         if (training_flag == PI_FULL_TRAINING) {
1325                 if (params->base.dramtype == LPDDR4) {
1326                         training_flag = PI_WRITE_LEVELING |
1327                                         PI_READ_GATE_TRAINING |
1328                                         PI_READ_LEVELING | PI_WDQ_LEVELING;
1329                 } else if (params->base.dramtype == LPDDR3) {
1330                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
1331                                         PI_READ_GATE_TRAINING;
1332                 } else if (params->base.dramtype == DDR3) {
1333                         training_flag = PI_WRITE_LEVELING |
1334                                         PI_READ_GATE_TRAINING |
1335                                         PI_READ_LEVELING;
1336                 }
1337         }
1338
1339         /* ca training(LPDDR4,LPDDR3 support) */
1340         if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) {
1341                 ret = data_training_ca(chan, channel, params);
1342                 if (ret < 0) {
1343                         debug("%s: data training ca failed\n", __func__);
1344                         return ret;
1345                 }
1346         }
1347
1348         /* write leveling(LPDDR4,LPDDR3,DDR3 support) */
1349         if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) {
1350                 ret = data_training_wl(chan, channel, params);
1351                 if (ret < 0) {
1352                         debug("%s: data training wl failed\n", __func__);
1353                         return ret;
1354                 }
1355         }
1356
1357         /* read gate training(LPDDR4,LPDDR3,DDR3 support) */
1358         if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) {
1359                 ret = data_training_rg(chan, channel, params);
1360                 if (ret < 0) {
1361                         debug("%s: data training rg failed\n", __func__);
1362                         return ret;
1363                 }
1364         }
1365
1366         /* read leveling(LPDDR4,LPDDR3,DDR3 support) */
1367         if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) {
1368                 ret = data_training_rl(chan, channel, params);
1369                 if (ret < 0) {
1370                         debug("%s: data training rl failed\n", __func__);
1371                         return ret;
1372                 }
1373         }
1374
1375         /* wdq leveling(LPDDR4 support) */
1376         if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) {
1377                 ret = data_training_wdql(chan, channel, params);
1378                 if (ret < 0) {
1379                         debug("%s: data training wdql failed\n", __func__);
1380                         return ret;
1381                 }
1382         }
1383
1384         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1385         clrbits_le32(&denali_phy[927], (1 << 22));
1386
1387         return 0;
1388 }
1389
1390 static void set_ddrconfig(const struct chan_info *chan,
1391                           const struct rk3399_sdram_params *params,
1392                           unsigned char channel, u32 ddrconfig)
1393 {
1394         /* only need to set ddrconfig */
1395         struct rk3399_msch_regs *ddr_msch_regs = chan->msch;
1396         unsigned int cs0_cap = 0;
1397         unsigned int cs1_cap = 0;
1398
1399         cs0_cap = (1 << (params->ch[channel].cap_info.cs0_row
1400                         + params->ch[channel].cap_info.col
1401                         + params->ch[channel].cap_info.bk
1402                         + params->ch[channel].cap_info.bw - 20));
1403         if (params->ch[channel].cap_info.rank > 1)
1404                 cs1_cap = cs0_cap >> (params->ch[channel].cap_info.cs0_row
1405                                 - params->ch[channel].cap_info.cs1_row);
1406         if (params->ch[channel].cap_info.row_3_4) {
1407                 cs0_cap = cs0_cap * 3 / 4;
1408                 cs1_cap = cs1_cap * 3 / 4;
1409         }
1410
1411         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
1412         writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8),
1413                &ddr_msch_regs->ddrsize);
1414 }
1415
1416 static void dram_all_config(struct dram_info *dram,
1417                             const struct rk3399_sdram_params *params)
1418 {
1419         u32 sys_reg2 = 0;
1420         u32 sys_reg3 = 0;
1421         unsigned int channel, idx;
1422
1423         sys_reg2 |= SYS_REG_ENC_DDRTYPE(params->base.dramtype);
1424         sys_reg2 |= SYS_REG_ENC_NUM_CH(params->base.num_channels);
1425
1426         for (channel = 0, idx = 0;
1427              (idx < params->base.num_channels) && (channel < 2);
1428              channel++) {
1429                 const struct rk3399_sdram_channel *info = &params->ch[channel];
1430                 struct rk3399_msch_regs *ddr_msch_regs;
1431                 const struct rk3399_msch_timings *noc_timing;
1432
1433                 if (params->ch[channel].cap_info.col == 0)
1434                         continue;
1435                 idx++;
1436                 sys_reg2 |= SYS_REG_ENC_ROW_3_4(info->cap_info.row_3_4, channel);
1437                 sys_reg2 |= SYS_REG_ENC_CHINFO(channel);
1438                 sys_reg2 |= SYS_REG_ENC_RANK(info->cap_info.rank, channel);
1439                 sys_reg2 |= SYS_REG_ENC_COL(info->cap_info.col, channel);
1440                 sys_reg2 |= SYS_REG_ENC_BK(info->cap_info.bk, channel);
1441                 sys_reg2 |= SYS_REG_ENC_BW(info->cap_info.bw, channel);
1442                 sys_reg2 |= SYS_REG_ENC_DBW(info->cap_info.dbw, channel);
1443                 SYS_REG_ENC_CS0_ROW(info->cap_info.cs0_row, sys_reg2, sys_reg3, channel);
1444                 if (info->cap_info.cs1_row)
1445                         SYS_REG_ENC_CS1_ROW(info->cap_info.cs1_row, sys_reg2,
1446                                             sys_reg3, channel);
1447                 sys_reg3 |= SYS_REG_ENC_CS1_COL(info->cap_info.col, channel);
1448                 sys_reg3 |= SYS_REG_ENC_VERSION(DDR_SYS_REG_VERSION);
1449
1450                 ddr_msch_regs = dram->chan[channel].msch;
1451                 noc_timing = &params->ch[channel].noc_timings;
1452                 writel(noc_timing->ddrtiminga0,
1453                        &ddr_msch_regs->ddrtiminga0);
1454                 writel(noc_timing->ddrtimingb0,
1455                        &ddr_msch_regs->ddrtimingb0);
1456                 writel(noc_timing->ddrtimingc0.d32,
1457                        &ddr_msch_regs->ddrtimingc0);
1458                 writel(noc_timing->devtodev0,
1459                        &ddr_msch_regs->devtodev0);
1460                 writel(noc_timing->ddrmode.d32,
1461                        &ddr_msch_regs->ddrmode);
1462
1463                 /**
1464                  * rank 1 memory clock disable (dfi_dram_clk_disable = 1)
1465                  *
1466                  * The hardware for LPDDR4 with
1467                  * - CLK0P/N connect to lower 16-bits
1468                  * - CLK1P/N connect to higher 16-bits
1469                  *
1470                  * dfi dram clk is configured via CLK1P/N, so disabling
1471                  * dfi dram clk will disable the CLK1P/N as well for lpddr4.
1472                  */
1473                 if (params->ch[channel].cap_info.rank == 1 &&
1474                     params->base.dramtype != LPDDR4)
1475                         setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],
1476                                      1 << 17);
1477         }
1478
1479         writel(sys_reg2, &dram->pmugrf->os_reg2);
1480         writel(sys_reg3, &dram->pmugrf->os_reg3);
1481         rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,
1482                      params->base.stride << 10);
1483
1484         /* reboot hold register set */
1485         writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) |
1486                 PRESET_GPIO1_HOLD(1),
1487                 &dram->pmucru->pmucru_rstnhold_con[1]);
1488         clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);
1489 }
1490
1491 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
1492 static int default_data_training(struct dram_info *dram, u32 channel, u8 rank,
1493                                  struct rk3399_sdram_params *params)
1494 {
1495         u8 training_flag = PI_READ_GATE_TRAINING;
1496
1497         /*
1498          * LPDDR3 CA training msut be trigger before
1499          * other training.
1500          * DDR3 is not have CA training.
1501          */
1502
1503         if (params->base.dramtype == LPDDR3)
1504                 training_flag |= PI_CA_TRAINING;
1505
1506         return data_training(dram, channel, params, training_flag);
1507 }
1508
1509 static int switch_to_phy_index1(struct dram_info *dram,
1510                                 struct rk3399_sdram_params *params)
1511 {
1512         u32 channel;
1513         u32 *denali_phy;
1514         u32 ch_count = params->base.num_channels;
1515         int ret;
1516         int i = 0;
1517
1518         writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1,
1519                              1 << 4 | 1 << 2 | 1),
1520                         &dram->cic->cic_ctrl0);
1521         while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {
1522                 mdelay(10);
1523                 i++;
1524                 if (i > 10) {
1525                         debug("index1 frequency change overtime\n");
1526                         return -ETIME;
1527                 }
1528         }
1529
1530         i = 0;
1531         writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);
1532         while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {
1533                 mdelay(10);
1534                 i++;
1535                 if (i > 10) {
1536                         debug("index1 frequency done overtime\n");
1537                         return -ETIME;
1538                 }
1539         }
1540
1541         for (channel = 0; channel < ch_count; channel++) {
1542                 denali_phy = dram->chan[channel].publ->denali_phy;
1543                 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8);
1544                 ret = data_training(dram, channel, params, PI_FULL_TRAINING);
1545                 if (ret < 0) {
1546                         debug("index1 training failed\n");
1547                         return ret;
1548                 }
1549         }
1550
1551         return 0;
1552 }
1553
1554 #else
1555
1556 struct rk3399_sdram_params lpddr4_timings[] = {
1557         #include "sdram-rk3399-lpddr4-400.inc"
1558         #include "sdram-rk3399-lpddr4-800.inc"
1559 };
1560
1561 static void *get_denali_pi(const struct chan_info *chan,
1562                            struct rk3399_sdram_params *params, bool reg)
1563 {
1564         return reg ? &chan->pi->denali_pi : &params->pi_regs.denali_pi;
1565 }
1566
1567 static u32 lpddr4_get_phy(struct rk3399_sdram_params *params, u32 ctl)
1568 {
1569         u32 lpddr4_phy[] = {1, 0, 0xb};
1570
1571         return lpddr4_phy[ctl];
1572 }
1573
1574 static u32 lpddr4_get_ctl(struct rk3399_sdram_params *params, u32 phy)
1575 {
1576         u32 lpddr4_ctl[] = {1, 0, 2};
1577
1578         return lpddr4_ctl[phy];
1579 }
1580
1581 static u32 get_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf)
1582 {
1583         return ((readl(&pmusgrf->soc_con4) >> 10) & 0x1F);
1584 }
1585
1586 static void set_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf, u32 stride)
1587 {
1588         rk_clrsetreg(&pmusgrf->soc_con4, 0x1f << 10, stride << 10);
1589 }
1590
1591 static void set_cap_relate_config(const struct chan_info *chan,
1592                                   struct rk3399_sdram_params *params,
1593                                   unsigned int channel)
1594 {
1595         u32 *denali_ctl = chan->pctl->denali_ctl;
1596         u32 tmp;
1597         struct rk3399_msch_timings *noc_timing;
1598
1599         if (params->base.dramtype == LPDDR3) {
1600                 tmp = (8 << params->ch[channel].cap_info.bw) /
1601                         (8 << params->ch[channel].cap_info.dbw);
1602
1603                 /**
1604                  * memdata_ratio
1605                  * 1 -> 0, 2 -> 1, 4 -> 2
1606                  */
1607                 clrsetbits_le32(&denali_ctl[197], 0x7,
1608                                 (tmp >> 1));
1609                 clrsetbits_le32(&denali_ctl[198], 0x7 << 8,
1610                                 (tmp >> 1) << 8);
1611         }
1612
1613         noc_timing = &params->ch[channel].noc_timings;
1614
1615         /*
1616          * noc timing bw relate timing is 32 bit, and real bw is 16bit
1617          * actually noc reg is setting at function dram_all_config
1618          */
1619         if (params->ch[channel].cap_info.bw == 16 &&
1620             noc_timing->ddrmode.b.mwrsize == 2) {
1621                 if (noc_timing->ddrmode.b.burstsize)
1622                         noc_timing->ddrmode.b.burstsize -= 1;
1623                 noc_timing->ddrmode.b.mwrsize -= 1;
1624                 noc_timing->ddrtimingc0.b.burstpenalty *= 2;
1625                 noc_timing->ddrtimingc0.b.wrtomwr *= 2;
1626         }
1627 }
1628
1629 static u32 calculate_ddrconfig(struct rk3399_sdram_params *params, u32 channel)
1630 {
1631         unsigned int cs0_row = params->ch[channel].cap_info.cs0_row;
1632         unsigned int col = params->ch[channel].cap_info.col;
1633         unsigned int bw = params->ch[channel].cap_info.bw;
1634         u16  ddr_cfg_2_rbc[] = {
1635                 /*
1636                  * [6]    highest bit col
1637                  * [5:3]  max row(14+n)
1638                  * [2]    insertion row
1639                  * [1:0]  col(9+n),col, data bus 32bit
1640                  *
1641                  * highbitcol, max_row, insertion_row,  col
1642                  */
1643                 ((0 << 6) | (2 << 3) | (0 << 2) | 0), /* 0 */
1644                 ((0 << 6) | (2 << 3) | (0 << 2) | 1), /* 1 */
1645                 ((0 << 6) | (1 << 3) | (0 << 2) | 2), /* 2 */
1646                 ((0 << 6) | (0 << 3) | (0 << 2) | 3), /* 3 */
1647                 ((0 << 6) | (2 << 3) | (1 << 2) | 1), /* 4 */
1648                 ((0 << 6) | (1 << 3) | (1 << 2) | 2), /* 5 */
1649                 ((1 << 6) | (0 << 3) | (0 << 2) | 2), /* 6 */
1650                 ((1 << 6) | (1 << 3) | (0 << 2) | 2), /* 7 */
1651         };
1652         u32 i;
1653
1654         col -= (bw == 2) ? 0 : 1;
1655         col -= 9;
1656
1657         for (i = 0; i < 4; i++) {
1658                 if ((col == (ddr_cfg_2_rbc[i] & 0x3)) &&
1659                     (cs0_row <= (((ddr_cfg_2_rbc[i] >> 3) & 0x7) + 14)))
1660                         break;
1661         }
1662
1663         if (i >= 4)
1664                 i = -EINVAL;
1665
1666         return i;
1667 }
1668
1669 /**
1670  * read mr_num mode register
1671  * rank = 1: cs0
1672  * rank = 2: cs1
1673  */
1674 static int read_mr(struct rk3399_ddr_pctl_regs *ddr_pctl_regs, u32 rank,
1675                    u32 mr_num, u32 *buf)
1676 {
1677         s32 timeout = 100;
1678
1679         writel(((1 << 16) | (((rank == 2) ? 1 : 0) << 8) | mr_num) << 8,
1680                &ddr_pctl_regs->denali_ctl[118]);
1681
1682         while (0 == (readl(&ddr_pctl_regs->denali_ctl[203]) &
1683                         ((1 << 21) | (1 << 12)))) {
1684                 udelay(1);
1685
1686                 if (timeout <= 0) {
1687                         printf("%s: pctl timeout!\n", __func__);
1688                         return -ETIMEDOUT;
1689                 }
1690
1691                 timeout--;
1692         }
1693
1694         if (!(readl(&ddr_pctl_regs->denali_ctl[203]) & (1 << 12))) {
1695                 *buf = readl(&ddr_pctl_regs->denali_ctl[119]) & 0xFF;
1696         } else {
1697                 printf("%s: read mr failed with 0x%x status\n", __func__,
1698                        readl(&ddr_pctl_regs->denali_ctl[17]) & 0x3);
1699                 *buf = 0;
1700         }
1701
1702         setbits_le32(&ddr_pctl_regs->denali_ctl[205], (1 << 21) | (1 << 12));
1703
1704         return 0;
1705 }
1706
1707 static int lpddr4_mr_detect(struct dram_info *dram, u32 channel, u8 rank,
1708                             struct rk3399_sdram_params *params)
1709 {
1710         u64 cs0_cap;
1711         u32 stride;
1712         u32 cs = 0, col = 0, bk = 0, bw = 0, row_3_4 = 0;
1713         u32 cs0_row = 0, cs1_row = 0, ddrconfig = 0;
1714         u32 mr5, mr12, mr14;
1715         struct chan_info *chan = &dram->chan[channel];
1716         struct rk3399_ddr_pctl_regs *ddr_pctl_regs = chan->pctl;
1717         void __iomem *addr = NULL;
1718         int ret = 0;
1719         u32 val;
1720
1721         stride = get_ddr_stride(dram->pmusgrf);
1722
1723         if (params->ch[channel].cap_info.col == 0) {
1724                 ret = -EPERM;
1725                 goto end;
1726         }
1727
1728         cs = params->ch[channel].cap_info.rank;
1729         col = params->ch[channel].cap_info.col;
1730         bk = params->ch[channel].cap_info.bk;
1731         bw = params->ch[channel].cap_info.bw;
1732         row_3_4 = params->ch[channel].cap_info.row_3_4;
1733         cs0_row = params->ch[channel].cap_info.cs0_row;
1734         cs1_row = params->ch[channel].cap_info.cs1_row;
1735         ddrconfig = params->ch[channel].cap_info.ddrconfig;
1736
1737         /* 2GB */
1738         params->ch[channel].cap_info.rank = 2;
1739         params->ch[channel].cap_info.col = 10;
1740         params->ch[channel].cap_info.bk = 3;
1741         params->ch[channel].cap_info.bw = 2;
1742         params->ch[channel].cap_info.row_3_4 = 0;
1743         params->ch[channel].cap_info.cs0_row = 15;
1744         params->ch[channel].cap_info.cs1_row = 15;
1745         params->ch[channel].cap_info.ddrconfig = 1;
1746
1747         set_memory_map(chan, channel, params);
1748         params->ch[channel].cap_info.ddrconfig =
1749                         calculate_ddrconfig(params, channel);
1750         set_ddrconfig(chan, params, channel,
1751                       params->ch[channel].cap_info.ddrconfig);
1752         set_cap_relate_config(chan, params, channel);
1753
1754         cs0_cap = (1 << (params->ch[channel].cap_info.bw
1755                         + params->ch[channel].cap_info.col
1756                         + params->ch[channel].cap_info.bk
1757                         + params->ch[channel].cap_info.cs0_row));
1758
1759         if (params->ch[channel].cap_info.row_3_4)
1760                 cs0_cap = cs0_cap * 3 / 4;
1761
1762         if (channel == 0)
1763                 set_ddr_stride(dram->pmusgrf, 0x17);
1764         else
1765                 set_ddr_stride(dram->pmusgrf, 0x18);
1766
1767         /* read and write data to DRAM, avoid be optimized by compiler. */
1768         if (rank == 1)
1769                 addr = (void __iomem *)0x100;
1770         else if (rank == 2)
1771                 addr = (void __iomem *)(cs0_cap + 0x100);
1772
1773         val = readl(addr);
1774         writel(val + 1, addr);
1775
1776         read_mr(ddr_pctl_regs, rank, 5, &mr5);
1777         read_mr(ddr_pctl_regs, rank, 12, &mr12);
1778         read_mr(ddr_pctl_regs, rank, 14, &mr14);
1779
1780         if (mr5 == 0 || mr12 != 0x4d || mr14 != 0x4d) {
1781                 ret = -EINVAL;
1782                 goto end;
1783         }
1784 end:
1785         params->ch[channel].cap_info.rank = cs;
1786         params->ch[channel].cap_info.col = col;
1787         params->ch[channel].cap_info.bk = bk;
1788         params->ch[channel].cap_info.bw = bw;
1789         params->ch[channel].cap_info.row_3_4 = row_3_4;
1790         params->ch[channel].cap_info.cs0_row = cs0_row;
1791         params->ch[channel].cap_info.cs1_row = cs1_row;
1792         params->ch[channel].cap_info.ddrconfig = ddrconfig;
1793
1794         set_ddr_stride(dram->pmusgrf, stride);
1795
1796         return ret;
1797 }
1798
1799 static void set_lpddr4_dq_odt(const struct chan_info *chan,
1800                               struct rk3399_sdram_params *params, u32 ctl,
1801                               bool en, bool ctl_phy_reg, u32 mr5)
1802 {
1803         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1804         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1805         struct io_setting *io;
1806         u32 reg_value;
1807
1808         if (!en)
1809                 return;
1810
1811         io = lpddr4_get_io_settings(params, mr5);
1812
1813         reg_value = io->dq_odt;
1814
1815         switch (ctl) {
1816         case 0:
1817                 clrsetbits_le32(&denali_ctl[139], 0x7 << 24, reg_value << 24);
1818                 clrsetbits_le32(&denali_ctl[153], 0x7 << 24, reg_value << 24);
1819
1820                 clrsetbits_le32(&denali_pi[132], 0x7 << 0, (reg_value << 0));
1821                 clrsetbits_le32(&denali_pi[139], 0x7 << 16, (reg_value << 16));
1822                 clrsetbits_le32(&denali_pi[147], 0x7 << 0, (reg_value << 0));
1823                 clrsetbits_le32(&denali_pi[154], 0x7 << 16, (reg_value << 16));
1824                 break;
1825         case 1:
1826                 clrsetbits_le32(&denali_ctl[140], 0x7 << 0, reg_value << 0);
1827                 clrsetbits_le32(&denali_ctl[154], 0x7 << 0, reg_value << 0);
1828
1829                 clrsetbits_le32(&denali_pi[129], 0x7 << 16, (reg_value << 16));
1830                 clrsetbits_le32(&denali_pi[137], 0x7 << 0, (reg_value << 0));
1831                 clrsetbits_le32(&denali_pi[144], 0x7 << 16, (reg_value << 16));
1832                 clrsetbits_le32(&denali_pi[152], 0x7 << 0, (reg_value << 0));
1833                 break;
1834         case 2:
1835         default:
1836                 clrsetbits_le32(&denali_ctl[140], 0x7 << 8, (reg_value << 8));
1837                 clrsetbits_le32(&denali_ctl[154], 0x7 << 8, (reg_value << 8));
1838
1839                 clrsetbits_le32(&denali_pi[127], 0x7 << 0, (reg_value << 0));
1840                 clrsetbits_le32(&denali_pi[134], 0x7 << 16, (reg_value << 16));
1841                 clrsetbits_le32(&denali_pi[142], 0x7 << 0, (reg_value << 0));
1842                 clrsetbits_le32(&denali_pi[149], 0x7 << 16, (reg_value << 16));
1843                 break;
1844         }
1845 }
1846
1847 static void set_lpddr4_ca_odt(const struct chan_info *chan,
1848                               struct rk3399_sdram_params *params, u32 ctl,
1849                               bool en, bool ctl_phy_reg, u32 mr5)
1850 {
1851         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1852         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1853         struct io_setting *io;
1854         u32 reg_value;
1855
1856         if (!en)
1857                 return;
1858
1859         io = lpddr4_get_io_settings(params, mr5);
1860
1861         reg_value = io->ca_odt;
1862
1863         switch (ctl) {
1864         case 0:
1865                 clrsetbits_le32(&denali_ctl[139], 0x7 << 28, reg_value << 28);
1866                 clrsetbits_le32(&denali_ctl[153], 0x7 << 28, reg_value << 28);
1867
1868                 clrsetbits_le32(&denali_pi[132], 0x7 << 4, reg_value << 4);
1869                 clrsetbits_le32(&denali_pi[139], 0x7 << 20, reg_value << 20);
1870                 clrsetbits_le32(&denali_pi[147], 0x7 << 4, reg_value << 4);
1871                 clrsetbits_le32(&denali_pi[154], 0x7 << 20, reg_value << 20);
1872                 break;
1873         case 1:
1874                 clrsetbits_le32(&denali_ctl[140], 0x7 << 4, reg_value << 4);
1875                 clrsetbits_le32(&denali_ctl[154], 0x7 << 4, reg_value << 4);
1876
1877                 clrsetbits_le32(&denali_pi[129], 0x7 << 20, reg_value << 20);
1878                 clrsetbits_le32(&denali_pi[137], 0x7 << 4, reg_value << 4);
1879                 clrsetbits_le32(&denali_pi[144], 0x7 << 20, reg_value << 20);
1880                 clrsetbits_le32(&denali_pi[152], 0x7 << 4, reg_value << 4);
1881                 break;
1882         case 2:
1883         default:
1884                 clrsetbits_le32(&denali_ctl[140], 0x7 << 12, (reg_value << 12));
1885                 clrsetbits_le32(&denali_ctl[154], 0x7 << 12, (reg_value << 12));
1886
1887                 clrsetbits_le32(&denali_pi[127], 0x7 << 4, reg_value << 4);
1888                 clrsetbits_le32(&denali_pi[134], 0x7 << 20, reg_value << 20);
1889                 clrsetbits_le32(&denali_pi[142], 0x7 << 4, reg_value << 4);
1890                 clrsetbits_le32(&denali_pi[149], 0x7 << 20, reg_value << 20);
1891                 break;
1892         }
1893 }
1894
1895 static void set_lpddr4_MR3(const struct chan_info *chan,
1896                            struct rk3399_sdram_params *params, u32 ctl,
1897                            bool ctl_phy_reg, u32 mr5)
1898 {
1899         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1900         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1901         struct io_setting *io;
1902         u32 reg_value;
1903
1904         io = lpddr4_get_io_settings(params, mr5);
1905
1906         reg_value = ((io->pdds << 3) | 1);
1907
1908         switch (ctl) {
1909         case 0:
1910                 clrsetbits_le32(&denali_ctl[138], 0xFFFF, reg_value);
1911                 clrsetbits_le32(&denali_ctl[152], 0xFFFF, reg_value);
1912
1913                 clrsetbits_le32(&denali_pi[131], 0xFFFF << 16, reg_value << 16);
1914                 clrsetbits_le32(&denali_pi[139], 0xFFFF, reg_value);
1915                 clrsetbits_le32(&denali_pi[146], 0xFFFF << 16, reg_value << 16);
1916                 clrsetbits_le32(&denali_pi[154], 0xFFFF, reg_value);
1917                 break;
1918         case 1:
1919                 clrsetbits_le32(&denali_ctl[138], 0xFFFF << 16,
1920                                 reg_value << 16);
1921                 clrsetbits_le32(&denali_ctl[152], 0xFFFF << 16,
1922                                 reg_value << 16);
1923
1924                 clrsetbits_le32(&denali_pi[129], 0xFFFF, reg_value);
1925                 clrsetbits_le32(&denali_pi[136], 0xFFFF << 16, reg_value << 16);
1926                 clrsetbits_le32(&denali_pi[144], 0xFFFF, reg_value);
1927                 clrsetbits_le32(&denali_pi[151], 0xFFFF << 16, reg_value << 16);
1928                 break;
1929         case 2:
1930         default:
1931                 clrsetbits_le32(&denali_ctl[139], 0xFFFF, reg_value);
1932                 clrsetbits_le32(&denali_ctl[153], 0xFFFF, reg_value);
1933
1934                 clrsetbits_le32(&denali_pi[126], 0xFFFF << 16, reg_value << 16);
1935                 clrsetbits_le32(&denali_pi[134], 0xFFFF, reg_value);
1936                 clrsetbits_le32(&denali_pi[141], 0xFFFF << 16, reg_value << 16);
1937                 clrsetbits_le32(&denali_pi[149], 0xFFFF, reg_value);
1938                 break;
1939         }
1940 }
1941
1942 static void set_lpddr4_MR12(const struct chan_info *chan,
1943                             struct rk3399_sdram_params *params, u32 ctl,
1944                             bool ctl_phy_reg, u32 mr5)
1945 {
1946         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1947         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1948         struct io_setting *io;
1949         u32 reg_value;
1950
1951         io = lpddr4_get_io_settings(params, mr5);
1952
1953         reg_value = io->ca_vref;
1954
1955         switch (ctl) {
1956         case 0:
1957                 clrsetbits_le32(&denali_ctl[140], 0xFFFF << 16,
1958                                 reg_value << 16);
1959                 clrsetbits_le32(&denali_ctl[154], 0xFFFF << 16,
1960                                 reg_value << 16);
1961
1962                 clrsetbits_le32(&denali_pi[132], 0xFF << 8, reg_value << 8);
1963                 clrsetbits_le32(&denali_pi[139], 0xFF << 24, reg_value << 24);
1964                 clrsetbits_le32(&denali_pi[147], 0xFF << 8, reg_value << 8);
1965                 clrsetbits_le32(&denali_pi[154], 0xFF << 24, reg_value << 24);
1966                 break;
1967         case 1:
1968                 clrsetbits_le32(&denali_ctl[141], 0xFFFF, reg_value);
1969                 clrsetbits_le32(&denali_ctl[155], 0xFFFF, reg_value);
1970
1971                 clrsetbits_le32(&denali_pi[129], 0xFF << 24, reg_value << 24);
1972                 clrsetbits_le32(&denali_pi[137], 0xFF << 8, reg_value << 8);
1973                 clrsetbits_le32(&denali_pi[144], 0xFF << 24, reg_value << 24);
1974                 clrsetbits_le32(&denali_pi[152], 0xFF << 8, reg_value << 8);
1975                 break;
1976         case 2:
1977         default:
1978                 clrsetbits_le32(&denali_ctl[141], 0xFFFF << 16,
1979                                 reg_value << 16);
1980                 clrsetbits_le32(&denali_ctl[155], 0xFFFF << 16,
1981                                 reg_value << 16);
1982
1983                 clrsetbits_le32(&denali_pi[127], 0xFF << 8, reg_value << 8);
1984                 clrsetbits_le32(&denali_pi[134], 0xFF << 24, reg_value << 24);
1985                 clrsetbits_le32(&denali_pi[142], 0xFF << 8, reg_value << 8);
1986                 clrsetbits_le32(&denali_pi[149], 0xFF << 24, reg_value << 24);
1987                 break;
1988         }
1989 }
1990
1991 static void set_lpddr4_MR14(const struct chan_info *chan,
1992                             struct rk3399_sdram_params *params, u32 ctl,
1993                             bool ctl_phy_reg, u32 mr5)
1994 {
1995         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1996         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1997         struct io_setting *io;
1998         u32 reg_value;
1999
2000         io = lpddr4_get_io_settings(params, mr5);
2001
2002         reg_value = io->dq_vref;
2003
2004         switch (ctl) {
2005         case 0:
2006                 clrsetbits_le32(&denali_ctl[142], 0xFFFF << 16,
2007                                 reg_value << 16);
2008                 clrsetbits_le32(&denali_ctl[156], 0xFFFF << 16,
2009                                 reg_value << 16);
2010
2011                 clrsetbits_le32(&denali_pi[132], 0xFF << 16, reg_value << 16);
2012                 clrsetbits_le32(&denali_pi[140], 0xFF << 0, reg_value << 0);
2013                 clrsetbits_le32(&denali_pi[147], 0xFF << 16, reg_value << 16);
2014                 clrsetbits_le32(&denali_pi[155], 0xFF << 0, reg_value << 0);
2015                 break;
2016         case 1:
2017                 clrsetbits_le32(&denali_ctl[143], 0xFFFF, reg_value);
2018                 clrsetbits_le32(&denali_ctl[157], 0xFFFF, reg_value);
2019
2020                 clrsetbits_le32(&denali_pi[130], 0xFF << 0, reg_value << 0);
2021                 clrsetbits_le32(&denali_pi[137], 0xFF << 16, reg_value << 16);
2022                 clrsetbits_le32(&denali_pi[145], 0xFF << 0, reg_value << 0);
2023                 clrsetbits_le32(&denali_pi[152], 0xFF << 16, reg_value << 16);
2024                 break;
2025         case 2:
2026         default:
2027                 clrsetbits_le32(&denali_ctl[143], 0xFFFF << 16,
2028                                 reg_value << 16);
2029                 clrsetbits_le32(&denali_ctl[157], 0xFFFF << 16,
2030                                 reg_value << 16);
2031
2032                 clrsetbits_le32(&denali_pi[127], 0xFF << 16, reg_value << 16);
2033                 clrsetbits_le32(&denali_pi[135], 0xFF << 0, reg_value << 0);
2034                 clrsetbits_le32(&denali_pi[142], 0xFF << 16, reg_value << 16);
2035                 clrsetbits_le32(&denali_pi[150], 0xFF << 0, reg_value << 0);
2036                 break;
2037         }
2038 }
2039
2040 static void lpddr4_copy_phy(struct dram_info *dram,
2041                             struct rk3399_sdram_params *params, u32 phy,
2042                             struct rk3399_sdram_params *timings,
2043                             u32 channel)
2044 {
2045         u32 *denali_ctl, *denali_phy;
2046         u32 *denali_phy_params;
2047         u32 speed = 0;
2048         u32 ctl, mr5;
2049
2050         denali_ctl = dram->chan[channel].pctl->denali_ctl;
2051         denali_phy = dram->chan[channel].publ->denali_phy;
2052         denali_phy_params = timings->phy_regs.denali_phy;
2053
2054         /* switch index */
2055         clrsetbits_le32(&denali_phy_params[896], 0x3 << 8, phy << 8);
2056         writel(denali_phy_params[896], &denali_phy[896]);
2057
2058         /* phy_pll_ctrl_ca, phy_pll_ctrl */
2059         writel(denali_phy_params[911], &denali_phy[911]);
2060
2061         /* phy_low_freq_sel */
2062         clrsetbits_le32(&denali_phy[913], 0x1,
2063                         denali_phy_params[913] & 0x1);
2064
2065         /* phy_grp_slave_delay_x, phy_cslvl_dly_step */
2066         writel(denali_phy_params[916], &denali_phy[916]);
2067         writel(denali_phy_params[917], &denali_phy[917]);
2068         writel(denali_phy_params[918], &denali_phy[918]);
2069
2070         /* phy_adrz_sw_wraddr_shift_x  */
2071         writel(denali_phy_params[512], &denali_phy[512]);
2072         clrsetbits_le32(&denali_phy[513], 0xffff,
2073                         denali_phy_params[513] & 0xffff);
2074         writel(denali_phy_params[640], &denali_phy[640]);
2075         clrsetbits_le32(&denali_phy[641], 0xffff,
2076                         denali_phy_params[641] & 0xffff);
2077         writel(denali_phy_params[768], &denali_phy[768]);
2078         clrsetbits_le32(&denali_phy[769], 0xffff,
2079                         denali_phy_params[769] & 0xffff);
2080
2081         writel(denali_phy_params[544], &denali_phy[544]);
2082         writel(denali_phy_params[545], &denali_phy[545]);
2083         writel(denali_phy_params[546], &denali_phy[546]);
2084         writel(denali_phy_params[547], &denali_phy[547]);
2085
2086         writel(denali_phy_params[672], &denali_phy[672]);
2087         writel(denali_phy_params[673], &denali_phy[673]);
2088         writel(denali_phy_params[674], &denali_phy[674]);
2089         writel(denali_phy_params[675], &denali_phy[675]);
2090
2091         writel(denali_phy_params[800], &denali_phy[800]);
2092         writel(denali_phy_params[801], &denali_phy[801]);
2093         writel(denali_phy_params[802], &denali_phy[802]);
2094         writel(denali_phy_params[803], &denali_phy[803]);
2095
2096         /*
2097          * phy_adr_master_delay_start_x
2098          * phy_adr_master_delay_step_x
2099          * phy_adr_master_delay_wait_x
2100          */
2101         writel(denali_phy_params[548], &denali_phy[548]);
2102         writel(denali_phy_params[676], &denali_phy[676]);
2103         writel(denali_phy_params[804], &denali_phy[804]);
2104
2105         /* phy_adr_calvl_dly_step_x */
2106         writel(denali_phy_params[549], &denali_phy[549]);
2107         writel(denali_phy_params[677], &denali_phy[677]);
2108         writel(denali_phy_params[805], &denali_phy[805]);
2109
2110         /*
2111          * phy_clk_wrdm_slave_delay_x
2112          * phy_clk_wrdqz_slave_delay_x
2113          * phy_clk_wrdqs_slave_delay_x
2114          */
2115         copy_to_reg((u32 *)&denali_phy[59], (u32 *)&denali_phy_params[59],
2116                     (63 - 58) * 4);
2117         copy_to_reg((u32 *)&denali_phy[187], (u32 *)&denali_phy_params[187],
2118                     (191 - 186) * 4);
2119         copy_to_reg((u32 *)&denali_phy[315], (u32 *)&denali_phy_params[315],
2120                     (319 - 314) * 4);
2121         copy_to_reg((u32 *)&denali_phy[443], (u32 *)&denali_phy_params[443],
2122                     (447 - 442) * 4);
2123
2124         /*
2125          * phy_dqs_tsel_wr_timing_x 8bits denali_phy_84/212/340/468 offset_8
2126          * dqs_tsel_wr_end[7:4] add half cycle
2127          * phy_dq_tsel_wr_timing_x 8bits denali_phy_83/211/339/467 offset_8
2128          * dq_tsel_wr_end[7:4] add half cycle
2129          */
2130         writel(denali_phy_params[83] + (0x10 << 16), &denali_phy[83]);
2131         writel(denali_phy_params[84] + (0x10 << 8), &denali_phy[84]);
2132         writel(denali_phy_params[85], &denali_phy[85]);
2133
2134         writel(denali_phy_params[211] + (0x10 << 16), &denali_phy[211]);
2135         writel(denali_phy_params[212] + (0x10 << 8), &denali_phy[212]);
2136         writel(denali_phy_params[213], &denali_phy[213]);
2137
2138         writel(denali_phy_params[339] + (0x10 << 16), &denali_phy[339]);
2139         writel(denali_phy_params[340] + (0x10 << 8), &denali_phy[340]);
2140         writel(denali_phy_params[341], &denali_phy[341]);
2141
2142         writel(denali_phy_params[467] + (0x10 << 16), &denali_phy[467]);
2143         writel(denali_phy_params[468] + (0x10 << 8), &denali_phy[468]);
2144         writel(denali_phy_params[469], &denali_phy[469]);
2145
2146         /*
2147          * phy_gtlvl_resp_wait_cnt_x
2148          * phy_gtlvl_dly_step_x
2149          * phy_wrlvl_resp_wait_cnt_x
2150          * phy_gtlvl_final_step_x
2151          * phy_gtlvl_back_step_x
2152          * phy_rdlvl_dly_step_x
2153          *
2154          * phy_master_delay_step_x
2155          * phy_master_delay_wait_x
2156          * phy_wrlvl_dly_step_x
2157          * phy_rptr_update_x
2158          * phy_wdqlvl_dly_step_x
2159          */
2160         writel(denali_phy_params[87], &denali_phy[87]);
2161         writel(denali_phy_params[88], &denali_phy[88]);
2162         writel(denali_phy_params[89], &denali_phy[89]);
2163         writel(denali_phy_params[90], &denali_phy[90]);
2164
2165         writel(denali_phy_params[215], &denali_phy[215]);
2166         writel(denali_phy_params[216], &denali_phy[216]);
2167         writel(denali_phy_params[217], &denali_phy[217]);
2168         writel(denali_phy_params[218], &denali_phy[218]);
2169
2170         writel(denali_phy_params[343], &denali_phy[343]);
2171         writel(denali_phy_params[344], &denali_phy[344]);
2172         writel(denali_phy_params[345], &denali_phy[345]);
2173         writel(denali_phy_params[346], &denali_phy[346]);
2174
2175         writel(denali_phy_params[471], &denali_phy[471]);
2176         writel(denali_phy_params[472], &denali_phy[472]);
2177         writel(denali_phy_params[473], &denali_phy[473]);
2178         writel(denali_phy_params[474], &denali_phy[474]);
2179
2180         /*
2181          * phy_gtlvl_lat_adj_start_x
2182          * phy_gtlvl_rddqs_slv_dly_start_x
2183          * phy_rdlvl_rddqs_dq_slv_dly_start_x
2184          * phy_wdqlvl_dqdm_slv_dly_start_x
2185          */
2186         writel(denali_phy_params[80], &denali_phy[80]);
2187         writel(denali_phy_params[81], &denali_phy[81]);
2188
2189         writel(denali_phy_params[208], &denali_phy[208]);
2190         writel(denali_phy_params[209], &denali_phy[209]);
2191
2192         writel(denali_phy_params[336], &denali_phy[336]);
2193         writel(denali_phy_params[337], &denali_phy[337]);
2194
2195         writel(denali_phy_params[464], &denali_phy[464]);
2196         writel(denali_phy_params[465], &denali_phy[465]);
2197
2198         /*
2199          * phy_master_delay_start_x
2200          * phy_sw_master_mode_x
2201          * phy_rddata_en_tsel_dly_x
2202          */
2203         writel(denali_phy_params[86], &denali_phy[86]);
2204         writel(denali_phy_params[214], &denali_phy[214]);
2205         writel(denali_phy_params[342], &denali_phy[342]);
2206         writel(denali_phy_params[470], &denali_phy[470]);
2207
2208         /*
2209          * phy_rddqz_slave_delay_x
2210          * phy_rddqs_dqz_fall_slave_delay_x
2211          * phy_rddqs_dqz_rise_slave_delay_x
2212          * phy_rddqs_dm_fall_slave_delay_x
2213          * phy_rddqs_dm_rise_slave_delay_x
2214          * phy_rddqs_gate_slave_delay_x
2215          * phy_wrlvl_delay_early_threshold_x
2216          * phy_write_path_lat_add_x
2217          * phy_rddqs_latency_adjust_x
2218          * phy_wrlvl_delay_period_threshold_x
2219          * phy_wrlvl_early_force_zero_x
2220          */
2221         copy_to_reg((u32 *)&denali_phy[64], (u32 *)&denali_phy_params[64],
2222                     (67 - 63) * 4);
2223         clrsetbits_le32(&denali_phy[68], 0xfffffc00,
2224                         denali_phy_params[68] & 0xfffffc00);
2225         copy_to_reg((u32 *)&denali_phy[69], (u32 *)&denali_phy_params[69],
2226                     (79 - 68) * 4);
2227         copy_to_reg((u32 *)&denali_phy[192], (u32 *)&denali_phy_params[192],
2228                     (195 - 191) * 4);
2229         clrsetbits_le32(&denali_phy[196], 0xfffffc00,
2230                         denali_phy_params[196] & 0xfffffc00);
2231         copy_to_reg((u32 *)&denali_phy[197], (u32 *)&denali_phy_params[197],
2232                     (207 - 196) * 4);
2233         copy_to_reg((u32 *)&denali_phy[320], (u32 *)&denali_phy_params[320],
2234                     (323 - 319) * 4);
2235         clrsetbits_le32(&denali_phy[324], 0xfffffc00,
2236                         denali_phy_params[324] & 0xfffffc00);
2237         copy_to_reg((u32 *)&denali_phy[325], (u32 *)&denali_phy_params[325],
2238                     (335 - 324) * 4);
2239
2240         copy_to_reg((u32 *)&denali_phy[448], (u32 *)&denali_phy_params[448],
2241                     (451 - 447) * 4);
2242         clrsetbits_le32(&denali_phy[452], 0xfffffc00,
2243                         denali_phy_params[452] & 0xfffffc00);
2244         copy_to_reg((u32 *)&denali_phy[453], (u32 *)&denali_phy_params[453],
2245                     (463 - 452) * 4);
2246
2247         /* phy_two_cyc_preamble_x */
2248         clrsetbits_le32(&denali_phy[7], 0x3 << 24,
2249                         denali_phy_params[7] & (0x3 << 24));
2250         clrsetbits_le32(&denali_phy[135], 0x3 << 24,
2251                         denali_phy_params[135] & (0x3 << 24));
2252         clrsetbits_le32(&denali_phy[263], 0x3 << 24,
2253                         denali_phy_params[263] & (0x3 << 24));
2254         clrsetbits_le32(&denali_phy[391], 0x3 << 24,
2255                         denali_phy_params[391] & (0x3 << 24));
2256
2257         /* speed */
2258         if (timings->base.ddr_freq < 400 * MHz)
2259                 speed = 0x0;
2260         else if (timings->base.ddr_freq < 800 * MHz)
2261                 speed = 0x1;
2262         else if (timings->base.ddr_freq < 1200 * MHz)
2263                 speed = 0x2;
2264
2265         /* phy_924 phy_pad_fdbk_drive */
2266         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
2267         /* phy_926 phy_pad_data_drive */
2268         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
2269         /* phy_927 phy_pad_dqs_drive */
2270         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
2271         /* phy_928 phy_pad_addr_drive */
2272         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
2273         /* phy_929 phy_pad_clk_drive */
2274         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
2275         /* phy_935 phy_pad_cke_drive */
2276         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
2277         /* phy_937 phy_pad_rst_drive */
2278         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
2279         /* phy_939 phy_pad_cs_drive */
2280         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
2281
2282         read_mr(dram->chan[channel].pctl, 1, 5, &mr5);
2283         set_ds_odt(&dram->chan[channel], timings, true, mr5);
2284
2285         ctl = lpddr4_get_ctl(timings, phy);
2286         set_lpddr4_dq_odt(&dram->chan[channel], timings, ctl, true, true, mr5);
2287         set_lpddr4_ca_odt(&dram->chan[channel], timings, ctl, true, true, mr5);
2288         set_lpddr4_MR3(&dram->chan[channel], timings, ctl, true, mr5);
2289         set_lpddr4_MR12(&dram->chan[channel], timings, ctl, true, mr5);
2290         set_lpddr4_MR14(&dram->chan[channel], timings, ctl, true, mr5);
2291
2292         /*
2293          * if phy_sw_master_mode_x not bypass mode,
2294          * clear phy_slice_pwr_rdc_disable.
2295          * note: need use timings, not ddr_publ_regs
2296          */
2297         if (!((denali_phy_params[86] >> 8) & (1 << 2))) {
2298                 clrbits_le32(&denali_phy[10], 1 << 16);
2299                 clrbits_le32(&denali_phy[138], 1 << 16);
2300                 clrbits_le32(&denali_phy[266], 1 << 16);
2301                 clrbits_le32(&denali_phy[394], 1 << 16);
2302         }
2303
2304         /*
2305          * when PHY_PER_CS_TRAINING_EN=1, W2W_DIFFCS_DLY_Fx can't
2306          * smaller than 8
2307          * NOTE: need use timings, not ddr_publ_regs
2308          */
2309         if ((denali_phy_params[84] >> 16) & 1) {
2310                 if (((readl(&denali_ctl[217 + ctl]) >> 16) & 0x1f) < 8)
2311                         clrsetbits_le32(&denali_ctl[217 + ctl],
2312                                         0x1f << 16, 8 << 16);
2313         }
2314 }
2315
2316 static void lpddr4_set_phy(struct dram_info *dram,
2317                            struct rk3399_sdram_params *params, u32 phy,
2318                            struct rk3399_sdram_params *timings)
2319 {
2320         u32 channel;
2321
2322         for (channel = 0; channel < 2; channel++)
2323                 lpddr4_copy_phy(dram, params, phy, timings, channel);
2324 }
2325
2326 static int lpddr4_set_ctl(struct dram_info *dram,
2327                           struct rk3399_sdram_params *params, u32 ctl, u32 hz)
2328 {
2329         u32 channel;
2330         int ret_clk, ret;
2331
2332         /* cci idle req stall */
2333         writel(0x70007, &dram->grf->soc_con0);
2334
2335         /* enable all clk */
2336         setbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2337
2338         /* idle */
2339         setbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2340         while ((readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2341                != (0x3 << 18))
2342                 ;
2343
2344         /* change freq */
2345         writel((((0x3 << 4) | (1 << 2) | 1) << 16) |
2346                 (ctl << 4) | (1 << 2) | 1, &dram->cic->cic_ctrl0);
2347         while (!(readl(&dram->cic->cic_status0) & (1 << 2)))
2348                 ;
2349
2350         ret_clk = clk_set_rate(&dram->ddr_clk, hz);
2351         if (ret_clk < 0) {
2352                 printf("%s clk set failed %d\n", __func__, ret_clk);
2353                 return ret_clk;
2354         }
2355
2356         writel(0x20002, &dram->cic->cic_ctrl0);
2357         while (!(readl(&dram->cic->cic_status0) & (1 << 0)))
2358                 ;
2359
2360         /* deidle */
2361         clrbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2362         while (readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2363                 ;
2364
2365         /* clear enable all clk */
2366         clrbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2367
2368         /* lpddr4 ctl2 can not do training, all training will fail */
2369         if (!(params->base.dramtype == LPDDR4 && ctl == 2)) {
2370                 for (channel = 0; channel < 2; channel++) {
2371                         if (!(params->ch[channel].cap_info.col))
2372                                 continue;
2373                         ret = data_training(dram, channel, params,
2374                                                      PI_FULL_TRAINING);
2375                         if (ret)
2376                                 printf("%s: channel %d training failed!\n",
2377                                        __func__, channel);
2378                         else
2379                                 debug("%s: channel %d training pass\n",
2380                                       __func__, channel);
2381                 }
2382         }
2383
2384         return 0;
2385 }
2386
2387 static int lpddr4_set_rate(struct dram_info *dram,
2388                            struct rk3399_sdram_params *params)
2389 {
2390         u32 ctl;
2391         u32 phy;
2392
2393         for (ctl = 0; ctl < 2; ctl++) {
2394                 phy = lpddr4_get_phy(params, ctl);
2395
2396                 lpddr4_set_phy(dram, params, phy, &lpddr4_timings[ctl]);
2397                 lpddr4_set_ctl(dram, params, ctl,
2398                                lpddr4_timings[ctl].base.ddr_freq);
2399
2400                 debug("%s: change freq to %d mhz %d, %d\n", __func__,
2401                       lpddr4_timings[ctl].base.ddr_freq / MHz, ctl, phy);
2402         }
2403
2404         return 0;
2405 }
2406 #endif /* CONFIG_RAM_RK3399_LPDDR4 */
2407
2408 static unsigned char calculate_stride(struct rk3399_sdram_params *params)
2409 {
2410         unsigned int stride = params->base.stride;
2411         unsigned int channel, chinfo = 0;
2412         unsigned int ch_cap[2] = {0, 0};
2413         u64 cap;
2414
2415         for (channel = 0; channel < 2; channel++) {
2416                 unsigned int cs0_cap = 0;
2417                 unsigned int cs1_cap = 0;
2418                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
2419
2420                 if (cap_info->col == 0)
2421                         continue;
2422
2423                 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col +
2424                                  cap_info->bk + cap_info->bw - 20));
2425                 if (cap_info->rank > 1)
2426                         cs1_cap = cs0_cap >> (cap_info->cs0_row
2427                                               - cap_info->cs1_row);
2428                 if (cap_info->row_3_4) {
2429                         cs0_cap = cs0_cap * 3 / 4;
2430                         cs1_cap = cs1_cap * 3 / 4;
2431                 }
2432                 ch_cap[channel] = cs0_cap + cs1_cap;
2433                 chinfo |= 1 << channel;
2434         }
2435
2436         /* stride calculation for 1 channel */
2437         if (params->base.num_channels == 1 && chinfo & 1)
2438                 return 0x17;    /* channel a */
2439
2440         /* stride calculation for 2 channels, default gstride type is 256B */
2441         if (ch_cap[0] == ch_cap[1]) {
2442                 cap = ch_cap[0] + ch_cap[1];
2443                 switch (cap) {
2444                 /* 512MB */
2445                 case 512:
2446                         stride = 0;
2447                         break;
2448                 /* 1GB */
2449                 case 1024:
2450                         stride = 0x5;
2451                         break;
2452                 /*
2453                  * 768MB + 768MB same as total 2GB memory
2454                  * useful space: 0-768MB 1GB-1792MB
2455                  */
2456                 case 1536:
2457                 /* 2GB */
2458                 case 2048:
2459                         stride = 0x9;
2460                         break;
2461                 /* 1536MB + 1536MB */
2462                 case 3072:
2463                         stride = 0x11;
2464                         break;
2465                 /* 4GB */
2466                 case 4096:
2467                         stride = 0xD;
2468                         break;
2469                 default:
2470                         printf("%s: Unable to calculate stride for ", __func__);
2471                         print_size((cap * (1 << 20)), " capacity\n");
2472                         break;
2473                 }
2474         }
2475
2476         sdram_print_stride(stride);
2477
2478         return stride;
2479 }
2480
2481 static void clear_channel_params(struct rk3399_sdram_params *params, u8 channel)
2482 {
2483         params->ch[channel].cap_info.rank = 0;
2484         params->ch[channel].cap_info.col = 0;
2485         params->ch[channel].cap_info.bk = 0;
2486         params->ch[channel].cap_info.bw = 32;
2487         params->ch[channel].cap_info.dbw = 32;
2488         params->ch[channel].cap_info.row_3_4 = 0;
2489         params->ch[channel].cap_info.cs0_row = 0;
2490         params->ch[channel].cap_info.cs1_row = 0;
2491         params->ch[channel].cap_info.ddrconfig = 0;
2492 }
2493
2494 static int pctl_init(struct dram_info *dram, struct rk3399_sdram_params *params)
2495 {
2496         int channel;
2497         int ret;
2498
2499         for (channel = 0; channel < 2; channel++) {
2500                 const struct chan_info *chan = &dram->chan[channel];
2501                 struct rk3399_cru *cru = dram->cru;
2502                 struct rk3399_ddr_publ_regs *publ = chan->publ;
2503
2504                 phy_pctrl_reset(cru, channel);
2505                 phy_dll_bypass_set(publ, params->base.ddr_freq);
2506
2507                 ret = pctl_cfg(dram, chan, channel, params);
2508                 if (ret < 0) {
2509                         printf("%s: pctl config failed\n", __func__);
2510                         return ret;
2511                 }
2512
2513                 /* start to trigger initialization */
2514                 pctl_start(dram, channel);
2515         }
2516
2517         return 0;
2518 }
2519
2520 static int sdram_init(struct dram_info *dram,
2521                       struct rk3399_sdram_params *params)
2522 {
2523         unsigned char dramtype = params->base.dramtype;
2524         unsigned int ddr_freq = params->base.ddr_freq;
2525         int channel, ch, rank;
2526         int ret;
2527
2528         debug("Starting SDRAM initialization...\n");
2529
2530         if ((dramtype == DDR3 && ddr_freq > 933) ||
2531             (dramtype == LPDDR3 && ddr_freq > 933) ||
2532             (dramtype == LPDDR4 && ddr_freq > 800)) {
2533                 debug("SDRAM frequency is to high!");
2534                 return -E2BIG;
2535         }
2536
2537         for (ch = 0; ch < 2; ch++) {
2538                 params->ch[ch].cap_info.rank = 2;
2539                 for (rank = 2; rank != 0; rank--) {
2540                         ret = pctl_init(dram, params);
2541                         if (ret < 0) {
2542                                 printf("%s: pctl init failed\n", __func__);
2543                                 return ret;
2544                         }
2545
2546                         /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
2547                         if (dramtype == LPDDR3)
2548                                 udelay(10);
2549
2550                         params->ch[ch].cap_info.rank = rank;
2551
2552                         ret = dram->ops->data_training(dram, ch, rank, params);
2553                         if (!ret) {
2554                                 debug("%s: data trained for rank %d, ch %d\n",
2555                                       __func__, rank, ch);
2556                                 break;
2557                         }
2558                 }
2559                 /* Computed rank with associated channel number */
2560                 params->ch[ch].cap_info.rank = rank;
2561         }
2562
2563         params->base.num_channels = 0;
2564         for (channel = 0; channel < 2; channel++) {
2565                 const struct chan_info *chan = &dram->chan[channel];
2566                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
2567                 u8 training_flag = PI_FULL_TRAINING;
2568
2569                 if (cap_info->rank == 0) {
2570                         clear_channel_params(params, channel);
2571                         continue;
2572                 } else {
2573                         params->base.num_channels++;
2574                 }
2575
2576                 debug("Channel ");
2577                 debug(channel ? "1: " : "0: ");
2578
2579                 /* LPDDR3 should have write and read gate training */
2580                 if (params->base.dramtype == LPDDR3)
2581                         training_flag = PI_WRITE_LEVELING |
2582                                         PI_READ_GATE_TRAINING;
2583
2584                 if (params->base.dramtype != LPDDR4) {
2585                         ret = data_training(dram, channel, params,
2586                                             training_flag);
2587                         if (!ret) {
2588                                 debug("%s: data train failed for channel %d\n",
2589                                       __func__, ret);
2590                                 continue;
2591                         }
2592                 }
2593
2594                 sdram_print_ddr_info(cap_info, &params->base);
2595
2596                 set_ddrconfig(chan, params, channel, cap_info->ddrconfig);
2597         }
2598
2599         if (params->base.num_channels == 0) {
2600                 printf("%s: ", __func__);
2601                 sdram_print_dram_type(params->base.dramtype);
2602                 printf(" - %dMHz failed!\n", params->base.ddr_freq);
2603                 return -EINVAL;
2604         }
2605
2606         params->base.stride = calculate_stride(params);
2607         dram_all_config(dram, params);
2608         dram->ops->set_rate(dram, params);
2609
2610         debug("Finish SDRAM initialization...\n");
2611         return 0;
2612 }
2613
2614 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev)
2615 {
2616 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
2617         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2618         int ret;
2619
2620         ret = dev_read_u32_array(dev, "rockchip,sdram-params",
2621                                  (u32 *)&plat->sdram_params,
2622                                  sizeof(plat->sdram_params) / sizeof(u32));
2623         if (ret) {
2624                 printf("%s: Cannot read rockchip,sdram-params %d\n",
2625                        __func__, ret);
2626                 return ret;
2627         }
2628         ret = regmap_init_mem(dev_ofnode(dev), &plat->map);
2629         if (ret)
2630                 printf("%s: regmap failed %d\n", __func__, ret);
2631
2632 #endif
2633         return 0;
2634 }
2635
2636 #if CONFIG_IS_ENABLED(OF_PLATDATA)
2637 static int conv_of_platdata(struct udevice *dev)
2638 {
2639         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2640         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
2641         int ret;
2642
2643         ret = regmap_init_mem_platdata(dev, dtplat->reg,
2644                                        ARRAY_SIZE(dtplat->reg) / 2,
2645                                        &plat->map);
2646         if (ret)
2647                 return ret;
2648
2649         return 0;
2650 }
2651 #endif
2652
2653 static const struct sdram_rk3399_ops rk3399_ops = {
2654 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
2655         .data_training = default_data_training,
2656         .set_rate = switch_to_phy_index1,
2657 #else
2658         .data_training = lpddr4_mr_detect,
2659         .set_rate = lpddr4_set_rate,
2660 #endif
2661 };
2662
2663 static int rk3399_dmc_init(struct udevice *dev)
2664 {
2665         struct dram_info *priv = dev_get_priv(dev);
2666         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2667         int ret;
2668 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
2669         struct rk3399_sdram_params *params = &plat->sdram_params;
2670 #else
2671         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
2672         struct rk3399_sdram_params *params =
2673                                         (void *)dtplat->rockchip_sdram_params;
2674
2675         ret = conv_of_platdata(dev);
2676         if (ret)
2677                 return ret;
2678 #endif
2679
2680         priv->ops = &rk3399_ops;
2681         priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC);
2682         priv->grf = syscon_get_first_range(ROCKCHIP_SYSCON_GRF);
2683         priv->pmu = syscon_get_first_range(ROCKCHIP_SYSCON_PMU);
2684         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
2685         priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF);
2686         priv->pmucru = rockchip_get_pmucru();
2687         priv->cru = rockchip_get_cru();
2688         priv->chan[0].pctl = regmap_get_range(plat->map, 0);
2689         priv->chan[0].pi = regmap_get_range(plat->map, 1);
2690         priv->chan[0].publ = regmap_get_range(plat->map, 2);
2691         priv->chan[0].msch = regmap_get_range(plat->map, 3);
2692         priv->chan[1].pctl = regmap_get_range(plat->map, 4);
2693         priv->chan[1].pi = regmap_get_range(plat->map, 5);
2694         priv->chan[1].publ = regmap_get_range(plat->map, 6);
2695         priv->chan[1].msch = regmap_get_range(plat->map, 7);
2696
2697         debug("con reg %p %p %p %p %p %p %p %p\n",
2698               priv->chan[0].pctl, priv->chan[0].pi,
2699               priv->chan[0].publ, priv->chan[0].msch,
2700               priv->chan[1].pctl, priv->chan[1].pi,
2701               priv->chan[1].publ, priv->chan[1].msch);
2702         debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p, pmu %p\n", priv->cru,
2703               priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru, priv->pmu);
2704
2705 #if CONFIG_IS_ENABLED(OF_PLATDATA)
2706         ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk);
2707 #else
2708         ret = clk_get_by_index(dev, 0, &priv->ddr_clk);
2709 #endif
2710         if (ret) {
2711                 printf("%s clk get failed %d\n", __func__, ret);
2712                 return ret;
2713         }
2714
2715         ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz);
2716         if (ret < 0) {
2717                 printf("%s clk set failed %d\n", __func__, ret);
2718                 return ret;
2719         }
2720
2721         ret = sdram_init(priv, params);
2722         if (ret < 0) {
2723                 printf("%s DRAM init failed %d\n", __func__, ret);
2724                 return ret;
2725         }
2726
2727         return 0;
2728 }
2729 #endif
2730
2731 static int rk3399_dmc_probe(struct udevice *dev)
2732 {
2733 #if defined(CONFIG_TPL_BUILD) || \
2734         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2735         if (rk3399_dmc_init(dev))
2736                 return 0;
2737 #else
2738         struct dram_info *priv = dev_get_priv(dev);
2739
2740         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
2741         debug("%s: pmugrf = %p\n", __func__, priv->pmugrf);
2742         priv->info.base = CONFIG_SYS_SDRAM_BASE;
2743         priv->info.size =
2744                 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2);
2745 #endif
2746         return 0;
2747 }
2748
2749 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info)
2750 {
2751         struct dram_info *priv = dev_get_priv(dev);
2752
2753         *info = priv->info;
2754
2755         return 0;
2756 }
2757
2758 static struct ram_ops rk3399_dmc_ops = {
2759         .get_info = rk3399_dmc_get_info,
2760 };
2761
2762 static const struct udevice_id rk3399_dmc_ids[] = {
2763         { .compatible = "rockchip,rk3399-dmc" },
2764         { }
2765 };
2766
2767 U_BOOT_DRIVER(dmc_rk3399) = {
2768         .name = "rockchip_rk3399_dmc",
2769         .id = UCLASS_RAM,
2770         .of_match = rk3399_dmc_ids,
2771         .ops = &rk3399_dmc_ops,
2772 #if defined(CONFIG_TPL_BUILD) || \
2773         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2774         .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata,
2775 #endif
2776         .probe = rk3399_dmc_probe,
2777         .priv_auto_alloc_size = sizeof(struct dram_info),
2778 #if defined(CONFIG_TPL_BUILD) || \
2779         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2780         .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat),
2781 #endif
2782 };