1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 #ifndef RAMINIT_COMMON_H 4 #define RAMINIT_COMMON_H 5 6 #include <device/dram/ddr3.h> 7 #include <stdint.h> 8 9 #define BASEFREQ 133 10 #define tDLLK 512 11 12 #define NUM_CHANNELS 2 13 #define NUM_SLOTRANKS 4 14 #define NUM_SLOTS 2 15 #define NUM_LANES 9 16 17 /* IOSAV_n_SP_CMD_CTRL DRAM commands */ 18 #define IOSAV_MRS (0xf000) 19 #define IOSAV_PRE (0xf002) 20 #define IOSAV_ZQCS (0xf003) 21 #define IOSAV_ACT (0xf006) 22 #define IOSAV_RD (0xf105) 23 #define IOSAV_NOP_ALT (0xf107) 24 #define IOSAV_WR (0xf201) 25 #define IOSAV_NOP (0xf207) 26 27 /* IOSAV_n_SUBSEQ_CTRL data direction */ 28 #define SSQ_NA 0 /* Non-data */ 29 #define SSQ_RD 1 /* Read */ 30 #define SSQ_WR 2 /* Write */ 31 #define SSQ_RW 3 /* Read and write */ 32 33 struct iosav_ssq { 34 /* IOSAV_n_SP_CMD_CTRL */ 35 union { 36 struct { 37 u32 command : 16; /* [15.. 0] */ 38 u32 ranksel_ap : 2; /* [17..16] */ 39 u32 : 14; 40 }; 41 u32 raw; 42 } sp_cmd_ctrl; 43 44 /* IOSAV_n_SUBSEQ_CTRL */ 45 union { 46 struct { 47 u32 cmd_executions : 9; /* [ 8.. 0] */ 48 u32 : 1; 49 u32 cmd_delay_gap : 5; /* [14..10] */ 50 u32 : 1; 51 u32 post_ssq_wait : 9; /* [24..16] */ 52 u32 : 1; 53 u32 data_direction : 2; /* [27..26] */ 54 u32 : 4; 55 }; 56 u32 raw; 57 } subseq_ctrl; 58 59 /* IOSAV_n_SP_CMD_ADDR */ 60 union { 61 struct { 62 u32 address : 16; /* [15.. 0] */ 63 u32 rowbits : 3; /* [18..16] */ 64 u32 : 1; 65 u32 bank : 3; /* [22..20] */ 66 u32 : 1; 67 u32 rank : 2; /* [25..24] */ 68 u32 : 6; 69 }; 70 u32 raw; 71 } sp_cmd_addr; 72 73 /* IOSAV_n_ADDR_UPDATE */ 74 union { 75 struct { 76 u32 inc_addr_1 : 1; /* [ 0.. 0] */ 77 u32 inc_addr_8 : 1; /* [ 1.. 1] */ 78 u32 inc_bank : 1; /* [ 2.. 2] */ 79 u32 inc_rank : 2; /* [ 4.. 3] */ 80 u32 addr_wrap : 5; /* [ 9.. 5] */ 81 u32 lfsr_upd : 2; /* [11..10] */ 82 u32 upd_rate : 4; /* [15..12] */ 83 u32 lfsr_xors : 2; /* [17..16] */ 84 u32 : 14; 85 }; 86 u32 raw; 87 } addr_update; 88 }; 89 90 union gdcr_rx_reg { 91 struct { 92 u32 rcven_pi_code : 6; /* [ 5.. 0] */ 93 u32 : 2; 94 u32 rx_dqs_p_pi_code : 7; /* [14.. 8] */ 95 u32 : 1; 96 u32 rcven_logic_delay : 3; /* [18..16] */ 97 u32 : 1; 98 u32 rx_dqs_n_pi_code : 7; /* [26..20] */ 99 u32 : 5; 100 }; 101 u32 raw; 102 }; 103 104 union gdcr_tx_reg { 105 struct { 106 u32 tx_dq_pi_code : 6; /* [ 5.. 0] */ 107 u32 : 2; 108 u32 tx_dqs_pi_code : 6; /* [13.. 8] */ 109 u32 : 1; 110 u32 tx_dqs_logic_delay : 3; /* [17..15] */ 111 u32 : 1; 112 u32 tx_dq_logic_delay : 1; /* [19..19] */ 113 u32 : 12; 114 }; 115 u32 raw; 116 }; 117 118 union gdcr_cmd_pi_coding_reg { 119 struct { 120 u32 cmd_pi_code : 6; /* [ 5.. 0] */ 121 u32 ctl_pi_code_d0 : 6; /* [11.. 6] */ 122 u32 cmd_logic_delay : 1; /* [12..12] */ 123 u32 cmd_phase_delay : 1; /* [13..13] */ 124 u32 cmd_xover_enable : 1; /* [14..14] */ 125 u32 ctl_logic_delay_d0 : 1; /* [15..15] */ 126 u32 ctl_phase_delay_d0 : 1; /* [16..16] */ 127 u32 ctl_xover_enable_d0 : 1; /* [17..17] */ 128 u32 ctl_pi_code_d1 : 6; /* [23..18] */ 129 u32 ctl_logic_delay_d1 : 1; /* [24..24] */ 130 u32 ctl_phase_delay_d1 : 1; /* [25..25] */ 131 u32 ctl_xover_enable_d1 : 1; /* [26..26] */ 132 u32 : 5; 133 }; 134 u32 raw; 135 }; 136 137 union gdcr_training_mod_reg { 138 struct { 139 u32 receive_enable_mode : 1; /* [ 0.. 0] */ 140 u32 write_leveling_mode : 1; /* [ 1.. 1] */ 141 u32 training_rank_sel : 2; /* [ 3.. 2] */ 142 u32 enable_dqs_wl : 4; /* [ 7.. 4] */ 143 u32 dqs_logic_delay_wl : 1; /* [ 8.. 8] */ 144 u32 dq_dqs_training_res : 1; /* [ 9.. 9] */ 145 u32 : 4; 146 u32 delay_dq : 1; /* [14..14] */ 147 u32 odt_always_on : 1; /* [15..15] */ 148 u32 : 4; 149 u32 force_drive_enable : 1; /* [20..20] */ 150 u32 dft_tx_pi_clk_view : 1; /* [21..21] */ 151 u32 dft_tx_pi_clk_swap : 1; /* [22..22] */ 152 u32 early_odt_en : 1; /* [23..23] */ 153 u32 vref_gen_ctl : 6; /* [29..24] */ 154 u32 ext_vref_sel : 1; /* [30..30] */ 155 u32 tx_fifo_always_on : 1; /* [31..31] */ 156 }; 157 u32 raw; 158 }; 159 160 union comp_ofst_1_reg { 161 struct { 162 u32 dq_odt_down : 3; /* [ 2.. 0] */ 163 u32 dq_odt_up : 3; /* [ 5.. 3] */ 164 u32 clk_odt_down : 3; /* [ 8.. 6] */ 165 u32 clk_odt_up : 3; /* [11.. 9] */ 166 u32 dq_drv_down : 3; /* [14..12] */ 167 u32 dq_drv_up : 3; /* [17..15] */ 168 u32 clk_drv_down : 3; /* [20..18] */ 169 u32 clk_drv_up : 3; /* [23..21] */ 170 u32 ctl_drv_down : 3; /* [26..24] */ 171 u32 ctl_drv_up : 3; /* [29..27] */ 172 u32 : 2; 173 }; 174 u32 raw; 175 }; 176 177 union tc_dbp_reg { 178 struct { 179 u32 tRCD : 4; /* [ 3.. 0] */ 180 u32 tRP : 4; /* [ 7.. 4] */ 181 u32 tAA : 4; /* [11.. 8] */ 182 u32 tCWL : 4; /* [15..12] */ 183 u32 tRAS : 8; /* [23..16] */ 184 u32 : 8; 185 }; 186 u32 raw; 187 }; 188 189 union tc_rap_reg { 190 struct { 191 u32 tRRD : 4; /* [ 3.. 0] */ 192 u32 tRTP : 4; /* [ 7.. 4] */ 193 u32 tCKE : 4; /* [11.. 8] */ 194 u32 tWTR : 4; /* [15..12] */ 195 u32 tFAW : 8; /* [23..16] */ 196 u32 tWR : 5; /* [28..24] */ 197 u32 dis_3st : 1; /* [29..29] */ 198 u32 tCMD : 2; /* [31..30] */ 199 }; 200 u32 raw; 201 }; 202 203 union tc_rwp_reg { 204 struct { 205 u32 tRRDR : 3; /* [ 2.. 0] */ 206 u32 : 1; 207 u32 tRRDD : 3; /* [ 6.. 4] */ 208 u32 : 1; 209 u32 tWWDR : 3; /* [10.. 8] */ 210 u32 : 1; 211 u32 tWWDD : 3; /* [14..12] */ 212 u32 : 1; 213 u32 tRWDRDD : 3; /* [18..16] */ 214 u32 : 1; 215 u32 tWRDRDD : 3; /* [22..20] */ 216 u32 : 1; 217 u32 tRWSR : 3; /* [26..24] */ 218 u32 dec_wrd : 1; /* [27..27] */ 219 u32 : 4; 220 }; 221 u32 raw; 222 }; 223 224 union tc_othp_reg { 225 struct { 226 u32 tXPDLL : 5; /* [ 4.. 0] */ 227 u32 tXP : 3; /* [ 7.. 5] */ 228 u32 tAONPD : 4; /* [11.. 8] */ 229 u32 tCPDED : 2; /* [13..12] */ 230 u32 tPRPDEN : 2; /* [15..14] */ 231 u32 odt_delay_d0 : 2; /* [17..16] */ 232 u32 odt_delay_d1 : 2; /* [19..18] */ 233 u32 : 12; 234 }; 235 u32 raw; 236 }; 237 238 union tc_dtp_reg { 239 struct { 240 u32 : 12; 241 u32 overclock_tXP : 1; /* [12..12] */ 242 u32 overclock_tXPDLL : 1; /* [13..13] */ 243 u32 : 18; 244 }; 245 u32 raw; 246 }; 247 248 union tc_rfp_reg { 249 struct { 250 u32 oref_ri : 8; /* [ 7.. 0] */ 251 u32 refresh_high_wm : 4; /* [11.. 8] */ 252 u32 refresh_panic_wm : 4; /* [15..12] */ 253 u32 refresh_2x_control : 2; /* [17..16] */ 254 u32 : 14; 255 }; 256 u32 raw; 257 }; 258 259 union tc_rftp_reg { 260 struct { 261 u32 tREFI : 16; /* [15.. 0] */ 262 u32 tRFC : 9; /* [24..16] */ 263 u32 tREFIx9 : 7; /* [31..25] */ 264 }; 265 u32 raw; 266 }; 267 268 union tc_srftp_reg { 269 struct { 270 u32 tXSDLL : 12; /* [11.. 0] */ 271 u32 tXS_offset : 4; /* [15..12] */ 272 u32 tZQOPER : 10; /* [25..16] */ 273 u32 : 2; 274 u32 tMOD : 4; /* [31..28] */ 275 }; 276 u32 raw; 277 }; 278 279 typedef struct ramctr_timing_st ramctr_timing; 280 281 void iosav_write_sequence(const int ch, const struct iosav_ssq *seq, const unsigned int length); 282 void iosav_run_queue(const int ch, const u8 loops, const u8 as_timer); 283 void wait_for_iosav(int channel); 284 void iosav_run_once_and_wait(const int ch); 285 286 void iosav_write_zqcs_sequence(int channel, int slotrank, u32 gap, u32 post, u32 wrap); 287 void iosav_write_prea_sequence(int channel, int slotrank, u32 post, u32 wrap); 288 void iosav_write_read_mpr_sequence( 289 int channel, int slotrank, u32 tMOD, u32 loops, u32 gap, u32 loops2, u32 post2); 290 void iosav_write_prea_act_read_sequence(ramctr_timing *ctrl, int channel, int slotrank); 291 void iosav_write_jedec_write_leveling_sequence( 292 ramctr_timing *ctrl, int channel, int slotrank, int bank, u32 mr1reg); 293 void iosav_write_misc_write_sequence(ramctr_timing *ctrl, int channel, int slotrank, 294 u32 gap0, u32 loops0, u32 gap1, u32 loops2, u32 wrap2); 295 void iosav_write_command_training_sequence( 296 ramctr_timing *ctrl, int channel, int slotrank, unsigned int address); 297 void iosav_write_data_write_sequence(ramctr_timing *ctrl, int channel, int slotrank); 298 void iosav_write_aggressive_write_read_sequence(ramctr_timing *ctrl, int channel, int slotrank); 299 void iosav_write_memory_test_sequence(ramctr_timing *ctrl, int channel, int slotrank); 300 301 /* FIXME: Vendor BIOS uses 64 but our algorithms are less 302 performant and even 1 seems to be enough in practice. */ 303 #define NUM_PATTERNS 4 304 305 /* 306 * WARNING: Do not forget to increase MRC_CACHE_VERSION when the saved data is changed! 307 */ 308 #define MRC_CACHE_VERSION 5 309 310 enum power_down_mode { 311 PDM_NONE = 0, 312 PDM_APD = 1, 313 PDM_PPD = 2, 314 PDM_APD_PPD = 3, 315 PDM_DLL_OFF = 6, 316 PDM_APD_DLL_OFF = 7, 317 }; 318 319 typedef struct odtmap_st { 320 u16 rttwr; 321 u16 rttnom; 322 } odtmap; 323 324 /* WARNING: Do not forget to increase MRC_CACHE_VERSION when this struct is changed! */ 325 typedef struct dimm_info_st { 326 struct dimm_attr_ddr3_st dimm[NUM_CHANNELS][NUM_SLOTS]; 327 } dimm_info; 328 329 /* WARNING: Do not forget to increase MRC_CACHE_VERSION when this struct is changed! */ 330 struct ram_rank_timings { 331 /* ROUNDT_LAT register: One byte per slotrank */ 332 u8 roundtrip_latency; 333 334 /* IO_LATENCY register: One nibble per slotrank */ 335 u8 io_latency; 336 337 /* Phase interpolator coding for command and control */ 338 int pi_coding; 339 340 struct ram_lane_timings { 341 /* GDCR RX timings */ 342 u16 rcven; 343 u8 rx_dqs_p; 344 u8 rx_dqs_n; 345 346 /* GDCR TX timings */ 347 int tx_dq; 348 u16 tx_dqs; 349 } lanes[NUM_LANES]; 350 }; 351 352 /* WARNING: Do not forget to increase MRC_CACHE_VERSION when this struct is changed! */ 353 typedef struct ramctr_timing_st { 354 u16 spd_crc[NUM_CHANNELS][NUM_SLOTS]; 355 356 /* CPUID value */ 357 u32 cpu; 358 359 /* DDR base_freq = 100 Mhz / 133 Mhz */ 360 u8 base_freq; 361 362 /* Frequency index */ 363 u32 FRQ; 364 365 u16 cas_supported; 366 /* Latencies are in units of ns, scaled by x256 */ 367 u32 tCK; 368 u32 tAA; 369 u32 tWR; 370 u32 tRCD; 371 u32 tRRD; 372 u32 tRP; 373 u32 tRAS; 374 u32 tRFC; 375 u32 tWTR; 376 u32 tRTP; 377 u32 tFAW; 378 u32 tCWL; 379 u32 tCMD; 380 /* Latencies in terms of clock cycles 381 They are saved separately as they are needed for DRAM MRS commands */ 382 u8 CAS; /* CAS read latency */ 383 u8 CWL; /* CAS write latency */ 384 385 u32 tREFI; 386 u32 tMOD; 387 u32 tXSOffset; 388 u32 tWLO; 389 u32 tCKE; 390 u32 tXPDLL; 391 u32 tXP; 392 u32 tAONPD; 393 394 /* Bits [0..11] of PM_DLL_CONFIG: Master DLL wakeup delay timer */ 395 u16 mdll_wake_delay; 396 397 u8 rankmap[NUM_CHANNELS]; 398 int ref_card_offset[NUM_CHANNELS]; 399 u32 mad_dimm[NUM_CHANNELS]; 400 int channel_size_mb[NUM_CHANNELS]; 401 u32 cmd_stretch[NUM_CHANNELS]; 402 403 int pi_code_offset; 404 int pi_coding_threshold; 405 406 bool ecc_supported; 407 bool ecc_forced; 408 bool ecc_enabled; 409 int lanes; /* active lanes: 8 or 9 */ 410 int edge_offset[3]; 411 int tx_dq_offset[3]; 412 413 int extended_temperature_range; 414 int auto_self_refresh; 415 416 int rank_mirror[NUM_CHANNELS][NUM_SLOTRANKS]; 417 418 struct ram_rank_timings timings[NUM_CHANNELS][NUM_SLOTRANKS]; 419 420 dimm_info info; 421 } ramctr_timing; 422 423 #define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0) 424 425 #define FOR_ALL_LANES for (lane = 0; lane < ctrl->lanes; lane++) 426 #define FOR_ALL_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++) 427 #define FOR_ALL_POPULATED_RANKS for (slotrank = 0; slotrank < NUM_SLOTRANKS; slotrank++) if (ctrl->rankmap[channel] & (1 << slotrank)) 428 #define FOR_ALL_POPULATED_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++) if (ctrl->rankmap[channel]) 429 #define MAX_EDGE_TIMING 71 430 #define MAX_TX_DQ 127 431 #define MAX_TX_DQS 511 432 #define MAX_RCVEN 127 433 #define MAX_CAS 18 434 #define MIN_CAS 4 435 436 /* 437 * 1 QCLK (quadrature clock) is one half of a full clock cycle (tCK). 438 * In addition, 64 PI (phase interpolator) ticks are equal to 1 QCLK. 439 * Logic delay values in I/O register bitfields are expressed in QCLKs. 440 */ 441 #define QCLK_PI 64 442 443 #define MAKE_ERR ((channel << 16) | (slotrank << 8) | 1) 444 #define GET_ERR_CHANNEL(x) (x >> 16) 445 446 void dram_mrscommands(ramctr_timing *ctrl); 447 void program_timings(ramctr_timing *ctrl, int channel); 448 void dram_find_common_params(ramctr_timing *ctrl); 449 void dram_xover(ramctr_timing *ctrl); 450 void dram_timing_regs(ramctr_timing *ctrl); 451 void dram_dimm_mapping(ramctr_timing *ctrl); 452 void dram_dimm_set_mapping(ramctr_timing *ctrl, int training); 453 void dram_zones(ramctr_timing *ctrl, int training); 454 void dram_memorymap(ramctr_timing *ctrl, int me_uma_size); 455 void dram_jedecreset(ramctr_timing *ctrl); 456 int receive_enable_calibration(ramctr_timing *ctrl); 457 int write_training(ramctr_timing *ctrl); 458 int command_training(ramctr_timing *ctrl); 459 int read_mpr_training(ramctr_timing *ctrl); 460 int aggressive_read_training(ramctr_timing *ctrl); 461 int aggressive_write_training(ramctr_timing *ctrl); 462 void normalize_training(ramctr_timing *ctrl); 463 int channel_test(ramctr_timing *ctrl); 464 void set_scrambling_seed(ramctr_timing *ctrl); 465 void set_wmm_behavior(const u32 cpu); 466 void prepare_training(ramctr_timing *ctrl); 467 void set_read_write_timings(ramctr_timing *ctrl); 468 void set_normal_operation(ramctr_timing *ctrl); 469 void final_registers(ramctr_timing *ctrl); 470 void restore_timings(ramctr_timing *ctrl); 471 int try_init_dram_ddr3(ramctr_timing *ctrl, int fast_boot, int s3resume, int me_uma_size); 472 473 void channel_scrub(ramctr_timing *ctrl); 474 bool get_host_ecc_cap(void); 475 bool get_host_ecc_forced(void); 476 477 #endif 478