1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * AMD Address Translation Library
4 *
5 * umc.c : Unified Memory Controller (UMC) topology helpers
6 *
7 * Copyright (c) 2023, Advanced Micro Devices, Inc.
8 * All Rights Reserved.
9 *
10 * Author: Yazen Ghannam <[email protected]>
11 */
12
13 #include "internal.h"
14
15 /*
16 * MI300 has a fixed, model-specific mapping between a UMC instance and
17 * its related Data Fabric Coherent Station instance.
18 *
19 * The MCA_IPID_UMC[InstanceId] field holds a unique identifier for the
20 * UMC instance within a Node. Use this to find the appropriate Coherent
21 * Station ID.
22 *
23 * Redundant bits were removed from the map below.
24 */
25 static const u16 umc_coh_st_map[32] = {
26 0x393, 0x293, 0x193, 0x093,
27 0x392, 0x292, 0x192, 0x092,
28 0x391, 0x291, 0x191, 0x091,
29 0x390, 0x290, 0x190, 0x090,
30 0x793, 0x693, 0x593, 0x493,
31 0x792, 0x692, 0x592, 0x492,
32 0x791, 0x691, 0x591, 0x491,
33 0x790, 0x690, 0x590, 0x490,
34 };
35
36 #define UMC_ID_MI300 GENMASK(23, 12)
get_coh_st_inst_id_mi300(struct atl_err * err)37 static u8 get_coh_st_inst_id_mi300(struct atl_err *err)
38 {
39 u16 umc_id = FIELD_GET(UMC_ID_MI300, err->ipid);
40 u8 i;
41
42 for (i = 0; i < ARRAY_SIZE(umc_coh_st_map); i++) {
43 if (umc_id == umc_coh_st_map[i])
44 break;
45 }
46
47 WARN_ON_ONCE(i >= ARRAY_SIZE(umc_coh_st_map));
48
49 return i;
50 }
51
52 /* XOR the bits in @val. */
bitwise_xor_bits(u16 val)53 static u16 bitwise_xor_bits(u16 val)
54 {
55 u16 tmp = 0;
56 u8 i;
57
58 for (i = 0; i < 16; i++)
59 tmp ^= (val >> i) & 0x1;
60
61 return tmp;
62 }
63
64 struct xor_bits {
65 bool xor_enable;
66 u16 col_xor;
67 u32 row_xor;
68 };
69
70 #define NUM_BANK_BITS 4
71 #define NUM_COL_BITS 5
72 #define NUM_SID_BITS 2
73
74 static struct {
75 /* UMC::CH::AddrHashBank */
76 struct xor_bits bank[NUM_BANK_BITS];
77
78 /* UMC::CH::AddrHashPC */
79 struct xor_bits pc;
80
81 /* UMC::CH::AddrHashPC2 */
82 u8 bank_xor;
83 } addr_hash;
84
85 static struct {
86 u8 bank[NUM_BANK_BITS];
87 u8 col[NUM_COL_BITS];
88 u8 sid[NUM_SID_BITS];
89 u8 num_row_lo;
90 u8 num_row_hi;
91 u8 row_lo;
92 u8 row_hi;
93 u8 pc;
94 } bit_shifts;
95
96 #define MI300_UMC_CH_BASE 0x90000
97 #define MI300_ADDR_CFG (MI300_UMC_CH_BASE + 0x30)
98 #define MI300_ADDR_SEL (MI300_UMC_CH_BASE + 0x40)
99 #define MI300_COL_SEL_LO (MI300_UMC_CH_BASE + 0x50)
100 #define MI300_ADDR_SEL_2 (MI300_UMC_CH_BASE + 0xA4)
101 #define MI300_ADDR_HASH_BANK0 (MI300_UMC_CH_BASE + 0xC8)
102 #define MI300_ADDR_HASH_PC (MI300_UMC_CH_BASE + 0xE0)
103 #define MI300_ADDR_HASH_PC2 (MI300_UMC_CH_BASE + 0xE4)
104
105 #define ADDR_HASH_XOR_EN BIT(0)
106 #define ADDR_HASH_COL_XOR GENMASK(13, 1)
107 #define ADDR_HASH_ROW_XOR GENMASK(31, 14)
108 #define ADDR_HASH_BANK_XOR GENMASK(5, 0)
109
110 #define ADDR_CFG_NUM_ROW_LO GENMASK(11, 8)
111 #define ADDR_CFG_NUM_ROW_HI GENMASK(15, 12)
112
113 #define ADDR_SEL_BANK0 GENMASK(3, 0)
114 #define ADDR_SEL_BANK1 GENMASK(7, 4)
115 #define ADDR_SEL_BANK2 GENMASK(11, 8)
116 #define ADDR_SEL_BANK3 GENMASK(15, 12)
117 #define ADDR_SEL_BANK4 GENMASK(20, 16)
118 #define ADDR_SEL_ROW_LO GENMASK(27, 24)
119 #define ADDR_SEL_ROW_HI GENMASK(31, 28)
120
121 #define COL_SEL_LO_COL0 GENMASK(3, 0)
122 #define COL_SEL_LO_COL1 GENMASK(7, 4)
123 #define COL_SEL_LO_COL2 GENMASK(11, 8)
124 #define COL_SEL_LO_COL3 GENMASK(15, 12)
125 #define COL_SEL_LO_COL4 GENMASK(19, 16)
126
127 #define ADDR_SEL_2_BANK5 GENMASK(4, 0)
128 #define ADDR_SEL_2_CHAN GENMASK(15, 12)
129
130 /*
131 * Read UMC::CH::AddrHash{Bank,PC,PC2} registers to get XOR bits used
132 * for hashing.
133 *
134 * Also, read UMC::CH::Addr{Cfg,Sel,Sel2} and UMC::CH:ColSelLo registers to
135 * get the values needed to reconstruct the normalized address. Apply additional
136 * offsets to the raw register values, as needed.
137 *
138 * Do this during module init, since the values will not change during run time.
139 *
140 * These registers are instantiated for each UMC across each AMD Node.
141 * However, they should be identically programmed due to the fixed hardware
142 * design of MI300 systems. So read the values from Node 0 UMC 0 and keep a
143 * single global structure for simplicity.
144 */
get_umc_info_mi300(void)145 int get_umc_info_mi300(void)
146 {
147 u32 temp;
148 int ret;
149 u8 i;
150
151 for (i = 0; i < NUM_BANK_BITS; i++) {
152 ret = amd_smn_read(0, MI300_ADDR_HASH_BANK0 + (i * 4), &temp);
153 if (ret)
154 return ret;
155
156 addr_hash.bank[i].xor_enable = FIELD_GET(ADDR_HASH_XOR_EN, temp);
157 addr_hash.bank[i].col_xor = FIELD_GET(ADDR_HASH_COL_XOR, temp);
158 addr_hash.bank[i].row_xor = FIELD_GET(ADDR_HASH_ROW_XOR, temp);
159 }
160
161 ret = amd_smn_read(0, MI300_ADDR_HASH_PC, &temp);
162 if (ret)
163 return ret;
164
165 addr_hash.pc.xor_enable = FIELD_GET(ADDR_HASH_XOR_EN, temp);
166 addr_hash.pc.col_xor = FIELD_GET(ADDR_HASH_COL_XOR, temp);
167 addr_hash.pc.row_xor = FIELD_GET(ADDR_HASH_ROW_XOR, temp);
168
169 ret = amd_smn_read(0, MI300_ADDR_HASH_PC2, &temp);
170 if (ret)
171 return ret;
172
173 addr_hash.bank_xor = FIELD_GET(ADDR_HASH_BANK_XOR, temp);
174
175 ret = amd_smn_read(0, MI300_ADDR_CFG, &temp);
176 if (ret)
177 return ret;
178
179 bit_shifts.num_row_hi = FIELD_GET(ADDR_CFG_NUM_ROW_HI, temp);
180 bit_shifts.num_row_lo = 10 + FIELD_GET(ADDR_CFG_NUM_ROW_LO, temp);
181
182 ret = amd_smn_read(0, MI300_ADDR_SEL, &temp);
183 if (ret)
184 return ret;
185
186 bit_shifts.bank[0] = 5 + FIELD_GET(ADDR_SEL_BANK0, temp);
187 bit_shifts.bank[1] = 5 + FIELD_GET(ADDR_SEL_BANK1, temp);
188 bit_shifts.bank[2] = 5 + FIELD_GET(ADDR_SEL_BANK2, temp);
189 bit_shifts.bank[3] = 5 + FIELD_GET(ADDR_SEL_BANK3, temp);
190 /* Use BankBit4 for the SID0 position. */
191 bit_shifts.sid[0] = 5 + FIELD_GET(ADDR_SEL_BANK4, temp);
192 bit_shifts.row_lo = 12 + FIELD_GET(ADDR_SEL_ROW_LO, temp);
193 bit_shifts.row_hi = 24 + FIELD_GET(ADDR_SEL_ROW_HI, temp);
194
195 ret = amd_smn_read(0, MI300_COL_SEL_LO, &temp);
196 if (ret)
197 return ret;
198
199 bit_shifts.col[0] = 2 + FIELD_GET(COL_SEL_LO_COL0, temp);
200 bit_shifts.col[1] = 2 + FIELD_GET(COL_SEL_LO_COL1, temp);
201 bit_shifts.col[2] = 2 + FIELD_GET(COL_SEL_LO_COL2, temp);
202 bit_shifts.col[3] = 2 + FIELD_GET(COL_SEL_LO_COL3, temp);
203 bit_shifts.col[4] = 2 + FIELD_GET(COL_SEL_LO_COL4, temp);
204
205 ret = amd_smn_read(0, MI300_ADDR_SEL_2, &temp);
206 if (ret)
207 return ret;
208
209 /* Use BankBit5 for the SID1 position. */
210 bit_shifts.sid[1] = 5 + FIELD_GET(ADDR_SEL_2_BANK5, temp);
211 bit_shifts.pc = 5 + FIELD_GET(ADDR_SEL_2_CHAN, temp);
212
213 return 0;
214 }
215
216 /*
217 * MI300 systems report a DRAM address in MCA_ADDR for DRAM ECC errors. This must
218 * be converted to the intermediate normalized address (NA) before translating to a
219 * system physical address.
220 *
221 * The DRAM address includes bank, row, and column. Also included are bits for
222 * pseudochannel (PC) and stack ID (SID).
223 *
224 * Abbreviations: (S)tack ID, (P)seudochannel, (R)ow, (B)ank, (C)olumn, (Z)ero
225 *
226 * The MCA address format is as follows:
227 * MCA_ADDR[27:0] = {S[1:0], P[0], R[14:0], B[3:0], C[4:0], Z[0]}
228 *
229 * Additionally, the PC and Bank bits may be hashed. This must be accounted for before
230 * reconstructing the normalized address.
231 */
232 #define MI300_UMC_MCA_BANK GENMASK(9, 6)
233 #define MI300_UMC_MCA_ROW GENMASK(24, 10)
234 #define MI300_UMC_MCA_PC BIT(25)
235 #define MI300_UMC_MCA_SID GENMASK(27, 26)
236
convert_dram_to_norm_addr_mi300(unsigned long addr)237 static unsigned long convert_dram_to_norm_addr_mi300(unsigned long addr)
238 {
239 u16 i, col, row, bank, pc, sid;
240 u32 temp;
241
242 col = FIELD_GET(MI300_UMC_MCA_COL, addr);
243 bank = FIELD_GET(MI300_UMC_MCA_BANK, addr);
244 row = FIELD_GET(MI300_UMC_MCA_ROW, addr);
245 pc = FIELD_GET(MI300_UMC_MCA_PC, addr);
246 sid = FIELD_GET(MI300_UMC_MCA_SID, addr);
247
248 /* Calculate hash for each Bank bit. */
249 for (i = 0; i < NUM_BANK_BITS; i++) {
250 if (!addr_hash.bank[i].xor_enable)
251 continue;
252
253 temp = bitwise_xor_bits(col & addr_hash.bank[i].col_xor);
254 temp ^= bitwise_xor_bits(row & addr_hash.bank[i].row_xor);
255 bank ^= temp << i;
256 }
257
258 /* Calculate hash for PC bit. */
259 if (addr_hash.pc.xor_enable) {
260 temp = bitwise_xor_bits(col & addr_hash.pc.col_xor);
261 temp ^= bitwise_xor_bits(row & addr_hash.pc.row_xor);
262 /* Bits SID[1:0] act as Bank[5:4] for PC hash, so apply them here. */
263 temp ^= bitwise_xor_bits((bank | sid << NUM_BANK_BITS) & addr_hash.bank_xor);
264 pc ^= temp;
265 }
266
267 /* Reconstruct the normalized address starting with NA[4:0] = 0 */
268 addr = 0;
269
270 /* Column bits */
271 for (i = 0; i < NUM_COL_BITS; i++) {
272 temp = (col >> i) & 0x1;
273 addr |= temp << bit_shifts.col[i];
274 }
275
276 /* Bank bits */
277 for (i = 0; i < NUM_BANK_BITS; i++) {
278 temp = (bank >> i) & 0x1;
279 addr |= temp << bit_shifts.bank[i];
280 }
281
282 /* Row lo bits */
283 for (i = 0; i < bit_shifts.num_row_lo; i++) {
284 temp = (row >> i) & 0x1;
285 addr |= temp << (i + bit_shifts.row_lo);
286 }
287
288 /* Row hi bits */
289 for (i = 0; i < bit_shifts.num_row_hi; i++) {
290 temp = (row >> (i + bit_shifts.num_row_lo)) & 0x1;
291 addr |= temp << (i + bit_shifts.row_hi);
292 }
293
294 /* PC bit */
295 addr |= pc << bit_shifts.pc;
296
297 /* SID bits */
298 for (i = 0; i < NUM_SID_BITS; i++) {
299 temp = (sid >> i) & 0x1;
300 addr |= temp << bit_shifts.sid[i];
301 }
302
303 pr_debug("Addr=0x%016lx", addr);
304 pr_debug("Bank=%u Row=%u Column=%u PC=%u SID=%u", bank, row, col, pc, sid);
305
306 return addr;
307 }
308
309 /*
310 * When a DRAM ECC error occurs on MI300 systems, it is recommended to retire
311 * all memory within that DRAM row. This applies to the memory with a DRAM
312 * bank.
313 *
314 * To find the memory addresses, loop through permutations of the DRAM column
315 * bits and find the System Physical address of each. The column bits are used
316 * to calculate the intermediate Normalized address, so all permutations should
317 * be checked.
318 *
319 * See amd_atl::convert_dram_to_norm_addr_mi300() for MI300 address formats.
320 */
321 #define MI300_NUM_COL BIT(HWEIGHT(MI300_UMC_MCA_COL))
_retire_row_mi300(struct atl_err * a_err)322 static void _retire_row_mi300(struct atl_err *a_err)
323 {
324 unsigned long addr;
325 struct page *p;
326 u8 col;
327
328 for (col = 0; col < MI300_NUM_COL; col++) {
329 a_err->addr &= ~MI300_UMC_MCA_COL;
330 a_err->addr |= FIELD_PREP(MI300_UMC_MCA_COL, col);
331
332 addr = amd_convert_umc_mca_addr_to_sys_addr(a_err);
333 if (IS_ERR_VALUE(addr))
334 continue;
335
336 addr = PHYS_PFN(addr);
337
338 /*
339 * Skip invalid or already poisoned pages to avoid unnecessary
340 * error messages from memory_failure().
341 */
342 p = pfn_to_online_page(addr);
343 if (!p)
344 continue;
345
346 if (PageHWPoison(p))
347 continue;
348
349 memory_failure(addr, 0);
350 }
351 }
352
353 /*
354 * In addition to the column bits, the row[13] bit should also be included when
355 * calculating addresses affected by a physical row.
356 *
357 * Instead of running through another loop over a single bit, just run through
358 * the column bits twice and flip the row[13] bit in-between.
359 *
360 * See MI300_UMC_MCA_ROW for the row bits in MCA_ADDR_UMC value.
361 */
retire_row_mi300(struct atl_err * a_err)362 static void retire_row_mi300(struct atl_err *a_err)
363 {
364 _retire_row_mi300(a_err);
365 a_err->addr ^= MI300_UMC_MCA_ROW13;
366 _retire_row_mi300(a_err);
367 }
368
amd_retire_dram_row(struct atl_err * a_err)369 void amd_retire_dram_row(struct atl_err *a_err)
370 {
371 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
372 return retire_row_mi300(a_err);
373 }
374 EXPORT_SYMBOL_GPL(amd_retire_dram_row);
375
get_addr(unsigned long addr)376 static unsigned long get_addr(unsigned long addr)
377 {
378 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
379 return convert_dram_to_norm_addr_mi300(addr);
380
381 return addr;
382 }
383
384 #define MCA_IPID_INST_ID_HI GENMASK_ULL(47, 44)
get_die_id(struct atl_err * err)385 static u8 get_die_id(struct atl_err *err)
386 {
387 /*
388 * AMD Node ID is provided in MCA_IPID[InstanceIdHi], and this
389 * needs to be divided by 4 to get the internal Die ID.
390 */
391 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) {
392 u8 node_id = FIELD_GET(MCA_IPID_INST_ID_HI, err->ipid);
393
394 return node_id >> 2;
395 }
396
397 /*
398 * For CPUs, this is the AMD Node ID modulo the number
399 * of AMD Nodes per socket.
400 */
401 return topology_amd_node_id(err->cpu) % topology_amd_nodes_per_pkg();
402 }
403
404 #define UMC_CHANNEL_NUM GENMASK(31, 20)
get_coh_st_inst_id(struct atl_err * err)405 static u8 get_coh_st_inst_id(struct atl_err *err)
406 {
407 if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous)
408 return get_coh_st_inst_id_mi300(err);
409
410 return FIELD_GET(UMC_CHANNEL_NUM, err->ipid);
411 }
412
convert_umc_mca_addr_to_sys_addr(struct atl_err * err)413 unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err)
414 {
415 u8 socket_id = topology_physical_package_id(err->cpu);
416 u8 coh_st_inst_id = get_coh_st_inst_id(err);
417 unsigned long addr = get_addr(err->addr);
418 u8 die_id = get_die_id(err);
419 unsigned long ret_addr;
420
421 pr_debug("socket_id=0x%x die_id=0x%x coh_st_inst_id=0x%x addr=0x%016lx",
422 socket_id, die_id, coh_st_inst_id, addr);
423
424 ret_addr = prm_umc_norm_to_sys_addr(socket_id, err->ipid, addr);
425 if (!IS_ERR_VALUE(ret_addr))
426 return ret_addr;
427
428 return norm_to_sys_addr(socket_id, die_id, coh_st_inst_id, addr);
429 }
430