1 /***********************license start***********************************
2 * Copyright (c) 2003-2017 Cavium Inc. ([email protected]). All rights
3 * reserved.
4 *
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of Cavium Inc. nor the names of
19 * its contributors may be used to endorse or promote products
20 * derived from this software without specific prior written
21 * permission.
22 *
23 * This Software, including technical data, may be subject to U.S. export
24 * control laws, including the U.S. Export Administration Act and its
25 * associated regulations, and may be subject to export or import
26 * regulations in other countries.
27 *
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
31 * TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
32 * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
33 * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
34 * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
35 * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
36 * QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK
37 * ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
39 #include <bdk.h>
40 #include <string.h>
41 #include "libbdk-arch/bdk-csrs-dtx.h"
42 #include "libbdk-arch/bdk-csrs-gser.h"
43 #include "libbdk-arch/bdk-csrs-gic.h"
44 #include "libbdk-arch/bdk-csrs-pem.h"
45 #include "libbdk-arch/bdk-csrs-pcierc.h"
46 #include "libbdk-arch/bdk-csrs-sli.h"
47 #include "libbdk-arch/bdk-csrs-rst.h"
48 #include "libbdk-hal/bdk-pcie.h"
49 #include "libbdk-hal/bdk-config.h"
50 #include "libbdk-hal/bdk-utils.h"
51 #include "libbdk-hal/if/bdk-if.h"
52 #include "libbdk-hal/bdk-qlm.h"
53 #include "libbdk-hal/device/bdk-device.h"
54 #include "libbdk-hal/bdk-ecam.h"
55
56 /**
57 * Return the number of possible PCIe ports on a node. The actual number
58 * of configured ports may be less and may also be disjoint.
59 *
60 * @param node Node to query
61 *
62 * @return Number of PCIe ports that are possible
63 */
bdk_pcie_get_num_ports(bdk_node_t node)64 int bdk_pcie_get_num_ports(bdk_node_t node)
65 {
66 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
67 return 6;
68 else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
69 return 4;
70 else if (CAVIUM_IS_MODEL(CAVIUM_CN81XX))
71 return 3;
72 else
73 return 0;
74 }
75
76
77 /**
78 * Given a PCIe port, determine which SLI controls its memory regions
79 *
80 * @param node Node for the PCIe port
81 * @param pcie_port The PCIe port
82 * @param sli The SLI index is written to this integer pointer
83 * @param sli_group The index of the PCIe port on the SLI is returned here. This is a sequencial
84 * number for each PCIe on an SLI. Use this to index SLI regions.
85 */
__bdk_pcie_get_sli(bdk_node_t node,int pcie_port,int * sli,int * sli_group)86 static void __bdk_pcie_get_sli(bdk_node_t node, int pcie_port, int *sli, int *sli_group)
87 {
88 /* This mapping should be determined by find the SLI number on the
89 same ECAM bus as the PCIERC bridge. That is fairly complex, so it is
90 hardcoded for now */
91 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
92 {
93 /* Ports 0-2 goto SLI0, ports 3-5 goto SLI1 */
94 *sli = (pcie_port >= 3) ? 1 : 0;
95 *sli_group = pcie_port - *sli * 3;
96 return;
97 }
98 else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX) || CAVIUM_IS_MODEL(CAVIUM_CN81XX))
99 {
100 /* Only one SLI */
101 *sli = 0;
102 *sli_group = pcie_port;
103 return;
104 }
105 else
106 bdk_fatal("Unable to determine SLI for PCIe port. Update __bdk_pcie_get_sli()\n");
107 }
108
109 /**
110 * Return the Core physical base address for PCIe MEM access. Memory is
111 * read/written as an offset from this address.
112 *
113 * @param node Node to use in a Numa setup
114 * @param pcie_port PCIe port the memory is on
115 * @param mem_type Type of memory
116 *
117 * @return 64bit physical address for read/write
118 */
bdk_pcie_get_base_address(bdk_node_t node,int pcie_port,bdk_pcie_mem_t mem_type)119 uint64_t bdk_pcie_get_base_address(bdk_node_t node, int pcie_port, bdk_pcie_mem_t mem_type)
120 {
121 /* See __bdk_pcie_sli_initialize() for a description about how SLI regions work */
122 int sli;
123 int sli_group;
124 __bdk_pcie_get_sli(node, pcie_port, &sli, &sli_group);
125 int region = (sli_group << 6) | (mem_type << 4);
126 union bdk_sli_s2m_op_s s2m_op;
127 s2m_op.u = 0;
128 s2m_op.s.io = 1;
129 s2m_op.s.node = node;
130 s2m_op.s.did_hi = 0x8 + sli;
131 s2m_op.s.region = region;
132 return s2m_op.u;
133 }
134
135 /**
136 * Size of the Mem address region returned at address
137 * bdk_pcie_get_base_address()
138 *
139 * @param node Node to use in a Numa setup
140 * @param pcie_port PCIe port the IO is for
141 * @param mem_type Type of memory
142 *
143 * @return Size of the Mem window
144 */
bdk_pcie_get_base_size(bdk_node_t node,int pcie_port,bdk_pcie_mem_t mem_type)145 uint64_t bdk_pcie_get_base_size(bdk_node_t node, int pcie_port, bdk_pcie_mem_t mem_type)
146 {
147 return 1ull << 36;
148 }
149
150 /**
151 * @INTERNAL
152 * Initialize the RC config space CSRs
153 *
154 * @param pcie_port PCIe port to initialize
155 */
__bdk_pcie_rc_initialize_config_space(bdk_node_t node,int pcie_port)156 static void __bdk_pcie_rc_initialize_config_space(bdk_node_t node, int pcie_port)
157 {
158 int sli;
159 int sli_group;
160 __bdk_pcie_get_sli(node, pcie_port, &sli, &sli_group);
161
162 /* The reset default for config retries is too short. Set it to 48ms, which
163 is what the Octeon SDK team is using. There is no documentation about
164 where they got the 48ms number */
165 int cfg_retry = 48 * 1000000 / (bdk_clock_get_rate(node, BDK_CLOCK_SCLK) >> 16);
166 if (cfg_retry >= 0x10000)
167 cfg_retry = 0xfffff;
168 BDK_CSR_MODIFY(c, node, BDK_PEMX_CTL_STATUS(pcie_port),
169 c.cn83xx.cfg_rtry = cfg_retry);
170
171
172 /* Max Payload Size (PCIE*_CFG030[MPS]) */
173 /* Max Read Request Size (PCIE*_CFG030[MRRS]) */
174 /* Relaxed-order, no-snoop enables (PCIE*_CFG030[RO_EN,NS_EN] */
175 /* Error Message Enables (PCIE*_CFG030[CE_EN,NFE_EN,FE_EN,UR_EN]) */
176 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG030(pcie_port),
177 c.s.mps = 1; /* Support 256 byte MPS */
178 c.s.mrrs = 0x5; /* Support 4KB MRRS */
179 c.s.ro_en = 1; /* Enable relaxed order processing. This will allow devices to affect read response ordering */
180 c.s.ns_en = 1; /* Enable no snoop processing. Not used */
181 c.s.ce_en = 1; /* Correctable error reporting enable. */
182 c.s.nfe_en = 1; /* Non-fatal error reporting enable. */
183 c.s.fe_en = 1; /* Fatal error reporting enable. */
184 c.s.ur_en = 1); /* Unsupported request reporting enable. */
185
186 /* Configure the PCIe slot number if specified */
187 int slot_num = bdk_config_get_int(BDK_CONFIG_PCIE_PHYSICAL_SLOT, node, pcie_port);
188 if (slot_num != -1)
189 {
190 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG028(pcie_port),
191 c.s.si = 1); /* Slot Implemented*/
192 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG033(pcie_port),
193 c.s.ps_num = slot_num);
194 }
195
196 /* Disable ECRC Generation as not all card support it. The OS can enable it
197 later if desired (PCIE*_CFG070[GE,CE]) */
198 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG070(pcie_port),
199 c.s.ge = 0; /* ECRC generation disable. */
200 c.s.ce = 0); /* ECRC check disable. */
201
202 /* Access Enables (PCIE*_CFG001[MSAE,ME]) */
203 /* ME and MSAE should always be set. */
204 /* Interrupt Disable (PCIE*_CFG001[I_DIS]) */
205 /* System Error Message Enable (PCIE*_CFG001[SEE]) */
206 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG001(pcie_port),
207 c.s.msae = 1; /* Memory space enable. */
208 c.s.me = 1; /* Bus master enable. */
209 c.s.i_dis = 1; /* INTx assertion disable. */
210 c.s.see = 1); /* SERR# enable */
211
212 /* Advanced Error Recovery Message Enables */
213 /* (PCIE*_CFG066,PCIE*_CFG067,PCIE*_CFG069) */
214 BDK_CSR_WRITE(node, BDK_PCIERCX_CFG066(pcie_port), 0);
215 /* Use BDK_PCIERCX_CFG067 hardware default */
216 BDK_CSR_WRITE(node, BDK_PCIERCX_CFG069(pcie_port), 0);
217
218
219 /* Active State Power Management (PCIE*_CFG032[ASLPC]) */
220 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG032(pcie_port),
221 c.s.aslpc = 0); /* Active state Link PM control. */
222
223 /* Link Width Mode (PCIERCn_CFG452[LME]) - Set during bdk_pcie_rc_initialize_link() */
224 /* Primary Bus Number (PCIERCn_CFG006[PBNUM]) */
225 /* Use bus numbers as follows:
226 0 - 31: Reserved for internal ECAM
227 32 - 87: First PCIe on SLI
228 88 - 143: Second PCIe on SLI
229 144 - 199: Third PCIe on SLI
230 200 - 255: Fourth PCIe on SLI
231 Start bus = 32 + pcie * 56 */
232 const int BUSSES_PER_PCIE = 56;
233 int bus = 32 + sli_group * BUSSES_PER_PCIE;
234 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG006(pcie_port),
235 c.s.pbnum = 0;
236 c.s.sbnum = bus;
237 c.s.subbnum = bus + BUSSES_PER_PCIE - 1);
238
239 /* Memory-mapped I/O BAR (PCIERCn_CFG008) */
240 uint64_t mem_base = bdk_pcie_get_base_address(node, pcie_port, BDK_PCIE_MEM_NORMAL);
241 uint64_t mem_limit = mem_base + bdk_pcie_get_base_size(node, pcie_port, BDK_PCIE_MEM_NORMAL) - 1;
242 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG008(pcie_port),
243 c.s.mb_addr = mem_base >> 16;
244 c.s.ml_addr = mem_limit >> 16);
245
246 /* Prefetchable BAR (PCIERCn_CFG009,PCIERCn_CFG010,PCIERCn_CFG011) */
247 uint64_t prefetch_base = bdk_pcie_get_base_address(node, pcie_port, BDK_PCIE_MEM_PREFETCH);
248 uint64_t prefetch_limit = prefetch_base + bdk_pcie_get_base_size(node, pcie_port, BDK_PCIE_MEM_PREFETCH) - 1;
249 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG009(pcie_port),
250 c.s.lmem_base = prefetch_base >> 16;
251 c.s.lmem_limit = prefetch_limit >> 16);
252 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG010(pcie_port),
253 c.s.umem_base = prefetch_base >> 32);
254 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG011(pcie_port),
255 c.s.umem_limit = prefetch_limit >> 32);
256
257 /* System Error Interrupt Enables (PCIERCn_CFG035[SECEE,SEFEE,SENFEE]) */
258 /* PME Interrupt Enables (PCIERCn_CFG035[PMEIE]) */
259 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG035(pcie_port),
260 c.s.secee = 1; /* System error on correctable error enable. */
261 c.s.sefee = 1; /* System error on fatal error enable. */
262 c.s.senfee = 1; /* System error on non-fatal error enable. */
263 c.s.pmeie = 1); /* PME interrupt enable. */
264
265 /* Advanced Error Recovery Interrupt Enables */
266 /* (PCIERCn_CFG075[CERE,NFERE,FERE]) */
267 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG075(pcie_port),
268 c.s.cere = 1; /* Correctable error reporting enable. */
269 c.s.nfere = 1; /* Non-fatal error reporting enable. */
270 c.s.fere = 1); /* Fatal error reporting enable. */
271
272 /* Make sure the PEM agrees with GSERX about the speed its going to try */
273 BDK_CSR_INIT(pem_cfg, node, BDK_PEMX_CFG(pcie_port));
274 switch (pem_cfg.cn83xx.md)
275 {
276 case 0: /* Gen 1 */
277 /* Set the target link speed */
278 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG040(pcie_port),
279 c.s.tls = 1);
280 break;
281 case 1: /* Gen 2 */
282 /* Set the target link speed */
283 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG040(pcie_port),
284 c.s.tls = 2);
285 break;
286 case 2: /* Gen 3 */
287 /* Set the target link speed */
288 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG040(pcie_port),
289 c.s.tls = 3);
290 break;
291 default:
292 bdk_error("N%d.PCIe%d: Unexpected rate of %d\n", node, pcie_port, pem_cfg.cn83xx.md);
293 break;
294 }
295
296 BDK_CSR_INIT(pemx_cfg, node, BDK_PEMX_CFG(pcie_port));
297 BDK_CSR_INIT(cfg452, node, BDK_PCIERCX_CFG452(pcie_port));
298 BDK_CSR_INIT(cfg031, node, BDK_PCIERCX_CFG031(pcie_port));
299 int lme = cfg452.s.lme;
300 int mlw = cfg031.s.mlw;
301
302 /* Link Width Mode (PCIERCn_CFG452[LME]) */
303 if (CAVIUM_IS_MODEL(CAVIUM_CN88XX))
304 {
305 lme = (pemx_cfg.cn88xx.lanes8) ? 0xf : 0x7;
306 mlw = (pemx_cfg.cn88xx.lanes8) ? 8 : 4;
307 }
308 /* CN83XX can support 8 lanes on QLM0+1 or QLM2+3. 4 lanes on DLM5+6 */
309 if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
310 {
311 switch (pcie_port)
312 {
313 case 0: /* PEM0 on QLM0-1 */
314 lme = (pemx_cfg.cn83xx.lanes8) ? 0xf : 0x7;
315 mlw = (pemx_cfg.cn83xx.lanes8) ? 8 : 4;
316 break;
317 case 1: /* PEM1 on QLM1 */
318 lme = 0x7;
319 mlw = 4;
320 break;
321 case 2: /* PEM2 on QLM2-3 or DLM4 */
322 {
323 BDK_CSR_INIT(pemx_qlm, node, BDK_PEMX_QLM(pcie_port));
324 if (pemx_qlm.s.pem_bdlm) /* PEM2 is on DLM4 */
325 {
326 lme = 0x3;
327 mlw = 2;
328 }
329 else /* PEM2 is on QLM2 */
330 {
331 lme = (pemx_cfg.cn83xx.lanes8) ? 0xf : 0x7;
332 mlw = (pemx_cfg.cn83xx.lanes8) ? 8 : 4;
333 }
334 break;
335 }
336 case 3: /* PEM3 on QLM3 or DLM5-6 */
337 {
338 BDK_CSR_INIT(pemx_qlm, node, BDK_PEMX_QLM(pcie_port));
339 if (pemx_qlm.s.pem_bdlm) /* PEM3 is on DLM5-6 */
340 {
341 lme = (pemx_cfg.cn83xx.lanes8) ? 0x7 : 0x3;
342 mlw = (pemx_cfg.cn83xx.lanes8) ? 4 : 2;
343 }
344 else /* PEM3 is on QLM3 */
345 {
346 lme = 0x7;
347 mlw = 4;
348 }
349 break;
350 }
351 }
352 }
353 /* CN80XX only supports 1 lane on PEM0 */
354 if (cavium_is_altpkg(CAVIUM_CN81XX) && (pcie_port == 0))
355 {
356 lme = 1;
357 mlw = 1;
358 }
359
360 /* Allow override of hardware max link width */
361 int max_width = bdk_config_get_int(BDK_CONFIG_PCIE_WIDTH, node, pcie_port);
362 switch (max_width)
363 {
364 case 1:
365 lme = 1;
366 mlw = 1;
367 break;
368 case 2:
369 lme = 3;
370 mlw = 2;
371 break;
372 case 4:
373 lme = 7;
374 mlw = 4;
375 break;
376 case 8:
377 lme = 0xf;
378 mlw = 8;
379 break;
380 case 16:
381 lme = 0x1f;
382 mlw = 16;
383 break;
384 default:
385 /* No change */
386 break;
387 }
388 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG452(pcie_port),
389 c.s.lme = lme);
390 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG031(pcie_port),
391 c.s.mlw = mlw);
392
393 /* Errata PEM-25990 - Disable ASLPMS */
394 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG031(pcie_port),
395 c.s.aslpms = 0);
396
397 /* Errata PEM-26189 - PEM EQ Preset Removal */
398 /* CFG554.PRV default changed from 16'h7ff to 16'h593. Should be
399 safe to apply to CN88XX, CN81XX, and CN83XX */
400 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG554(pcie_port),
401 c.s.prv = bdk_config_get_int(BDK_CONFIG_PCIE_PRESET_REQUEST_VECTOR, node, pcie_port));
402
403 /* Errata PEM-26189 - Disable the 2ms timer on all chips */
404 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG554(pcie_port),
405 c.s.p23td = 1);
406
407 /* Errata PEM-21178 - Change the CFG[089-092] LxUTP and LxDTP defaults.
408 Should be safe to apply to CN88XX, CN81XX, and CN83XX */
409 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG089(pcie_port),
410 c.s.l0dtp = 0x7;
411 c.s.l0utp = 0x7;
412 c.cn83xx.l1dtp = 0x7;
413 c.s.l1utp = 0x7);
414 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG090(pcie_port),
415 c.s.l2dtp = 0x7;
416 c.s.l2utp = 0x7;
417 c.s.l3dtp = 0x7;
418 c.s.l3utp = 0x7);
419 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG091(pcie_port),
420 c.s.l4dtp = 0x7;
421 c.s.l4utp = 0x7;
422 c.s.l5dtp = 0x7;
423 c.s.l5utp = 0x7);
424 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG092(pcie_port),
425 c.s.l6dtp = 0x7;
426 c.s.l6utp = 0x7;
427 c.s.l7dtp = 0x7;
428 c.s.l7utp = 0x7);
429
430 /* (ECAM-27114) PCIERC has incorrect device code */
431 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG002(pcie_port),
432 c.s.sc = 0x4;
433 c.s.bcc = 0x6);
434
435 /* Errata PCIE-29440 - Atomic Egress ATOM_OP/ATOM_OP_EP not implemented
436 correctly */
437 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG038(pcie_port),
438 c.s.atom_op =0x1;
439 c.s.atom_op_eb=0);
440
441 /* Errata PCIE-29566 PEM Link Hangs after going into L1 */
442 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG548(pcie_port),
443 c.s.grizdnc = 0x0);
444 }
445
446 /**
447 * Get the PCIe LTSSM state for the given port
448 *
449 * @param node Node to query
450 * @param pcie_port PEM to query
451 *
452 * @return LTSSM state
453 */
__bdk_pcie_rc_get_ltssm_state(bdk_node_t node,int pcie_port)454 static int __bdk_pcie_rc_get_ltssm_state(bdk_node_t node, int pcie_port)
455 {
456 /* LTSSM state is in debug select 0 */
457 BDK_CSR_WRITE(node, BDK_DTX_PEMX_SELX(pcie_port, 0), 0);
458 BDK_CSR_WRITE(node, BDK_DTX_PEMX_ENAX(pcie_port, 0), 0xfffffffffull);
459 /* Read the value */
460 uint64_t debug = BDK_CSR_READ(node, BDK_DTX_PEMX_DATX(pcie_port, 0));
461 /* Disable the PEM from driving OCLA signals */
462 BDK_CSR_WRITE(node, BDK_DTX_PEMX_ENAX(pcie_port, 0), 0);
463 if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
464 return bdk_extract(debug, 0, 6); /* DBGSEL = 0x0, bits[5:0] */
465 else
466 return bdk_extract(debug, 3, 6); /* DBGSEL = 0x0, bits[8:3] */
467 }
468
469 /**
470 * Get the PCIe LTSSM state for the given port
471 *
472 * @param node Node to query
473 * @param pcie_port PEM to query
474 *
475 * @return LTSSM state
476 */
ltssm_string(int ltssm)477 static const char *ltssm_string(int ltssm)
478 {
479 switch (ltssm)
480 {
481 case 0x00: return "DETECT_QUIET";
482 case 0x01: return "DETECT_ACT";
483 case 0x02: return "POLL_ACTIVE";
484 case 0x03: return "POLL_COMPLIANCE";
485 case 0x04: return "POLL_CONFIG";
486 case 0x05: return "PRE_DETECT_QUIET";
487 case 0x06: return "DETECT_WAIT";
488 case 0x07: return "CFG_LINKWD_START";
489 case 0x08: return "CFG_LINKWD_ACEPT";
490 case 0x09: return "CFG_LANENUM_WAIT";
491 case 0x0A: return "CFG_LANENUM_ACEPT";
492 case 0x0B: return "CFG_COMPLETE";
493 case 0x0C: return "CFG_IDLE";
494 case 0x0D: return "RCVRY_LOCK";
495 case 0x0E: return "RCVRY_SPEED";
496 case 0x0F: return "RCVRY_RCVRCFG";
497 case 0x10: return "RCVRY_IDLE";
498 case 0x11: return "L0";
499 case 0x12: return "L0S";
500 case 0x13: return "L123_SEND_EIDLE";
501 case 0x14: return "L1_IDLE";
502 case 0x15: return "L2_IDLE";
503 case 0x16: return "L2_WAKE";
504 case 0x17: return "DISABLED_ENTRY";
505 case 0x18: return "DISABLED_IDLE";
506 case 0x19: return "DISABLED";
507 case 0x1A: return "LPBK_ENTRY";
508 case 0x1B: return "LPBK_ACTIVE";
509 case 0x1C: return "LPBK_EXIT";
510 case 0x1D: return "LPBK_EXIT_TIMEOUT";
511 case 0x1E: return "HOT_RESET_ENTRY";
512 case 0x1F: return "HOT_RESET";
513 case 0x20: return "RCVRY_EQ0";
514 case 0x21: return "RCVRY_EQ1";
515 case 0x22: return "RCVRY_EQ2";
516 case 0x23: return "RCVRY_EQ3";
517 default: return "Unknown";
518 }
519 }
520
521 /**
522 * During PCIe link initialization we need to make config request to the attached
523 * device to verify its speed and width. These config access happen very early
524 * after the device is taken out of reset, so may fail for some amount of time.
525 * This function automatically retries these config accesses. The normal builtin
526 * hardware retry isn't enough for this very early access.
527 *
528 * @param node Note to read from
529 * @param pcie_port PCIe port to read from
530 * @param bus PCIe bus number
531 * @param dev PCIe device
532 * @param func PCIe function on the device
533 * @param reg Register to read
534 *
535 * @return Config register value, or all ones on failure
536 */
cfg_read32_retry(bdk_node_t node,int pcie_port,int bus,int dev,int func,int reg)537 static uint32_t cfg_read32_retry(bdk_node_t node, int pcie_port, int bus, int dev, int func, int reg)
538 {
539 /* Read the PCI config register until we get a valid value. Some cards
540 require time after link up to return data. Wait at most 3 seconds */
541 uint64_t timeout = bdk_clock_get_count(BDK_CLOCK_TIME) + bdk_clock_get_rate(bdk_numa_local(), BDK_CLOCK_TIME) * 3;
542 uint32_t val;
543 do
544 {
545 /* Read PCI capability pointer */
546 val = bdk_pcie_config_read32(node, pcie_port, bus, dev, func, reg);
547 /* Check the read succeeded */
548 if (val != 0xffffffff)
549 return val;
550 /* Failed, wait a little and try again */
551 bdk_wait_usec(10000);
552 bdk_watchdog_poke();
553 } while (bdk_clock_get_count(BDK_CLOCK_TIME) < timeout);
554
555 BDK_TRACE(PCIE, "N%d.PCIe%d: Config read failed, can't communicate with device\n",
556 node, pcie_port);
557 return 0xffffffff;
558 }
559
560 /**
561 * Initialize a host mode PCIe link. This function assumes the PEM has already
562 * been taken out of reset and configure. It brings up the link and checks that
563 * the negotiated speed and width is correct for the configured PEM and the
564 * device plugged into it. Note that the return code will signal a retry needed
565 * for some link failures. The caller is responsible for PEM reset and retry.
566 *
567 * @param node Node the PEM is on
568 * @param pcie_port PCIe port to initialize link on
569 *
570 * @return Zero on success
571 * Negative on failures where retries are not needed
572 * Positive if a retry is needed to fix a failure
573 */
__bdk_pcie_rc_initialize_link(bdk_node_t node,int pcie_port)574 static int __bdk_pcie_rc_initialize_link(bdk_node_t node, int pcie_port)
575 {
576 #define LTSSM_HISTORY_SIZE 64 /* Number of LTSSM transitions to record, must be a power of 2 */
577 uint8_t ltssm_history[LTSSM_HISTORY_SIZE];
578 int ltssm_history_loc;
579 bool do_retry_speed = false;
580
581 BDK_TRACE(PCIE, "N%d.PCIe%d: Checking the PEM is out of reset\n", node, pcie_port);
582 if (BDK_CSR_WAIT_FOR_FIELD(node, BDK_PEMX_ON(pcie_port), pemoor, ==, 1, 100000))
583 {
584 printf("N%d.PCIe%d: PEM in reset, skipping.\n", node, pcie_port);
585 return -1;
586 }
587
588 /* Determine the maximum link speed and width */
589 BDK_CSR_INIT(pciercx_cfg031, node, BDK_PCIERCX_CFG031(pcie_port));
590 int max_gen = pciercx_cfg031.s.mls; /* Max speed of PEM from config (1-3) */
591 int max_width = pciercx_cfg031.s.mlw; /* Max lane width of PEM (1-8) */
592 BDK_TRACE(PCIE, "N%d.PCIe%d: Link supports up to %d lanes, speed gen%d\n",
593 node, pcie_port, max_width, max_gen);
594
595 /* Record starting LTSSM state for debug */
596 memset(ltssm_history, -1, sizeof(ltssm_history));
597 ltssm_history[0] = __bdk_pcie_rc_get_ltssm_state(node, pcie_port);
598 ltssm_history_loc = 0;
599
600 /* Bring up the link */
601 BDK_TRACE(PCIE, "N%d.PCIe%d: Enabling the link\n", node, pcie_port);
602 BDK_CSR_MODIFY(c, node, BDK_PEMX_CTL_STATUS(pcie_port), c.cn83xx.lnk_enb = 1);
603
604 if (bdk_config_get_int(BDK_CONFIG_PCIE_SKIP_LINK_TRAIN, node, pcie_port)) {
605 BDK_TRACE(PCIE, "N%d.PCIe%d: Skipping link configuration\n", node, pcie_port);
606 return 0;
607 }
608
609 retry_speed:
610 /* Clear RC Correctable Error Status Register */
611 BDK_CSR_WRITE(node, BDK_PCIERCX_CFG068(pcie_port), -1);
612
613 /* Wait for the link to come up and link training to be complete */
614 BDK_TRACE(PCIE, "N%d.PCIe%d: Waiting for link\n", node, pcie_port);
615
616 uint64_t clock_rate = bdk_clock_get_rate(bdk_numa_local(), BDK_CLOCK_TIME);
617 uint64_t hold_time = clock_rate / 5; /* 200ms */
618 uint64_t bounce_allow_time = clock_rate / 100; /* 10ms */
619 uint64_t timeout = bdk_clock_get_count(BDK_CLOCK_TIME) + clock_rate; /* Timeout = 1s */
620 uint64_t good_time = 0; /* Records when the link first went good */
621 BDK_CSR_DEFINE(pciercx_cfg032, BDK_PCIERCX_CFG032(pcie_port));
622 bool link_up;
623 bool is_loop_done;
624 do
625 {
626 /* Read link state */
627 pciercx_cfg032.u = BDK_CSR_READ(node, BDK_PCIERCX_CFG032(pcie_port));
628
629 /* Record LTSSM state for debug */
630 int ltssm_state = __bdk_pcie_rc_get_ltssm_state(node, pcie_port);
631 if (ltssm_history[ltssm_history_loc] != ltssm_state)
632 {
633 ltssm_history_loc = (ltssm_history_loc + 1) & (LTSSM_HISTORY_SIZE - 1);
634 ltssm_history[ltssm_history_loc] = ltssm_state;
635 }
636
637 /* Check if the link is up */
638 uint64_t current_time = bdk_clock_get_count(BDK_CLOCK_TIME);
639 link_up = (pciercx_cfg032.s.dlla && !pciercx_cfg032.s.lt);
640 if (link_up)
641 {
642 /* Is this the first link up? */
643 if (!good_time)
644 {
645 /* Mark the time when the link transitioned to good */
646 good_time = current_time;
647 }
648 else
649 {
650 /* Check for a link error */
651 BDK_CSR_INIT(cfg068, node, BDK_PCIERCX_CFG068(pcie_port));
652 if (cfg068.s.res)
653 {
654 /* Ignore errors before we've been stable for bounce_allow_time */
655 if (good_time + bounce_allow_time <= current_time)
656 {
657 BDK_TRACE(PCIE, "N%d.PCIe%d: Link errors after link up\n", node, pcie_port);
658 return 1; /* Link error, signal a retry */
659 }
660 else
661 {
662 /* Clear RC Correctable Error Status Register */
663 BDK_CSR_WRITE(node, BDK_PCIERCX_CFG068(pcie_port), -1);
664 BDK_TRACE(PCIE, "N%d.PCIe%d: Ignored error during settling time\n", node, pcie_port);
665 }
666 }
667 }
668 }
669 else if (good_time)
670 {
671 if (good_time + bounce_allow_time <= current_time)
672 {
673 /* We allow bounces for bounce_allow_time after the link is good.
674 Once this time passes any bounce requires a retry */
675 BDK_TRACE(PCIE, "N%d.PCIe%d: Link bounce detected\n", node, pcie_port);
676 return 1; /* Link bounce, signal a retry */
677 }
678 else
679 {
680 BDK_TRACE(PCIE, "N%d.PCIe%d: Ignored bounce during settling time\n", node, pcie_port);
681 }
682 }
683
684 /* Determine if we've hit the timeout */
685 is_loop_done = (current_time >= timeout);
686 /* Determine if we've had a good link for the required hold time */
687 is_loop_done |= link_up && (good_time + hold_time <= current_time);
688 } while (!is_loop_done);
689
690 /* Trace the LTSSM state */
691 BDK_TRACE(PCIE, "N%d.PCIe%d: LTSSM History\n", node, pcie_port);
692 for (int i = 0; i < LTSSM_HISTORY_SIZE; i++)
693 {
694 ltssm_history_loc = (ltssm_history_loc + 1) & (LTSSM_HISTORY_SIZE - 1);
695 if (ltssm_history[ltssm_history_loc] != 0xff)
696 BDK_TRACE(PCIE, "N%d.PCIe%d: %s\n",
697 node, pcie_port, ltssm_string(ltssm_history[ltssm_history_loc]));
698 }
699
700 if (!link_up)
701 {
702 BDK_TRACE(PCIE, "N%d.PCIe%d: Link down, Data link layer %s(DLLA=%d), Link training %s(LT=%d), LTSSM %s\n",
703 node, pcie_port,
704 pciercx_cfg032.s.dlla ? "active" : "down", pciercx_cfg032.s.dlla,
705 pciercx_cfg032.s.lt ? "active" : "complete", pciercx_cfg032.s.lt,
706 ltssm_string(__bdk_pcie_rc_get_ltssm_state(node, pcie_port)));
707 return 1; /* Link down, signal a retry */
708 }
709
710 /* Report the negotiated link speed and width */
711 int neg_gen = pciercx_cfg032.s.ls; /* Current speed of PEM (1-3) */
712 int neg_width = pciercx_cfg032.s.nlw; /* Current lane width of PEM (1-8) */
713 BDK_TRACE(PCIE, "N%d.PCIe%d: Link negotiated %d lanes, speed gen%d\n",
714 node, pcie_port, neg_width, neg_gen);
715
716 /* Determine PCIe bus number the directly attached device uses */
717 BDK_CSR_INIT(pciercx_cfg006, node, BDK_PCIERCX_CFG006(pcie_port));
718 int bus = pciercx_cfg006.s.sbnum;
719
720 int dev_gen = 1; /* Device max speed (1-3) */
721 int dev_width = 1; /* Device max lane width (1-16) */
722 BDK_TRACE(PCIE, "N%d.PCIe%d: Reading device max speed and width\n",
723 node, pcie_port);
724
725 /* Read PCI capability pointer */
726 uint32_t cap = cfg_read32_retry(node, pcie_port, bus, 0, 0, 0x34);
727
728 /* Check if we were able to read capabilities pointer */
729 if (cap == 0xffffffff)
730 return 1; /* Signal retry needed */
731
732 /* Read device max speed and width */
733 int cap_next = cap & 0xff;
734 while (cap_next)
735 {
736 cap = cfg_read32_retry(node, pcie_port, bus, 0, 0, cap_next);
737 if (cap == 0xffffffff)
738 return 1; /* Signal retry needed */
739
740 /* Is this a PCIe capability (0x10)? */
741 if ((cap & 0xff) == 0x10)
742 {
743 BDK_TRACE(PCIE, "N%d.PCIe%d: Found PCIe capability at offset 0x%x\n",
744 node, pcie_port, cap_next);
745 /* Offset 0xc contains the max link info */
746 cap = cfg_read32_retry(node, pcie_port, bus, 0, 0, cap_next + 0xc);
747 if (cap == 0xffffffff)
748 return 1; /* Signal retry needed */
749 dev_gen = cap & 0xf; /* Max speed of PEM from config (1-3) */
750 dev_width = (cap >> 4) & 0x3f; /* Max lane width of PEM (1-16) */
751 BDK_TRACE(PCIE, "N%d.PCIe%d: Device supports %d lanes, speed gen%d\n",
752 node, pcie_port, dev_width, dev_gen);
753 break;
754 }
755 /* Move to next capability */
756 cap_next = (cap >> 8) & 0xff;
757 }
758
759 /* Desired link speed and width is either limited by the device or our PEM
760 configuration. Choose the most restrictive limit */
761 int desired_gen = (dev_gen < max_gen) ? dev_gen : max_gen;
762 int desired_width = (dev_width < max_width) ? dev_width : max_width;
763
764 /* We need a change if we don't match the desired speed or width. Note that
765 we allow better than expected in case the device lied about its
766 capabilities */
767 bool need_speed_change = (neg_gen < desired_gen);
768 bool need_lane_change = (neg_width < desired_width);
769
770 if (need_lane_change)
771 {
772 /* We didn't get the maximum number of lanes */
773 BDK_TRACE(PCIE, "N%d.PCIe%d: Link width (%d) less that supported (%d)\n",
774 node, pcie_port, neg_width, desired_width);
775 return 2; /* Link wrong width, signal a retry */
776 }
777 else if (need_speed_change)
778 {
779 if (do_retry_speed)
780 {
781 BDK_TRACE(PCIE, "N%d.PCIe%d: Link speed (gen%d) less that supported (gen%d)\n",
782 node, pcie_port, neg_gen, desired_gen);
783 return 1; /* Link at width, but speed low. Request a retry */
784 }
785 else
786 {
787 /* We didn't get the maximum speed. Request a speed change */
788 BDK_TRACE(PCIE, "N%d.PCIe%d: Link speed (gen%d) less that supported (gen%d), requesting a speed change\n",
789 node, pcie_port, neg_gen, desired_gen);
790 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG515(pcie_port),
791 c.s.dsc = 1);
792 bdk_wait_usec(100000);
793 do_retry_speed = true;
794 goto retry_speed;
795 }
796 }
797 else
798 {
799 BDK_TRACE(PCIE, "N%d.PCIe%d: Link at best speed and width\n", node, pcie_port);
800 /* For gen3 links check if we are getting errors over the link */
801 if (neg_gen == 3)
802 {
803 /* Read RC Correctable Error Status Register */
804 BDK_CSR_INIT(cfg068, node, BDK_PCIERCX_CFG068(pcie_port));
805 if (cfg068.s.res)
806 {
807 BDK_TRACE(PCIE, "N%d.PCIe%d: Link reporting error status\n", node, pcie_port);
808 return 1; /* Getting receiver errors, request a retry */
809 }
810 }
811 return 0; /* Link at correct speed and width */
812 }
813 }
814
815 /**
816 * Setup the SLI memory mapped regions to allow access to PCIe by the cores
817 * using addresses returned by bdk_pcie_get_base_address().
818 *
819 * @param node Node to configure
820 * @param pcie_port PCIe port to configure
821 */
__bdk_pcie_sli_initialize(bdk_node_t node,int pcie_port)822 static void __bdk_pcie_sli_initialize(bdk_node_t node, int pcie_port)
823 {
824 int sli;
825 int sli_group;
826 __bdk_pcie_get_sli(node, pcie_port, &sli, &sli_group);
827
828 /* Setup store merge timer */
829 BDK_CSR_MODIFY(c, node, BDK_SLIX_S2M_CTL(sli),
830 c.s.max_word = 0; /* Allow 16 words to combine */
831 c.s.timer = 50); /* Wait up to 50 cycles for more data */
832
833 /* There are 256 regions per SLI. We need four regions per PCIe port to
834 support config, IO, normal, and prefetchable regions. The 256 regions
835 are shared across PCIe, so we need three groups of these (one group
836 for each PCIe). The setup is:
837 SLI bit[7:6]: PCIe port, relative to SLI (max of 4)
838 SLI bit[5:4]: Region. See bdk_pcie_mem_t enumeration
839 SLI bit[3:0]: Address extension from 32 bits to 36 bits
840 */
841 for (bdk_pcie_mem_t mem_region = BDK_PCIE_MEM_CONFIG; mem_region <= BDK_PCIE_MEM_IO; mem_region++)
842 {
843 /* Use top two bits for PCIe port, next two bits for memory region */
844 int sli_region = sli_group << 6;
845 /* Use next two bits for mem region type */
846 sli_region |= mem_region << 4;
847 /* Figure out the hardware setting for each region */
848 int ctype = 3;
849 int nmerge = 1;
850 int ordering = 0;
851 switch (mem_region)
852 {
853 case BDK_PCIE_MEM_CONFIG: /* Config space */
854 ctype = 1; /* Config space */
855 nmerge = 1; /* No merging allowed */
856 ordering = 0; /* NO "relaxed ordering" or "no snoop" */
857 break;
858 case BDK_PCIE_MEM_NORMAL: /* Memory, not prefetchable */
859 ctype = 0; /* Memory space */
860 nmerge = 1; /* No merging allowed */
861 ordering = 0; /* NO "relaxed ordering" or "no snoop" */
862 break;
863 case BDK_PCIE_MEM_PREFETCH: /* Memory, prefetchable */
864 ctype = 0; /* Memory space */
865 nmerge = 0; /* Merging allowed */
866 ordering = 1; /* Yes "relaxed ordering" and "no snoop" */
867 break;
868 case BDK_PCIE_MEM_IO: /* IO */
869 ctype = 2; /* I/O space */
870 nmerge = 1; /* No merging allowed */
871 ordering = 0; /* NO "relaxed ordering" or "no snoop" */
872 break;
873 }
874 /* Use the lower order bits to work as an address extension, allowing
875 each PCIe port to map a total of 36 bits (32bit each region, 16
876 regions) */
877 int epf = sli_group;
878 if (CAVIUM_IS_MODEL(CAVIUM_CN83XX) || CAVIUM_IS_MODEL(CAVIUM_CN81XX)) {
879 BDK_CSR_INIT(lmac_const0,node,BDK_SLIX_LMAC_CONST0X(sli,pcie_port));
880 epf = lmac_const0.s.epf;
881 }
882 for (int r = sli_region; r < sli_region + 16; r++)
883 {
884 uint64_t address = 0;
885 /* Address only applies to memory space */
886 if (mem_region == BDK_PCIE_MEM_NORMAL)
887 {
888 /* Normal starts at bus address 0 */
889 address = r - sli_region;
890 } else if (mem_region == BDK_PCIE_MEM_PREFETCH)
891 {
892 /* Normal starts at bus address 0x10.0000.0000 */
893 address = r - sli_region + 16;
894 }
895 BDK_CSR_MODIFY(c, node, BDK_SLIX_S2M_REGX_ACC(sli, r),
896 c.s.ctype = ctype;
897 c.s.zero = 0;
898 c.cn83xx.epf = epf; /* Superimposed onto c.cn81xx.mac. EPF value works for both */
899 c.s.nmerge = nmerge;
900 c.s.wtype = ordering;
901 c.s.rtype = ordering;
902 c.s.ba = address);
903 }
904 }
905
906 /* Setup MAC control */
907 BDK_CSR_MODIFY(c, node, BDK_SLIX_M2S_MACX_CTL(sli, sli_group),
908 c.s.waitl_com = 1; /* Improves ordering in Ali flash testing */
909 c.s.ctlp_ro = 1;
910 c.s.ptlp_ro = 1;
911 c.s.wind_d = 1;
912 c.s.bar0_d = 1;
913 c.s.wait_com = (bdk_config_get_int(BDK_CONFIG_PCIE_ORDERING) == 1));
914 }
915
916
917 /**
918 * Perform a complete PCIe RC reset. This is documented in the HRM as issuing a
919 * fundamental reset
920 *
921 * @param node Node to reset
922 * @param pcie_port PCIe port to reset
923 *
924 * @return Zero on success, negative on failure
925 */
__bdk_pcie_rc_reset(bdk_node_t node,int pcie_port)926 static int __bdk_pcie_rc_reset(bdk_node_t node, int pcie_port)
927 {
928 /* Find which QLM/DLM is associated with this PCIe port */
929 int qlm = bdk_qlm_get_qlm_num(node, BDK_IF_PCIE, pcie_port, 0);
930 if (qlm < 0)
931 return -1;
932
933 /* Check if this PCIe port combines two QLM/DLM */
934 BDK_CSR_INIT(pemx_cfg, node, BDK_PEMX_CFG(pcie_port));
935 int is_dual = CAVIUM_IS_MODEL(CAVIUM_CN81XX) ? pemx_cfg.cn81xx.lanes4 : pemx_cfg.cn83xx.lanes8;
936
937 BDK_TRACE(PCIE, "N%d.PCIe%d: Performing PCIe fundamental reset\n", node, pcie_port);
938
939 /* Host software may want to issue a fundamental reset to the PCIe bus.
940 Software should perform the following steps:
941 1. Write PEM(0..1)_ON[PEMON] = 0. */
942 BDK_CSR_MODIFY(c, node, BDK_PEMX_ON(pcie_port),
943 c.s.pemon = 0);
944 /* 2. Write RST_SOFT_PRST(0..3)[SOFT_PRST] = 1.
945 - This reassertion of [SOFT_PRST] causes the chip to drive PERSTn_L
946 low (if RST_CTL(0..3)[RST_DRV] = 1). */
947 BDK_CSR_MODIFY(c, node, BDK_RST_SOFT_PRSTX(pcie_port),
948 c.s.soft_prst = 1);
949 /* 3. Read RST_SOFT_PRST(0..3). This ensures the PCIe bus is now in reset.
950 - Note that PCIERCn_CFGn registers cannot be accessed when
951 RST_SOFT_PRST(0..3)[SOFT_PRST] = 1. */
952 BDK_CSR_READ(node, BDK_RST_SOFT_PRSTX(pcie_port));
953 /* 4. Write GSER(0..8)_PHY_CTL[PHY_RESET] = 1.
954 - This puts the PHY in reset. */
955 BDK_CSR_MODIFY(c, node, BDK_GSERX_PHY_CTL(qlm),
956 c.s.phy_reset = 1);
957 if (is_dual)
958 BDK_CSR_MODIFY(c, node, BDK_GSERX_PHY_CTL(qlm + 1),
959 c.s.phy_reset = 1);
960 /* Wait 10 us before proceeding to step 5. */
961 bdk_wait_usec(10);
962 /* 5. Write GSERx_PHY_CTL[PHY_RESET] = 0 */
963 BDK_CSR_MODIFY(c, node, BDK_GSERX_PHY_CTL(qlm),
964 c.s.phy_reset = 0);
965 if (is_dual)
966 BDK_CSR_MODIFY(c, node, BDK_GSERX_PHY_CTL(qlm + 1),
967 c.s.phy_reset = 0);
968
969 /* Turn on PEM clocks */
970 if (CAVIUM_IS_MODEL(CAVIUM_CN83XX))
971 BDK_CSR_MODIFY(c, node, BDK_PEMX_CLK_EN(pcie_port),
972 c.cn83xx.pceclk_gate = 0;
973 c.cn83xx.csclk_gate = 0);
974
975 /* 6. Wait 2 ms or more before taking the PCIe port out of reset. */
976 bdk_wait_usec(2000);
977
978 /* To take PCIe port out of reset, perform the following steps: */
979 /* 1. Write PEM(0..1)_ON[PEMON] = 1. */
980 BDK_CSR_MODIFY(c, node, BDK_PEMX_ON(pcie_port),
981 c.s.pemon = 1);
982 /* 2. Write RST_SOFT_PRST(0..3)[SOFT_PRST] = 0. */
983 /* 3. After RST_CTL(0..3)[RST_DONE], perform any configuration as the
984 PCIe MAC has been reset. Set the PEM(0..1)_CTL_STATUS[LNK_ENB] = 1. */
985 /* These steps are executed when we bring the link up. See
986 bdk_pcie_rc_initialize() */
987 return 0;
988 }
989
990 /**
991 * Before PCIe link can be brought up a number of steps must be performed to
992 * reset the PEM, take the PEM out of reset, initialize the PEM, initialize
993 * RC config space, and initialize SLI. These steps must be performed every
994 * time the PEM is reset, which may be repeated if the PCIe link doesn't come
995 * up at the desired speed and width.
996 *
997 * @param node Node to initialize
998 * @param pcie_port PCIe port to initialize
999 *
1000 * @return Zero on success, negative on failure
1001 */
__bdk_pcie_rc_pre_link_init(bdk_node_t node,int pcie_port)1002 static int __bdk_pcie_rc_pre_link_init(bdk_node_t node, int pcie_port)
1003 {
1004 /* Make sure the PEM and GSER do a full reset before starting PCIe */
1005 if (__bdk_pcie_rc_reset(node, pcie_port))
1006 {
1007 bdk_error("N%d.PCIe%d: Reset failed.\n", node, pcie_port);
1008 return -1;
1009 }
1010
1011 /* Bring the PCIe out of reset */
1012 BDK_TRACE(PCIE, "N%d.PCIe%d: Taking port out of reset\n", node, pcie_port);
1013 BDK_CSR_WRITE(node, BDK_RST_SOFT_PRSTX(pcie_port), 0);
1014
1015 /* Check and make sure PCIe came out of reset. If it doesn't the board
1016 probably hasn't wired the clocks up and the interface should be
1017 skipped */
1018 BDK_TRACE(PCIE, "N%d.PCIe%d: Waiting for reset to complete\n", node, pcie_port);
1019 if (BDK_CSR_WAIT_FOR_FIELD(node, BDK_RST_CTLX(pcie_port), rst_done, ==, 1, 10000))
1020 {
1021 printf("N%d.PCIe%d: Stuck in reset, skipping.\n", node, pcie_port);
1022 return -1;
1023 }
1024
1025 /* Check BIST status */
1026 BDK_TRACE(PCIE, "N%d.PCIe%d: Checking BIST\n", node, pcie_port);
1027 BDK_CSR_INIT(pemx_bist_status, node, BDK_PEMX_BIST_STATUS(pcie_port));
1028 if (pemx_bist_status.u)
1029 bdk_warn("N%d.PCIe%d: BIST FAILED (0x%016llx)\n", node, pcie_port, pemx_bist_status.u);
1030
1031 /* Initialize the config space CSRs */
1032 BDK_TRACE(PCIE, "N%d.PCIe%d: Setting up internal config space\n", node, pcie_port);
1033 __bdk_pcie_rc_initialize_config_space(node, pcie_port);
1034
1035 /* Enable gen2 speed selection */
1036 BDK_TRACE(PCIE, "N%d.PCIe%d: Enabling dynamic speed changes\n", node, pcie_port);
1037 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG515(pcie_port),
1038 c.s.dsc = 1);
1039
1040 /* Setup the SLI windows to allow access to this PCIe from the core */
1041 BDK_TRACE(PCIE, "N%d.PCIe%d: Initializing SLI\n", node, pcie_port);
1042 __bdk_pcie_sli_initialize(node, pcie_port);
1043 return 0;
1044 }
1045
1046 /**
1047 * Initialize a PCIe port for use in host(RC) mode. It doesn't
1048 * enumerate the bus.
1049 *
1050 * @param pcie_port PCIe port to initialize
1051 *
1052 * @return Zero on success
1053 */
bdk_pcie_rc_initialize(bdk_node_t node,int pcie_port)1054 int bdk_pcie_rc_initialize(bdk_node_t node, int pcie_port)
1055 {
1056 const int MAX_RETRIES = 2; /* Total of 3 attempts: First + 2 retries */
1057 int retry_count = 0;
1058 int result= -1,i;
1059 bdk_pemx_bar1_indexx_t bar1_idx;
1060
1061 /* Make sure we aren't trying to setup a target mode interface in host
1062 mode. Sadly this bit is RAZ for CN88XX and CN81XX because the hardware
1063 team removed it. So much for backward compatibility */
1064 BDK_CSR_INIT(pemx_cfg, node, BDK_PEMX_CFG(pcie_port));
1065 int host_mode = CAVIUM_IS_MODEL(CAVIUM_CN83XX) ? pemx_cfg.cn83xx.hostmd : 1;
1066 if (!host_mode)
1067 {
1068 printf("N%d.PCIe%d: Port in endpoint mode.\n", node, pcie_port);
1069 return -1;
1070 }
1071
1072 while (retry_count <= MAX_RETRIES)
1073 {
1074 if (retry_count)
1075 BDK_TRACE(PCIE, "N%d.PCIe%d: Starting link retry %d\n", node, pcie_port, retry_count);
1076 /* Perform init that must be done after PEM reset, but before link */
1077 if (__bdk_pcie_rc_pre_link_init(node, pcie_port))
1078 return -1;
1079
1080 if (retry_count == MAX_RETRIES)
1081 {
1082 BDK_CSR_INIT(pciercx_cfg031, node, BDK_PCIERCX_CFG031(pcie_port));
1083 /* Drop speed to gen2 if link bouncing */
1084 /* Result =-1 PEM in reset */
1085 /* Result = 0: link speed and width ok no retry needed */
1086 /* Result = 1: Link errors or speed change needed */
1087 /* Result = 2: lane width error */
1088 if ((pciercx_cfg031.s.mls == 3) && (result != 2))
1089 {
1090 BDK_TRACE(PCIE, "N%d.PCIe%d: Dropping speed to gen2\n", node, pcie_port);
1091 pciercx_cfg031.s.mls = 2;
1092 BDK_CSR_WRITE(node, BDK_PCIERCX_CFG031(pcie_port), pciercx_cfg031.u);
1093 /* Set the target link speed */
1094 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG040(pcie_port),
1095 c.s.tls = 2);
1096 }
1097 }
1098 /* Bring the link up */
1099 result = __bdk_pcie_rc_initialize_link(node, pcie_port);
1100 if (result == 0)
1101 {
1102 BDK_TRACE(PCIE, "N%d.PCIe%d: Link does not need a retry\n", node, pcie_port);
1103 break;
1104 }
1105 else if (result > 0)
1106 {
1107 if (retry_count >= MAX_RETRIES)
1108 {
1109 BDK_TRACE(PCIE, "N%d.PCIe%d: Link requested a retry, but hit the max retries\n", node, pcie_port);
1110 /* If the link is down, report failure */
1111 BDK_CSR_INIT(pciercx_cfg032, node, BDK_PCIERCX_CFG032(pcie_port));
1112 bool link_up = (pciercx_cfg032.s.dlla && !pciercx_cfg032.s.lt);
1113 if (!link_up)
1114 result = -1;
1115 }
1116 else
1117 BDK_TRACE(PCIE, "N%d.PCIe%d: Link requested a retry\n", node, pcie_port);
1118 }
1119 if (result < 0)
1120 {
1121 int ltssm_state = __bdk_pcie_rc_get_ltssm_state(node, pcie_port);
1122 printf("N%d.PCIe%d: Link timeout, probably the slot is empty (LTSSM %s)\n",
1123 node, pcie_port, ltssm_string(ltssm_state));
1124 return -1;
1125 }
1126 retry_count++;
1127 bdk_watchdog_poke();
1128 }
1129
1130 /* Errata PCIE-28816: Link retrain initiated at GEN1 can cause PCIE
1131 link to hang. For Gen1 links we must disable equalization */
1132 BDK_CSR_INIT(pciercx_cfg032, node, BDK_PCIERCX_CFG032(pcie_port));
1133 if (pciercx_cfg032.s.ls == 1)
1134 {
1135 BDK_TRACE(PCIE, "N%d.PCIe%d: Disabling equalization for Gen1 link\n", node, pcie_port);
1136 BDK_CSR_MODIFY(c, node, BDK_PCIERCX_CFG548(pcie_port),
1137 c.s.ed = 1);
1138 }
1139
1140 BDK_TRACE(PCIE, "N%d.PCIe%d: Setting up internal BARs\n", node, pcie_port);
1141 /* Disable BAR0 */
1142 BDK_CSR_WRITE(node, BDK_PEMX_P2N_BAR0_START(pcie_port), -1);
1143 /* BAR1 Starting at address 0 */
1144 BDK_CSR_WRITE(node, BDK_PEMX_P2N_BAR1_START(pcie_port), 0);
1145 /* Set BAR2 to cover all memory starting at address 0 */
1146 BDK_CSR_WRITE(node, BDK_PEMX_P2N_BAR2_START(pcie_port), 0);
1147 /* Setup BAR attributes */
1148 BDK_CSR_MODIFY(c, node, BDK_PEMX_BAR_CTL(pcie_port),
1149 c.cn83xx.bar1_siz = 1; /* 64MB BAR1 */
1150 c.s.bar2_enb = 1; /* BAR2 is enabled */
1151 c.s.bar2_cax = 0); /* Cache in L2 */
1152
1153 /* Allow devices that truncate the bus address to 32-bits to reach the GITS_TRANSLATER */
1154 bar1_idx.u = 0;
1155 bar1_idx.s.addr_idx = bdk_numa_get_address(node, BDK_GITS_TRANSLATER) >> 22;
1156 bar1_idx.s.addr_v = 1;
1157
1158 BDK_CSR_WRITE(node, BDK_PEMX_BAR1_INDEXX(pcie_port, 0), bar1_idx.u);
1159
1160 /* The rest of the windows map linearly to match the BAR2 translation. */
1161 for (i = 1; i < 16; i++)
1162 {
1163 bar1_idx.s.addr_idx = i;
1164 BDK_CSR_WRITE(node, BDK_PEMX_BAR1_INDEXX(pcie_port, i), bar1_idx.u);
1165 }
1166
1167 /* Display the link status */
1168 printf("N%d.PCIe%d: Link active, %d lanes, speed gen%d\n",
1169 node, pcie_port, pciercx_cfg032.s.nlw, pciercx_cfg032.s.ls);
1170
1171 return 0;
1172
1173 }
1174
1175 /**
1176 * Return PCIe state
1177 *
1178 * @param pcie_port PCIe port to query
1179 *
1180 * @return True if port is up and running
1181 */
bdk_pcie_is_running(bdk_node_t node,int pcie_port)1182 int bdk_pcie_is_running(bdk_node_t node, int pcie_port)
1183 {
1184 BDK_CSR_INIT(pemx_on, node, BDK_PEMX_ON(pcie_port));
1185 BDK_CSR_INIT(rst_soft_prstx, node, BDK_RST_SOFT_PRSTX(pcie_port));
1186 BDK_CSR_INIT(pciercx_cfg032, node, BDK_PCIERCX_CFG032(pcie_port));
1187
1188 if (!pemx_on.s.pemon || rst_soft_prstx.s.soft_prst)
1189 return 0;
1190
1191 return bdk_config_get_int(BDK_CONFIG_PCIE_SKIP_LINK_TRAIN, node, pcie_port) ||
1192 (pciercx_cfg032.s.dlla && !pciercx_cfg032.s.lt);
1193 }
1194
1195 /**
1196 * Shutdown a PCIe port and put it in reset
1197 *
1198 * @param pcie_port PCIe port to shutdown
1199 *
1200 * @return Zero on success
1201 */
bdk_pcie_rc_shutdown(bdk_node_t node,int pcie_port)1202 int bdk_pcie_rc_shutdown(bdk_node_t node, int pcie_port)
1203 {
1204 /* Check that the controller is out of reset */
1205 BDK_CSR_INIT(rst_ctlx, node, BDK_RST_CTLX(pcie_port));
1206 if (!rst_ctlx.s.rst_done)
1207 goto skip_idle_wait;
1208
1209 /* Check if link is up */
1210 BDK_CSR_INIT(pciercx_cfg032, node, BDK_PCIERCX_CFG032(pcie_port));
1211 if ((pciercx_cfg032.s.dlla == 0) || (pciercx_cfg032.s.lt == 1))
1212 goto skip_idle_wait;
1213 #if 0 // FIXME
1214 /* Wait for all pending operations to complete */
1215 if (BDK_CSR_WAIT_FOR_FIELD(node, BDK_PEMX_CPL_LUT_VALID(pcie_port), tag, ==, 0, 2000))
1216 printf("N%d.PCIe%d: Shutdown timeout\n", node, pcie_port);
1217 #endif
1218 skip_idle_wait:
1219 /* Bring down the link */
1220 BDK_CSR_MODIFY(c, node, BDK_PEMX_CTL_STATUS(pcie_port), c.cn83xx.lnk_enb = 0);
1221 /* Force reset */
1222 __bdk_pcie_rc_reset(node, pcie_port);
1223 return 0;
1224 }
1225
1226 /**
1227 * @INTERNAL
1228 * Build a PCIe config space request address for a device
1229 *
1230 * @param pcie_port PCIe port to access
1231 * @param bus Sub bus
1232 * @param dev Device ID
1233 * @param fn Device sub function
1234 * @param reg Register to access
1235 *
1236 * @return 64bit IO address
1237 */
pcie_build_config_addr(bdk_node_t node,int pcie_port,int bus,int dev,int fn,int reg)1238 uint64_t pcie_build_config_addr(bdk_node_t node, int pcie_port, int bus, int dev, int fn, int reg)
1239 {
1240 int num_pems = bdk_pcie_get_num_ports(node);
1241 if (pcie_port < num_pems)
1242 {
1243 /* Errata (SLI-22555) ECAM to off-chip PCI misroutes address. Use
1244 the SLI regions instead of ECAMs for config space access */
1245 uint64_t address = bdk_pcie_get_base_address(node, pcie_port, BDK_PCIE_MEM_CONFIG);
1246 /* Display the link status */
1247 address += (uint64_t)bus << 24; /* Bus is bits 31:24 */
1248 address += dev << 19; /* device+func is bits 23:16 */
1249 address += fn << 16;
1250 address += reg; /* Offset is bits 11:0 */
1251 return address;
1252 }
1253 else if (pcie_port >= 100)
1254 {
1255 bdk_device_t device;
1256 memset(&device, 0, sizeof(device));
1257 device.node = node;
1258 device.ecam = pcie_port - 100;
1259 device.bus = bus;
1260 device.dev = dev;
1261 device.func = fn;
1262 return __bdk_ecam_build_address(&device, reg);
1263 }
1264 return 0;
1265 }
1266