1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4 *
5 */
6
7 /* Support for NVIDIA specific attributes. */
8
9 #include <linux/module.h>
10 #include <linux/topology.h>
11
12 #include "arm_cspmu.h"
13
14 #define NV_PCIE_PORT_COUNT 10ULL
15 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
16
17 #define NV_NVL_C2C_PORT_COUNT 2ULL
18 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
19
20 #define NV_CNVL_PORT_COUNT 4ULL
21 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
22
23 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
24
25 #define NV_PRODID_MASK GENMASK(31, 0)
26
27 #define NV_FORMAT_NAME_GENERIC 0
28
29 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
30
31 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
32 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
33
34 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
35 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
39
40 struct nv_cspmu_ctx {
41 const char *name;
42 u32 filter_mask;
43 u32 filter_default_val;
44 struct attribute **event_attr;
45 struct attribute **format_attr;
46 };
47
48 static struct attribute *scf_pmu_event_attrs[] = {
49 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
50
51 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
52 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
53 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
54 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
55
56 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
57 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
58
59 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
60
61 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
62 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
63 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
64
65 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
66 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
67 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
68 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
69 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
70 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
71 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
72
73 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
74
75 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
76 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
77 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
78 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
79 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
80
81 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
82 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
83 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
84 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
85 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
86 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
87
88 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
89
90 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
91
92 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
93 NULL,
94 };
95
96 static struct attribute *mcf_pmu_event_attrs[] = {
97 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
98 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
99 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
100 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
101 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
102 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
103 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
104 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
105 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
106 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
107 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
108 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
109 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
110 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
111 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
112 NULL,
113 };
114
115 static struct attribute *generic_pmu_event_attrs[] = {
116 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
117 NULL,
118 };
119
120 static struct attribute *scf_pmu_format_attrs[] = {
121 ARM_CSPMU_FORMAT_EVENT_ATTR,
122 NULL,
123 };
124
125 static struct attribute *pcie_pmu_format_attrs[] = {
126 ARM_CSPMU_FORMAT_EVENT_ATTR,
127 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
128 NULL,
129 };
130
131 static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
132 ARM_CSPMU_FORMAT_EVENT_ATTR,
133 ARM_CSPMU_FORMAT_ATTR(port, "config1:0-1"),
134 NULL,
135 };
136
137 static struct attribute *cnvlink_pmu_format_attrs[] = {
138 ARM_CSPMU_FORMAT_EVENT_ATTR,
139 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
140 NULL,
141 };
142
143 static struct attribute *generic_pmu_format_attrs[] = {
144 ARM_CSPMU_FORMAT_EVENT_ATTR,
145 ARM_CSPMU_FORMAT_FILTER_ATTR,
146 NULL,
147 };
148
149 static struct attribute **
nv_cspmu_get_event_attrs(const struct arm_cspmu * cspmu)150 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
151 {
152 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
153
154 return ctx->event_attr;
155 }
156
157 static struct attribute **
nv_cspmu_get_format_attrs(const struct arm_cspmu * cspmu)158 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
159 {
160 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
161
162 return ctx->format_attr;
163 }
164
165 static const char *
nv_cspmu_get_name(const struct arm_cspmu * cspmu)166 nv_cspmu_get_name(const struct arm_cspmu *cspmu)
167 {
168 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
169
170 return ctx->name;
171 }
172
nv_cspmu_event_filter(const struct perf_event * event)173 static u32 nv_cspmu_event_filter(const struct perf_event *event)
174 {
175 const struct nv_cspmu_ctx *ctx =
176 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
177
178 const u32 filter_val = event->attr.config1 & ctx->filter_mask;
179
180 if (filter_val == 0)
181 return ctx->filter_default_val;
182
183 return filter_val;
184 }
185
186 enum nv_cspmu_name_fmt {
187 NAME_FMT_GENERIC,
188 NAME_FMT_SOCKET
189 };
190
191 struct nv_cspmu_match {
192 u32 prodid;
193 u32 prodid_mask;
194 u64 filter_mask;
195 u32 filter_default_val;
196 const char *name_pattern;
197 enum nv_cspmu_name_fmt name_fmt;
198 struct attribute **event_attr;
199 struct attribute **format_attr;
200 };
201
202 static const struct nv_cspmu_match nv_cspmu_match[] = {
203 {
204 .prodid = 0x103,
205 .prodid_mask = NV_PRODID_MASK,
206 .filter_mask = NV_PCIE_FILTER_ID_MASK,
207 .filter_default_val = NV_PCIE_FILTER_ID_MASK,
208 .name_pattern = "nvidia_pcie_pmu_%u",
209 .name_fmt = NAME_FMT_SOCKET,
210 .event_attr = mcf_pmu_event_attrs,
211 .format_attr = pcie_pmu_format_attrs
212 },
213 {
214 .prodid = 0x104,
215 .prodid_mask = NV_PRODID_MASK,
216 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK,
217 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
218 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
219 .name_fmt = NAME_FMT_SOCKET,
220 .event_attr = mcf_pmu_event_attrs,
221 .format_attr = nvlink_c2c_pmu_format_attrs
222 },
223 {
224 .prodid = 0x105,
225 .prodid_mask = NV_PRODID_MASK,
226 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK,
227 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
228 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
229 .name_fmt = NAME_FMT_SOCKET,
230 .event_attr = mcf_pmu_event_attrs,
231 .format_attr = nvlink_c2c_pmu_format_attrs
232 },
233 {
234 .prodid = 0x106,
235 .prodid_mask = NV_PRODID_MASK,
236 .filter_mask = NV_CNVL_FILTER_ID_MASK,
237 .filter_default_val = NV_CNVL_FILTER_ID_MASK,
238 .name_pattern = "nvidia_cnvlink_pmu_%u",
239 .name_fmt = NAME_FMT_SOCKET,
240 .event_attr = mcf_pmu_event_attrs,
241 .format_attr = cnvlink_pmu_format_attrs
242 },
243 {
244 .prodid = 0x2CF,
245 .prodid_mask = NV_PRODID_MASK,
246 .filter_mask = 0x0,
247 .filter_default_val = 0x0,
248 .name_pattern = "nvidia_scf_pmu_%u",
249 .name_fmt = NAME_FMT_SOCKET,
250 .event_attr = scf_pmu_event_attrs,
251 .format_attr = scf_pmu_format_attrs
252 },
253 {
254 .prodid = 0,
255 .prodid_mask = 0,
256 .filter_mask = NV_GENERIC_FILTER_ID_MASK,
257 .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
258 .name_pattern = "nvidia_uncore_pmu_%u",
259 .name_fmt = NAME_FMT_GENERIC,
260 .event_attr = generic_pmu_event_attrs,
261 .format_attr = generic_pmu_format_attrs
262 },
263 };
264
nv_cspmu_format_name(const struct arm_cspmu * cspmu,const struct nv_cspmu_match * match)265 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
266 const struct nv_cspmu_match *match)
267 {
268 char *name;
269 struct device *dev = cspmu->dev;
270
271 static atomic_t pmu_generic_idx = {0};
272
273 switch (match->name_fmt) {
274 case NAME_FMT_SOCKET: {
275 const int cpu = cpumask_first(&cspmu->associated_cpus);
276 const int socket = cpu_to_node(cpu);
277
278 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
279 socket);
280 break;
281 }
282 case NAME_FMT_GENERIC:
283 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
284 atomic_fetch_inc(&pmu_generic_idx));
285 break;
286 default:
287 name = NULL;
288 break;
289 }
290
291 return name;
292 }
293
nv_cspmu_init_ops(struct arm_cspmu * cspmu)294 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
295 {
296 u32 prodid;
297 struct nv_cspmu_ctx *ctx;
298 struct device *dev = cspmu->dev;
299 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
300 const struct nv_cspmu_match *match = nv_cspmu_match;
301
302 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
303 if (!ctx)
304 return -ENOMEM;
305
306 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
307
308 /* Find matching PMU. */
309 for (; match->prodid; match++) {
310 const u32 prodid_mask = match->prodid_mask;
311
312 if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
313 break;
314 }
315
316 ctx->name = nv_cspmu_format_name(cspmu, match);
317 ctx->filter_mask = match->filter_mask;
318 ctx->filter_default_val = match->filter_default_val;
319 ctx->event_attr = match->event_attr;
320 ctx->format_attr = match->format_attr;
321
322 cspmu->impl.ctx = ctx;
323
324 /* NVIDIA specific callbacks. */
325 impl_ops->event_filter = nv_cspmu_event_filter;
326 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
327 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
328 impl_ops->get_name = nv_cspmu_get_name;
329
330 return 0;
331 }
332
333 /* Match all NVIDIA Coresight PMU devices */
334 static const struct arm_cspmu_impl_match nv_cspmu_param = {
335 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA,
336 .module = THIS_MODULE,
337 .impl_init_ops = nv_cspmu_init_ops
338 };
339
nvidia_cspmu_init(void)340 static int __init nvidia_cspmu_init(void)
341 {
342 int ret;
343
344 ret = arm_cspmu_impl_register(&nv_cspmu_param);
345 if (ret)
346 pr_err("nvidia_cspmu backend registration error: %d\n", ret);
347
348 return ret;
349 }
350
nvidia_cspmu_exit(void)351 static void __exit nvidia_cspmu_exit(void)
352 {
353 arm_cspmu_impl_unregister(&nv_cspmu_param);
354 }
355
356 module_init(nvidia_cspmu_init);
357 module_exit(nvidia_cspmu_exit);
358
359 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver");
360 MODULE_LICENSE("GPL v2");
361